diff --git a/.dockerignore b/.dockerignore index 4d9a61d4f..114f349c0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,76 @@ -.github -.idea -zz_* +# Exclude version control folders and configuration +.git/ +.gitignore +.github/ + +# Generated or build-related directories +.generated/ vendor/ +pkg/client/ +docker/mongodb-enterprise-tests scripts/ -.git/ +samples/ +.multi_cluster_local_test_files/ + +# Python-related ignore patterns +__pycache__/ +*.pyc +*.pyo +*.pyd +*.toml + +# Virtual environment and dependency files +venv/ +.env + +# Ignore temporary and system files +*.swp +*.bak +*.tmp +.DS_Store +Thumbs.db + +# Testing-related files +*.test.* +*_test.go +cover.out + +# Compiled and object files +*.o +*.so +*.dylib + +# Logs and binary artifacts +logs/ +*.log +*.csv +*.tgz + +# Tools and binaries bin/ -testbin/ -.mypy_cache/ -main -__debug_bin -# allow agent LICENSE -!scripts/dev/templates/agent/LICENSE +tools.go + +# Kubernetes and operator-related artifacts +deploy/ +config/manifests/ +helm_chart/templates/ +helm_chart/crds/ +public/ + +# Documentation files +docs/ +*.md + +# IDE and editor config files +*.iml +*.idea/ +.vscode/ + +# Data and build artifacts +blobs/ +ssdlc-report/ + +# Dockerfiles or specific project directories +# (Add Docker-specific exclusion patterns here if known) + +# By excluding directories, specific files inside them are also excluded, so listing every specific file isn't necessary unless needed. diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..c9f27ddb1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[{Makefile,go.mod,go.sum,*.go}] +indent_style = tab +indent_size = 4 + +[*.py] +indent_style = space +indent_size = 4 diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml new file mode 100644 index 000000000..c98e43fdd --- /dev/null +++ b/.evergreen-functions.yml @@ -0,0 +1,760 @@ +variables: + - &e2e_include_expansions_in_env + include_expansions_in_env: + - ARTIFACTORY_PASSWORD + - ARTIFACTORY_USERNAME + - GRS_PASSWORD + - GRS_USERNAME + - OVERRIDE_VERSION_ID + - PKCS11_URI + - branch_name + - build_id + - build_variant + - distro + - e2e_cloud_qa_apikey_owner_ubi_cloudqa + - e2e_cloud_qa_orgid_owner_ubi_cloudqa + - e2e_cloud_qa_user_owner_ubi_cloudqa + - ecr_registry + - ecr_registry_needs_auth + - execution + - github_commit + - image_name + - include_tags + - is_patch + - mms_eng_test_aws_access_key + - mms_eng_test_aws_region + - mms_eng_test_aws_secret + - openshift_token + - openshift_url + - otel_collector_endpoint + - otel_parent_id + - otel_trace_id + - pin_tag_at + - registry + - requester + - skip_tags + - task_name + - triggered_by_git_tag + - version_id + - workdir + +functions: + + ### Setup Functions ### + + setup_context: &setup_context # Running the first switch is important to fill the workdir and other important initial env vars + command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + <<: *e2e_include_expansions_in_env + script: | + echo "Initializing context files" + cp scripts/dev/contexts/evg-private-context scripts/dev/contexts/private-context + scripts/dev/switch_context.sh root-context + echo "Finished initializing to the root context" + + switch_context: &switch_context + command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + <<: *e2e_include_expansions_in_env + add_to_path: + - ${workdir}/bin + - ${workdir}/google-cloud-sdk/bin + script: | + echo "Switching context" + scripts/dev/switch_context.sh "${build_variant}" + echo "Finished switching context" + + python_venv: &python_venv + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + command: scripts/dev/recreate_python_venv.sh + + "clone": + - command: subprocess.exec + type: setup + params: + command: "mkdir -p src/github.com/10gen" + - command: git.get_project + type: setup + params: + directory: src/github.com/10gen/ops-manager-kubernetes + - command: subprocess.exec + type: setup + params: + command: "git config --global user.name 'Evergreen'" + - command: subprocess.exec + type: setup + params: + command: "git config --global user.email 'kubernetes-hosted-team@mongodb.com'" + - *setup_context + + setup_kubectl: &setup_kubectl + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/setup_kubectl.sh + + setup_jq: &setup_jq + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/setup_jq.sh + + setup_shellcheck: + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/setup_shellcheck.sh + + setup_aws: &setup_aws + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/setup_aws.sh + + setup_gcloud_cli: + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - GCP_SERVICE_ACCOUNT_JSON_FOR_SNIPPETS_TESTS + add_to_path: + - ${workdir}/google-cloud-sdk/bin + binary: scripts/evergreen/setup_gcloud_cli.sh + + setup_mongosh: + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/google-cloud-sdk/bin + binary: scripts/evergreen/setup_mongosh.sh + + # configures Docker size, installs the Kind binary (if necessary) + setup_kind: &setup_kind + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/setup_kind.sh + + setup_docker_datadir: &setup_docker_datadir + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/configure-docker-datadir.sh + + setup_preflight: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/setup_preflight.sh + + setup_prepare_openshift_bundles: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/setup_yq.sh + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/setup_prepare_openshift_bundles.sh + + install_olm: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/operator-sdk/install-olm.sh + + prepare_openshift_bundles_for_e2e: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh + + setup_docker_sbom: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/setup_docker_sbom.sh + + # Logs into all used registries + configure_docker_auth: &configure_docker_auth + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/dev/configure_docker_auth.sh + + lint_repo: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/setup_yq.sh + - command: subprocess.exec + type: test + params: + add_to_path: + - ${workdir}/bin + - ${workdir}/venv/bin + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/check_precommit.sh + + # Configures docker authentication to ECR and RH registries. + setup_building_host: + - *switch_context + - *setup_aws + - *configure_docker_auth + - *setup_docker_datadir + - *python_venv + + prune_docker_resources: + - command: subprocess.exec + type: setup + params: + command: "docker system prune -a -f" + + # the task configures the set of tools necessary for any task working with K8 cluster: + # installs kubectl, jq, kind (if necessary), configures docker authentication + download_kube_tools: + - *switch_context + - *setup_kubectl + - *setup_jq + # we need aws to configure docker authentication + - *setup_aws + - *configure_docker_auth + - *setup_kind + + teardown_kubernetes_environment: + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + scripts/evergreen/teardown_kubernetes_environment.sh + + # Makes sure a kubectl context is defined. + setup_kubernetes_environment_p: &setup_kubernetes_environment_p + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/setup_kubernetes_environment.sh + + setup_kubernetes_environment: + - *setup_kubernetes_environment_p + # After setting up KUBE, we need to update the KUBECONFIG and other env vars. + - *switch_context + + # cleanup_exec_environment is a very generic name when the only thing this function + # does is to clean the logs directory. In the future, more "commands" can be + # added to it with more clearing features, when needed. + cleanup_exec_environment: + - command: shell.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + rm -rf logs + + quay_login: + - command: subprocess.exec + type: setup + params: + command: "docker login quay.io -u ${quay_prod_username} -p ${quay_prod_robot_token}" + + setup_cloud_qa: + - *switch_context + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/e2e/setup_cloud_qa.py create + # The additional switch is needed, since we now have created the needed OM exports. + - *switch_context + + teardown_cloud_qa: + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/e2e/setup_cloud_qa.py delete + + ### Publish and release image ### + + # Tags and pushes an image into an external Docker registry. The source image + # needs to exist before it can be pushed to a remote registry. + # It is expected that IMAGE_SOURCE is accessible with no authentication (like a + # local image), and the IMAGE_TARGET will be authenticated with DOCKER_* series of + # environment variables. + release_docker_image_to_registry: + - command: subprocess.exec + type: system + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + include_expansions_in_env: + - tag_source + - tag_dest + - image_source + - image_target + - docker_username + - docker_password + binary: scripts/evergreen/tag_push_docker_image.sh + + # + # Performs some AWS cleanup + # + prepare_aws: &prepare_aws + command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/prepare_aws.sh + + build-dockerfiles: + - command: subprocess.exec + type: setup + params: + add_to_path: + - ${workdir}/bin + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/run_python.sh scripts/update_supported_dockerfiles.py + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - triggered_by_git_tag + # if you ever change the target folder structure, the same needs to be reflected in PCT + command: "tar -czvf ./public/dockerfiles-${triggered_by_git_tag}.tgz ./public/dockerfiles" + + enable_QEMU: + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + echo "Enabling QEMU building for Docker" + docker run --rm --privileged 268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/multiarch/qemu-user-static --reset -p yes + + upload_dockerfiles: + - command: s3.put + params: + aws_key: ${enterprise_aws_access_key_id} + aws_secret: ${enterprise_aws_secret_access_key} + local_file: src/github.com/10gen/ops-manager-kubernetes/public/dockerfiles-${triggered_by_git_tag}.tgz + remote_file: bundles/dockerfiles-${triggered_by_git_tag}.tgz + bucket: operator-e2e-bundles + permissions: public-read + content_type: application/x-binary + + # upload_e2e_logs has the responsibility of dumping as much information as + # possible into the S3 bucket that corresponds to this ${version}. The + # Kubernetes cluster where the test finished running, should still be + # reachable. Note that after a timeout, Evergreen kills the running process + # and any running container in the host (which kills Kind). + upload_e2e_logs: + - command: s3.put + params: + aws_key: ${enterprise_aws_access_key_id} + aws_secret: ${enterprise_aws_secret_access_key} + local_files_include_filter: + - src/github.com/10gen/ops-manager-kubernetes/logs/* + remote_file: logs/${task_id}/${execution}/ + bucket: operator-e2e-artifacts + permissions: public-read + content_type: text/plain + - command: attach.xunit_results + params: + file: "src/github.com/10gen/ops-manager-kubernetes/logs/myreport.xml" + + upload_code_snippets_logs: + - command: s3.put + params: + aws_key: ${enterprise_aws_access_key_id} + aws_secret: ${enterprise_aws_secret_access_key} + local_files_include_filter: + - src/github.com/10gen/ops-manager-kubernetes/public/architectures/**/*.log + - src/github.com/10gen/ops-manager-kubernetes/public/architectures/**/*.out + remote_file: logs/${task_id}/${execution}/ + bucket: operator-e2e-artifacts + permissions: public-read + content_type: text/plain + + preflight_image: + - *switch_context + - command: subprocess.exec + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + include_expansions_in_env: + - image_version + - rh_pyxis + binary: scripts/evergreen/run_python.sh scripts/preflight_images.py --image ${image_name} --submit "${preflight_submit}" + + build_multi_cluster_binary: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh + + build_and_push_appdb_database: + - command: subprocess.exec + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes/docker/mongodb-enterprise-appdb-database + binary: ./build_and_push_appdb_database_images.sh + add_to_path: + - ${workdir}/bin + - ${workdir} + + pipeline: + - *switch_context + - command: shell.exec + type: setup + params: + shell: bash + script: | + # Docker Hub workaround + # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror + docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use + docker buildx inspect --bootstrap + - command: ec2.assume_role + display_name: Assume IAM role with permissions to pull Kondukto API token + params: + role_arn: ${kondukto_role_arn} + - command: shell.exec + display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file + params: + silent: true + shell: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + set -e + # use AWS CLI to get the Kondukto API token from AWS Secrets Manager + kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text) + # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file + echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env + - command: subprocess.exec + retry_on_failure: true + type: setup + params: + shell: bash + <<: *e2e_include_expansions_in_env + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/run_python.sh pipeline.py --include ${image_name} --parallel --sign + + teardown_cloud_qa_all: + - *switch_context + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py delete_all + + # Updates current expansions with variables from release.json file. + # Use e.g. ${mongoDbOperator} afterwards. + update_evergreen_expansions: + - command: subprocess.exec + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: "scripts/evergreen/generate_evergreen_expansions.sh" + - command: expansions.update + params: + file: "src/github.com/10gen/ops-manager-kubernetes/evergreen_expansions.yaml" + + # Uploads openshift bundle specified by bundle_file_name argument. + upload_openshift_bundle: + - command: s3.put + params: + aws_key: ${enterprise_aws_access_key_id} + aws_secret: ${enterprise_aws_secret_access_key} + local_file: src/github.com/10gen/ops-manager-kubernetes/bundle/${bundle_file_name} + remote_file: bundles/${bundle_file_name} + bucket: operator-e2e-bundles + permissions: public-read + content_type: application/x-binary + + prepare_openshift_bundles: + - command: subprocess.exec + type: setup + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + command: scripts/evergreen/operator-sdk/prepare-openshift-bundles.sh + + # Performs some AWS cleanup + cleanup_aws: + - *setup_jq + - *setup_aws + - *prepare_aws + - command: subprocess.exec + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + add_to_path: + - ${workdir}/bin + # Below script deletes agent images created for an Evergreen patch older than 1 day + command: scripts/evergreen/run_python.sh scripts/evergreen/periodic-cleanup-aws.py + + ### Test Functions ### + + # + # e2e_test is the main function used to run the e2e tests. It expects Ops + # Manager to be running (local to the Kubernetes cluster or Cloud Manager) and + # its configuration to exist in a ${workdir}/.ops-manager-env file. + # + # The e2e script will run all the tasks that are needed by the e2e tests like + # fetching the OM API credentials to use and create the Secret and ConfigMap + # objects that are required. + # + # At this point, the Kubernetes environment should be configured already + # (kubectl configuration points to the Kubernetes cluster where we run the tests). + # + # Please note: There are many ENV variables passed to the `e2e` script, so try + # to not add more. If this is required, discuss your use case with the team first. + # + e2e_test: + - command: subprocess.exec + type: test + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - otel_parent_id + - branch_name + - github_commit + - revision + - github_pr_number + - project_identifier + - revision_order_id + add_to_path: + - ${workdir}/bin + binary: scripts/evergreen/e2e/e2e.sh + + e2e_test_perf: + - command: subprocess.exec + type: test + params: + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - otel_parent_id + - branch_name + - github_commit + - revision + - github_pr_number + - project_identifier + - revision_order_id + add_to_path: + - ${workdir}/bin + env: + PERF_TASK_DEPLOYMENTS: ${PERF_TASK_DEPLOYMENTS} + PERF_TASK_REPLICAS: ${PERF_TASK_REPLICAS} + TEST_NAME_OVERRIDE: ${TEST_NAME_OVERRIDE} + binary: scripts/evergreen/e2e/e2e.sh + + test_golang_unit: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + make test-race + - command: gotest.parse_files + params: + files: [ "src/github.com/10gen/ops-manager-kubernetes/*.suite", "src/github.com/10gen/ops-manager-kubernetes/public/tools/multicluster/*.suite", "src/github.com/10gen/ops-manager-kubernetes/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/*.suite" ] + + test_python_unit: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + make python-tests + + test_sboms: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + make sbom-tests + + generate_perf_tests_tasks: + - *switch_context + - command: shell.exec + type: setup + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + source .generated/context.export.env + scripts/evergreen/run_python.sh scripts/evergreen/e2e/performance/create_variants.py ${variant} ${size}> evergreen_tasks.json + echo "tasks to run:" + cat evergreen_tasks.json + - command: generate.tasks + params: + files: + - evergreen_tasks.json + + ### Other ### + + run_retry_script: + - command: shell.exec + type: test + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - EVERGREEN_API_KEY + - EVERGREEN_USER + - evergreen_retry + env: + EVERGREEN_RETRY: ${evergreen_retry} + script: | + scripts/evergreen/retry-evergreen.sh ${version_id} + + # This is a generic function for conditionally running given task. + # It works by appending to if returns no error. + # + # It has 3 input parameters: + # - condition_script: path to the script that will be executed. + # Error code == 0 resulting from the scripts indicates that should be added dynamically to + # Error code != 0 means that the task will not be executed + # - variant: variant to which task will be appended + # - task: task name to be executed + # + # Example usage: + # - func: run_task_conditionally + # vars: + # condition_script: scripts/evergreen/should_prepare_openshift_bundles.sh + # variant: prepare_openshift_bundles + # task: prepare_and_upload_openshift_bundles + run_task_conditionally: + - command: shell.exec + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + script: | + if ${condition_script}; then + echo "Adding ${task} task to ${variant} variant" + scripts/evergreen/add_evergreen_task.sh ${variant} ${task} + else + echo "skipping task ${task} due to ${condition_script} result: $?" + fi + - command: generate.tasks + params: + files: + - evergreen_tasks.json + optional: true + + # + # Code snippet test automation + # + + sample_commit_output: + - command: github.generate_token + params: + expansion_name: GH_TOKEN + - command: subprocess.exec + params: + include_expansions_in_env: + - GH_TOKEN + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/code_snippets/sample_commit_output.sh + + gke_multi_cluster_snippets: + - *switch_context + - command: shell.exec + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - version_id + - code_snippets_teardown + - code_snippets_reset + - task_name + script: | + ./scripts/code_snippets/gke_multi_cluster_test.sh + + + gke_multi_cluster_no_mesh_snippets: + - *switch_context + - command: shell.exec + params: + shell: bash + working_dir: src/github.com/10gen/ops-manager-kubernetes + include_expansions_in_env: + - version_id + - code_snippets_teardown + - code_snippets_reset + - task_name + script: | + ./scripts/code_snippets/gke_multi_cluster_no_mesh_test.sh diff --git a/.evergreen-periodic-builds.yaml b/.evergreen-periodic-builds.yaml new file mode 100644 index 000000000..129234f1c --- /dev/null +++ b/.evergreen-periodic-builds.yaml @@ -0,0 +1,200 @@ +include: + - filename: .evergreen-functions.yml + - filename: .evergreen-tasks.yml + +parameters: + - key: pin_tag_at + value: 00:00 + description: Pin tags at this time of the day. Midnight by default. + +variables: + - &setup_group + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: setup_docker_sbom + - func: download_kube_tools + - func: setup_building_host + - func: quay_login + - func: switch_context + +tasks: + - name: periodic_build_operator + commands: + - func: pipeline + vars: + image_name: operator-daily + + - name: periodic_teardown_aws + commands: + - func: cleanup_aws + + - name: periodic_teardown_cloudqa + commands: + - func: teardown_cloud_qa_all + + - name: periodic_build_init_appdb + commands: + - func: pipeline + vars: + image_name: init-appdb-daily + + - name: periodic_build_init_database + commands: + - func: pipeline + vars: + image_name: init-database-daily + + - name: periodic_build_init_opsmanager + commands: + - func: pipeline + vars: + image_name: init-ops-manager-daily + + - name: periodic_build_database + commands: + - func: pipeline + vars: + image_name: database-daily + + - name: periodic_build_sbom_cli + commands: + - func: pipeline + vars: + image_name: cli + + - name: periodic_build_ops_manager_6 + commands: + - func: pipeline + vars: + image_name: ops-manager-6-daily + + - name: periodic_build_ops_manager_7 + commands: + - func: pipeline + vars: + image_name: ops-manager-7-daily + + - name: periodic_build_ops_manager_8 + commands: + - func: pipeline + vars: + image_name: ops-manager-8-daily + + # the periodic agent builds are more commented in the pipeline.py file. + # The gist is - we want to split up the periodic build on as many machines as possible + # To speed up the builds as we have too many agents due to the matrix build. + # For now its one without operator suffix and the last 3. This only works as long as we + # only have operator versions we support (minor version), as soon as we have multiple patch versions - + # this won't work anymore and we will need a dynamic solution. + - name: periodic_build_agent + exec_timeout_secs: 43200 + commands: + - func: enable_QEMU + - func: pipeline + vars: + image_name: mongodb-agent-daily + + - name: periodic_build_agent_1 + exec_timeout_secs: 43200 + commands: + - func: enable_QEMU + - func: pipeline + vars: + image_name: mongodb-agent-1-daily + + - name: periodic_build_agent_2 + exec_timeout_secs: 43200 + commands: + - func: enable_QEMU + - func: pipeline + vars: + image_name: mongodb-agent-2-daily + + - name: periodic_build_agent_3 + exec_timeout_secs: 43200 + commands: + - func: enable_QEMU + - func: pipeline + vars: + image_name: mongodb-agent-3-daily + + - name: periodic_build_community_operator + commands: + - func: enable_QEMU + - func: pipeline + vars: + image_name: mongodb-kubernetes-operator-daily + + - name: periodic_build_readiness_probe + commands: + - func: pipeline + vars: + image_name: mongodb-kubernetes-readinessprobe-daily + + - name: periodic_build_version_upgrade_post_start_hook + commands: + - func: pipeline + vars: + image_name: mongodb-kubernetes-operator-version-upgrade-post-start-hook-daily + + - name: periodic_build_appdb_database + commands: + - func: build_and_push_appdb_database + +task_groups: + - name: periodic_build_task_group + max_hosts: -1 + <<: *setup_group + tasks: + - periodic_build_operator + - periodic_build_readiness_probe + - periodic_build_version_upgrade_post_start_hook + - periodic_build_init_appdb + - periodic_build_init_database + - periodic_build_init_opsmanager + - periodic_build_ops_manager_6 + - periodic_build_ops_manager_7 + - periodic_build_ops_manager_8 + - periodic_build_database + - periodic_build_community_operator + - periodic_build_appdb_database + - periodic_build_sbom_cli + - periodic_build_agent + - periodic_build_agent_1 + - periodic_build_agent_2 + - periodic_build_agent_3 + + - name: periodic_teardown_task_group + <<: *setup_group + tasks: + - periodic_teardown_aws + - periodic_teardown_cloudqa + +buildvariants: + - name: periodic_teardown + display_name: periodic_teardown + tags: [ "periodic_teardown" ] + run_on: + - ubuntu2204-small + tasks: + - name: periodic_teardown_task_group + + - name: periodic_build + display_name: periodic_build + tags: [ "periodic_build" ] + run_on: + - ubuntu2204-large + tasks: + - name: periodic_build_task_group + + - name: preflight_release_images_check_only + display_name: preflight_release_images_check_only + tags: [ "periodic_build" ] + depends_on: + - name: "*" + variant: periodic_build + run_on: + - rhel90-large + tasks: + - name: preflight_images_task_group diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml new file mode 100644 index 000000000..df8be4339 --- /dev/null +++ b/.evergreen-tasks.yml @@ -0,0 +1,1206 @@ +task_groups: + - name: preflight_images_task_group + max_hosts: -1 + tasks: + - preflight_images + - preflight_official_database_image + - preflight_mongodb_agent_image + - preflight_ops_manager + +tasks: + - name: preflight_images + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: operator + - func: preflight_image + vars: + image_name: init-appdb + - func: preflight_image + vars: + image_name: init-database + - func: preflight_image + vars: + image_name: init-ops-manager + - func: preflight_image + vars: + image_name: database + + - name: preflight_ops_manager + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: ops-manager + + - name: preflight_official_database_image + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: mongodb-enterprise-server + + - name: preflight_mongodb_agent_image + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: mongodb-agent + + - name: preflight_om_image + tags: [ "image_preflight" ] + commands: + - func: clone + - func: python_venv + - func: setup_preflight + - func: preflight_image + vars: + image_name: ops-manager + + - name: gke_multi_cluster_snippets + tags: [ "code_snippets" ] + commands: + - func: gke_multi_cluster_snippets + - func: sample_commit_output + + - name: gke_multi_cluster_no_mesh_snippets + tags: [ "code_snippets" ] + commands: + - func: gke_multi_cluster_no_mesh_snippets + - func: sample_commit_output + +## Below are only e2e runs for .evergreen.yml ## + + - name: e2e_multiple_cluster_failures + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_standalone_custom_podspec + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_external_access + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_users_schema_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_crd_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_config_map + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_standalone_groups + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_recovery + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_partial_crd + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_clusterwide + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_proxy + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_multi_namespaces + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_upgrade_replica_set + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_upgrade_sharded_cluster + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_upgrade_ops_manager + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_operator_upgrade_appdb_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_olm_operator_upgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_olm_operator_webhooks + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_olm_operator_upgrade_with_resources + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_backup_delete_sts_and_log_rotation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_backup_kmip + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_liveness_probe + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_mongodb_validation_webhook + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_mongodb_roles_validation_webhook + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_recovery + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_config_map + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_replica_set_groups + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_pv + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_pv_multiple + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_exposed_externally + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_readiness_probe + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_migration + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_replication_state_awareness + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_statefulset_status + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_update_delete_parallel + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_sc_additional_certs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_sharded_cluster_certs_prefix + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_migration + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_rs_additional_certs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_rs_intermediate_ca + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_rs_external_access + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_pv + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_recovery + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_secret + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_scale_shards + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_shard_overrides + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_statefulset_status + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_agent_flags + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_agent_flags + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_agent_flags_and_readinessProbe + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_all_mongodb_resources_parallel + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_upgrade_downgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_upgrade_downgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_custom_podspec + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_custom_sa + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_report_pending_pods + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_mongod_options + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_mongod_options_and_log_rotation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_custom_podspec + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_upgrade_downgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_standalone_no_tls_no_status_is_set + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_default + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_override + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ignore_unknown_users + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_process_hostnames + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_process_hostnames + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_member_options + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_allow + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_prefer + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_require + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_certs_secret_prefix + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_tls_certs_top_level_prefix + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_disable_tls_scale_up + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_replica_set_tls_require_to_allow + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_sharded_cluster_tls_require_custom_ca + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_require_and_disable + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_tls_multiple_different_ssl_configs + commands: + - func: "e2e_test" + + - name: e2e_replica_set_tls_require_upgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_rs + tags: [ "patch-run" ] + # longer timeout than usual as this test tests recovery from bad states which can take some time + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_sc + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_users_addition_removal + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_user_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_configure_all_options_rs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_tls_x509_configure_all_options_sc + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_scram_sha_256_user_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_scram_sha_256_user_first + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_scram_sha_1_user_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_scram_sha_256_user_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_scram_sha_1_user_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_scram_sha_1_upgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_scram_sha_1_upgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_x509_to_scram_transition + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_x509_to_scram_transition + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_internal_cluster_transition + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_ldap + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap_user_to_dn_mapping + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap_agent_auth + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap_agent_client_certs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_custom_roles + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_pv_resize + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_pv_resize + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_update_roles_no_privileges + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap_group_dn + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_ldap_group_dn_with_x509_agent + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_feature_controls_authentication + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_scram_sha_and_x509 + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_scram_sha_and_x509 + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_replica_set_scram_x509_internal_cluster + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_replica_set_scram_x509_ic_manual_certs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_sharded_cluster_scram_x509_ic_manual_certs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # TODO: not used in any variant + - name: e2e_sharded_cluster_scram_x509_internal_cluster + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_configure_tls_and_x509_simultaneously_rs + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_configure_tls_and_x509_simultaneously_sc + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_configure_tls_and_x509_simultaneously_st + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + # E2E tests for Ops Manager (sorted alphabetically): + - name: e2e_om_appdb_external_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_flags_and_config + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_monitoring_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_multi_change + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_scale_up_down + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_scram + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_upgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_appdb_validation + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_external_connectivity + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_weak_password + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_multiple + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_backup + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_backup_sharded_cluster + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_backup_liveness_probe + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_backup_light + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_backup_tls + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_backup_s3_tls + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_backup_tls_custom_ca + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_backup_restore + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_queryable_backup + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_feature_controls + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_scale + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_upgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_multi_cluster_appdb_state_operator_upgrade_downgrade + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_prometheus + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_pod_spec + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_validation_webhook + tags: [ "patch-run" ] + commands: + - func: "e2e_test" + + - name: e2e_om_ops_manager_https_enabled + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_https_enabled_hybrid + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_https_enabled_prefix + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_https_enabled_internet_mode + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_jvm_params + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_migration + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_localmode + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_enable_local_mode_running_om + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_remotemode + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_localmode_multiple_pv + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_secure_config + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set_migration + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set_member_options + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set_scale_up + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_scale_up_cluster + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_scale_up_cluster_new_cluster + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_scale_down_cluster + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set_scale_down + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set_deletion + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_mtls_test + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_scram + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_sts_override + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_tls_with_scram + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_enable_tls + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_upgrade_downgrade + tags: [ "patch-run" ] + commands: + - func: e2e_test + + # TODO: not used in any variant + - name: e2e_multi_cluster_tls_cert_rotation + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_tls_no_mesh + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_backup_restore + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_appdb_s3_based_backup_restore + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_backup_restore_no_mesh + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_appdb_validation + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_om_validation + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_appdb + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_tls_with_x509 + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_with_ldap + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_with_ldap_custom_roles + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_specific_namespaces + tags: [ "patch-run" ] + commands: + - func: e2e_test + + # TODO: not used in any variant + - name: e2e_multi_cluster_clusterwide + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_disaster_recovery + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_multi_disaster_recovery + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_2_clusters_replica_set + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_2_clusters_clusterwide + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_recover + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_recover_network_partition + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_recover_clusterwide + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_agent_flags + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_replica_set_ignore_unknown_users + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_validation + tags: [ "patch-run" ] + exec_timeout_secs: 1000 + commands: + - func: e2e_test + + - name: e2e_om_update_before_reconciliation + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_vault_setup + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_vault_setup_tls + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_vault_setup_om + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_vault_setup_om_backup + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_om_ops_manager_backup_restore_minio + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_appdb_disaster_recovery + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_disaster_recovery + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_om_networking_clusterwide + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_om_appdb_no_mesh + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_pvc_resize + tags: [ "patch-run" ] + commands: + - func: e2e_test + + # this test is run, with an operator with race enabled + - name: e2e_om_reconcile_race_with_telemetry + tags: [ "patch-run" ] + commands: + - func: e2e_test + + # TODO: not used in any variant + - name: e2e_om_reconcile_perf + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_geo_sharding + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_scaling + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_scaling_all_shard_overrides + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_simplest + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_simplest_no_mesh + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_external_access_no_ext_domain + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_tls_no_mesh + tags: [ "patch-run" ] + commands: + - func: e2e_test + + + - name : e2e_multi_cluster_sharded_snippets + tags: [ "patch-run" ] + commands: + - func: e2e_test + + - name: e2e_multi_cluster_sharded_tls + tags: [ "patch-run" ] + commands: + - func: e2e_test diff --git a/.evergreen.yml b/.evergreen.yml new file mode 100644 index 000000000..54ad536f8 --- /dev/null +++ b/.evergreen.yml @@ -0,0 +1,1812 @@ +# 2h timeout for all the tasks +exec_timeout_secs: 7200 + +include: + - filename: .evergreen-functions.yml + - filename: .evergreen-tasks.yml + +variables: + - &ops_manager_60_latest 6.0.27 # The order/index is important, since these are anchors. Please do not change + + - &ops_manager_70_latest 7.0.15 # The order/index is important, since these are anchors. Please do not change + + - &ops_manager_80_latest 8.0.6 # The order/index is important, since these are anchors. Please do not change + + # The dependency unification between static and non-static is intentional here. + # Even though some images are exclusive, in EVG they all are built once and in parallel. + # It is not worth the effort of splitting them out. + # Once Static Containers are default, this piece can be cleaned up. + - &base_om6_dependency + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + - &base_no_om_image_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + - &setup_group + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + + - &setup_group_multi_cluster + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: download_kube_tools + - func: setup_building_host + - func: build_multi_cluster_binary + + - &setup_and_teardown_group_gke_code_snippets + setup_group: + - func: clone + - func: setup_gcloud_cli + - func: setup_mongosh + - func: download_kube_tools + - func: build_multi_cluster_binary + teardown_group: + - func: upload_code_snippets_logs + + - &setup_and_teardown_task_cloudqa + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + - func: setup_cloud_qa + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment + - func: teardown_cloud_qa + + - &setup_and_teardown_task + setup_task_can_fail_task: true + setup_task: + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + teardown_task_can_fail_task: true + teardown_task: + - func: upload_e2e_logs + - func: teardown_kubernetes_environment + + - &teardown_group + teardown_group: + - func: prune_docker_resources + - func: run_retry_script + + - &base_om7_dependency + depends_on: + - name: build_om_images + variant: build_om70_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + + - &base_om8_dependency + depends_on: + - name: build_om_images + variant: build_om80_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + +parameters: + - key: evergreen_retry + value: "true" + description: set this to false to suppress retries on failure + + - key: pin_tag_at + value: 10:00 + description: Pin tags at this time of the day. Midnight by default for periodic and 10 for releases. + + - key: OVERRIDE_VERSION_ID + value: "" + description: "Patch id to reuse images from other Evergreen build" + + - key: code_snippets_teardown + value: "true" + description: set this to false if you would like to keep the clusters created during code snippets tests + + - key: code_snippets_reset + value: "false" + description: set this to true if you would like to delete the resources created in the code snippet tests, but keep the clusters + + +# Triggered manually or by PCT. +patch_aliases: + - alias: "periodic_builds" + variant_tags: [ "periodic_build" ] + task: ".*" + - alias: "periodic_teardowns" + variant_tags: [ "periodic_teardown" ] + task: ".*" + - alias: "release_agent" + variant_tags: [ "release_agent" ] + task: ".*" + - alias: "release_all_agents_manually" + variant_tags: [ "release_all_agents_manually" ] + task: ".*" + - alias: "release" + variant_tags: [ "release" ] + task_tags: [ "image_release", "image_preflight", "openshift_bundles", "code_snippets" ] + - alias: "smoke_test_release" + variant_tags: [ "e2e_smoke_release_test_suite" ] + task_tags: [ "patch-run" ] + - alias: "patch-run-cloudqa" + variant_tags: [ "cloudqa" ] + task: ".*" + +# Triggered whenever the GitHub PR is created +github_pr_aliases: + - variant_tags: [ "unit_tests" ] + task_tags: [ "unit_tests" ] + - variant_tags: [ "e2e_test_suite" ] + task_tags: [ "patch-run" ] + - variant_tags: [ "auto_bump" ] + task_tags: [ "patch-run" ] + +# Allows to see evergreen checks in GitHub commits +# https://github.com/10gen/ops-manager-kubernetes/commits/master/ +github_checks_aliases: + - variant: ".*" + task: ".*" + +# Triggered on git tag +git_tag_aliases: + - git_tag: "^(\\d+\\.)?(\\d+\\.)?(\\d+)$" + variant_tags: [ "release" ] + task_tags: [ "image_release", "image_preflight", "openshift_bundles", "code_snippets" ] + +tasks: + + - name: unit_tests_golang + tags: [ "unit_tests" ] + commands: + - func: "test_golang_unit" + + - name: unit_tests_python + tags: [ "unit_tests" ] + commands: + - func: "test_python_unit" + + - name: sbom_tests + tags: [ "unit_tests" ] + # The SBOM tests run only on commit builds. Running this on patches might cause false-positive failures + # because certain images might not be there yet. Such situation happens for OM image upgrades for example. + # See https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#limiting-when-a-task-or-variant-will-run + patchable: false + commands: + - func: "test_sboms" + + - name: lint_repo + tags: [ "unit_tests" ] + commands: + - func: lint_repo + + - name: release_operator + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: operator + + # Releases init images to Quay + - name: release_init_appdb + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-appdb + + - name: release_init_database + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-database + + - name: release_init_ops_manager + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: init-ops-manager + + - name: release_agent_operator_release + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: pipeline + vars: + image_name: agent + + - name: upload_dockerfiles + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: build-dockerfiles + - func: upload_dockerfiles + + # pct only triggers this variant once a new agent image is out + - name: release_agent + # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) + allowed_requesters: [ "patch", "github_pr" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: agent-pct + include_tags: release + + - name: run_precommit_and_push + tags: ["patch-run"] + commands: + - func: clone + - func: python_venv + - func: download_kube_tools + - func: setup_shellcheck + - command: github.generate_token + params: + expansion_name: GH_TOKEN + - command: subprocess.exec + type: setup + params: + include_expansions_in_env: + - GH_TOKEN + working_dir: src/github.com/10gen/ops-manager-kubernetes + binary: scripts/evergreen/precommit_bump.sh + + # Pct only triggers this variant once a new agent image is out + # these releases the agent with the operator suffix (not patch id) on ecr to allow for digest pinning to pass. + # For this to work, we rely on skip_tags which is used to determine whether + # we want to release on quay or not, in this case - ecr instead. + # We rely on the init_database from ecr for the agent x operator images. + # This runs on agent releases that are not concurrent with operator releases. + - name: release_agents_on_ecr_conditional + # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) + allowed_requesters: [ "patch", "github_pr" ] + commands: + - func: clone + - func: run_task_conditionally + vars: + condition_script: scripts/evergreen/should_release_agents_on_ecr.sh + variant: init_release_agents_on_ecr + task: release_agents_on_ecr + + - name: release_agents_on_ecr + # this enables us to run this variant either manually (patch) which pct does or during an OM bump (github_pr) + allowed_requesters: [ "patch", "github_pr" ] + priority: 70 + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-pct + skip_tags: release + + - name: release_all_agents_on_ecr + # this enables us to run this manually (patch) and release all agent versions to ECR + # it's needed during operator new version release process - e2e tests (especially olm tests) + # will look for agent with new operator version suffix, but during PR checks we only build + # agent versions for most recent major OM versions and the tests will fail. Before running the PR + # we have to manually release all agents to ECR by triggering this patch + allowed_requesters: [ "patch" ] + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent-pct + skip_tags: release + all_agents: true + + - name: build_test_image + commands: + - func: clone + - func: setup_building_host + - func: build_multi_cluster_binary + - func: pipeline + vars: + image_name: test + + - name: build_operator_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + skip_tags: ubuntu,release + distro: ubi + image_name: operator + + - name: build_init_om_images_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-ops-manager + skip_tags: ubuntu,release + + - name: build_init_appdb_images_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-appdb + skip_tags: ubuntu,release + + - name: build_agent_images_ubi + depends_on: + - name: build_init_database_image_ubi + variant: init_test_run + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: agent + skip_tags: ubuntu,release + + - name: build_init_database_image_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: init-database + skip_tags: ubuntu,release + + - name: build_database_image_ubi + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: database + skip_tags: ubuntu,release + + - name: prepare_aws + priority: 59 + commands: + - func: clone + - func: setup_jq + - func: setup_aws + - func: prepare_aws + + - name: run_retry_script + tags: [ "patch-run" ] + commands: + - func: run_retry_script + + - name: generate_perf_tasks_one_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_operator_perf_one_thread + size: small + + - name: generate_perf_tasks_10_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_operator_perf + size: small + + - name: generate_perf_tasks_30_thread + commands: + - func: clone + - func: generate_perf_tests_tasks + vars: + variant: e2e_operator_perf_thirty + size: small + + - name: release_database + tags: [ "image_release" ] + allowed_requesters: [ "patch", "github_tag" ] + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: database + + - name: build_om_images + commands: + - func: clone + - func: setup_building_host + - func: pipeline + vars: + image_name: ops-manager + skip_tags: release + + - name: publish_ops_manager + commands: + - func: clone + - func: setup_building_host + - func: quay_login + - func: setup_docker_sbom + - func: pipeline + vars: + image_name: ops-manager + include_tags: release + + - name: prepare_and_upload_openshift_bundles_for_e2e + commands: + - func: clone + - func: setup_building_host + - func: download_kube_tools + - func: setup_prepare_openshift_bundles + - func: prepare_openshift_bundles_for_e2e + + - name: prepare_and_upload_openshift_bundles + tags: [ "openshift_bundles" ] + commands: + - func: clone + - func: setup_aws + - func: configure_docker_auth + - func: quay_login + - func: setup_prepare_openshift_bundles + - func: prepare_openshift_bundles + - func: update_evergreen_expansions + - func: upload_openshift_bundle + vars: + # mongoDbOperator expansion is added in update_evergreen_expansions func from release.json + bundle_file_name: "operator-certified-${mongodbOperator}.tgz" + + - name: run_conditionally_prepare_and_upload_openshift_bundles + tags: [ "openshift_bundles" ] + commands: + - func: clone + - func: run_task_conditionally + vars: + condition_script: scripts/evergreen/should_prepare_openshift_bundles.sh + variant: prepare_openshift_bundles + task: prepare_and_upload_openshift_bundles + +task_groups: + - name: unit_task_group + max_hosts: -1 + setup_group_can_fail_task: true + setup_group: + - func: clone + - func: python_venv + - func: download_kube_tools + - func: setup_shellcheck + tasks: + - lint_repo + - unit_tests_golang + - unit_tests_python + - sbom_tests + + - name: gke_code_snippets_task_group + <<: *setup_and_teardown_group_gke_code_snippets + max_hosts: -1 + tasks: + - gke_multi_cluster_snippets + - gke_multi_cluster_no_mesh_snippets + + # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant + - name: e2e_mdb_kind_cloudqa_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + # e2e_kube_only_task_group + - e2e_crd_validation + - e2e_replica_set_config_map + - e2e_replica_set_exposed_externally + - e2e_replica_set_pv + - e2e_replica_set_pv_multiple + - e2e_replica_set_schema_validation + - e2e_replica_set_statefulset_status + - e2e_sharded_cluster_pv + - e2e_sharded_cluster_recovery + - e2e_sharded_cluster_schema_validation + - e2e_sharded_cluster_statefulset_status + - e2e_standalone_config_map + - e2e_standalone_recovery + - e2e_standalone_schema_validation + - e2e_users_schema_validation + - e2e_replica_set_tls_default + - e2e_replica_set_tls_override + - e2e_replica_set_ignore_unknown_users + # e2e_core_task_group + - e2e_all_mongodb_resources_parallel + - e2e_multiple_cluster_failures + - e2e_replica_set_custom_podspec + - e2e_replica_set_custom_sa + - e2e_replica_set_report_pending_pods + - e2e_replica_set_recovery + - e2e_replica_set_upgrade_downgrade + - e2e_replica_set_agent_flags_and_readinessProbe + - e2e_sharded_cluster + - e2e_sharded_cluster_secret + - e2e_sharded_cluster_upgrade_downgrade + - e2e_sharded_cluster_custom_podspec + - e2e_sharded_cluster_agent_flags + - e2e_standalone_upgrade_downgrade + - e2e_standalone_custom_podspec + - e2e_standalone_agent_flags + - e2e_replica_set_process_hostnames + - e2e_replica_set_member_options + # e2e_tls_task_group + - e2e_replica_set_tls_allow + - e2e_replica_set_tls_prefer + - e2e_replica_set_tls_require_upgrade + - e2e_tls_rs_additional_certs + - e2e_tls_sc_additional_certs + - e2e_tls_rs_intermediate_ca + - e2e_tls_sharded_cluster_certs_prefix + - e2e_sharded_cluster_migration + - e2e_standalone_no_tls_no_status_is_set + - e2e_feature_controls_authentication + - e2e_replica_set + - e2e_replica_set_liveness_probe + - e2e_replica_set_mongod_options + - e2e_replica_set_readiness_probe + - e2e_replica_set_update_delete_parallel + - e2e_sharded_cluster_scale_shards + - e2e_sharded_cluster_shard_overrides + - e2e_sharded_cluster_mongod_options_and_log_rotation + - e2e_tls_rs_external_access + - e2e_replica_set_tls_require + - e2e_replica_set_tls_certs_secret_prefix + - e2e_disable_tls_scale_up + - e2e_replica_set_tls_require_and_disable + # e2e_x509_task_group + - e2e_configure_tls_and_x509_simultaneously_st + - e2e_configure_tls_and_x509_simultaneously_rs + - e2e_configure_tls_and_x509_simultaneously_sc + - e2e_tls_x509_rs + - e2e_tls_x509_sc + - e2e_tls_x509_configure_all_options_rs + - e2e_tls_x509_configure_all_options_sc + - e2e_tls_x509_user_connectivity + - e2e_tls_x509_users_addition_removal + - e2e_replica_set_tls_process_hostnames + # e2e_ldap_task_group + - e2e_replica_set_ldap + - e2e_sharded_cluster_ldap + - e2e_replica_set_ldap_tls + - e2e_replica_set_ldap_user_to_dn_mapping + # e2e_replica_set_ldap_agent_auth + - e2e_replica_set_ldap_agent_client_certs + - e2e_replica_set_custom_roles + - e2e_replica_set_update_roles_no_privileges + - e2e_replica_set_ldap_group_dn + - e2e_replica_set_ldap_group_dn_with_x509_agent + # e2e_scram_sha_task_group_with_manually_generated_certs + - e2e_replica_set_scram_sha_256_user_connectivity + - e2e_replica_set_scram_sha_256_user_first + - e2e_replica_set_scram_sha_1_user_connectivity + - e2e_replica_set_scram_sha_1_upgrade + - e2e_replica_set_scram_x509_ic_manual_certs + - e2e_sharded_cluster_scram_sha_1_upgrade + - e2e_sharded_cluster_scram_sha_256_user_connectivity + - e2e_sharded_cluster_scram_sha_1_user_connectivity + - e2e_sharded_cluster_scram_x509_ic_manual_certs + - e2e_sharded_cluster_external_access + # e2e_auth_transitions_task_group + - e2e_replica_set_scram_sha_and_x509 + - e2e_replica_set_x509_to_scram_transition + - e2e_sharded_cluster_scram_sha_and_x509 + - e2e_sharded_cluster_x509_to_scram_transition + - e2e_sharded_cluster_internal_cluster_transition + # e2e_webhook_validation_task_group + - e2e_mongodb_validation_webhook + - e2e_mongodb_roles_validation_webhook + - e2e_vault_setup + - e2e_vault_setup_tls + # this test will either migrate from static to non-static or the other way around + - e2e_replica_set_migration + - e2e_replica_set_pv_resize + - e2e_sharded_cluster_pv_resize + <<: *teardown_group + + # this task group contains just a one task, which is smoke testing whether the operator + # works correctly when we run it without installing webhook cluster roles + - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_replica_set_agent_flags_and_readinessProbe + <<: *teardown_group + + # This task group mostly duplicates tests running in e2e_mdb_kind_ubi_cloudqa + # This is mostly a smoke test, so we execute only a few essential ones. + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + # OM tests are also run on the same Openshift cluster, so we use 1 max host to not + # allow the helm installations to interfere with each other during the setup of the tests. + max_hosts: 1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_crd_validation + - e2e_replica_set_scram_sha_256_user_connectivity + - e2e_sharded_cluster_scram_sha_256_user_connectivity + - e2e_replica_set_pv + - e2e_sharded_cluster_pv + + - name: e2e_custom_domain_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_replica_set + + # e2e_operator_task_group includes the tests for the specific Operator configuration/behavior. They may deal with + # cluster-wide resources so should be run in isolated K8s clusters only (Kind or Minikube). + - name: e2e_operator_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_operator_upgrade_replica_set + - e2e_operator_upgrade_sharded_cluster + - e2e_operator_upgrade_ops_manager + - e2e_operator_proxy + - e2e_operator_partial_crd + - e2e_operator_clusterwide + - e2e_operator_multi_namespaces + - e2e_operator_upgrade_appdb_tls + <<: *teardown_group + + # e2e_operator_race_with_telemetry_task_group includes the tests for testing the operator with race detector enabled + # additionally, it sends telemetry to cloud-dev; more here: https://wiki.corp.mongodb.com/display/MMS/Telemetry + - name: e2e_operator_race_with_telemetry_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + - e2e_om_reconcile_race_with_telemetry + <<: *teardown_group + + # e2e_operator_task_group includes the tests for the specific Operator configuration/behavior. They may deal with + # cluster-wide resources so should be run in isolated K8s clusters only (Kind or Minikube). + - name: e2e_static_operator_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_operator_upgrade_replica_set + - e2e_operator_upgrade_sharded_cluster + - e2e_operator_upgrade_ops_manager + - e2e_operator_proxy + - e2e_operator_partial_crd + - e2e_operator_clusterwide + - e2e_operator_multi_namespaces + - e2e_operator_upgrade_appdb_tls + <<: *teardown_group + + - name: e2e_multi_cluster_kind_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_multi_cluster_replica_set + - e2e_multi_cluster_replica_set_migration + - e2e_multi_cluster_replica_set_member_options + - e2e_multi_cluster_recover + - e2e_multi_cluster_recover_clusterwide + - e2e_multi_cluster_specific_namespaces + - e2e_multi_cluster_scram + - e2e_multi_cluster_tls_with_x509 + - e2e_multi_cluster_tls_no_mesh + - e2e_multi_cluster_enable_tls + # e2e_multi_cluster_with_ldap + # e2e_multi_cluster_with_ldap_custom_roles + - e2e_multi_cluster_mtls_test + - e2e_multi_cluster_replica_set_deletion + - e2e_multi_cluster_replica_set_scale_up + - e2e_multi_cluster_scale_up_cluster + - e2e_multi_cluster_scale_up_cluster_new_cluster + - e2e_multi_cluster_replica_set_scale_down + - e2e_multi_cluster_scale_down_cluster + - e2e_multi_sts_override + - e2e_multi_cluster_tls_with_scram + - e2e_multi_cluster_upgrade_downgrade + - e2e_multi_cluster_backup_restore + - e2e_multi_cluster_backup_restore_no_mesh + - e2e_multi_cluster_disaster_recovery + - e2e_multi_cluster_multi_disaster_recovery + - e2e_multi_cluster_recover_network_partition + - e2e_multi_cluster_validation + - e2e_multi_cluster_agent_flags + - e2e_multi_cluster_replica_set_ignore_unknown_users + - e2e_multi_cluster_pvc_resize + - e2e_multi_cluster_sharded_geo_sharding + - e2e_multi_cluster_sharded_scaling + - e2e_multi_cluster_sharded_scaling_all_shard_overrides + - e2e_multi_cluster_sharded_simplest + - e2e_multi_cluster_sharded_snippets + - e2e_multi_cluster_sharded_simplest_no_mesh + - e2e_multi_cluster_sharded_external_access_no_ext_domain + - e2e_multi_cluster_sharded_tls_no_mesh + - e2e_multi_cluster_sharded_tls + - e2e_multi_cluster_sharded_disaster_recovery + - e2e_sharded_cluster + - e2e_sharded_cluster_agent_flags + - e2e_sharded_cluster_custom_podspec + - e2e_sharded_cluster_mongod_options_and_log_rotation + - e2e_sharded_cluster_migration + - e2e_sharded_cluster_pv + - e2e_sharded_cluster_pv_resize + - e2e_sharded_cluster_recovery + - e2e_sharded_cluster_scale_shards + - e2e_sharded_cluster_shard_overrides + - e2e_sharded_cluster_secret + - e2e_sharded_cluster_statefulset_status + - e2e_sharded_cluster_upgrade_downgrade + - e2e_configure_tls_and_x509_simultaneously_sc + - e2e_sharded_cluster_tls_require_custom_ca + - e2e_tls_sharded_cluster_certs_prefix + - e2e_tls_sc_additional_certs + - e2e_tls_x509_configure_all_options_sc + - e2e_tls_x509_sc + + <<: *teardown_group + + - name: e2e_multi_cluster_2_clusters_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task_cloudqa + tasks: + - e2e_multi_cluster_2_clusters_replica_set + - e2e_multi_cluster_2_clusters_clusterwide + <<: *teardown_group + + - name: e2e_multi_cluster_om_appdb_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + # Dedicated AppDB Multi-Cluster tests + - e2e_multi_cluster_appdb_validation + - e2e_multi_cluster_om_validation + - e2e_multi_cluster_appdb + - e2e_multi_cluster_appdb_s3_based_backup_restore + - e2e_multi_cluster_appdb_disaster_recovery + - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure + - e2e_multi_cluster_om_networking_clusterwide + - e2e_multi_cluster_om_appdb_no_mesh + # Reused OM tests with AppDB Multi-Cluster topology + - e2e_operator_upgrade_appdb_tls + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_upgrade + - e2e_om_appdb_monitoring_tls + - e2e_om_ops_manager_backup + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_secure_config + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_migration + - e2e_om_localmode + - e2e_om_ops_manager_enable_local_mode_running_om + - e2e_om_localmode_multiple_pv + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_https_enabled + - e2e_om_ops_manager_https_enabled_hybrid + - e2e_om_ops_manager_https_enabled_internet_mode + - e2e_om_ops_manager_https_enabled_prefix + - e2e_om_ops_manager_scale + - e2e_om_ops_manager_upgrade + - e2e_om_update_before_reconciliation + - e2e_om_feature_controls + - e2e_multi_cluster_appdb_state_operator_upgrade_downgrade + - e2e_multi_cluster_sharded_disaster_recovery + # disabled tests: + # - e2e_om_multiple # multi-cluster failures in EVG + # - e2e_om_appdb_scale_up_down # test not "reused" for multi-cluster appdb + <<: *teardown_group + + - name: e2e_static_multi_cluster_om_appdb_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + # Dedicated AppDB Multi-Cluster tests + - e2e_multi_cluster_appdb_validation + - e2e_multi_cluster_om_validation + - e2e_multi_cluster_appdb + - e2e_multi_cluster_appdb_s3_based_backup_restore + - e2e_multi_cluster_appdb_disaster_recovery + - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure + - e2e_multi_cluster_om_networking_clusterwide + - e2e_multi_cluster_om_appdb_no_mesh + # Reused OM tests with AppDB Multi-Cluster topology + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_upgrade + - e2e_om_appdb_monitoring_tls + - e2e_om_ops_manager_backup + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_secure_config + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_migration + # Deactivated tests + # - e2e_om_localmode + # - e2e_om_localmode_multiple_pv + # - e2e_om_ops_manager_https_enabled + # - e2e_om_ops_manager_https_enabled_hybrid + # - e2e_om_ops_manager_https_enabled_internet_mode + - e2e_om_ops_manager_https_enabled_prefix + - e2e_om_ops_manager_enable_local_mode_running_om + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_scale + - e2e_om_ops_manager_upgrade + - e2e_multi_cluster_appdb_state_operator_upgrade_downgrade + - e2e_om_update_before_reconciliation + - e2e_om_feature_controls + - e2e_multi_cluster_sharded_disaster_recovery + <<: *teardown_group + + # Dedicated task group for deploying OM Multi-Cluster when the operator is in the central cluster + # that is not in the mesh + - name: e2e_multi_cluster_om_operator_not_in_mesh_task_group + max_hosts: -1 + <<: *setup_group_multi_cluster + <<: *setup_and_teardown_task + tasks: + - e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking + <<: *teardown_group + + + # This task group runs on Kind clusters. In theory ALL Ops Manager tests should be added here + # including the cluster-wide resources changing. Uses Cloud-qa. TODO why Cloud-qa? those are OM tests + - name: e2e_ops_manager_kind_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_appdb_external_connectivity + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_monitoring_tls + - e2e_om_appdb_multi_change + - e2e_om_appdb_scale_up_down + - e2e_om_appdb_upgrade + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_migration + - e2e_om_localmode + - e2e_om_ops_manager_enable_local_mode_running_om + - e2e_om_localmode_multiple_pv + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_restore_minio + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_https_enabled + - e2e_om_ops_manager_https_enabled_hybrid + - e2e_om_ops_manager_https_enabled_internet_mode + - e2e_om_ops_manager_https_enabled_prefix + - e2e_om_ops_manager_scale + - e2e_om_ops_manager_upgrade + - e2e_om_validation_webhook + - e2e_om_ops_manager_secure_config + - e2e_om_update_before_reconciliation + - e2e_om_multiple + - e2e_om_feature_controls + - e2e_vault_setup_om + - e2e_vault_setup_om_backup + <<: *teardown_group + + - name: e2e_static_ops_manager_kind_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_appdb_external_connectivity + - e2e_om_appdb_flags_and_config + - e2e_om_appdb_monitoring_tls + - e2e_om_appdb_multi_change + - e2e_om_appdb_scale_up_down + - e2e_om_appdb_upgrade + - e2e_om_appdb_validation + - e2e_om_appdb_scram + - e2e_om_external_connectivity + - e2e_om_jvm_params + - e2e_om_weak_password + - e2e_om_ops_manager_backup_delete_sts_and_log_rotation + - e2e_om_ops_manager_backup_light + - e2e_om_ops_manager_backup_tls + - e2e_om_ops_manager_backup_s3_tls + - e2e_om_ops_manager_backup_restore_minio + - e2e_om_ops_manager_backup_tls_custom_ca + - e2e_om_ops_manager_backup_liveness_probe + - e2e_om_ops_manager_backup_sharded_cluster + - e2e_om_ops_manager_backup_kmip + - e2e_om_ops_manager_pod_spec + - e2e_om_ops_manager_scale + - e2e_om_ops_manager_upgrade + - e2e_om_validation_webhook + - e2e_om_ops_manager_secure_config + - e2e_om_update_before_reconciliation + - e2e_om_multiple + - e2e_om_feature_controls + - e2e_vault_setup_om + - e2e_vault_setup_om_backup + # Deactivated tests + # - e2e_om_localmode_multiple_pv + # - e2e_om_localmode + # - e2e_om_ops_manager_https_enabled_hybrid + # - e2e_om_ops_manager_https_enabled_internet_mode + # - e2e_om_ops_manager_https_enabled + # - e2e_om_ops_manager_https_enabled_prefix + <<: *teardown_group + + - name: e2e_smoke_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_backup + <<: *teardown_group + + - name: e2e_ops_manager_kind_5_0_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_remotemode + - e2e_om_ops_manager_backup_restore + - e2e_om_ops_manager_queryable_backup + - e2e_om_ops_manager_backup + <<: *teardown_group + + # Tests features only supported on OM60 + - name: e2e_ops_manager_kind_6_0_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_prometheus + <<: *teardown_group + + # Tests features only supported on OM60 + - name: e2e_static_ops_manager_kind_6_0_only_task_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + tasks: + - e2e_om_ops_manager_prometheus + <<: *teardown_group + + - name: e2e_kind_olm_group + max_hosts: -1 + <<: *setup_group + <<: *setup_and_teardown_task + setup_task_can_fail_task: true + setup_task: + # Even if the two first tasks are already part of setup_and_teardown_task, + # we need to repeat them because we are overriding the setup_task variable of the YAML anchor + - func: cleanup_exec_environment + - func: configure_docker_auth + - func: setup_kubernetes_environment + - func: setup_prepare_openshift_bundles + - func: install_olm + tasks: + - e2e_olm_operator_upgrade + - e2e_olm_operator_upgrade_with_resources + <<: *teardown_group + +buildvariants: + + ## Unit tests + lint build variant + + - name: unit_tests + display_name: "unit_tests" + tags: [ "unit_tests" ] + run_on: + - ubuntu2204-small + tasks: + - name: "unit_task_group" + + ## Build variants for E2E tests + + # The pattern for naming build variants for E2E tests: + # e2e___[_] + # where is any of mdb|om|operator + # where is any of kind|openshift + # where is any of ubuntu|ubi + # where denotes the OM version tested (e.g. om50, om60, cloudqa) - used only for MDB tests + + ## MongoDB build variants + - name: e2e_mdb_kind_ubi_cloudqa + display_name: e2e_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_cloudqa_task_group + + - name: e2e_custom_domain_mdb_kind_ubi_cloudqa + display_name: e2e_custom_domain_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_custom_domain_task_group + + - name: e2e_static_mdb_kind_ubi_cloudqa + display_name: e2e_static_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_cloudqa_task_group + + - name: e2e_static_custom_domain_mdb_kind_ubi_cloudqa + display_name: e2e_static_custom_domain_mdb_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + tasks: + - name: e2e_custom_domain_task_group + + - name: e2e_mdb_openshift_ubi_cloudqa + display_name: e2e_mdb_openshift_ubi_cloudqa + tags: [ "e2e_openshift_test_suite", "cloudqa" ] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + run_on: + - ubuntu2204-small + tasks: + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + + # This name is on purpose reversed from e2e_static_openshift to e2e_openshift_static. + # That is because we run a regex + # in evergreen for all variants matching e2e_static-*, but we do not want to run openshift variants on every pr. + - name: e2e_openshift_static_mdb_ubi_cloudqa + display_name: e2e_openshift_static_mdb_ubi_cloudqa + tags: [ "e2e_openshift_test_suite", "cloudqa" ] + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + run_on: + - ubuntu2204-small + tasks: + - name: e2e_mdb_openshift_ubi_cloudqa_task_group + + ## Ops Manager build variants + + # Isolated Ops Manager Tests for 6.0 version + - name: e2e_om60_kind_ubi + display_name: e2e_om60_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group + - name: e2e_ops_manager_kind_5_0_only_task_group + - name: e2e_ops_manager_kind_6_0_only_task_group + + # Isolated Ops Manager Tests for 6.0 version + - name: e2e_static_om60_kind_ubi + display_name: e2e_static_om60_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_static_ops_manager_kind_only_task_group + - name: e2e_static_ops_manager_kind_6_0_only_task_group + + - name: e2e_om70_kind_ubi + display_name: e2e_om70_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group + - name: e2e_ops_manager_kind_5_0_only_task_group + - name: e2e_ops_manager_kind_6_0_only_task_group + + - name: e2e_static_om70_kind_ubi + display_name: e2e_static_om70_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_static_ops_manager_kind_only_task_group + - name: e2e_static_ops_manager_kind_6_0_only_task_group + + - name: e2e_om80_kind_ubi + display_name: e2e_om80_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_ops_manager_kind_only_task_group + - name: e2e_ops_manager_kind_5_0_only_task_group + - name: e2e_ops_manager_kind_6_0_only_task_group + + - name: e2e_static_om80_kind_ubi + display_name: e2e_static_om80_kind_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om8_dependency + tasks: + - name: e2e_static_ops_manager_kind_only_task_group + - name: e2e_static_ops_manager_kind_6_0_only_task_group + + - name: e2e_operator_race_ubi_with_telemetry + display_name: e2e_operator_race_ubi_with_telemetry + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: e2e_operator_race_with_telemetry_task_group + + - name: e2e_smoke + display_name: e2e_smoke + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_test_image + variant: init_test_run + tasks: + - name: e2e_smoke_task_group + + - name: e2e_static_smoke + display_name: e2e_static_smoke + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_test_image + variant: init_test_run + tasks: + - name: e2e_smoke_task_group + + - name: e2e_smoke_release + display_name: e2e_smoke_release + tags: [ "e2e_smoke_release_test_suite" ] + run_on: + - ubuntu2204-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: build_test_image + variant: init_test_run + tasks: + - name: e2e_smoke_task_group + + - name: e2e_static_smoke_release + display_name: e2e_static_smoke_release + tags: [ "e2e_smoke_release_test_suite" ] + run_on: + - ubuntu2204-large + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: build_test_image + variant: init_test_run + tasks: + - name: e2e_smoke_task_group + + - name: e2e_multi_cluster_kind + display_name: e2e_multi_cluster_kind + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_kind_task_group + + - name: e2e_static_multi_cluster_kind + display_name: e2e_static_multi_cluster_kind + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_kind_task_group + + - name: e2e_multi_cluster_2_clusters + display_name: e2e_multi_cluster_2_clusters + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_2_clusters_task_group + + - name: e2e_static_multi_cluster_2_clusters + display_name: e2e_static_multi_cluster_2_clusters + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_2_clusters_task_group + + - name: e2e_multi_cluster_om_appdb + display_name: e2e_multi_cluster_om_appdb + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_multi_cluster_om_appdb_task_group + + - name: e2e_static_multi_cluster_om_appdb + display_name: e2e_static_multi_cluster_om_appdb + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om6_dependency + tasks: + - name: e2e_static_multi_cluster_om_appdb_task_group + + - name: e2e_multi_cluster_om_operator_not_in_mesh + display_name: e2e_multi_cluster_om_operator_not_in_mesh + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + <<: *base_om7_dependency + tasks: + - name: e2e_multi_cluster_om_operator_not_in_mesh_task_group + + ## Operator tests build variants + + - name: e2e_operator_kind_ubi_cloudqa + display_name: e2e_operator_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_operator_task_group + + - name: e2e_static_operator_kind_ubi_cloudqa + display_name: e2e_static_operator_kind_ubi_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_static_operator_task_group + + - name: e2e_operator_no_webhook_roles_cloudqa + display_name: e2e_operator_no_webhook_roles_cloudqa + tags: [ "e2e_test_suite", "cloudqa" ] + run_on: + - ubuntu2204-large + <<: *base_no_om_image_dependency + tasks: + - name: e2e_mdb_kind_no_webhook_roles_cloudqa_task_group + + - name: e2e_kind_olm_ubi + display_name: e2e_kind_olm_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: prepare_and_upload_openshift_bundles_for_e2e + variant: init_tests_with_olm + tasks: + - name: e2e_kind_olm_group + + - name: e2e_static_kind_olm_ubi + display_name: e2e_static_kind_olm_ubi + tags: [ "e2e_test_suite" ] + run_on: + - ubuntu2204-large + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: prepare_and_upload_openshift_bundles_for_e2e + variant: init_tests_with_olm + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + tasks: + - name: e2e_kind_olm_group + + ## Manual (patch) E2E tests not run for every PR and commit + + - name: e2e_operator_perf + display_name: e2e_operator_perf + tags: [ "e2e_perf_test_suite" ] + allowed_requesters: [ "patch" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: generate_perf_tasks_10_thread + + - name: e2e_operator_perf_one_thread + display_name: e2e_operator_perf_one_thread + tags: [ "e2e_perf_test_suite" ] + allowed_requesters: [ "patch" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: generate_perf_tasks_one_thread + + - name: e2e_operator_perf_thirty + display_name: e2e_operator_perf_thirty + tags: [ "e2e_perf_test_suite" ] + allowed_requesters: [ "patch" ] + run_on: + - ubuntu1804-xlarge + <<: *base_om7_dependency + tasks: + - name: generate_perf_tasks_30_thread + + ### Prerequisites for E2E test suite + + - name: init_test_run + display_name: init_test_run + max_hosts: -1 + run_on: + - ubuntu2204-small + tasks: + - name: build_operator_ubi + - name: build_test_image + - name: build_init_appdb_images_ubi + - name: build_init_om_images_ubi + - name: build_init_database_image_ubi + - name: build_database_image_ubi + - name: build_agent_images_ubi + - name: prepare_aws + + - name: init_release_agents_on_ecr + display_name: init_release_agents_on_ecr + # We want that to run first and finish asap. Digest pinning depends on this to succeed. + priority: 70 + run_on: + - ubuntu2204-large + tasks: + - name: release_agents_on_ecr_conditional + + - name: run_pre_commit + priority: 70 + display_name: run_pre_commit + allowed_requesters: [ "patch", "github_pr" ] + tags: [ "auto_bump" ] + run_on: + - ubuntu2204-small + tasks: + - name: run_precommit_and_push + + - name: init_tests_with_olm + display_name: init_tests_with_olm + depends_on: + - name: build_om_images + variant: build_om60_images + - name: build_operator_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + - name: build_test_image + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_agent_images_ubi + variant: init_test_run + - name: release_agents_on_ecr_conditional + variant: init_release_agents_on_ecr + run_on: + - ubuntu2204-small + tasks: + - name: prepare_and_upload_openshift_bundles_for_e2e + + ### End of build variants for E2E + + ### Variants that run on each commit + + - name: preflight_release_images_check_only + display_name: preflight_release_images_check_only + run_on: + - rhel90-large + tasks: + - name: preflight_images_task_group + + - name: build_om60_images + display_name: build_om60_images + run_on: + - ubuntu2204-small + tasks: + - name: build_om_images + + - name: preflight_om60_images + display_name: preflight_om60_images + run_on: + - rhel90-large + tasks: + - name: preflight_om_image + + - name: build_om70_images + display_name: build_om70_images + run_on: + - ubuntu2204-small + tasks: + - name: build_om_images + + - name: preflight_om70_images + display_name: preflight_om70_images + run_on: + - rhel90-large + tasks: + - name: preflight_om_image + + - name: build_om80_images + display_name: build_om80_images + run_on: + - ubuntu2204-small + tasks: + - name: build_om_images + + - name: preflight_om80_images + display_name: preflight_om80_images + run_on: + - rhel90-large + tasks: + - name: preflight_om_image + + ### Release build variants + + ## Adds versions as supported in the supported versions Database. + - name: release_images + display_name: release_images + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + max_hosts: -1 + run_on: + - ubuntu2204-large + depends_on: + - name: build_operator_ubi + variant: init_test_run + - name: build_init_om_images_ubi + variant: init_test_run + - name: build_init_appdb_images_ubi + variant: init_test_run + - name: build_init_database_image_ubi + variant: init_test_run + - name: build_database_image_ubi + variant: init_test_run + tasks: + - name: release_operator + - name: release_init_appdb + - name: release_init_database + - name: release_init_ops_manager + - name: release_database + # Once we release the operator, we will also release the init databases, we require them to be out first + # such that we can reference them and retrieve those binaries. + # Since we immediately run daily rebuild after creating the image, we can ensure that the init_database is out + # such that the agent image build can use it. + - name: release_agent_operator_release + depends_on: + - name: release_init_database + + - name: preflight_release_images + display_name: preflight_release_images + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + run_on: + - rhel90-large + expansions: + preflight_submit: true + tasks: + - name: preflight_images_task_group + + - name: prepare_openshift_bundles + display_name: prepare_openshift_bundles + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + depends_on: + - name: "*" + variant: release_images + - name: "*" + variant: preflight_release_images + run_on: + - ubuntu2204-large + tasks: + - name: run_conditionally_prepare_and_upload_openshift_bundles + + - name: upload_dockerfiles + display_name: upload_dockerfiles + tags: [ "release" ] + allowed_requesters: [ "patch", "github_tag" ] + # CLOUDP-182323 This ensures that there will be Operator Dockerfile available in S3 before + # syncing and uploading the bundle TAR archive. + depends_on: + - name: release_operator + variant: release_images + run_on: + - ubuntu2204-small + tasks: + - name: upload_dockerfiles + + # It will be called by pct while bumping the agent cloud manager image + - name: release_agent + display_name: (Static Containers) Release Agent matrix + tags: [ "release_agent" ] + run_on: + - ubuntu2204-large + depends_on: + - variant: init_release_agents_on_ecr + name: '*' + - variant: e2e_multi_cluster_kind + name: '*' + - variant: e2e_static_multi_cluster_2_clusters + name: '*' + - variant: e2e_mdb_kind_ubi_cloudqa + name: '*' + - variant: e2e_static_mdb_kind_ubi_cloudqa + name: '*' + tasks: + - name: release_agent + + # Only called manually, It's used for testing the task release_agents_on_ecr in case the release.json + # has not changed, and you still want to push the images to ecr. + - name: manual_ecr_release_agent + display_name: Manual Agent Release for all versions + tags: [ "release_all_agents_manually" ] + run_on: + - ubuntu2204-large + tasks: + - name: release_all_agents_on_ecr + + # These variants are used to test the code snippets and each one can be used in patches + # Prerelease is especially used when the repo is tagged + # More details in the TD: https://docs.google.com/document/d/1fuTxfRtP8QPtn7sKYxQM_AGcD6xycTZH8svngGxyKhc/edit?tab=t.0#bookmark=id.e8uva0393mbe + - name: public_gke_code_snippets + display_name: public_gke_code_snippets + allowed_requesters: ["patch"] + run_on: + - ubuntu2204-small + tasks: + - name: gke_code_snippets_task_group + + - name: prerelease_gke_code_snippets + display_name: prerelease_gke_code_snippets + tags: [ "release" ] + allowed_requesters: ["patch", "github_tag"] + depends_on: + - variant: release_images + name: '*' + patch_optional: true + run_on: + - ubuntu2204-small + tasks: + - name: gke_code_snippets_task_group + + - name: private_gke_code_snippets + display_name: private_gke_code_snippets + allowed_requesters: ["patch"] + run_on: + - ubuntu2204-small + <<: *base_om8_dependency + tasks: + - name: gke_code_snippets_task_group + + ### Build variants for manual patch only + + - name: publish_om60_images + display_name: publish_om60_images + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - ubuntu2204-large + depends_on: + - variant: e2e_om60_kind_ubi + name: '*' + - variant: e2e_static_om60_kind_ubi + name: '*' + tasks: + - name: publish_ops_manager + - name: release_agent + + - name: publish_om70_images + display_name: publish_om70_images + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - ubuntu2204-large + depends_on: + - variant: e2e_om70_kind_ubi + name: '*' + - variant: e2e_static_om70_kind_ubi + name: '*' + tasks: + - name: publish_ops_manager + - name: release_agent + + - name: publish_om80_images + display_name: publish_om80_images + allowed_requesters: [ "patch", "github_pr" ] + run_on: + - ubuntu2204-large + depends_on: + - variant: e2e_om80_kind_ubi + name: '*' + - variant: e2e_static_om80_kind_ubi + name: '*' + tasks: + - name: publish_ops_manager + - name: release_agent diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..36873ea16 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E203, E266, E501, W503 +max-line-length = 120 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 000000000..84227c765 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +if [ -f "${PROJECT_DIR}"/venv/bin/activate ]; then + source "${PROJECT_DIR}"/venv/bin/activate +fi + +if [[ -z "${EVERGREEN_MODE:-}" ]]; then + # According to the latest SSDLC recommendations, the CI needs to always check all the files. Not just delta. + git_last_changed=$(git ls-tree -r origin/master --name-only) +else + git_last_changed=$(git diff --cached --name-only --diff-filter=ACM origin/master) +fi + +mkdir -p "$(go env GOPATH)/bin" + +# Generates a yaml file to install the operator from the helm sources. +function generate_standalone_yaml() { + HELM_OPTS=$@ + + charttmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'charttmpdir') + charttmpdir=${charttmpdir}/chart + mkdir -p "${charttmpdir}" + + FILES=( + "${charttmpdir}/enterprise-operator/templates/operator-roles.yaml" + "${charttmpdir}/enterprise-operator/templates/database-roles.yaml" + "${charttmpdir}/enterprise-operator/templates/operator-sa.yaml" + "${charttmpdir}/enterprise-operator/templates/operator.yaml" + ) + + # generate normal public example + helm template --namespace mongodb -f helm_chart/values.yaml helm_chart --output-dir "${charttmpdir}" ${HELM_OPTS[@]} + cat "${FILES[@]}" >public/mongodb-enterprise.yaml + cat "helm_chart/crds/"* >public/crds.yaml + + # generate openshift public example + rm -rf "${charttmpdir:?}/*" + helm template --namespace mongodb -f helm_chart/values.yaml helm_chart --output-dir "${charttmpdir}" --values helm_chart/values-openshift.yaml ${HELM_OPTS[@]} + cat "${FILES[@]}" >public/mongodb-enterprise-openshift.yaml + + # generate openshift files for kustomize used for generating OLM bundle + rm -rf "${charttmpdir:?}/*" + helm template --namespace mongodb -f helm_chart/values.yaml helm_chart --output-dir "${charttmpdir}" --values helm_chart/values-openshift.yaml \ + --set operator.webhook.registerConfiguration=false --set operator.webhook.installClusterRole=false ${HELM_OPTS[@]} + + # update kustomize files for OLM bundle with files generated for openshift + cp "${charttmpdir}/enterprise-operator/templates/operator.yaml" config/manager/manager.yaml + cp "${charttmpdir}/enterprise-operator/templates/database-roles.yaml" config/rbac/database-roles.yaml + cp "${charttmpdir}/enterprise-operator/templates/operator-roles.yaml" config/rbac/operator-roles.yaml + + # generate multi-cluster public example + rm -rf "${charttmpdir:?}/*" + helm template --namespace mongodb -f helm_chart/values.yaml helm_chart --output-dir "${charttmpdir}" --values helm_chart/values-multi-cluster.yaml ${HELM_OPTS[@]} + cat "${FILES[@]}" >public/mongodb-enterprise-multi-cluster.yaml + +} + +function python_formatting() { + # installing Black + if ! command -v "black" >/dev/null; then + pip install -r requirements.txt + fi + + echo "formatting isort" + isort . + echo "formatting black" + black . +} + +function generate_manifests() { + make manifests + + git add config/crd/bases + git add helm_chart/crds + git add public/crds.yaml +} + +function update_values_yaml_files() { + # ensure that all helm values files are up to date. + # shellcheck disable=SC2154 + python scripts/evergreen/release/update_helm_values_files.py + + # commit any changes we made + git add helm_chart/values.yaml + git add helm_chart/values-openshift.yaml + + # these can change if the version of community operator is different + git add go.mod + git add go.sum +} + +function update_release_json() { + # ensure that release.json is up 2 date + # shellcheck disable=SC2154 + python scripts/evergreen/release/update_release.py + + # commit any changes we made + git add release.json +} + +function pre_commit() { + # Update release.json first in case there is a newer version + update_release_json + # Generate operator manifests (CRDs, etc) + generate_manifests + # We need to generate the values files first + update_values_yaml_files + # The values files are used for generating the standalone yaml + generate_standalone_yaml + # Run black on python files that have changed + python_formatting + + source scripts/evergreen/lint_code.sh + + echo 'regenerating licenses.csv' + scripts/evergreen/update_licenses.sh + git add licenses.csv + + if echo "$git_last_changed" | grep -q 'public/tools/multicluster'; then + echo 'regenerating multicluster RBAC public example' + pushd public/tools/multicluster + EXPORT_RBAC_SAMPLES="true" go test -run TestPrintingOutRolesServiceAccountsAndRoleBindings + popd + git add public/samples/multi-cluster-cli-gitops + fi + + if find . -name "Makefile" | grep -v vendor | xargs grep "\${"; then + echo 'ERROR: Makefiles should NEVER contain curly brackets variables' + exit 1 + fi + + # Makes sure there are not erroneous kubebuilder annotations that can + # end up in CRDs as descriptions. + if grep "// kubebuilder" ./* -r --exclude-dir=vendor --include=\*.go; then + echo "Found an erroneous kubebuilder annotation" + exit 1 + fi + + start_shellcheck + +} + +# Function to run shellcheck on a single file +run_shellcheck() { + local file="$1" + echo "Running shellcheck on $file" + if ! shellcheck -x "$file" -e SC2154 -e SC1091 -e SC1090 -e SC2148 -o require-variable-braces -P "scripts"; then + echo "shellcheck failed on $file" + exit 1 + fi +} + +start_shellcheck() { + files_1=$(find scripts -type f -name "*.sh") + files_2=$(find scripts/dev/contexts -type f) + files_3=$(find scripts/funcs -type f) + files_4=$(find public/architectures -type f -name "*.sh") + files=$(echo -e "$files_1\n$files_2\n$files_3\n$files_4") + # Process each file in parallel + for file in $files; do + run_shellcheck "$file" & + done + + # Wait for all background jobs + for job in $(jobs -p); do + wait "$job" || exit 1 + done + +} + +cmd=${1:-"pre-commit"} + +if [[ "${cmd}" == "generate_standalone_yaml" ]]; then + shift 1 + generate_standalone_yaml "$@" +elif [[ "${cmd}" == "pre-commit" ]]; then + pre_commit +elif [[ "${cmd}" == "shellcheck" ]]; then + start_shellcheck +fi diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index db61cf612..196ae8c7a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,3 @@ -* @mircea-cosbuc @lsierant @nammn @Julien-Ben @MaciejKaras @lucian-tosa @fealebenpae @m1kola \ No newline at end of file +* @10gen/kubernetes-hosted + +helm_chart/crds/ @dan-mckean diff --git a/.github/PULL_REQUEST_TEMPLATE/internal_change.md b/.github/PULL_REQUEST_TEMPLATE/internal_change.md new file mode 100644 index 000000000..0336bf54a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/internal_change.md @@ -0,0 +1,3 @@ +# Summary + +*Enter your issue summary here.* diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md new file mode 100644 index 000000000..a8748f9f0 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/release.md @@ -0,0 +1,44 @@ +# Release + +*For in-depth information of how to perform any of these steps, please read +refer to [Kubernetes Enterprise Operator Release Guide](https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Enterprise+Operator+Release+Guide) document.* + +# Pre-merging tasks + +*We'll make sure all the following tasks are completed before merging.* + +## Project Tasks + +- [ ] Update any finished ticket's `Fix Version` to this version. +- [ ] Prepare release notes in [public repo](https://github.com/mongodb/mongodb-enterprise-kubernetes/releases/new). +- [ ] Prepare release notes in [DOCSP](https://jira.mongodb.org/secure/CreateIssueDetails!init.jspa?pid=14181&issuetype=3&summary=[MEKO]%20Kubernetes%20Enterprise%20Operator%20x.y.z%20Release%20Notes). + +## Versioning + +- [ ] Ensure that all of versions in release.json are correct and are properly reflected in `values.yaml` and `values-openshift.yaml`. + +## Public repo + +- [ ] All public samples are up-to-date. + +# Post-merging tasks + +*After merging this PR make sure you complete the following tasks.* + +- [ ] Unblock relevant release tasks from Evergreen's Waterfall. +- [ ] Update public repo contents. + +## Openshift/RedHat Changes + +*Refer to +[Red Hat Image and Container Certification Process](https://wiki.corp.mongodb.com/display/MMS/Red+Hat+Image+and+Container+Certification+Process) +for more infomation on how to do this* + +- [ ] New version has been pushed to Operatorhub.io. +- [ ] New version has been pushed to Openshift Marketplace. + +# Following day + +- [ ] Check [#k8s-operator-daily-builds](https://mongodb.slack.com/archives/C01HYH2KUJ1) to see if this version was pushed correctly. +- [ ] Publish public repo tag/release. +- [ ] Ask DOCS team to publish Release Notes. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..e442424cb --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,23 @@ +# Summary + + + +## Proof of Work + + + +## Checklist +- [ ] Have you linked a jira ticket and/or is the ticket in the title? +- [ ] Have you checked whether your jira ticket required DOCSP changes? +- [ ] Have you checked for release_note changes? + +## Reminder (Please remove this when merging) +- Please try to Approve or Reject Changes the PR, keep PRs in review as short as possible +- Our Short Guide for PRs: [Link](https://docs.google.com/document/d/1T93KUtdvONq43vfTfUt8l92uo4e4SEEvFbIEKOxGr44/edit?tab=t.0) +- Remember the following Communication Standards - use comment prefixes for clarity: + * **blocking**: Must be addressed before approval. + * **follow-up**: Can be addressed in a later PR or ticket. + * **q**: Clarifying question. + * **nit**: Non-blocking suggestions. + * **note**: Side-note, non-actionable. Example: Praise + * --> no prefix is considered a question diff --git a/.gitignore b/.gitignore index 0229263df..2038ad0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,103 +1,94 @@ -# Temporary Build Files -build/_output -build/_test -# Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode -### Emacs ### -# -*- mode: gitignore; -*- -*~ -\#*\# -/.emacs.desktop -/.emacs.desktop.lock -*.elc -auto-save-list -tramp -.\#* -# Org-mode -.org-id-locations -*_archive -# flymake-mode -*_flymake.* -# eshell files -/eshell/history -/eshell/lastdir -# elpa packages -/elpa/ -# reftex files -*.rel -# AUCTeX auto folder -/auto/ -# cask packages -.cask/ -dist/ -# Flycheck -flycheck_*.el -# server auth directory -/server/ -# projectiles files -.projectile -projectile-bookmarks.eld -# directory configuration .dir-locals.el -# saveplace -places -# url cache -url/cache/ -# cedet -ede-projects.el -# smex -smex-items -# company-statistics -company-statistics-cache.el -# anaconda-mode -anaconda-mode/ -### Go ### -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib -# Test binary, build with 'go test -c' -*.test -# Output of the go coverage tool, specifically when used with LiteIDE -*.out -### Vim ### -# swap -.sw[a-p] -.*.sw[a-p] -# session -Session.vim -# temporary -.netrwhist -# auto-generated tag files -tags -### VisualStudioCode ### -.vscode/* -.history -# End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode -*mypy_cache -bin/ -venv/ -local-config.json -.idea -vendor -__pycache__ -Dockerfile -Dockerfile_python_formatting -logs/* -testbin/bin -# OSX Trash +.idea/ + +/pkg/client +/vendor/ +public/tools/multicluster/vendor +docker/mongodb-enterprise-operator/content/mongodb-enterprise-operator +docker/mongodb-enterprise-database/content/mongodb-mms-automation-agent-version.properties +docker/mongodb-enterprise-database/content/readinessprobe +docker/mongodb-enterprise-ops-manager/scripts/mmsconfiguration +docker/mongodb-enterprise-tests/public + +my-* +.vscode +.env +data +cache +**/__pycache__ +**/myenv +**/venv*/ +**/env*/ +*.bak +**/pytest_cache/ +tmp/* +bin/* +**/*.iml +helm_out +.redo-last-namespace +exports.do +__debug_bin* +.generated/ +scripts/dev/contexts/private-context +scripts/dev/contexts/private-context-* + +# These files get generated by emacs and sometimes they are still present when committing +**/flycheck_* + +public/support/*.gz +public/support/logs* +public/architectures/**/log +public/architectures/**/*.run.log +public/architectures/**/.generated +public/architectures/**/certs/* +public/architectures/**/secrets/* + +docker/mongodb-enterprise-appdb/content/readinessprobe +ops-manager-kubernetes +docker/mongodb-enterprise-operator/Dockerfile +docker/mongodb-enterprise-database/Dockerfile +docker/mongodb-enterprise-ops-manager/Dockerfile +docker/mongodb-enterprise-init-database/Dockerfile +docker/mongodb-enterprise-init-ops-manager/Dockerfile +docker/mongodb-enterprise-operator/content/mongodb-enterprise-operator.tar +docker/mongodb-enterprise-tests/helm_chart/ +docker/mongodb-enterprise-tests/public/ +docker/mongodb-enterprise-tests/requirements.txt +docker/mongodb-enterprise-tests/.flake8 +docker/mongodb-enterprise-tests/.pylintrc +docker/mongodb-enterprise-tests/pyproject.toml + +bundle + .DS_Store +cover.out +result.suite +# loadtesting binary +production_notes/cmd/runtest/runtest +production_notes/helm_charts/opsmanager/charts/ +multi_cluster/examples/secret_mirror/secret_mirror +multi_cluster/examples/service_account/service_account +multi_cluster/examples/service_account_using_go/service_account_using_go + +_.yaml + +# Things we built or download for the e2e tests & local dev +multi-cluster-kube-config-creator +multi-cluster-kube-config-creator_linux +istio-* +.multi_cluster_local_test_files +csi-driver-host-path-* +downloadIstioCandidate.sh -# ignore files generated by sonar -Dockerfile.ubi-* -Dockerfile.ubuntu-* -diagnostics +# ignore symlink to ~/.operator-dev +.operator-dev +tmp -!test/test-app/Dockerfile +licenses_full.csv +licenses_stderr +docker/mongodb-enterprise-tests/.test_identifiers* -Pipfile -Pipfile.lock -.community-operator-dev -*.iml +logs-debug/ +/ssdlc-report/* +.gocache/ diff --git a/.golangci.yml b/.golangci.yml index 795e08728..1096d29be 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,55 +7,109 @@ # configure golangci-lint # see https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml -issues: - exclude-rules: - - path: _test\.go - linters: - - dupl - - gosec - - goconst - - golint - text: "underscore" - - path: ^pkg\/util\/envvar - linters: - - forbidigo - - path: ^cmd\/(readiness|versionhook|manager)\/main\.go$ - linters: - - forbidigo +version: "2" +run: + # Number of operating system threads (`GOMAXPROCS`) that can execute golangci-lint simultaneously. + # Default: 0 (automatically set to match Linux container CPU quota and + # fall back to the number of logical CPUs in the machine) + concurrency: 4 + # Timeout for total work, e.g. 30s, 5m, 5m30s. + # If the value is lower or equal to 0, the timeout is disabled. + # Default: 0 (disabled) + timeout: 5m + modules-download-mode: mod + go: 1.24 linters: enable: - - govet + - dupl - errcheck - - staticcheck - - unused - - gosimple + - forbidigo + - goconst + - gosec + - govet - ineffassign - - typecheck - rowserrcheck - - gosec + - staticcheck - unconvert - - forbidigo -linters-settings: - gosec: - excludes: - - G115 - forbidigo: - forbid: - - p: os\.(Getenv|LookupEnv|Environ|ExpandEnv) - pkg: os - msg: "Reading environemnt variables here is prohibited. Please read environment variables in the main package." - - p: os\.(Clearenv|Unsetenv|Setenv) - msg: "Modifying environemnt variables is prohibited." - pkg: os - - p: envvar\.(Read.*?|MergeWithOverride|GetEnvOrDefault) - pkg: github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar - msg: "Using this envvar package here is prohibited. Please work with environment variables in the main package." - # Rules with the `pkg` depend on it - analyze-types: true - -run: - modules-download-mode: mod - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 5m - # default concurrency is a available CPU number - concurrency: 4 + - unused + settings: + forbidigo: + forbid: + - pattern: os\.(Getenv|LookupEnv|Environ|ExpandEnv) + pkg: os + msg: "Reading environemnt variables here is prohibited. Please read environment variables in the main package." + - pattern: os\.(Clearenv|Unsetenv|Setenv) + pkg: os + msg: "Modifying environemnt variables is prohibited." + - pattern: env\.(Read.*?|EnsureVar) + pkg: github.com/10gen/ops-manager-kubernetes/pkg/util/env + msg: "Using this env package here is prohibited. Please work with environment variables in the main package." + # Rules with the `pkg` depend on it + analyze-types: true + staticcheck: + # STxxxx checks in https://staticcheck.io/docs/configuration/options/#checks + # Default: ["*"] + checks: + - all + - -ST1000 + - -ST1003 + - -ST1020 + - -ST1021 + - -ST1022 + - -ST1023 + - -QF1003 + exclusions: + # Mode of the generated files analysis. + # + # - `strict`: sources are excluded by strictly following the Go generated file convention. + # Source files that have lines matching only the following regular expression will be excluded: `^// Code generated .* DO NOT EDIT\.$` + # This line must appear before the first non-comment, non-blank text in the file. + # https://go.dev/s/generatedcode + # - `lax`: sources are excluded if they contain lines like `autogenerated file`, `code generated`, `do not edit`, etc. + # - `disable`: disable the generated files exclusion. + # + # Default: lax + generated: lax + presets: + - common-false-positives + - legacy + rules: + - linters: + - dupl + - goconst + - gosec + - errcheck + path: _test\.go + - linters: + - forbidigo + path: ^pkg\/util\/env + - linters: + - forbidigo + path: ^main.go$ +formatters: + enable: + - gci + - gofmt + - gofumpt + settings: + gci: + # Section configuration to compare against. + # Section names are case-insensitive and may contain parameters in (). + # The default order of sections is `standard > default > custom > blank > dot > alias > localmodule`, + # If `custom-order` is `true`, it follows the order of `sections` option. + # Default: ["standard", "default"] + sections: + - standard # Standard section: captures all standard packages. + - default # Default section: contains all imports that could not be matched to another section type. + - prefix(github.com/10gen/ops-manager-kubernetes) # Custom section: groups all imports with the specified Prefix. + - prefix(github.com/mongodb/mongodb-kubernetes-operator) # Custom section: groups all imports with the specified Prefix. + - blank # Blank section: contains all blank imports. This section is not present unless explicitly enabled. + - dot # Dot section: contains all dot imports. This section is not present unless explicitly enabled. + - alias # Alias section: contains all alias imports. This section is not present unless explicitly enabled. + - localmodule # Local module section: contains all local packages. This section is not present unless explicitly enabled. + gofumpt: + # Module path which contains the source code being formatted. + # Default: "" + module-path: github.com/10gen/ops-manager-kubernetes + exclusions: + generated: lax diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..cc3929d41 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,10 @@ +[GENERAL] +max-line-length=120 + +[MESSAGES CONTROL] +disable= + C0114, # missing-module-docstring + C0115, # missing-class-docstring + C0116, # missing-docstring + R0201, # no-self-use + W0621, # redefined-outer-name diff --git a/.rsyncignore b/.rsyncignore new file mode 100644 index 000000000..3848c7cdf --- /dev/null +++ b/.rsyncignore @@ -0,0 +1,2 @@ +public/ +.git diff --git a/.rsyncinclude b/.rsyncinclude new file mode 100644 index 000000000..41f55daff --- /dev/null +++ b/.rsyncinclude @@ -0,0 +1 @@ +/scripts/dev/contexts/private-context diff --git a/Makefile b/Makefile index 6f1811c8f..c006caac2 100644 --- a/Makefile +++ b/Makefile @@ -1,226 +1,342 @@ SHELL := /bin/bash -MONGODB_COMMUNITY_CONFIG ?= $(HOME)/.community-operator-dev/config.json +all: manager -# Image URL to use all building/pushing image targets -REPO_URL := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) -OPERATOR_IMAGE := $(shell jq -r .operator_image < $(MONGODB_COMMUNITY_CONFIG)) -NAMESPACE := $(shell jq -r .namespace < $(MONGODB_COMMUNITY_CONFIG)) -UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < $(MONGODB_COMMUNITY_CONFIG)) -READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < $(MONGODB_COMMUNITY_CONFIG)) -REGISTRY := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) -AGENT_IMAGE_NAME := $(shell jq -r .agent_image < $(MONGODB_COMMUNITY_CONFIG)) -HELM_CHART ?= ./helm-charts/charts/community-operator - -STRING_SET_VALUES := --set namespace=$(NAMESPACE),versionUpgradeHook.name=$(UPGRADE_HOOK_IMG),readinessProbe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operatorImageName=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.versionUpgradeHook=$(REGISTRY),registry.readinessProbe=$(REGISTRY),registry.operator=$(REGISTRY),versionUpgradeHook.version=latest,readinessProbe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) -STRING_SET_VALUES_LOCAL := $(STRING_SET_VALUES) --set operator.replicas=0 - -DOCKERFILE ?= operator -# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:crdVersions=v1" -RELEASE_NAME_HELM ?= mongodb-kubernetes-operator -TEST_NAMESPACE ?= $(NAMESPACE) +export MAKEFLAGS="-j 16" # enable parallelism + +usage: + @ echo "Development utility to work with Operator on daily basis. Just edit your configuration in '~/.operator-dev/contexts', " + @ echo "switch to it using 'make switch', make sure Ops Manager is running (use 'make om') " + @ echo "and call 'make' to start the Kubernetes cluster and the Operator" + @ echo + @ echo "More information can be found by the link https://wiki.corp.mongodb.com/display/MMS/Setting+up+local+development+and+E2E+testing" + @ echo + @ echo "Usage:" + @ echo " prerequisites: installs the command line applications necessary for working with this tool and adds git pre-commit hook." + @ echo " switch: switch current dev context, e.g 'make switch context=kops'. Note, that it switches" + @ echo " kubectl context as well and sets the current namespace to the one configured as the default" + @ echo " one" + @ echo " operator: build and push Operator image, deploy it to the Kubernetes cluster" + @ echo " Use the 'debug' flag to build and deploy the Operator in debug mode - you need" + @ echo " to ensure the 30042 port on the K8s node is open" + @ echo " Use the 'watch_namespace' flag to specify a namespace to watch or leave empty to watch project namespace." + @ echo " database: build and push Database image" + @ echo " full: ('make' is an alias for this command) ensures K8s cluster is up, cleans Kubernetes" + @ echo " resources, build-push-deploy operator, push-deploy database, create secrets, " + @ echo " config map, resources etc" + @ echo " appdb: build and push AppDB image. Specify 'om_version' in format '4.2.1' to provide the already released Ops Manager" + @ echo " version which will be used to find the matching tag and find the Automation Agent version. Add 'om_branch' " + @ echo " if Ops Manager is not released yet and you want to have some git branch as the source " + @ echo " parameters in ~/operator-dev/om" + @ echo " reset: cleans all Operator related state from Kubernetes and Ops Manager. Pass the 'light=true'" + @ echo " to perform a \"light\" cleanup - delete only Mongodb resources" + @ echo " e2e: runs the e2e test, e.g. 'make e2e test=e2e_sharded_cluster_pv'. The Operator is redeployed before" + @ echo " the test, the namespace is cleaned. The e2e app image is built and pushed. Use a 'light=true'" + @ echo " in case you are developing tests and not changing the application code - this will allow to" + @ echo " avoid rebuilding Operator/Database/Init images. Use 'debug=true' to run operator in debug mode." + @ echo " Use a 'local=true' to run the test locally using 'pytest'." + @ echo " Use a 'skip=true' to skip cleaning resources (this may help developing long-running tests like for Ops Manager)" + @ echo " Sometimes you may need to pass some custom configuration, this can be done this way:" + @ echo " make e2e test=e2e_om_ops_manager_upgrade CUSTOM_OM_VERSION=4.2.8" + @ echo " recreate-e2e-kops: deletes and creates a specified e2e cluster 'cluster' using kops (note, that you don't need to switch to the correct" + @ echo " kubectl context - the script will handle everything). Pass the flag 'imsure=yes' to make it work." + @ echo " Pass 'cluster' parameter for a cluster name if it's different from default ('e2e.mongokubernetes.com')" + @ echo " Possible values are: 'e2e.om.mongokubernetes.com', 'e2e.multinamespace.mongokubernetes.com'" + @ echo " recreate-e2e-openshift: deletes and creates an e2e Openshift cluster" + @ echo " recreate-e2e-multicluster-kind Recreates local (Kind-based) development environment for running tests" + @ echo " log: reads the Operator log" + @ echo " status: prints the current context and the state of Kubernetes cluster" + @ echo " dashboard: opens the Kubernetes dashboard. Make sure the cluster was installed using current Makefile as" + @ echo " dashboard is not installed by default and the script ensures it's installed and permissions" + @ echo " are configured." + @ echo " open-automation-config/ac: displays the contents of the Automation Config in in $EDITOR using ~/.operator-dev configuration" + + +# install all necessary software, must be run only once +prerequisites: + @ scripts/dev/install.sh + +precommit: + @ EVERGREEN_MODE=true .githooks/pre-commit + +switch: + @ scripts/dev/switch_context.sh $(context) $(additional_override) + +# builds the Operator binary file and docker image and pushes it to the remote registry if using a remote registry. Deploys it to +# k8s cluster +operator: configure-operator build-and-push-operator-image + @ $(MAKE) deploy-operator + +# build-push, (todo) restart database +database: aws_login + @ scripts/evergreen/run_python.sh pipeline.py --include database + +# ensures cluster is up, cleans Kubernetes + OM, build-push-deploy operator, +# push-deploy database, create secrets, config map, resources etc +full: build-and-push-images + @ $(MAKE) deploy-and-configure-operator + +# build-push appdb image +appdb: aws_login + @ scripts/evergreen/run_python.sh pipeline.py --include appdb + +# runs the e2e test: make e2e test=e2e_sharded_cluster_pv. The Operator is redeployed before the test, the namespace is cleaned. +# The e2e test image is built and pushed together with all main ones (operator, database, init containers) +# Use 'light=true' parameter to skip images rebuilding - use this mode when you are focused on e2e tests development only +e2e: build-and-push-test-image + @ if [[ -z "$(skip)" ]]; then \ + $(MAKE) reset; \ + fi + @ if [[ -z "$(light)" ]]; then \ + $(MAKE) build-and-push-images; \ + fi + @ scripts/dev/launch_e2e.sh -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif +e2e-telepresence: build-and-push-test-image + telepresence connect --context $(test_pod_cluster); scripts/dev/launch_e2e.sh; telepresence quit -BASE_GO_PACKAGE = github.com/mongodb/mongodb-kubernetes-operator -GO_LICENSES = go-licenses -DISALLOWED_LICENSES = restricted # found reciprocal MPL-2.0 +# deletes and creates a kops e2e cluster +recreate-e2e-kops: + @ scripts/dev/recreate_e2e_kops.sh $(imsure) $(cluster) -all: manager +# clean all kubernetes cluster resources and OM state +reset: + go run scripts/dev/reset.go -##@ Development +status: + @ scripts/dev/status -fmt: ## Run go fmt against code - go fmt ./... +# opens the automation config in your editor +open-automation-config: ac +ac: + @ scripts/dev/print_automation_config.sh -vet: ## Run go vet against code - go vet ./... +############################################################################### +# Internal Targets +# These won't do anything bad if you call them, they just aren't the ones that +# were designed to be helpful by themselves. Anything below won't be documented +# in the usage target above. +############################################################################### -generate: controller-gen ## Generate code - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." +# dev note on '&> /dev/null || true': if the 'aws_login' is run in parallel (e.g. 'make' launches builds for images +# in parallel and both call 'aws_login') then Docker login may return an error "Error saving credentials:..The +# specified item already exists in the keychain". Seems this allows to ignore the error +aws_login: + @ scripts/dev/configure_docker_auth.sh + +# cleans up aws resources, including s3 buckets which are older than 5 hours +aws_cleanup: + @ scripts/evergreen/prepare_aws.sh + +build-and-push-operator-image: aws_login + @ scripts/evergreen/run_python.sh pipeline.py --include operator-quick + +build-and-push-database-image: aws_login + @ scripts/dev/build_push_database_image -$(GO_LICENSES): - @if ! which $@ &> /dev/null; then \ - go install github.com/google/go-licenses@latest; \ +build-and-push-test-image: aws_login build-multi-cluster-binary + @ if [[ -z "$(local)" ]]; then \ + scripts/evergreen/run_python.sh pipeline.py --include test; \ fi -licenses.csv: go.mod $(GO_LICENSES) ## Track licenses in a CSV file - @echo "Tracking licenses into file $@" - @echo "========================================" - GOOS=linux GOARCH=amd64 $(GO_LICENSES) csv --include_tests $(BASE_GO_PACKAGE)/... > $@ - -# We only check that go.mod is NOT newer than licenses.csv because the CI -# tends to generate slightly different results, so content comparison wouldn't work -licenses-tracked: ## Checks license.csv is up to date - @if [ go.mod -nt licenses.csv ]; then \ - echo "License.csv is stale! Please run 'make licenses.csv' and commit"; exit 1; \ - else echo "License.csv OK (up to date)"; fi - -.PHONY: check-licenses-compliance -check-licenses-compliance: licenses.csv ## Check licenses are compliant with our restrictions - @echo "Checking licenses not to be: $(DISALLOWED_LICENSES)" - @echo "============================================" - GOOS=linux GOARCH=amd64 $(GO_LICENSES) check --include_tests $(BASE_GO_PACKAGE)/... \ - --disallowed_types $(DISALLOWED_LICENSES) - @echo "--------------------" - @echo "Licenses check: PASS" - -.PHONY: check-licenses -check-licenses: licenses-tracked check-licenses-compliance ## Check license tracking & compliance - -TEST ?= ./pkg/... ./api/... ./cmd/... ./controllers/... ./test/e2e/util/mongotester/... -test: generate fmt vet manifests ## Run unit tests - go test $(options) $(TEST) -coverprofile cover.out - -manager: generate fmt vet ## Build operator binary - go build -o bin/manager ./cmd/manager/main.go - -run: install ## Run the operator against the configured Kubernetes cluster in ~/.kube/config - eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ - go run ./cmd/manager/main.go - -debug: install install-rbac ## Run the operator in debug mode with dlv - eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ - dlv debug ./cmd/manager/main.go +build-multi-cluster-binary: + scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh -CONTROLLER_GEN = $(shell pwd)/bin/controller-gen -controller-gen: ## Download controller-gen locally if necessary - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0) +# builds all app images in parallel +# note that we cannot build both appdb and database init images in parallel as they change the same docker file +build-and-push-images: build-and-push-operator-image appdb-init-image om-init-image database operator-image database-init-image + @ $(MAKE) agent-image -# Try to use already installed helm from PATH -ifeq (ok,$(shell test -f "$$(which helm)" && echo ok)) - HELM=$(shell which helm) -else - HELM=/usr/local/bin/helm -endif +# builds all init images +build-and-push-init-images: appdb-init-image om-init-image database-init-image -helm: ## Download helm locally if necessary - $(call install-helm) +database-init-image: + @ scripts/evergreen/run_python.sh pipeline.py --include init-database -install-prerequisites-macos: ## installs prerequisites for macos development - scripts/dev/install_prerequisites.sh +appdb-init-image: + @ scripts/evergreen/run_python.sh pipeline.py --include init-appdb -##@ Installation/Uninstallation +# Not setting a parallel-factor will default to 0 which will lead to using all CPUs, that can cause docker to die. +# Here we are defaulting to 6, a higher value might work for you. +agent-image: + @ scripts/evergreen/run_python.sh pipeline.py --include agent --all-agents --parallel --parallel-factor 6 -install: manifests helm install-crd ## Install CRDs into a cluster +agent-image-slow: + @ scripts/evergreen/run_python.sh pipeline.py --include agent --parallel-factor 1 -install-crd: - kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +operator-image: + @ scripts/evergreen/run_python.sh pipeline.py --include operator -install-chart: uninstall-crd - $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) $(HELM_CHART) --namespace $(NAMESPACE) --create-namespace +om-init-image: + @ scripts/evergreen/run_python.sh pipeline.py --include init-ops-manager -install-chart-local-operator: uninstall-crd - $(HELM) upgrade --install $(STRING_SET_VALUES_LOCAL) $(RELEASE_NAME_HELM) $(HELM_CHART) --namespace $(NAMESPACE) --create-namespace +om-image: + @ scripts/evergreen/run_python.sh pipeline.py --include ops-manager -prepare-local-dev: generate-env-file install-chart-local-operator install-rbac setup-sas +configure-operator: + @ scripts/dev/configure_operator.sh -# patches all sas to use the local-image-registry -setup-sas: - scripts/dev/setup_sa.sh +deploy-and-configure-operator: deploy-operator configure-operator -install-chart-with-tls-enabled: - $(HELM) upgrade --install --set createResource=true $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) $(HELM_CHART) --namespace $(NAMESPACE) --create-namespace +cert: + @ openssl req -nodes -new -x509 -keyout ca-tls.key -out ca-tls.crt -extensions v3_ca -days 3650 + @ mv ca-tls.key ca-tls.crt docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ + @ cat docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.crt \ + docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb-download.crt \ + > docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls-full-chain.crt -install-rbac: - $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml $(HELM_CHART) | kubectl apply -f - - $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl apply -f - +.PHONY: recreate-e2e-multicluster-kind +recreate-e2e-multicluster-kind: + scripts/dev/recreate_kind_clusters.sh -uninstall-crd: - kubectl delete crd --ignore-not-found mongodbcommunity.mongodbcommunity.mongodb.com +#################################### +## operator-sdk provided Makefile ## +#################################### +# +# The next section is the Makefile provided by operator-sdk. +# We'll start moving to use some of the targets provided by it, like +# `manifests` and `bundle`. For now we'll try to maintain both. -uninstall-chart: - $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) -uninstall-rbac: - $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml $(HELM_CHART) | kubectl delete -f - - $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl delete -f - +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 1.15.0 -uninstall: manifests helm uninstall-chart uninstall-crd ## Uninstall CRDs from a cluster +# EXPIRES sets a label to expire images (quay specific) +EXPIRES := --label quay.expires-after=48h -##@ Deployment +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=preview,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="preview,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif -deploy: manifests helm install-chart install-crd ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) -undeploy: uninstall-chart uninstall-crd ## UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= controller-bundle:$(VERSION) -manifests: controller-gen ## Generate manifests e.g. CRD, RBAC etc. - $(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./..." output:crd:artifacts:config=config/crd/bases - cp config/crd/bases/* $(HELM_CHART)/crds +# Image URL to use all building/pushing image targets +IMG ?= mongodb-enterprise-operator:latest +# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) +CRD_OPTIONS ?= "crd" -##@ E2E +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif -# Run e2e tests locally using go build while also setting up a proxy in the shell to allow -# the test to run as if it were inside the cluster. This enables mongodb connectivity while running locally. -# "MDB_LOCAL_OPERATOR=true" ensures the operator pod is not spun up while running the e2e test - since you're -# running it locally. -e2e-telepresence: cleanup-e2e install ## Run e2e tests locally using go build while also setting up a proxy e.g. make e2e-telepresence test=replica_set cleanup=true - export MDB_LOCAL_OPERATOR=true; \ - telepresence connect; \ - eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ - go test -v -timeout=30m -failfast $(options) ./test/e2e/$(test) ; \ - telepresence quit -e2e-k8s: cleanup-e2e install e2e-image ## Run e2e test by deploying test image in kubernetes, you can provide e2e.py flags e.g. make e2e-k8s test=replica_set e2eflags="--perform-cleanup". - python scripts/dev/e2e.py $(e2eflags) --test $(test) +# Run tests +ENVTEST_ASSETS_DIR=$(shell pwd)/testbin -e2e: cleanup-e2e install ## Run e2e test locally. e.g. make e2e test=replica_set cleanup=true - eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ - go test -v -short -timeout=30m -failfast $(options) ./test/e2e/$(test) +golang-tests: + scripts/evergreen/unit-tests.sh -e2e-gh: ## Trigger a Github Action of the given test - scripts/dev/run_e2e_gh.sh $(test) +golang-tests-race: + USE_RACE=true scripts/evergreen/unit-tests.sh -cleanup-e2e: ## Cleans up e2e test env - kubectl delete mdbc,all,secrets -l e2e-test=true -n ${TEST_NAMESPACE} || true - # Most of the tests use StatefulSets, which in turn use stable storage. In order to - # avoid interleaving tests with each other, we need to drop them all. - kubectl delete pvc --all -n $(NAMESPACE) || true - kubectl delete pv --all -n $(NAMESPACE) || true +sbom-tests: + @ scripts/evergreen/run_python.sh -m pytest generate_ssdlc_report_test.py -generate-env-file: ## generates a local-test.env for local testing - mkdir -p .community-operator-dev - { python scripts/dev/get_e2e_env_vars.py | tee >(cut -d' ' -f2 > .community-operator-dev/local-test.env) ;} > .community-operator-dev/local-test.export.env - . .community-operator-dev/local-test.export.env +python-tests: + @ scripts/evergreen/run_python.sh -m pytest pipeline_test.py + @ scripts/evergreen/run_python.sh -m pytest lib/sonar + @ scripts/evergreen/run_python.sh -m pytest scripts/evergreen/release/agent_matrix_test.py + @ scripts/evergreen/run_python.sh -m pytest docker/mongodb-enterprise-tests/kubeobject -##@ Image +generate-ssdlc-report: + @ scripts/evergreen/run_python.sh generate_ssdlc_report.py -operator-image: ## Build and push the operator image - python pipeline.py --image-name operator $(IMG_BUILD_ARGS) +# test-race runs golang test with race enabled +test-race: generate fmt vet manifests golang-tests-race -e2e-image: ## Build and push e2e test image - python pipeline.py --image-name e2e $(IMG_BUILD_ARGS) +test: generate fmt vet manifests golang-tests -agent-image: ## Build and push agent image - python pipeline.py --image-name agent $(IMG_BUILD_ARGS) +# all-tests will run golang and python tests without race (used locally) +all-tests: test python-tests -readiness-probe-image: ## Build and push readiness probe image - python pipeline.py --image-name readiness-probe $(IMG_BUILD_ARGS) +# Build manager binary +manager: generate fmt vet + GOOS=linux GOARCH=amd64 go build -o docker/mongodb-enterprise-operator/content/mongodb-enterprise-operator main.go -version-upgrade-post-start-hook-image: ## Build and push version upgrade post start hook image - python pipeline.py --image-name version-upgrade-hook $(IMG_BUILD_ARGS) +# Run against the configured Kubernetes cluster in ~/.kube/config +run: generate fmt vet manifests + go run ./main.go -all-images: operator-image e2e-image agent-image readiness-probe-image version-upgrade-post-start-hook-image ## create all required images +# Install CRDs into a cluster +install: manifests kustomize + $(KUSTOMIZE) build config/crd | kubectl apply -f - -define install-helm -@[ -f $(HELM) ] || { \ -set -e ;\ -TMP_DIR=$$(mktemp -d) ;\ -cd $$TMP_DIR ;\ -curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 ;\ -chmod 700 get_helm.sh ;\ -./get_helm.sh ;\ -rm -rf $(TMP_DIR) ;\ -} -endef +# Uninstall CRDs from a cluster +uninstall: manifests kustomize + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +# Deploy controller in the configured Kubernetes cluster in ~/.kube/config +deploy: manifests kustomize + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/default | kubectl apply -f - + +# UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config +undeploy: + $(KUSTOMIZE) build config/default | kubectl delete -f - + +# Generate manifests e.g. CRD, RBAC etc. +manifests: controller-gen + export PATH="$(PATH)"; export GOROOT=$(GOROOT); $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths=./... output:crd:artifacts:config=config/crd/bases + # copy the CRDs to the public folder + cp config/crd/bases/* helm_chart/crds/ + cat "helm_chart/crds/"* > public/crds.yaml + + +# Run go fmt against code +fmt: + go fmt ./... + +# Run go vet against code +vet: + go vet ./... + +# Generate code +generate: controller-gen + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +# Build the docker image +docker-build: test + docker build -t $(IMG) . + +# Push the docker image +docker-push: + docker push $(IMG) + +# Download controller-gen locally if necessary +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +controller-gen: + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0) + +# Download kustomize locally if necessary +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.4) -# go-install-tool will 'go install' any package $2 and install it to $1. +# go-install-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-install-tool @[ -f $(1) ] || { \ @@ -234,9 +350,30 @@ rm -rf $$TMP_DIR ;\ } endef -help: ## Show this help screen. - @echo 'Usage: make ... ' - @echo '' - @echo 'Available targets are:' - @echo '' - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +# Generate bundle manifests and metadata, then validate generated files. +.PHONY: bundle +bundle: manifests kustomize + # we want to create a file that only has the deployment. Note: this will not work if something is added + # after the deployment in openshift.yaml + operator-sdk generate kustomize manifests -q + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)\ + --channels=stable --default-channel=stable\ + --output-dir ./bundle/$(VERSION)/ + operator-sdk bundle validate ./bundle/$(VERSION) + + +# Build the bundle image. +.PHONY: bundle-build +bundle-build: + docker build $(EXPIRES) --platform linux/amd64 -f ./bundle/$(VERSION)/bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: dockerfiles +dockerfiles: + python scripts/update_supported_dockerfiles.py + tar -czvf ./public/dockerfiles-$(VERSION).tgz ./public/dockerfiles + +prepare-local-e2e: # prepares the local environment to run a local operator + scripts/dev/prepare_local_e2e_run.sh + +prepare-operator-configmap: # prepares the local environment to run a local operator + source scripts/dev/set_env_context.sh && source scripts/funcs/printing && source scripts/funcs/operator_deployment && prepare_operator_config_map "$(kubectl config current-context)" diff --git a/PROJECT b/PROJECT index fcd3ceff3..c76140b76 100644 --- a/PROJECT +++ b/PROJECT @@ -1,25 +1,36 @@ domain: mongodb.com -layout: -- go.kubebuilder.io/v3 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} -projectName: mko-v1 -repo: github.com/mongodb/mongodb-kubernetes-operator +layout: go.kubebuilder.io/v3 +projectName: mongodb-enterprise +repo: github.com/10gen/ops-manager-kubernetes resources: - api: crdVersion: v1 namespaced: true - group: mongodbcommunity - kind: MongoDBCommunity + controller: true + domain: mongodb.com + group: mongodb + kind: MongoDB + path: github.com/10gen/ops-manager-kubernetes/api/v1 version: v1 - api: crdVersion: v1 namespaced: true controller: true domain: mongodb.com - group: mongodbcommunity - kind: SimpleMongoDBCommunity - path: github.com/mongodb/mongodb-kubernetes-operator/api/v1alpha1 - version: v1alpha1 + group: mongodb + kind: MongoDBOpsManager + path: github.com/10gen/ops-manager-kubernetes/api/v1 + version: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: mongodb.com + group: mongodb + kind: MongoDBUser + path: github.com/10gen/ops-manager-kubernetes/api/v1 + version: v1 version: "3" +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} diff --git a/README.md b/README.md index 8af30d276..c4ab16f44 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,13 @@ -# MongoDB Community Kubernetes Operator # +# Ops Manager Operator # - + -This is a [Kubernetes Operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) which deploys MongoDB Community into Kubernetes clusters. +This is a Kubernetes Operator (https://coreos.com/operators/) to work +with Ops Manager and Kubernetes clusters. It allows you to easily add new +MongoDB deployments (standalones, replica sets, sharded clusters) to your Kubernetes cluster, configure them (modify, scale up/down, remove) and to manage them from your +Ops Manager installation. This provides the combined power of Kubernetes (native scheduling of applications to nodes, scaling, fault tolerance etc) with Ops Manager capabilities (monitoring, backup, upgrades etc) -If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes). +## Development -Here is a talk from MongoDB Live 2020 about the Community Operator: -* [Run it in Kubernetes! Community and Enterprise MongoDB in Containers](https://www.youtube.com/watch?v=2Xszdg-4T6A&t=1368s) - -> **Note** -> -> Hi, I'm Dan Mckean 👋 I'm the Product Manager for MongoDB's support of Kubernetes. -> -> The [Community Operator](https://github.com/mongodb/mongodb-kubernetes-operator) is something I inherited when I started, but it doesn't get as much attention from us as we'd like, and we're trying to understand how it's used in order to establish it's future. It will help us establish exactly what level of support we can offer, and what sort of timeframe we aim to provide support in 🙂 -> ->Here's a super short survey (it's much easier for us to review all the feedback that way!): [https://docs.google.com/forms/d/e/1FAIpQLSfwrwyxBSlUyJ6AmC-eYlgW_3JEdfA48SB2i5--_WpiynMW2w/viewform?usp=sf_link](https://docs.google.com/forms/d/e/1FAIpQLSfwrwyxBSlUyJ6AmC-eYlgW_3JEdfA48SB2i5--_WpiynMW2w/viewform?usp=sf_link) -> -> If you'd rather email me instead: [dan.mckean@mongodb.com](mailto:dan.mckean@mongodb.com?subject=MongoDB%20Community%20Operator%20feedback) - -## Table of Contents - -- [Documentation](#documentation) -- [Supported Features](#supported-features) - - [Planned Features](#planned-features) -- [Contribute](#contribute) -- [License](#license) - -## Documentation - -See the [documentation](docs) to learn how to: - -1. [Install or upgrade](docs/install-upgrade.md) the Operator. -1. [Deploy and configure](docs/deploy-configure.md) MongoDB resources. -1. [Configure Logging](docs/logging.md) of the MongoDB resource components. -1. [Create a database user](docs/users.md) with SCRAM authentication. -1. [Secure MongoDB resource connections](docs/secure.md) using TLS. - -*NOTE: [MongoDB Enterprise Kubernetes Operator](https://www.mongodb.com/docs/kubernetes-operator/master/) docs are for the enterprise operator use case and NOT for the community operator. In addition to the docs mentioned above, you can refer to this [blog post](https://www.mongodb.com/blog/post/run-secure-containerized-mongodb-deployments-using-the-mongo-db-community-kubernetes-oper) as well to learn more about community operator deployment* - -## Supported Features - -The MongoDB Community Kubernetes Operator supports the following features: - -- Create [replica sets](https://www.mongodb.com/docs/manual/replication/) -- Upgrade and downgrade MongoDB server version -- Scale replica sets up and down -- Read from and write to the replica set while scaling, upgrading, and downgrading. These operations are done in an "always up" manner. -- Report MongoDB server state via the [MongoDBCommunity resource](/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml) `status` field -- Use any of the available [Docker MongoDB images](https://hub.docker.com/_/mongo/) -- Connect to the replica set from inside the Kubernetes cluster (no external connectivity) -- Secure client-to-server and server-to-server connections with TLS -- Create users with [SCRAM](https://www.mongodb.com/docs/manual/core/security-scram/) authentication -- Create custom roles -- Enable a [metrics target that can be used with Prometheus](docs/prometheus/README.md) - -## Contribute - -Before you contribute to the MongoDB Community Kubernetes Operator, please read: - -- [MongoDB Community Kubernetes Operator Architecture](docs/architecture.md) -- [Contributing to MongoDB Community Kubernetes Operator](docs/contributing.md) - -Please file issues before filing PRs. For PRs to be accepted, contributors must sign our [CLA](https://www.mongodb.com/legal/contributor-agreement). - -Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). - -## Linting - -This project uses the following linters upon every Pull Request: - -* `gosec` is a tool that find security problems in the code -* `Black` is a tool that verifies if Python code is properly formatted -* `MyPy` is a Static Type Checker for Python -* `Kube-linter` is a tool that verified if all Kubernetes YAML manifests are formatted correctly -* `Go vet` A built-in Go static checker -* `Snyk` The vulnerability scanner - -## License - -Please see the [LICENSE](LICENSE.md) file. +Please see the [starting guide](https://wiki.corp.mongodb.com/display/MMS/Setting+up+local+development+and+E2E+testing). +>>>>>>> master-temp diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 000000000..dd38691fb --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,917 @@ +[//]: # (Consider renaming or removing the header for next release, otherwise it appears as duplicate in the published release, e.g: https://github.com/mongodb/mongodb-enterprise-kubernetes/releases/tag/1.22.0 ) + + +# MongoDB Enterprise Kubernetes Operator 1.33.0 + +## New Features +* **MongoDBOpsManager**, **AppDB**: Introduced support for OpsManager and Application Database deployments across multiple Kubernetes clusters without requiring a Service Mesh. + * New property [spec.applicationDatabase.externalAccess](TBD) used for common service configuration or in single cluster deployments + * Added support for existing, but unused property [spec.applicationDatabase.clusterSpecList.externalAccess](TBD) + * You can define annotations for external services managed by the operator that contain placeholders which will be automatically replaced to the proper values. To learn more please see the relevant documentation: + * AppDB: [spec.applicationDatabase.externalAccess.externalService.annotations](TBD) + * MongoDBOpsManager: Due to different way of configuring external service placeholders are not yet supported + * More details can be found in the [public documentation](TBD). + +## Bug Fixes +* Fixed a bug where workloads in the `static` container architecture were still downloading binaries. This occurred when the operator was running with the default container architecture set to `non-static`, but the workload was deployed with the `static` architecture using the `mongodb.com/v1.architecture: "static"` annotation. +* **AppDB**: Fixed an issue with wrong monitoring hostnames for `Application Database` deployed in multi-cluster mode. Monitoring agents should discover the correct hostnames and send data back to `Ops Manager`. The hostnames used for monitoring AppDB in Multi-Cluster deployments with a service mesh are `{resource_name}-db-{cluster_index}-{pod_index}-svc.{namespace}.svc.{cluster_domain}`. TLS certificate should be defined for these hostnames. + * **NOTE (Multi-Cluster)** This bug fix will result in the loss of historical monitoring data for multi-cluster AppDB members. If retaining this data is critical, please back it up before upgrading. This only affects monitoring data for multi-cluster AppDB deployments — it does not impact single-cluster AppDBs or any other MongoDB deployments managed by this Ops Manager instance. + * To export the monitoring data of AppDB members, please refer to the Ops Manager API reference https://www.mongodb.com/docs/ops-manager/current/reference/api/measures/get-host-process-system-measurements/ +* **OpsManager**: Fixed a bug where the `spec.clusterSpecList.externalConnectivity` field was not being used by the operator, but documented in Ops Manager API reference https://www.mongodb.com/docs/kubernetes-operator/current/reference/k8s-operator-om-specification/#mongodb-opsmgrkube-opsmgrkube.spec.clusterSpecList.externalConnectivity + + + +# MongoDB Enterprise Kubernetes Operator 1.32.0 + +## New Features +* **General Availability - Multi Cluster Sharded Clusters:** Support configuring highly available MongoDB Sharded Clusters across multiple Kubernetes clusters. + - `MongoDB` resources of type Sharded Cluster now support both single and multi cluster topologies. + - The implementation is backwards compatible with single cluster deployments of MongoDB Sharded Clusters, by defaulting `spec.topology` to `SingleCluster`. Existing `MongoDB` resources do not need to be modified to upgrade to this version of the operator. + - Introduced support for Sharded deployments across multiple Kubernetes clusters without requiring a Service Mesh - the is made possible by enabling all components of such a deployment (including mongos, config servers, and mongod) to be exposed externally to the Kubernetes clusters, which enables routing via external interfaces. + - More details can be found in the [public documentation](https://www.mongodb.com/docs/kubernetes-operator/current/reference/k8s-operator-specification/#sharded-cluster-settings). +* Adding opt-out anonymized telemetry to the operator. The data does not contain any Personally Identifiable Information +(PII) or even data that can be tied back to any specific customer or company. More can be read [public documentation](https://www.mongodb.com/docs/kubernetes-operator/current/reference/meko-telemetry), this link further elaborates on the following topics: + * What data is included in the telemetry + * How to disable telemetry + * What RBACs are added and why they are required +* **MongoDB**: To ensure the correctness of scaling operations, a new validation has been added to Sharded Cluster deployments. This validation restricts scaling different components in two directions simultaneously within a single change to the YAML file. For example, it is not allowed to add more nodes (scaling up) to shards while simultaneously removing (scaling down) config servers or mongos. This restriction also applies to multi-cluster deployments. A simple change that involves "moving" one node from one cluster to another—without altering the total number of members—will now be blocked. It is necessary to perform a scale-up operation first and then execute a separate change for scaling down. + +## Bug Fixes +* Fixes the bug when status of `MongoDBUser` was being set to `Updated` prematurely. For example, new users were not immediately usable following `MongoDBUser` creation despite the operator reporting `Updated` state. +* Fixed a bug causing cluster health check issues when ordering of users and tokens differed in Kubeconfig. +* Fixed a bug when deploying a Multi-Cluster sharded resource with an external access configuration could result in pods not being able to reach each others. +* Fixed a bug when setting `spec.fcv = AlwaysMatchVersion` and `agentAuth` to be `SCRAM` causes the operator to set the auth value to be `SCRAM-SHA-1` instead of `SCRAM-SHA-256`. + +# MongoDB Enterprise Kubernetes Operator 1.31.0 + +## Kubernetes versions +* The minimum supported Kubernetes version for this operator is 1.30 and OpenShift 4.17. + +## Bug Fixes +* Fixed handling proxy environment variables in the operator pod. The environment variables [`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`] when set on the operator pod, can now be propagated to the MongoDB agents by also setting the environment variable `MDB_PROPAGATE_PROXY_ENV=true`. + + +# MongoDB Enterprise Kubernetes Operator 1.30.0 + +## New Features + +* **MongoDB**: fixes and improvements to Multi-Cluster Sharded Cluster deployments (Public Preview) +* **MongoDB**: `spec.shardOverrides` field, which was added in 1.28.0 as part of Multi-Cluster Sharded Cluster Public Preview is now fully supported for single-cluster topologies and is the recommended way of customizing settings for specific shards. +* **MongoDB**: `spec.shardSpecificPodSpec` was deprecated. The recommended way of customizing specific shard settings is to use `spec.shardOverrides` for both Single and Multi Cluster topology. An example of how to migrate the settings to spec.shardOverrides is available [here](https://github.com/mongodb/mongodb-enterprise-kubernetes/blob/master/samples/sharded_multicluster/shardSpecificPodSpec_migration.yaml). + +## Bug Fixes +* **MongoDB**: Fixed placeholder name for `mongos` in Single Cluster Sharded with External Domain set. Previously it was called `mongodProcessDomain` and `mongodProcessFQDN` now they're called `mongosProcessDomain` and `mongosProcessFQDN`. +* **MongoDB**, **MongoDBMultiCluster**, **MongoDBOpsManager**: In case of losing one of the member clusters we no longer emit validation errors if the failed cluster still exists in the `clusterSpecList`. This allows easier reconfiguration of the deployments as part of disaster recovery procedure. + +## Kubernetes versions + * The minimum supported Kubernetes version for this operator is 1.29 and OpenShift 4.17. + +# MongoDB Enterprise Kubernetes Operator 1.29.0 + +## New Features +* **AppDB**: Added support for easy resize. More can be read in changelog 1.28.0 - "automated expansion of the pvc" + +## Bug Fixes + +* **MongoDB**, **AppDB**, **MongoDBMultiCluster**: Fixed a bug where specifying a fractional number for a storage volume's size such as `1.7Gi` can break the reconciliation loop for that resource with an error like `Can't execute update on forbidden fields` even if the underlying Persistence Volume Claim is deployed successfully. +* **MongoDB**, **MongoDBMultiCluster**, **OpsManager**, **AppDB**: Increased stability of deployments during TLS rotations. In scenarios where the StatefulSet of the deployment was reconciling and a TLS rotation happened, the deployment would reach a broken state. Deployments will now store the previous TLS certificate alongside the new one. + +# MongoDB Enterprise Kubernetes Operator 1.28.0 + +## New Features + +* **MongoDB**: public preview release of multi kubernetes cluster support for sharded clusters. This can be enabled by setting `spec.topology=MultiCluster` when creating `MongoDB` resource of `spec.type=ShardedCluster`. More details can be found [here](https://www.mongodb.com/docs/kubernetes-operator/master/multi-cluster-sharded-cluster/). +* **MongoDB**, **MongoDBMultiCluster**: support for automated expansion of the PVC. + More details can be found [here](https://www.mongodb.com/docs/kubernetes-operator/upcoming/tutorial/resize-pv-storage/). + **Note**: Expansion of the pvc is only supported if the storageClass supports expansion. + Please ensure that the storageClass supports in-place expansion without data-loss. + * **MongoDB** This can be done by increasing the size of the PVC in the CRD setting: + * one PVC - increase: `spec.persistence.single.storage` + * multiple PVCs - increase: `spec.persistence.multiple.(data/journal/logs).storage` + * **MongoDBMulti** This can be done by increasing the storage via the statefulset override: +```yaml + statefulSet: + spec: + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 2Gi # this is my increased storage + storageClass: +``` +* **MongoDB**, **MongoDBMultiCluster** **AppDB**: change default behaviour of setting featurecompatibilityversion (fcv) for the database. + * When upgrading mongoDB version the operator sets the FCV to the prior version we are upgrading from. This allows to + have sanity checks before setting the fcv to the upgraded version. More information can be found [here](https://www.mongodb.com/docs/kubernetes-operator/current/reference/k8s-operator-specification/#mongodb-setting-spec.featureCompatibilityVersion). + * To keep the prior behaviour to always use the mongoDB version as FCV; set `spec.featureCompatibilityVersion: "AlwaysMatchVersion"` +* Docker images are now built with `ubi9` as the base image with the exception of [mongodb-enterprise-database-ubi](quay.io/mongodb/mongodb-enterprise-database-ubi) which is still based on `ubi8` to support `MongoDB` workloads < 6.0.4. The `ubi8` image is only in use for the default non-static architecture. +For a full `ubi9` setup, the [Static Containers](https://www.mongodb.com/docs/kubernetes-operator/upcoming/tutorial/plan-k8s-op-container-images/#static-containers--public-preview-) architecture should be used instead. +* **OpsManager**: Introduced support for Ops Manager 8.0.0 +* **MongoDB**, **MongoDBMulti**: support for MongoDB 8.0.0 +## Bug Fixes + +* **MongoDB**, **AppDB**, **MongoDBMultiCluster**: Fixed a bug where the init container was not getting the default security context, which was flagged by security policies. +* **MongoDBMultiCluster**: Fixed a bug where resource validations were not performed as part of the reconcile loop. + +# MongoDB Enterprise Kubernetes Operator 1.27.0 + +## New Features + +* **MongoDB** Added Support for enabling LogRotation for MongoDB processes, MonitoringAgent and BackupAgent. More can be found in the following [documentation](LINK TO DOCS). + * `spec.agent.mongod.logRotation` to configure the mongoDB processes + * `spec.agent.mongod.auditLogRotation` to configure the mongoDB processes audit logs + * `spec.agent.backupAgent.logRotation` to configure the backup agent + * `spec.agent.monitoringAgent.logRotation` to configure the backup agent + * `spec.agent.readinessProbe.environmentVariables` to configure the environment variables the readinessProbe runs with. + That also applies to settings related to the logRotation, + the supported environment settings can be found [here](https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/docs/logging.md#readinessprobe). + * the same applies for AppDB: + * you can configure AppDB via `spec.applicationDatabase.agent.mongod.logRotation` + * Please Note: For shardedCluster we only support configuring logRotation under `spec.Agent` + and not per process type (mongos, configsrv etc.) + +* **Opsmanager** Added support for replacing the logback.xml which configures general logging settings like logRotation + * `spec.logging.LogBackAccessRef` points at a ConfigMap/key with the logback access configuration file to mount on the Pod + * the key of the configmap has to be `logback-access.xml` + * `spec.logging.LogBackRef` points at a ConfigMap/key with the logback access configuration file to mount on the Pod + * the key of the configmap has to be `logback.xml` + +## Deprecations + +* **AppDB** logRotate for appdb has been deprecated in favor for the new field + * this `spec.applicationDatabase.agent.logRotation` has been deprecated for `spec.applicationDatabase.agent.mongod.logRotation` + +## Bug Fixes + +* **Agent** launcher: under some resync scenarios we can have corrupted journal data in `/journal`. + The agent now makes sure that there are not conflicting journal data and prioritizes the data from `/data/journal`. + * To deactivate this behaviour set the environment variable in the operator `MDB_CLEAN_JOURNAL` + to any other value than 1. +* **MongoDB**, **AppDB**, **MongoDBMulti**: make sure to use external domains in the connectionString created if configured. + +* **MongoDB**: Removed panic response when configuring shorter horizon config compared to number of members. The operator now signals a +descriptive error in the status of the **MongoDB** resource. + +* **MongoDB**: Fixed a bug where creating a resource in a new project named as a prefix of another project would fail, preventing the `MongoDB` resource to be created. + +# MongoDB Enterprise Kubernetes Operator 1.26.0 + +## New Features + +* Added the ability to control how many reconciles can be performed in parallel by the operator. + This enables strongly improved cpu utilization and vertical scaling of the operator and will lead to quicker reconcile of all managed resources. + * It might lead to increased load on the Ops Manager and K8s API server in the same time window. + by setting `MDB_MAX_CONCURRENT_RECONCILES` for the operator deployment or `operator.maxConcurrentReconciles` in the operator's Helm chart. + If not provided, the default value is 1. + * Observe the operator's resource usage and adjust (`operator.resources.requests` and `operator.resources.limits`) if needed. + +## Helm Chart + +* New `operator.maxConcurrentReconciles` parameter. It controls how many reconciles can be performed in parallel by the operator. The default value is 1. +* New `operator.webhook.installClusterRole` parameter. It controls whether to install the cluster role allowing the operator to configure admission webhooks. It should be disabled when cluster roles are not allowed. Default value is true. + +## Bug Fixes + +* **MongoDB**: Fixed a bug where configuring a **MongoDB** with multiple entries in `spec.agent.startupOptions` would cause additional unnecessary reconciliation of the underlying `StatefulSet`. +* **MongoDB, MongoDBMultiCluster**: Fixed a bug where the operator wouldn't watch for changes in the X509 certificates configured for agent authentication. +* **MongoDB**: Fixed a bug where boolean flags passed to the agent cannot be set to `false` if their default value is `true`. + +# MongoDB Enterprise Kubernetes Operator 1.25.0 + +## New Features + +* **MongoDBOpsManager**: Added support for deploying Ops Manager Application on multiple Kubernetes clusters. See [documentation](LINK TO DOCS) for more information. +* (Public Preview) **MongoDB, OpsManager**: Introduced opt-in Static Architecture (for all types of deployments) that avoids pulling any binaries at runtime. + * This feature is recommended only for testing purposes, but will become the default in a later release. + * You can activate this mode by setting the `MDB_DEFAULT_ARCHITECTURE` environment variable at the Operator level to `static`. Alternatively, you can annotate a specific `MongoDB` or `OpsManager` Custom Resource with `mongodb.com/v1.architecture: "static"`. + * The Operator supports seamless migration between the Static and non-Static architectures. + * To learn more please see the relevant documentation: + * [Use Static Containers](https://www.mongodb.com/docs/kubernetes-operator/stable/tutorial/plan-k8s-op-considerations/#use-static-containers--beta-) + * [Migrate to Static Containers](https://www.mongodb.com/docs/kubernetes-operator/stable/tutorial/plan-k8s-op-container-images/#migrate-to-static-containers) +* **MongoDB**: Recover Resource Due to Broken Automation Configuration has been extended to all types of MongoDB resources, now including Sharded Clusters. For more information see https://www.mongodb.com/docs/kubernetes-operator/master/reference/troubleshooting/#recover-resource-due-to-broken-automation-configuration +* **MongoDB, MongoDBMultiCluster**: Placeholders in external services. + * You can now define annotations for external services managed by the operator that contain placeholders which will be automatically replaced to the proper values. + * Previously, the operator was configuring the same annotations for all external services created for each pod. Now, with placeholders the operator is able to customize + annotations in each service with values that are relevant and different for the particular pod. + * To learn more please see the relevant documentation: + * MongoDB: [spec.externalAccess.externalService.annotations](https://www.mongodb.com/docs/kubernetes-operator/stable/reference/k8s-operator-specification/#mongodb-setting-spec.externalAccess.externalService.annotations) + * MongoDBMultiCluster: [spec.externalAccess.externalService.annotations](https://www.mongodb.com/docs/kubernetes-operator/stable/reference/k8s-operator-multi-cluster-specification/#mongodb-setting-spec.externalAccess.externalService.annotations) +* `kubectl mongodb`: + * Added printing build info when using the plugin. + * `setup` command: + * Added `--image-pull-secrets` parameter. If specified, created service accounts will reference the specified secret on `ImagePullSecrets` field. + * Improved handling of configurations when the operator is installed in a separate namespace than the resources it's watching and when the operator is watching more than one namespace. + * Optimized roles and permissions setup in member clusters, using a single service account per cluster with correctly configured Role and RoleBinding (no ClusterRoles necessary) for each watched namespace. +* **OpsManager**: Added the `spec.internalConnectivity` field to allow overrides for the service used by the operator to ensure internal connectivity to the `OpsManager` pods. +* Extended the existing event based reconciliation by a time-based one, that is triggered every 24 hours. This ensures all Agents are always upgraded on timely manner. +* OpenShift / OLM Operator: Removed the requirement for cluster-wide permissions. Previously, the operator needed these permissions to configure admission webhooks. Now, webhooks are automatically configured by [OLM](https://olm.operatorframework.io/docs/advanced-tasks/adding-admission-and-conversion-webhooks/). +* Added optional `MDB_WEBHOOK_REGISTER_CONFIGURATION` environment variable for the operator. It controls whether the operator should perform automatic admission webhook configuration. Default: true. It's set to false for OLM and OpenShift deployments. + +## Breaking Change + +* **MongoDBOpsManager** Stopped testing against Ops Manager 5.0. While it may continue to work, we no longer officially support Ops Manager 5 and customers should move to a later version. + +## Helm Chart + +* New `operator.webhook.registerConfiguration` parameter. It controls whether the operator should perform automatic admission webhook configuration (by setting `MDB_WEBHOOK_REGISTER_CONFIGURATION` environment variable for the operator). Default: true. It's set to false for OLM and OpenShift deployments. +* Changing the default `agent.version` to `107.0.0.8502-1`, that will change the default agent used in helm deployments. +* Added `operator.additionalArguments` (default: []) allowing to pass additional arguments for the operator binary. +* Added `operator.createResourcesServiceAccountsAndRoles` (default: true) to control whether to install roles and service accounts for MongoDB and Ops Manager resources. When `mongodb kubectl` plugin is used to configure the operator for multi-cluster deployment, it installs all necessary roles and service accounts. Therefore, in some cases it is required to not install those roles using the operator's helm chart to avoid clashes. + +## Bug Fixes + +* **MongoDBMultiCluster**: Fields `spec.externalAccess.externalDomain` and `spec.clusterSpecList[*].externalAccess.externalDomains` were reported as required even though they weren't +used. Validation was triggered prematurely when structure `spec.externalAccess` was defined. Now, uniqueness of external domains will only be checked when the external domains are +actually defined in `spec.externalAccess.externalDomain` or `spec.clusterSpecList[*].externalAccess.externalDomains`. +* **MongoDB**: Fixed a bug where upon deleting a **MongoDB** resource the `controlledFeature` policies are not unset on the related OpsManager/CloudManager instance, making cleanup in the UI impossible in the case of losing the kubernetes operator. +* **OpsManager**: The `admin-key` Secret is no longer deleted when removing the OpsManager Custom Resource. This enables easier Ops Manager re-installation. +* **MongoDB ReadinessProbe** Fixed the misleading error message of the readinessProbe: `"... kubelet Readiness probe failed:..."`. This affects all mongodb deployments. +* **Operator**: Fixed cases where sometimes while communicating with Opsmanager the operator skipped TLS verification, even if it was activated. + +## Improvements + +**Kubectl plugin**: The released plugin binaries are now signed, the signatures are published with the [release assets](https://github.com/mongodb/mongodb-enterprise-kubernetes/releases). Our public key is available at [this address](https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem). They are also notarized for MacOS. +**Released Images signed**: All container images published for the enterprise operator are cryptographically signed. This is visible on our Quay registry, and can be verified using our public key. It is available at [this address](https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem). + + +# MongoDB Enterprise Kubernetes Operator 1.24.0 + +## New Features +* **MongoDBOpsManager**: Added support for the upcoming 7.0.x series of Ops Manager Server. + +## Bug Fixes +* Fix a bug that prevented terminating backup correctly. + +# MongoDB Enterprise Kubernetes Operator 1.23.0 +## Warnings and Breaking Changes + +* Starting from 1.23 component image version numbers will be aligned to the MongoDB Enterprise Operator release tag. This allows clear identification of all images related to a specific version of the Operator. This affects the following images: + * `quay.io/mongodb/mongodb-enterprise-database-ubi` + * `quay.io/mongodb/mongodb-enterprise-init-database-ubi` + * `quay.io/mongodb/mongodb-enterprise-init-appdb-ubi` + * `quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi` +* Removed `spec.exposedExternally` in favor of `spec.externalAccess` from the MongoDB Customer Resource. `spec.exposedExternally` was deprecated in operator version 1.19. + +## Bug Fixes +* Fix a bug with scaling a multi-cluster replica-set in the case of losing connectivity to a member cluster. The fix addresses both the manual and automated recovery procedures. +* Fix of a bug where changing the names of the automation agent and MongoDB audit logs prevented them from being sent to Kubernetes pod logs. There are no longer restrictions on MongoDB audit log file names (mentioned in the previous release). +* New log types from the `mongodb-enterprise-database` container are now streamed to Kubernetes logs. + * New log types: + * agent-launcher-script + * monitoring-agent + * backup-agent + * The rest of available log types: + * automation-agent-verbose + * automation-agent-stderr + * automation-agent + * mongodb + * mongodb-audit +* **MongoDBUser** Fix a bug ignoring the `Spec.MongoDBResourceRef.Namespace`. This prevented storing the user resources in another namespace than the MongoDB resource. + +# MongoDB Enterprise Kubernetes Operator 1.22.0 +## Breaking Changes +* **All Resources**: The Operator no longer uses the "Reconciling" state. In most of the cases it has been replaced with "Pending" and a proper message + +## Deprecations +None + +## Bug Fixes +* **MongoDB**: Fix support for setting `autoTerminateOnDeletion=true` for sharded clusters. This setting makes sure that the operator stops and terminates the backup before the cleanup. + +## New Features +* **MongoDB**: An Automatic Recovery mechanism has been introduced for `MongoDB` resources and is turned on by default. If a Custom Resource remains in `Pending` or `Failed` state for a longer period of time (controlled by `MDB_AUTOMATIC_RECOVERY_BACKOFF_TIME_S` environment variable at the Operator Pod spec level, the default is 20 minutes) + the Automation Config is pushed to the Ops Manager. This helps to prevent a deadlock when an Automation Config can not be pushed because of the StatefulSet not being ready and the StatefulSet being not ready because of a broken Automation Config. + The behavior can be turned off by setting `MDB_AUTOMATIC_RECOVERY_ENABLE` environment variable to `false`. +* **MongoDB**: MongoDB audit logs can now be routed to Kubernetes pod logs. + * Ensure MongoDB audit logs are written to `/var/log/mongodb-mms-automation/mongodb-audit.log` file. Pod monitors this file and tails its content to k8s logs. + * Use the following example configuration in MongoDB resource to send audit logs to k8s logs: + ``` + spec: + additionalMongodConfig: + auditLog: + destination: file + format: JSON + path: /var/log/mongodb-mms-automation/mongodb-audit.log + ``` + * Audit log entries are tagged with the "mongodb-audit" key in pod logs. Extract audit log entries with the following example command: + ``` + kubectl logs -c mongodb-enterprise-database replica-set-0 | jq -r 'select(.logType == "mongodb-audit") | .contents' + ``` +* **MongoDBOpsManager**: Improved handling of unreachable clusters in AppDB Multi-Cluster resources + * In the last release, the operator required a healthy connection to the cluster to scale down processes, which could block the reconcile process if there was a full-cluster outage. + * Now, the operator will still successfully manage the remaining healthy clusters, as long as they have a majority of votes to elect a primary. + * The associated processes of an unreachable cluster are not automatically removed from the automation config and replica set configuration. These processes will only be removed under the following conditions: + * The corresponding cluster is deleted from `spec.applicationDatabase.clusterSpecList` or has zero members specified. + * When deleted, the operator scales down the replica set by removing processes tied to that cluster one at a time. +* **MongoDBOpsManager**: Add support for configuring [logRotate](https://www.mongodb.com/docs/ops-manager/current/reference/cluster-configuration/#mongodb-instances) on the automation-agent for appdb. +* **MongoDBOpsManager**: [systemLog](https://www.mongodb.com/docs/manual/reference/configuration-options/#systemlog-options) can now be configured to differ from the otherwise default of `/var/log/mongodb-mms-automation`. + +# MongoDB Enterprise Kubernetes Operator 1.21.0 +## Breaking changes +* The environment variable to track the operator namespace has been renamed from [CURRENT_NAMESPACE](https://github.com/mongodb/mongodb-enterprise-kubernetes/blob/master/mongodb-enterprise.yaml#L244) to `NAMESPACE`. If you set this variable manually via YAML files, you should update this environment variable name while upgrading the operator deployment. + +## Bug fixes +* Fixes a bug where passing the labels via statefulset override mechanism would not lead to an override on the actual statefulset. + +## New Feature +* Support for Label and Annotations Wrapper for the following CRDs: mongodb, mongodbmulti and opsmanager + * Additionally, to the `specWrapper` for `statefulsets` we now support overriding `metadata.Labels` and `metadata.Annotations` via the `MetadataWrapper`. + +# MongoDBOpsManager Resource + +## New Features +* Support configuring `OpsManager` with a highly available `applicationDatabase` across multiple Kubernetes clusters by introducing the following fields: + - `om.spec.applicationDatabase.topology` which can be one of `MultiCluster` and `SingleCluster`. + - `om.spec.applicationDatabase.clusterSpecList` for configuring the list of Kubernetes clusters which will have For extended considerations for the multi-cluster AppDB configuration, check [the official guide](https://www.mongodb.com/docs/kubernetes-operator/stable/tutorial/plan-om-resource.html#using-onprem-with-multi-kubernetes-cluster-deployments) and the `OpsManager` [resource specification](https://www.mongodb.com/docs/kubernetes-operator/stable/reference/k8s-operator-om-specification/#k8s-om-specification). +The implementation is backwards compatible with single cluster deployments of AppDB, by defaulting `om.spec.applicationDatabase.topology` to `SingleCluster`. Existing `OpsManager` resources do not need to be modified to upgrade to this version of the operator. +* Support for providing a list of custom certificates for S3 based backups via secret references `spec.backup.[]s3Stores.customCertificateSecretRefs` and `spec.backup.[]s3OpLogStores.customCertificateSecretRefs` + * The list consists of single certificate strings, each references a secret containing a certificate authority. + * We do not support adding multiple certificates in a chain. In that case, only the first certificate in the chain is imported. + * Note: + * If providing a list of `customCertificateSecretRefs`, then those certificates will be used instead of the default certificates setup in the JVM Trust Store (in Ops Manager or Cloud Manager). + * If none are provided, the default JVM Truststore certificates will be used instead. + +## Breaking changes +* The `appdb-ca` is no longer automatically added to the JVM Trust Store (in Ops Manager or Cloud Manager). Since a bug introduced in version `1.17.0`, automatically adding these certificates to the JVM Trust Store has no longer worked. + * This will only impact you if: + * You are using the same custom certificate for both appdb-ca and for your S3 compatible backup store + * AND: You are using an operator prior to `1.17.0` (where automated inclusion in the JVM Trust Store worked) OR had a workaround (such as mounting your own trust store to OM) + * If you do need to use the same custom certificate for both appdb-ca and for your S3 compatible backup store then you now need to utilise `spec.backup.[]s3Config.customCertificateSecretRefs` (introduced in this release and covered below in the release notes) to specify the certificate authority for use for backups. + * The `appdb-ca` is the certificate authority saved in the configmap specified under `om.spec.applicationDatabase.security.tls.ca`. + +## Bug fixes +* Allowed setting an arbitrary port number in `spec.externalConnectivity.port` when `LoadBalancer` service type is used for exposing Ops Manager instance externally. +* The operator is now able to import the `appdb-ca` which consists of a bundle of certificate authorities into the ops-manager JVM trust store. Previously, the keystore had 2 problems: + * It was immutable. + * Only the first certificate authority out of the bundle was imported into the trust store. + * Both could lead to certificates being rejected by Ops Manager during requests to it. + +## Deprecation +* The setting `spec.backup.[]s3Stores.customCertificate` and `spec.backup.[]s3OpLogStores.customCertificate` are being deprecated in favor of `spec.backup.[]s3OpLogStores.[]customCertificateSecretRefs` and `spec.backup.[]s3Stores.[]customCertificateSecretRefs` + * Previously, when enabling `customCertificate`, the operator would use the `appdb-ca` as the custom certificate. Currently, this should be explicitly set via `customCertificateSecretRefs`. + +## New Features +* Support for providing a list of custom certificates for S3 based backups via secret references `spec.backup.[]s3Stores.customCertificateSecretRefs` and `spec.backup.[]s3OpLogStores.customCertificateSecretRefs` + * The list consists of single certificate strings, each references a secret containing a certificate authority. + * We do not support adding multiple certificates in a chain. In that case, only the first certificate in the chain is imported. + * Note: + * If providing a list of `customCertificateSecretRefs`, then those certificates will be used instead of the default certificates setup in the JVM Trust Store (in Ops Manager or Cloud Manager). + * If none are provided, the default JVM Truststore certificates will be used instead. + +# MongoDB Enterprise Kubernetes Operator 1.20.1 + +This release fixes an issue that prevented upgrading the Kubernetes Operator to 1.20.0 in OpenShift. Upgrade to this release instead. + +## Helm Chart +Fixes a bug where the operator container image was referencing to the deprecated ubuntu image. This has been patched to refer to the `ubi` based images. + +# MongoDB Enterprise Kubernetes Operator 1.20.0 + +# MongoDBOpsManager Resource +* Added support for votes, priority and tags by introducing the `spec.applicationDatabase.memberConfig.votes`, `spec.applicationDatabase.memberConfig.priority` +and `spec.applicationDatabase.memberConfig.tags` field. +* Introduced automatic change of the AppDB's image version suffix `-ent` to `-ubi8`. + * This enables migration of AppDB images from the legacy repository (`quay.io/mongodb/mongodb-enterprise-appdb-database-ubi`) to the new official one (`quay.io/mongodb/mongodb-enterprise-server`) without changing the version in MongoDBOpsManager's `applicationDatabase.version` field. + * The change will result a rolling update of AppDB replica set pods to the new, official images (referenced in Helm Chart in `values.mongodb.name` field), which are functionally equivalent to the previous ones (the same MongoDB version). + * Suffix change occurs under specific circumstances: + * Helm setting for appdb image: `mongodb.name` will now default to `mongodb-enterprise-server`. + * The operator will automatically replace the suffix for image repositories + that end with `mongodb-enterprise-server`. + Operator will replace the suffix `-ent` with the value set in the environment variable + `MDB_IMAGE_TYPE`, which defaults to `-ubi8`. + For instance, the operator will migrate: + * `quay.io/mongodb/mongodb-enterprise-server:4.2.11-ent` to `quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8`. + * `MDB_IMAGE_TYPE=ubuntu2024 quay.io/mongodb/mongodb-enterprise-server:4.2.11-ent` to `quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubuntu2024`. + * The operator will do the automatic migration of suffixes only for images + that reference the name `mongodb-enterprise-server`. + It won't perform migration for any other image name, e.g.: + * `mongodb-enterprise-appdb-database-ubi:4.0.0-ent` will not be altered + * To stop the automatic suffix migration behavior, + set the following environment variable to true: `MDB_APPDB_ASSUME_OLD_FORMAT=true` + or alternatively in the following helm chart setting: `mongodb.appdbAssumeOldFormat=true` +* Added support for defining bare versions in `spec.applicationDatabase.version`. Previously, it was required to specify AppDB's version with `-ent` suffix. Currently, it is possible to specify a bare version, e.g. `6.0.5` and the operator will convert it to `6.0.5-${MDB_IMAGE_TYPE}`. The default for environment variable `MDB_IMAGE_TYPE` is `-ubi8`. + +## Bug fixes +* Fixed MongoDBMultiCluster not watching Ops Manager's connection configmap and secret. +* Fixed support for rotating the clusterfile secret, which is used for internal x509 authentication in MongoDB and MongoDBMultiCluster resources. + +## Helm Chart +* All images reference ubi variants by default (added suffix -ubi) + * quay.io/mongodb/mongodb-enterprise-database-ubi + * quay.io/mongodb/mongodb-enterprise-init-database-ubi + * quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + * quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi + * quay.io/mongodb/mongodb-enterprise-init-appdb-ubi + * quay.io/mongodb/mongodb-agent-ubi + * quay.io/mongodb/mongodb-enterprise-appdb-database-ubi +* Changed default AppDB repository to official MongoDB Enterprise repository in `values.mongodb.name` field: quay.io/mongodb/mongodb-enterprise-server. +* Introduced `values.mongodb.imageType` variable to specify a default image type suffix added to AppDB's version used by MongoDBOpsManager resource. + +## Breaking changes +* Removal of `appdb.connectionSpec.Project` since it has been deprecated for over 2 years. + +# MongoDB Enterprise Kubernetes Operator 1.19.0 + +## MongoDB Resource +* Added support for setting replica set member votes by introducing the `spec.memberOptions.[*].votes` field. +* Added support for setting replica set member priority by introducing the `spec.memberOptions.[*].priority` field. +* Added support for setting replica set member tags by introducing the `spec.memberOptions.[*].tags` field. + +## MongoDBMulti Resource +* Added support for setting replica set member votes by introducing the `spec.clusterSpecList.[*].memberOptions.[*].votes` field. +* Added support for setting replica set member priority by introducing the `spec.clusterSpecList.[*].memberOptions.[*].priority` field. +* Added support for setting replica set member tags by introducing the `spec.clusterSpecList.[*].memberOptions.[*].tags` field. + +## Improvements + +* New guidance for multi-Kubernetes-cluster deployments without a Service Mesh. It covers use of a Load Balancer Service +to expose ReplicaSet members on an externally reachable domain (`spec.externalAccess.externalDomain`). +This leverages setting the `process.hostname` field in the Automation Config. +[This tutorial](ttps://www.mongodb.com/docs/kubernetes-operator/v1.19/tutorial/proper_link) provides full guidance. +* `spec.security.authentication.ldap.transportSecurity`: "none" is now a valid configuration to use no transportSecurity. +* Allows you to configure `podSpec` per shard in a MongoDB Sharded cluster by specifying an array of `podSpecs` under `spec.shardSpecificPodSpec` for each shard. + +## Deprecations + +* Making the field orgId in the project configmap a requirement. **Note**: If explicitly an empty `orgId = ""` has been chosen then OM will try to create an ORG with the project name. +* Ubuntu-based images were deprecated in favor of UBI-based images in operator version 1.17.0. In the 1.19.0 release we are removing the support for Ubuntu-based images. The ubuntu based images won't be rebuilt daily with updates. Please upgrade to the UBI-based images by following these instructions: https://www.mongodb.com/docs/kubernetes-operator/master/tutorial/migrate-k8s-images/#migrate-k8s-images +* The `spec.exposedExternally` option becomes deprecated in favor of `spec.externalAccess`. The deprecated option will be removed in MongoDB Enterprise Operator 1.22.0. + +## Bug fixes +* Fixed handling of `WATCH_NAMESPACE='*'` environment variable for multi-cluster deployments with cluster-wide operator. In some specific circumstances, API clients for member clusters were configured incorrectly resulting in deployment errors. + * Example error in this case: + * `The secret object 'mdb-multi-rs-cert' does not contain all the valid certificates needed: secrets "mdb-multi-rs-cert-pem" already exists` + * These specific circumstances were: + * `WATCH_NAMESPACE='*'` environment variable passed to the operator deployment + * specific namespace set in kubeconfig for member clusters + * not using multi-cluster cli tool for configuring + * Possible workarounds: + * set WATCH_NAMESPACE environment variable to specific namespaces instead of '*' + * make sure that kubeconfigs for member clusters doesn't specify a namespace + +## Breaking changes +* Renaming of the multicluster CRD `MongoDBMulti` to `MongoDBMultiCluster` + +* The `spec.members` field is required to be set in case of MongoDB deployment of type `ReplicaSet`. +## Bug fixes +* Fixed a panic when `CertificatesSecretsPrefix` was set but no further `spec.security.tls` setting was set i.e. `tls.additionalCertificateDomains` or `tls.ca`. + +# MongoDB Enterprise Kubernetes Operator 1.18.0 + +## Improvements + +* Added support for the missing features for Ops Manager Backup configuration page. This includes: + * KMIP Backup Configuration support by introducing `spec.backup.encryption.kmip` in both OpsManager and MongoDB resources. + * Backup Assignment Labels settings in `spec.backup.[*].assignmentLabels` elements of the OpsManager resource. + * Backup Snapshot Schedule configuration via `spec.backup.snapshotSchedule` in the OpsManager resource. +* Added `SCRAM-SHA-1` support for both user and Agent authentication. Before enabling this capability, make sure you use both `MONGODB-CR` and `SCRAM-SHA-1` in the authentication modes. + +## Bug fixes +* Fixed liveness probe reporting positive result when the agent process was killed. This could cause database pods to run without automation agent. +* Fixed startup script in database pod, that could in some situations report errors on pod's restart. + +## Breaking changes and deprecations + +* The field `spec.security.tls.secretRef.prefix` has been removed from MongoDB and OpsManager resources. It was deprecated in the [MongoDB Enterprise +1.15.0](https://www.mongodb.com/docs/kubernetes-operator/master/release-notes/#k8s-op-full-1-15-0) and removed from the Operator runtime in +[1.17.0](https://www.mongodb.com/docs/kubernetes-operator/master/release-notes/#k8s-op-full-1-17-0). Before upgrading to this version, make +sure you migrated to the new TLS format using the following [Migration Guide](https://www.mongodb.com/docs/kubernetes-operator/v1.16/tutorial/migrate-to-new-tls-format/) before upgrading the Operator. + +# MongoDB Enterprise Kubernetes Operator 1.17.2 + +* Fixed the OpenShift installation problem mentioned in the Enterprise Operator 1.7.1 release notes. The OLM (Operator Lifecycle Manager) + upgrade graph will automatically skip the 1.7.1 release and perform an upgrade from 1.7.0 directly to this release. +* Adds startup probes for database and OpsManager resources with some defaults. This improves the reliability of upgrades by ensuring things occur in the correct order. Customers can also override probe configurations with `podTemplateSpec`. +# MongoDB Enterprise Kubernetes Operator 1.17.1 + +## Important OpenShift Warning +For OpenShift customers, we recommend that you do NOT upgrade to this release (version 1.17.1), and instead upgrade to version 1.17.2, which is due the week commencing 17th October 2022, or upgrade to later versions. + +This release has invalid `quay.io/mongodb/mongodb-agent-ubi` digests referenced in certified bundle's CSV. Installing it could result in ImagePullBackOff errors in AppDB pods (OpsManager's database). Errors will look similar to: +``` + Failed to pull image "quay.io/mongodb/mongodb-agent-ubi@sha256:a4cadf209ab87eb7d121ccd8b1503fa5d88be8866b5c3cb7897d14c36869abf6": rpc error: code = Unknown desc = reading manifest sha256:a4cadf209ab87eb7d121ccd8b1503fa5d88be8866b5c3cb7897d14c36869abf6 in quay.io/mongodb/mongodb-agent-ubi: manifest unknown: manifest unknown +``` +This affects only OpenShift deployments when installing/upgrading the Operator version 1.17.1 from the certified bundle (OperatorHub). + +The following workaround fixes the issue by replacing the invalid sha256 digests. + +### Workaround + +If you proceed to use the Operator version 1.17.1 in OpenShift, you must make the following changes. Update the Operator's Subscription with the following `spec.config.env`: +```yaml +spec: + config: + env: + - name: AGENT_IMAGE + value: >- + quay.io/mongodb/mongodb-agent-ubi@sha256:ffa842168cc0865bba022b414d49e66ae314bf2fd87288814903d5a430162620 + - name: RELATED_IMAGE_AGENT_IMAGE_11_0_5_6963_1 + value: >- + quay.io/mongodb/mongodb-agent-ubi@sha256:e7176c627ef5669be56e007a57a81ef5673e9161033a6966c6e13022d241ec9e + - name: RELATED_IMAGE_AGENT_IMAGE_11_12_0_7388_1 + value: >- + quay.io/mongodb/mongodb-agent-ubi@sha256:ffa842168cc0865bba022b414d49e66ae314bf2fd87288814903d5a430162620 + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_4_7554_1 + value: >- + quay.io/mongodb/mongodb-agent-ubi@sha256:3e07e8164421a6736b86619d9d72f721d4212acb5f178ec20ffec045a7a8f855 +``` + +**This workaround should be removed as soon as the new Operator version (>=1.17.2) is installed.** + +## MongoDB Operator + +## Improvements + +* The Red Hat certified operator now uses Quay as an image registry. New images will be automatically pulled upon the operator upgrade and no user action is required as a result of this change. + +## Breaking changes + +* Removed `operator.deployment_name` from the Helm chart. Parameter was used in an incorrect way and only for customising the name of the operator container. The name of the container is now set to `operator.name`. This is a breaking change only if `operator.deployment_name` was set to a different value than `operator.name` and if there is external tooling relying on this. Otherwise this change will be unnoticeable. + +# MongoDB Enterprise Kubernetes Operator 1.17.0 + +## Improvements + +* Introduced support for Ops Manager 6.0. +* For custom S3 compatible backends for the Oplog and Snapshot stores, it is now possible to specify the + `spec.backup.s3OpLogStores[n].s3RegionOverride` and the `spec.backup.s3Stores[n].s3RegionOverride` parameter. +* Improved security by introducing `readOnlyRootFilesystem` property to all deployed containers. This change also introduces a few additional volumes and volume mounts. +* Improved security by introducing `allowPrivilegeEscalation` set to `false` for all containers. + +## Breaking changes and deprecations + +* Ubuntu-based images are being deprecated in favor of the UBI-based images for new users, a migration guide for existing users will be published soon. + The Ubuntu-based images will no longer be made available as of version 1.19. All existing Ubuntu-based images will continue to be + supported until their version [End Of Life](https://www.mongodb.com/docs/kubernetes-operator/master/reference/support-lifecycle/). + It is highly recommended to switch into UBI-based images as soon as possible. +* Concatenated PEM format TLS certificates are not supported in Operator 1.17.0 and above. They were deprecated in + 1.13.0. Before upgrading to Operator 1.17.0, please confirm you have upgraded to the `Kubernetes TLS`. Please refer to the + Migration [Migration Guide](https://www.mongodb.com/docs/kubernetes-operator/v1.16/tutorial/migrate-to-new-tls-format/) before upgrading the Operator. +* Ops Manager 4.4 is [End of Life](https://www.mongodb.com/support-policy/lifecycles) and is no longer supported by the operator. If you're + using Ops Manager 4.4, please upgrade to a newer version prior to the operator upgrade. + +# MongoDB Enterprise Kubernetes Operator 1.16.4 + +## Security fixes + +* The operator and init-ops-manager binaries are built with Go 1.18.4 which addresses security issues. + +# MongoDB Enterprise Kubernetes Operator 1.16.3 + +## MongoDB Resource + +* Security Context are now defined only at Pod level (not both Pod and Container level as before). +* Added `timeoutMS`, `userCacheInvalidationInterval` fields to `spec.security.authentication.ldap` object. + +* Bug fixes + * Fixes ignored `additionalMongodConfig.net.tls.mode` for `mongos`, `configSrv` and `shard` objects when configuring ShardedCluster resource. + +# MongoDB Enterprise Kubernetes Operator 1.16.2 + +## MongoDB Resource + +* `spec.podSpec.podAntiAffinityTopologyKey` , `spec.podSpec.podAffinity` and `spec.podSpec.nodeAffinity` has been removed. Please use `spec.podSpec.podTemplate` override to set these fields. +* Wiredtiger cache computation has been removed. This was needed for server version `>=4.0.0 <4.0.9` and `<3.6.13`. These server version have reached EOL. Make sure to update your MDB deployment to a version later than `4.0.9` before upgrading the operator. + +## MongoDBOpsManager Resource + +* `spec.applicationDatabase.podSpec.podAntiAffinityTopologyKey` , `spec.applicationDatabase.podSpec.podAffinity` and `spec.applicationDatabase.podSpec.nodeAffinity` has been removed. Please use `spec.applicationDatabase.podSpec.podTemplate` override to set these fields. +# MongoDB Enterprise Kubernetes Operator 1.16.1 + +## MongoDB Resource + +* `spec.Service` has been deprecated. Please use `spec.statefulSet.spec.serviceName` to provide a custom service name. + +# MongoDB Enterprise Kubernetes Operator 1.15.3 + +## MongoDB Resource + +* `spec.security.tls.secretRef.name` has been removed. It was deprecated in operator version `v1.10.0`. Please use the field + `spec.security.certsSecretPrefix` to specify the secret name containing the certificate for Database. Make sure to create the secret containing the certificates accordingly. +* `spec.podSpec.cpu` and `spec.podSpec.memory` has been removed to override the CPU/Memory resources for the + database pod, you need to override them using the statefulset spec override under `spec.podSpec.podTemplate.spec.containers`. +* Custom labels specified under `metadata.labels` is propagated to the database StatefulSet and the PVC objects. + +## MongoDBOpsManager Resource +* `spec.applicationDatabase.security.tls.secretRef.name` has been removed. It was deprecated in operator version `v1.10.0`. Please use the field +`spec.applicationDatabase.security.certsSecretPrefix` to specify the secret name containing the certificate for AppDB. Make sure to create the secret containing the certificates accordingly. +* * Custom labels specified under `metadata.labels` is propagated to the OM, AppDB and BackupDaemon StatefulSet and the PVC objects. + +## MongoDBUser Resource +* Changes: + * Added the optional field `spec.connectionStringSecretName` to be able to provide a deterministic secret name for the user specific connection string secret generated by the operator. + +* `spec.applicationDatabase.podSpec.cpu` and `spec.applicationDatabase.podSpec.memory` has been removed to override the CPU/Memory resources for the + appDB pod, you need to override them using the statefulset spec override under `spec.applicationDatabase.podSpec.podTemplate.spec.containers`. +# MongoDB Enterprise Kubernetes Operator 1.15.2 +## MongoDBOpsManager Resource +* Bug Fix + * For enabling custom TLS certificates for S3 Oplog and Snapshot stores for backup. In additioning to setting `spec.security.tls.ca` and `spec.security.tls.secretRef`. The field `spec.backup.s3OpLogStores[n].customCertificate` / `spec.backup.s3Stores[n].customCertificate` needs to be set `true`. + +# MongoDB Enterprise Kubernetes Operator 1.15.1 + +## MongoDBOpsManager Resource + +* Bug fixes + * Fixes an issue that prevented the Operator to be upgraded when managing a TLS + enabled ApplicationDB, when the ApplicationDB TLS certificate is stored in a + `Secret` of type Opaque. + +# MongoDB Enterprise Kubernetes Operator 1.15.0 + + +## MongoDB Resource + +* Changes: + * The `spec.security.tls.enabled` and `spec.security.tls.secretRef.prefix` fields are now **deprecated** and will be removed in a future release. To enable TLS it is now sufficient to set the `spec.security.certsSecretPrefix` field. + +## MongoDBOpsManager Resource + +* Changes: + * A new field has been added: `spec.backup.queryableBackupSecretRef`. The secrets referenced by this field contains the certificates used to enable [Queryable Backups](https://docs.opsmanager.mongodb.com/current/tutorial/query-backup/) feature. + * Added support for configuring custom TLS certificates for the S3 Oplog and Snapshot Stores for backup. These can be configured with + `spec.security.tls.ca` and `spec.security.tls.secretRef`. + * It is possible to disable AppDB processes via the `spec.applicationDatabase.automationConfig.processes[n].disabled` field, this enables backing up the AppDB. + * The `spec.security.tls.enabled`, `spec.security.tls.secretRef.prefix`, `spec.applicationDatabase.security.tls.enabled` and `spec.applicationDatabase.security.tls.prefix` fields are now **deprecated** and will be removed in a future release. To enable TLS it is now sufficient to set the `spec.security.certsSecretPrefix` and/or `spec.applicationDatabase.security.certsSecretPrefix` field. + +*All the images can be found in:* + +https://quay.io/repository/mongodb (ubuntu-based) + +https://connect.redhat.com/ (rhel-based) + + +# MongoDB Enterprise Kubernetes Operator 1.14.0 + + +## MongoDB Resource +* Changes: + * A new field has been added: `spec.backup.autoTerminateOnDeletion`. AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, when the MongoDB Resource is deleted. +* Bug fixes + * Fixes an issue which would make a ShardedCluster Resource fail when disabling authentication. + +## MongoDBOpsManager Resource + +* Bug Fixes + * Fixes an issue where the operator would not properly trigger a reconciliation when rotating the AppDB TLS Certificate. + * Fixes an issue where a custom CA specified in the MongoDBOpsManager resource was not mounted into the Backup Daemon pod, + which prevented backups from working when Ops Manager was configured in hybrid mode and used a custom CA. +* Changes + * Added support for configuring S3 Oplog Stores using the `spec.backup.s3OpLogStores` field. + + +# MongoDB Enterprise Kubernetes Operator 1.13.0 + +## Kubernetes Operator +* Breaking Changes: + * The Operator no longer generates certificates for TLS resources. +* When deploying to multiple namespaces, imagePullSecrets has to be created only in the namespace where the Operator is installed. From here, the Operator will be sync this secret across all watched namespaces. +* The credentials secret used by the Operator now accepts the pair of fields `publicKey` and `privateKey`. These should be preferred to the existent `user` and `publicApiKey` when using Programmatic API Keys in Ops Manager. +* For TLS-enabled resources, the operator now watches the ConfigMap containing the Certificate Authority and the secret containg the TLS certificate. Changes to these resources now trigger a reconciliation of the related resource. +* The Operator can now watch over a list of Namespaces. To install the Operator in this mode, you need to set the value `operator.watchNamespace` to a comma-separated list of Namespaces. + The Helm install process will create Roles and Service Accounts required, in the Namespaces that the Operator will be watching. + +### Support for TLS certificates provided as kubernetes.io/tls secrets +* The operator now supports referencing TLS secrets of type kubernetes.io/tls + * This type of secrets contain a tls.crt and tls.key entry + * The operator can read these secrets and automatically generate a new one, containing the concatenation of tls.crt and tls.key + * This removes the need for a manual concatenation of the fields and enables users to natively reference secrets generated by tools such as cert-manager + +**Deprecation Notice** +The usage of generic secrets, manually created by concatenating certificate and private key, is now deprecated. + +## MongoDB Resource +* Breaking Changes: + * The field `spec.project` has been removed from MongoDB spec, this field has been deprecated since operator version `1.3.0`. Make sure to specify the project configmap name under `spec.opsManager.configMapRef.name` or ``spec.cloudManager.configMapRef.name`` before upgrading the operator. +* Changes: + * A new field has been added: `spec.security.certsSecretPrefix`. This string is now used to determine the name of the secrets containing various TLS certificates: + * For TLS member certificates, the secret name is `--cert` + * Note: If either `spec.security.tls.secretRef.name` or `spec.security.tls.secretRef.prefix` are specified, these will take precedence over the new field + * Note: if none of these three fields are specified, the secret name is `-cert` + * For agent certificates, if `spec.security.certsSecretPrefix` is specified, the secret name is`--agent-certs` + * Note: if `spec.authentication.agents.clientCertificateSecretRef` is specified, this will take precedence over the new field + * If none of these fields are set, the secret name is still `agent-certs` + * For internal cluster authentication certificates, if `spec.security.certsSecretPrefix` is specified, the secret name is `--clusterfile` + * Otherwise, it is still `-clusterfile` +* Bug fixes + * Fixes an issue where Sharded Cluster backups could not be correctly configured using the MongoDB CR. + * Fixes an issue where Backup Daemon fails to start after OpsManager version upgrade. + + +## MongoDBOpsManager Resource +* Operator will report status of FileSystemSnaphot store names configured under `spec.backup.fileSystemStores` in OM CR. The FS however needs to be manually configured. +* It is now possible to disable creation of "LoadBalancer" Type service for queryable backup by setting `spec.backup.externalServiceEnabled` to `false` in OM CR. By default, the operator would create the LoadBalancer type service object. +* The operator will now automatically upgrade the used API Key to a programmatic one when deploying OM >= 5.0.0. It is now possible to upgrade from older versions of OM to OM 5.0 without manual intervention. +* A new field has been added: `spec.security.certSecretPrefix`. This is string is now used to determine the name of the secret containing the TLS certificate for OpsManager. + * If the existing field `spec.security.tls.secretRef.Name` is specified, it will take the precedence + * Please note that this field is now **deprecated** and will be removed in a future release + * Otherwise, if `spec.security.certSecretPrefix` is specified, the secret name will be `--cert` + +## MongoDBUser Resource +* Breaking Changes: + * The field `spec.project` has been removed from User spec, this field has been deprecated since operator version `1.3.0`. Make sure to specify the MongoDB resource name under `spec.MongoDBResourceRef.name` before upgrading the operator. +## Miscellaneous +* Ops Manager versions 4.4.7, 4.4.9, 4.4.10, 4.4.11, 4.4.12 and 4.4.13 base images have been updated to Ubuntu 20.04. +* Ops Manager versions 4.4.16 and 5.0.1 are now supported + + +# MongoDB Enterprise Kubernetes Operator 1.12.0 + +## MongoDB Resource +* Bug Fixes + * Fixes a bug when an user could only specify `net.ssl.mode` and not `net.tls.mode` in the `spec.additionalMongodConfig` field. +* Changes + * If `spec.exposedExternally` is set to `false` after being set to `true`, the Operator will now delete the corresponding service + +## MongoDBOpsManager Resource +* Changes + * If `spec.externalConnectivity` is unset after being set, the Operator will now delete the corresponding service + * It is now possible to specify the number of backup daemon pods to deploy through the `spec.backup.members` field. The value defaults to 1 if not set. + + +## Miscellaneous +* Ops Manager versions 4.4.13, 4.4.14, 4.4.15 and 4.2.25 are now supported +* Ops Manager version 5.0.0 is now supported +* Ubuntu based operator images are now based on Ubuntu 20.04 instead of Ubuntu 16.04 +* Ubuntu based database images starting from 2.0.1 will be based on Ubuntu 18.04 instead of Ubuntu 16.04 + **NOTE: MongoDB 4.0.0 does not support Ubuntu 18.04 - If you want to use MongoDB 4.0.0, stay on previously released images** +* Ubuntu based Ops Manager images after 4.4.13 will be based on Ubuntu 20.04 instead of Ubuntu 16.04 + +* Newly released ubi images for Operator, Ops Manager and Database will be based un ubi-minimal instead of ubi +## Notes before upgrading to OpsManager 5.0.0 +* Before upgrading OpsManager to version 5.0.0 make sure the Operator is using a [programmatic API key](https://docs.mongodb.com/kubernetes-operator/stable/tutorial/create-operator-credentials/#create-k8s-credentials). This is only required when the OpsManager instance is managed by the Operator. +* You will find a small tutorial on how to do this in [upgrading-to-ops-manager-5.md](docs/upgrading-to-ops-manager-5.md) document. + +# MongoDB Enterprise Kubernetes Operator 1.11.0 + +## MongoDB Resource +* Bug fixes + * Fixes an issue with the `LivenessProbe` that could cause the database Pods to be restarted in the middle of a restore operation from Backup. + +## MongoDBOpsManager Resource +* Breaking Changes + *For a complete guide on how to safely upgrade, please check the [upgrade instructions](link here TODO)* + * Each Application Database pod consists now of three containers (`mongodb`, `mongodb-agent`, `mongodb-agent-monitoring`) and it does not bundle anymore a MongoDB version + * You can now use any version of MongoDB for the Application Database (we recommend to use the enterprise ones provided by MongoDB, see the *New Images* section) + * You need to make sure the MongoDB version used is [supported](https://docs.opsmanager.mongodb.com/current/reference/mongodb-compatibility/) by OpsManager + * `spec.applicationDatabase.version` is no longer optional. + * `spec.applicationDatabase.persistent` does not exist anymore, the Operator will now always use persistent volumes for the AppDB. + +## New Images + +* mongodb-agent 10.29.0.6830-1: + * Ubi: quay.io/mongodb/mongodb-agent-ubi:10.29.0.6830-1 + * Ubuntu: quay.io/mongodb/mongodb-agent:10.29.0.6830-1 + +* mongodb-enterprise-appdb-database + * Ubi: quay.io/mongodb/mongodb-enterprise-appdb-database-ubi + * Ubuntu: quay.io/mongodb/mongodb-enterprise-appdb-database + +* mongodb-enterprise-init-appdb 1.0.7 + * Ubi: quay.io/mongodb/mongodb-enterprise-init-appdb-ubi:1.0.7 + * Ubuntu: quay.io/mongodb/mongodb-enterprise-init-appdb:1.0.7 + +* mongodb-enterprise-init-database 1.0.3 + * Ubi: quay.io/mongodb/mongodb-enterprise-init-database-ubi:1.0.3 + * Ubuntu: quay.io/mongodb/mongodb-enterprise-init-database:1.0.3 + +# MongoDB Enterprise Kubernetes Operator 1.10.1 + +## Kubernetes Operator +* Changes + * Added a liveness probe to the Backup Daemon. + * Added a readiness probe to the Backup Daemon. + * The readiness probe on Database Pods is more strict when restarting a + Replica Set and will only set the Pod as "Ready" when the MongoDB server has + reached `PRIMARY` or `SECONDARY` states. + +## MongoDB Resource +* Changes + * Deprecated field `spec.security.tls.secretRef.name`, the field `spec.security.tls.secretRef.prefix` should now be used instead. + * Added field `spec.security.tls.secretRef.prefix`. This property should be used to specify the prefix of the secret which contains custom tls certificates. + +## MongoDBOpsManager Resource +* Changes + * A new status field for the OpsManager backup has been added: `Disabled`. This status will be displayed when `spec.backup.enabled` is set to `false` and no backup is configured in OpsManager + +## Miscellaneous +* Added a new value in openshift-values.yaml `operator_image_name` which allows the label selector of the webhook + to match the operator label. + + +## MongoDB Resource +* Changes + * Deprecated field `spec.security.tls.secretRef.name`, the field `spec.security.tls.secretRef.prefix` should now be used instead. + * Added field `spec.security.tls.secretRef.prefix`. This property should be used to specify the prefix of the secret which contains custom tls certificates. + + +# MongoDB Enterprise Kubernetes Operator 1.10.0 + +## Kubernetes Operator + +* Changes + * The CRDs have been updated to from `v1beta1` to `v1` version. This should not have any impact on Kubernetes clusters 1.16 and up. The CRDs won't be installable in clusters with versions older than 1.16. + +* Bug fixes + * Fixes an issue which made it not possible do have multiple ops-manager resources with the same name in different namespaces. + * Fixes an issue which made new MongoDB resources created with `spec.backup.mode=disabled` fail. + * Fixes an issue which made a Replica Set go to Fail state if, at the same time, the amount of members of a Replica Set are increased and TLS is disabled. + +## MongoDBOpsManager Resource + +* Known issues + * When using remote or hybrid mode, and `automation.versions.download.baseUrl` has been set, the property `automation.versions.download.baseUrl.allowOnlyAvailableBuilds` + needs to be set to `false`. This has been fixed in Ops Manager version 4.4.11. + + +# MongoDB Enterprise Kubernetes Operator 1.9.3 +## Kubernetes Operator + +* Changes + * The CRDs have been updated to from `v1beta1` to `v1` version. This should not have any impact on Kubernetes clusters 1.16 and up. The CRDs won't be installable in clusters with versions older than 1.16. + +* Bug fixes + * Fixes an issue which made it not possible do have multiple ops-manager resources with the same name in different namespaces. + * Fixes an issue which made new MongoDB resources created with `spec.backup.mode=disabled` fail. + * Fixes an issue which made a Replica Set go to Fail state if, at the same time, the amount of members of a Replica Set are increased and TLS is disabled. + +## MongoDBOpsManager Resource + +* Known issues + * When using remote or hybrid mode, and `automation.versions.download.baseUrl` has been set, the property `automation.versions.download.baseUrl.allowOnlyAvailableBuilds` + needs to be set to `false`. This has been fixed in Ops Manager version 4.4.11. + + +# MongoDB Enterprise Kubernetes Operator 1.9.3 +## Kubernetes Operator +* Bug fixes + * Fixes an issue which made it not possible do have multiple ops-manager resources with the same name in different namespaces + * Fixes an issue which made new MongoDB resources created with `spec.backup.mode=disabled` fail + * Fixes an issue which made a Replica Set go to Fail state if, at the same time, the amount of members of a Replica Set are increased and TLS is disabled. + +## MongoDBOpsManager Resource +* Known issues + * When using remote or hybrid mode, and `automation.versions.download.baseUrl` has been set, the property `automation.versions.download.baseUrl.allowOnlyAvailableBuilds` + needs to be set to `false`. This has been fixed in Ops Manager version 4.4.11. + + +# MongoDB Enterprise Kubernetes Operator 1.9.2 +## Miscellaneous +* Fix errors with CSV + + + +# MongoDB Enterprise Kubernetes Operator 1.9.1 +## Kubernetes Operator +* Bug fixes + * Fixes an issue where the service-account-name could not be specified in the StatefulSet podSpec override. + * Removed unnecessary `delete service` permission from operator role. + +## MongoDB Resource +* Bug fixes + * Fixes an issue where updating a role in `spec.security.authentication.roles` by removing the `privileges` array would cause the resource to enter a bad state + +## MongoDBOpsManager Resource +* Breaking Changes + * The new Application Database image `mongodb-enterprise-appdb:10.2.15.5958-1_4.2.11-ent` was released. The image needs + to be downloaded to the local repositories otherwise MongoDBOpsManager resource won't start. The image contains a new bundled MongoDB + `4.2.11-ent` instead of `4.2.2-ent`. +* Changes + * Ops Manager user now has "backup", "restore" and "hostManager" roles, allowing for backups/restores on the AppDB. + * If `spec.applicationDatabase.version` is omitted the Operator will use `4.2.11-ent` as a default MongoDB. + +# MongoDB Enterprise Kubernetes Operator 1.9.0 + +## Kubernetes Operator + +* Bug fixes + * Fixes an issue where connections were not closed leading to too many file + descriptors open. + +## MongoDB Resource +* Changes + * Continuous backups can now be configured with the MongoDB CRD. Set `spec.backup.enabled` to `true`. *Note*: You must have an Ops Manager resource already configured with backup. See [the docs](https://docs.mongodb.com/kubernetes-operator/master/tutorial/deploy-om-container/#id6) for more information. +## MongoDBOpsManager Resource + +* Changes + * A StatefulSet resource that holds the Ops Manager Backup Daemon will be + deleted and recreated in order to change the `matchLabels` attribute, + required for a new `Service` to allow for Queryable Backups feature to work. + This is a safe operation. + * Changed the way the Operator collects statuses of MongoDB Agents running in + Application Database Pods. + +# MongoDB Enterprise Kubernetes Operator 1.8.2 + +## MongoDBOpsManager Resource + +* Bug Fixes + * Fixes an issue when `MongoDBOpsManager` resource gets to `Failing` state when + both external connectivity and backups are enabled. + +## New Images + +* mongodb-enterprise-operator 1.8.2: + * Ubi: quay.io/mongodb/mongodb-enterprise-operator-ubi:1.8.2 + * Ubuntu: quay.io/mongodb/mongodb-enterprise-operator:1.8.2 diff --git a/api/v1/addtoscheme_mongodb_v1.go b/api/v1/addtoscheme_mongodb_v1.go new file mode 100644 index 000000000..81b01b1c5 --- /dev/null +++ b/api/v1/addtoscheme_mongodb_v1.go @@ -0,0 +1,6 @@ +package v1 + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, SchemeBuilder.AddToScheme) +} diff --git a/api/v1/apis.go b/api/v1/apis.go new file mode 100644 index 000000000..01683259a --- /dev/null +++ b/api/v1/apis.go @@ -0,0 +1,13 @@ +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// AddToSchemes may be used to add all resources defined in the project to a Scheme +var AddToSchemes runtime.SchemeBuilder + +// AddToScheme adds all Resources to the Scheme +func AddToScheme(s *runtime.Scheme) error { + return AddToSchemes.AddToScheme(s) +} diff --git a/api/v1/customresource_readwriter.go b/api/v1/customresource_readwriter.go new file mode 100644 index 000000000..7fc00dc08 --- /dev/null +++ b/api/v1/customresource_readwriter.go @@ -0,0 +1,16 @@ +package v1 + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// CustomResourceReadWriter is an interface for all Custom Resources with Status read/write capabilities +// TODO this may be a good candidate for further refactoring +// +kubebuilder:object:generate=false +type CustomResourceReadWriter interface { + client.Object + status.Reader + status.Writer +} diff --git a/api/v1/doc.go b/api/v1/doc.go index a6a3905a8..920e76ec9 100644 --- a/api/v1/doc.go +++ b/api/v1/doc.go @@ -1,4 +1,5 @@ -package v1 +// +k8s:deepcopy-gen=package,register -// +k8s:deepcopy-gen=package -// +versionName=v1 +// Package v1 is the v1 version of the API. +// +groupName=mongodb.com +package v1 diff --git a/api/v1/kmip.go b/api/v1/kmip.go new file mode 100644 index 000000000..ed19d8580 --- /dev/null +++ b/api/v1/kmip.go @@ -0,0 +1,55 @@ +package v1 + +import ( + "fmt" +) + +// KmipServerConfig contains the relevant configuration for KMIP integration. +type KmipServerConfig struct { + // KMIP Server url in the following format: hostname:port + // Valid examples are: + // 10.10.10.3:5696 + // my-kmip-server.mycorp.com:5696 + // kmip-svc.svc.cluster.local:5696 + // +kubebuilder:validation:Pattern=`[^\:]+:[0-9]{0,5}` + URL string `json:"url"` + + // CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + // used for KMIP authentication + CA string `json:"ca"` +} + +// KmipClientConfig contains the relevant configuration for KMIP integration. +type KmipClientConfig struct { + // A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + // The names are generated using the following pattern: + // KMIP Client Certificate (TLS Secret): + // --kmip-client + // KMIP Client Certificate Password: + // --kmip-client-password + // The expected key inside is called "password". + // +optional + ClientCertificatePrefix string `json:"clientCertificatePrefix"` +} + +func (k *KmipClientConfig) ClientCertificateSecretName(crName string) string { + if len(k.ClientCertificatePrefix) == 0 { + return fmt.Sprintf("%s-kmip-client", crName) + } + return fmt.Sprintf("%s-%s-kmip-client", k.ClientCertificatePrefix, crName) +} + +func (k *KmipClientConfig) ClientCertificatePasswordSecretName(crName string) string { + if len(k.ClientCertificatePrefix) == 0 { + return fmt.Sprintf("%s-kmip-client-password", crName) + } + return fmt.Sprintf("%s-%s-kmip-client-password", k.ClientCertificatePrefix, crName) +} + +func (k *KmipClientConfig) ClientCertificateSecretKeyName() string { + return "tls.crt" +} + +func (k *KmipClientConfig) ClientCertificatePasswordKeyName() string { + return "password" +} diff --git a/api/v1/mdb/doc.go b/api/v1/mdb/doc.go new file mode 100644 index 000000000..49d34fef6 --- /dev/null +++ b/api/v1/mdb/doc.go @@ -0,0 +1,4 @@ +package mdb + +// +k8s:deepcopy-gen=package +// +versionName=v1 diff --git a/api/v1/mdb/groupversion_info.go b/api/v1/mdb/groupversion_info.go new file mode 100644 index 000000000..ef921b6eb --- /dev/null +++ b/api/v1/mdb/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1 contains API Schema definitions for the mongodb v1 API group +// +kubebuilder:object:generate=true +// +groupName=mongodb.com +package mdb + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "mongodb.com", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1/mdb/mongodb_roles_validation.go b/api/v1/mdb/mongodb_roles_validation.go new file mode 100644 index 000000000..8d19538e1 --- /dev/null +++ b/api/v1/mdb/mongodb_roles_validation.go @@ -0,0 +1,316 @@ +package mdb + +// IMPORTANT: this package is intended to contain only "simple" validation—in +// other words, validation that is based only on the properties in the MongoDB +// resource. More complex validation, such as validation that needs to observe +// the state of the cluster, belongs somewhere else. + +import ( + "net" + + "github.com/blang/semver" + "golang.org/x/xerrors" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// Go doesn't allow us to define constant array, so we wrap it in a function + +// This is the list of valid actions for pivileges defined on the DB level +func validDbActions() []string { + return []string{ + "changeCustomData", + "changeOwnCustomData", + "changeOwnPassword", + "changePassword", + "createCollection", + "createIndex", + "createRole", + "createUser", + "dropCollection", + "dropRole", + "dropUser", + "emptycapped", + "enableProfiler", + "grantRole", + "killCursors", + "listCachedAndActiveUsers", + "revokeRole", + "setAuthenticationRestriction", + "unlock", + "viewRole", + "viewUser", + "find", + "insert", + "remove", + "update", + "bypassDocumentValidation", + "changeStream", + "planCacheRead", + "planCacheWrite", + "planCacheIndexFilter", + "storageDetails", + "enableSharding", + "getShardVersion", + "moveChunk", + "splitChunk", + "splitVector", + "collMod", + "compact", + "convertToCapped", + "dropDatabase", + "dropIndex", + "reIndex", + "renameCollectionSameDB", + "repairDatabase", + "collStats", + "dbHash", + "dbStats", + "indexStats", + "listCollections", + "listIndexes", + "validate", + } +} + +// This is the list of valid actions for pivileges defined on the Cluster level +func validClusterActions() []string { + return []string{ + "bypassWriteBlockingMode", + "checkMetadataConsistency", + "transitionFromDedicatedConfigServer", + "setUserWriteBlockMode", + "setFeatureCompatibilityVersion", + "setDefaultRWConcern", + "rotateCertificates", + "getClusterParameter", + "setClusterParameter", + "getDefaultRWConcern", + "transitionToDedicatedConfigServer", + "compact", + "useUUID", + "dropConnections", + "killAnyCursor", + "unlock", + "authSchemaUpgrade", + "cleanupOrphaned", + "cpuProfiler", + "inprog", + "invalidateUserCache", + "killop", + "appendOplogNote", + "replSetConfigure", + "replSetGetConfig", + "replSetGetStatus", + "replSetHeartbeat", + "replSetStateChange", + "resync", + "addShard", + "flushRouterConfig", + "getShardMap", + "listShards", + "removeShard", + "shardingState", + "applicationMessage", + "closeAllDatabases", + "connPoolSync", + "forceUUID", + "fsync", + "getParameter", + "hostInfo", + "logRotate", + "setParameter", + "shutdown", + "touch", + "impersonate", + "listSessions", + "killAnySession", + "connPoolStats", + "cursorInfo", + "diagLogging", + "getCmdLineOpts", + "getLog", + "listDatabases", + "netstat", + "serverStatus", + "top", + } +} + +func validateAuthenticationRestriction(ar AuthenticationRestriction) v1.ValidationResult { + clientSources := ar.ClientSource + serverAddresses := ar.ServerAddress + + // Validate all clientSources, they have to be either valid IP addresses or CIDR ranges + for _, clientSource := range clientSources { + if !isValidIp(clientSource) && !isValidCIDR(clientSource) { + return v1.ValidationError("clientSource %s is neither a valid IP address nor a valid CIDR range", clientSource) + } + } + + // validate all serveraddresses, they have to be either valid IP addresses or CIDR ranges + for _, serverAddress := range serverAddresses { + if !isValidIp(serverAddress) && !isValidCIDR(serverAddress) { + return v1.ValidationError("serverAddress %s is neither a valid IP address nor a valid CIDR range", serverAddress) + } + } + + return v1.ValidationSuccess() +} + +// isVersionAtLeast takes two strings representing version (in semver notation) +// and returns true if the first one is greater or equal the second +// false otherwise +func isVersionAtLeast(mdbVersion string, expectedVersion string) (bool, error) { + currentV, err := semver.Make(mdbVersion) + if err != nil { + return false, xerrors.Errorf("error parsing mdbVersion %s with semver: %w", mdbVersion, err) + } + expectedVersionSemver, err := semver.Make(expectedVersion) + if err != nil { + return false, xerrors.Errorf("error parsing mdbVersion %s with semver: %w", expectedVersion, err) + } + return currentV.GTE(expectedVersionSemver), nil +} + +func validateClusterPrivilegeActions(actions []string, mdbVersion string) v1.ValidationResult { + isAtLeastThreePointSix, err := isVersionAtLeast(mdbVersion, "3.6.0-0") + if err != nil { + return v1.ValidationError("Error when parsing version strings: %s", err) + } + isAtLeastFourPointTwo, err := isVersionAtLeast(mdbVersion, "4.2.0-0") + if err != nil { + return v1.ValidationError("Error when parsing version strings: %s", err) + } + invalidActionsForLessThanThreeSix := []string{"impersonate", "listSessions", "killAnySession", "useUUID", "forceUUID"} + invalidActionsForLessThanFourTwo := []string{"dropConnections", "killAnyCursor"} + + if !isAtLeastFourPointTwo { + // Return error if the privilege specifies actions that are not valid in MongoDB < 4.2 + if stringutil.ContainsAny(actions, invalidActionsForLessThanFourTwo...) { + return v1.ValidationError("Some of the provided actions are not valid for MongoDB %s", mdbVersion) + } + if !isAtLeastThreePointSix { + // Return error if the privilege specifies actions that are not valid in MongoDB < 3.6 + if stringutil.ContainsAny(actions, invalidActionsForLessThanThreeSix...) { + return v1.ValidationError("Some of the provided actions are not valid for MongoDB %s", mdbVersion) + } + } + } + + // Check that every action provided is valid + for _, action := range actions { + if !stringutil.Contains(validClusterActions(), action) { + return v1.ValidationError("%s is not a valid cluster action", action) + } + } + return v1.ValidationSuccess() +} + +func validateDbPrivilegeActions(actions []string, mdbVersion string) v1.ValidationResult { + isAtLeastThreePointSix, err := isVersionAtLeast(mdbVersion, "3.6.0-0") + if err != nil { + return v1.ValidationError("Error when parsing version strings: %s", err) + } + isAtLeastFourPointTwo, err := isVersionAtLeast(mdbVersion, "4.2.0-0") + if err != nil { + return v1.ValidationError("Error when parsing version strings: %s", err) + } + invalidActionsForLessThanThreeSix := []string{"setAuthenticationRestriction", "changeStream"} + + if !isAtLeastFourPointTwo { + // Return error if the privilege specifies actions that are not valid in MongoDB < 4.2 + if stringutil.Contains(actions, "listCachedAndActiveUsers") { + return v1.ValidationError("listCachedAndActiveUsers is not a valid action for MongoDB %s", mdbVersion) + } + if !isAtLeastThreePointSix { + // Return error if the privilege specifies actions that are not valid in MongoDB < 3.6 + if stringutil.ContainsAny(actions, invalidActionsForLessThanThreeSix...) { + return v1.ValidationError("Some of the provided actions are not valid for MongoDB %s", mdbVersion) + } + } + } + + // Check that every action provided is valid + for _, action := range actions { + if !stringutil.Contains(validDbActions(), action) { + return v1.ValidationError("%s is not a valid db action", action) + } + } + return v1.ValidationSuccess() +} + +func validatePrivilege(privilege Privilege, mdbVersion string) v1.ValidationResult { + if privilege.Resource.Cluster != nil { + if !*privilege.Resource.Cluster { + return v1.ValidationError("The only valid value for privilege.cluster, if set, is true") + } + if privilege.Resource.Collection != "" || privilege.Resource.Db != "" { + return v1.ValidationError("Cluster: true is not compatible with setting db/collection") + } + if res := validateClusterPrivilegeActions(privilege.Actions, mdbVersion); res.Level == v1.ErrorLevel { + return v1.ValidationError("Actions are not valid - %s", res.Msg) + } + } else { + if res := validateDbPrivilegeActions(privilege.Actions, mdbVersion); res.Level == v1.ErrorLevel { + return v1.ValidationError("Actions are not valid - %s", res.Msg) + } + } + + return v1.ValidationSuccess() +} + +func isValidIp(ip string) bool { + return net.ParseIP(ip) != nil +} + +func isValidCIDR(cidr string) bool { + _, _, err := net.ParseCIDR(cidr) + return err == nil +} + +func roleIsCorrectlyConfigured(role MongoDbRole, mdbVersion string) v1.ValidationResult { + // Extensive validation of the roles attribute + + if role.Role == "" { + return v1.ValidationError("Cannot create a role with an empty name") + } + if role.Db == "" { + return v1.ValidationError("Cannot create a role with an empty db") + } + + for _, inheritedRole := range role.Roles { + if inheritedRole.Role == "" { + return v1.ValidationError("Cannot inherit from a role with an empty name") + } + if inheritedRole.Db == "" { + return v1.ValidationError("Cannot inherit from a role with an empty db") + } + } + // authenticationRestrictions: + for _, ar := range role.AuthenticationRestrictions { + if res := validateAuthenticationRestriction(ar); res.Level == v1.ErrorLevel { + return v1.ValidationError("AuthenticationRestriction is invalid - %s", res.Msg) + } + } + + // privileges + for _, p := range role.Privileges { + if res := validatePrivilege(p, mdbVersion); res.Level == v1.ErrorLevel { + return v1.ValidationError("Privilege is invalid - %s", res.Msg) + } + } + + return v1.ValidationSuccess() +} + +func rolesAttributeisCorrectlyConfigured(d DbCommonSpec) v1.ValidationResult { + // Validate every single entry and return error on the first one that fails validation + for _, role := range d.Security.Roles { + if res := roleIsCorrectlyConfigured(role, d.Version); res.Level == v1.ErrorLevel { + return v1.ValidationError("Error validating role - %s", res.Msg) + } + } + return v1.ValidationSuccess() +} diff --git a/api/v1/mdb/mongodb_types.go b/api/v1/mdb/mongodb_types.go new file mode 100644 index 000000000..8c837c47a --- /dev/null +++ b/api/v1/mdb/mongodb_types.go @@ -0,0 +1,1625 @@ +package mdb + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/blang/semver" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/fcv" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func init() { + v1.SchemeBuilder.Register(&MongoDB{}, &MongoDBList{}) +} + +type LogLevel string + +type ResourceType string + +type TransportSecurity string + +const ( + Debug LogLevel = "DEBUG" + Info LogLevel = "INFO" + Warn LogLevel = "WARN" + Error LogLevel = "ERROR" + Fatal LogLevel = "FATAL" + + Standalone ResourceType = "Standalone" + ReplicaSet ResourceType = "ReplicaSet" + ShardedCluster ResourceType = "ShardedCluster" + + TransportSecurityNone TransportSecurity = "none" + TransportSecurityTLS TransportSecurity = "tls" + + ClusterTopologySingleCluster = "SingleCluster" + ClusterTopologyMultiCluster = "MultiCluster" + + LabelMongoDBResourceOwner = "mongodb.com/v1.mongodbResourceOwner" +) + +// MongoDB resources allow you to deploy Standalones, ReplicaSets or SharedClusters +// to your Kubernetes cluster + +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=mongodb,scope=Namespaced,shortName=mdb +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Current state of the MongoDB deployment." +// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.version",description="Version of MongoDB server." +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type",description="The type of MongoDB deployment. One of 'ReplicaSet', 'ShardedCluster' and 'Standalone'." +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="The time since the MongoDB resource was created." +type MongoDB struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + // +optional + Status MongoDbStatus `json:"status"` + Spec MongoDbSpec `json:"spec"` +} + +func (m *MongoDB) IsAgentImageOverridden() bool { + if m.Spec.PodSpec.IsAgentImageOverridden() { + return true + } + + if m.Spec.ShardPodSpec != nil && m.Spec.ShardPodSpec.IsAgentImageOverridden() { + return true + } + + if m.Spec.IsAgentImageOverridden() { + return true + } + + return false +} + +func isAgentImageOverriden(containers []corev1.Container) bool { + for _, c := range containers { + if c.Name == util.AgentContainerName && c.Image != "" { + return true + } + } + return false +} + +func (m *MongoDB) ForcedIndividualScaling() bool { + return false +} + +func (m *MongoDB) GetSpec() DbSpec { + return &m.Spec +} + +func (m *MongoDB) GetProjectConfigMapNamespace() string { + return m.GetNamespace() +} + +func (m *MongoDB) GetCredentialsSecretNamespace() string { + return m.GetNamespace() +} + +func (m *MongoDB) GetProjectConfigMapName() string { + return m.Spec.GetProject() +} + +func (m *MongoDB) GetCredentialsSecretName() string { + return m.Spec.Credentials +} + +func (m *MongoDB) GetSecurity() *Security { + return m.Spec.GetSecurity() +} + +func (m *MongoDB) GetConnectionSpec() *ConnectionSpec { + return &m.Spec.ConnectionSpec +} + +func (m *MongoDB) GetPrometheus() *mdbcv1.Prometheus { + return m.Spec.Prometheus +} + +func (m *MongoDB) GetBackupSpec() *Backup { + return m.Spec.Backup +} + +func (m *MongoDB) GetResourceType() ResourceType { + return m.Spec.ResourceType +} + +func (m *MongoDB) IsShardedCluster() bool { + return m.GetResourceType() == ShardedCluster +} + +func (m *MongoDB) GetResourceName() string { + return m.Name +} + +// GetSecretsMountedIntoDBPod returns a list of all the optional secret names that are used by this resource. +func (m *MongoDB) GetSecretsMountedIntoDBPod() []string { + secrets := []string{} + var tls string + if m.Spec.ResourceType == ShardedCluster { + for i := 0; i < m.Spec.ShardCount; i++ { + tls = m.GetSecurity().MemberCertificateSecretName(m.ShardRsName(i)) + if tls != "" { + secrets = append(secrets, tls) + } + } + tls = m.GetSecurity().MemberCertificateSecretName(m.ConfigRsName()) + if tls != "" { + secrets = append(secrets, tls) + } + tls = m.GetSecurity().MemberCertificateSecretName(m.ConfigRsName()) + if tls != "" { + secrets = append(secrets, tls) + } + } else { + tls = m.GetSecurity().MemberCertificateSecretName(m.Name) + if tls != "" { + secrets = append(secrets, tls) + } + } + agentCerts := m.GetSecurity().AgentClientCertificateSecretName(m.Name).Name + if agentCerts != "" { + secrets = append(secrets, agentCerts) + } + if m.Spec.Security.Authentication != nil && m.Spec.Security.Authentication.Ldap != nil { + secrets = append(secrets, m.Spec.GetSecurity().Authentication.Ldap.BindQuerySecretRef.Name) + if m.Spec.Security.Authentication != nil && m.Spec.Security.Authentication.Agents.AutomationPasswordSecretRef.Name != "" { + secrets = append(secrets, m.Spec.Security.Authentication.Agents.AutomationPasswordSecretRef.Name) + } + } + return secrets +} + +func (m *MongoDB) GetHostNameOverrideConfigmapName() string { + return fmt.Sprintf("%s-hostname-override", m.Name) +} + +type AdditionalMongodConfigType int + +const ( + StandaloneConfig = iota + ReplicaSetConfig + MongosConfig + ConfigServerConfig + ShardConfig +) + +func GetLastAdditionalMongodConfigByType(lastSpec *MongoDbSpec, configType AdditionalMongodConfigType) (*AdditionalMongodConfig, error) { + if lastSpec == nil { + return &AdditionalMongodConfig{}, nil + } + + switch configType { + case ReplicaSetConfig, StandaloneConfig: + return lastSpec.GetAdditionalMongodConfig(), nil + case MongosConfig: + return lastSpec.MongosSpec.GetAdditionalMongodConfig(), nil + case ConfigServerConfig: + return lastSpec.ConfigSrvSpec.GetAdditionalMongodConfig(), nil + case ShardConfig: + return lastSpec.ShardSpec.GetAdditionalMongodConfig(), nil + } + return &AdditionalMongodConfig{}, nil +} + +// GetLastAdditionalMongodConfigByType returns the last successfully achieved AdditionalMongodConfigType for the given component. +func (m *MongoDB) GetLastAdditionalMongodConfigByType(configType AdditionalMongodConfigType) (*AdditionalMongodConfig, error) { + if m.Spec.GetResourceType() == ShardedCluster { + panic(errors.Errorf("this method cannot be used from ShardedCluster controller; use non-method GetLastAdditionalMongodConfigByType and pass lastSpec from the deployment state.")) + } + lastSpec, err := m.GetLastSpec() + if err != nil || lastSpec == nil { + return &AdditionalMongodConfig{}, err + } + return GetLastAdditionalMongodConfigByType(lastSpec, configType) +} + +type ClusterSpecList []ClusterSpecItem + +// ClusterSpecItem is the mongodb multi-cluster spec that is specific to a +// particular Kubernetes cluster, this maps to the statefulset created in each cluster +type ClusterSpecItem struct { + // ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + // name should have a one on one mapping with the service-account created in the central cluster + // to talk to the workload clusters. + ClusterName string `json:"clusterName,omitempty"` + // this is an optional service, it will get the name "-service" in case not provided + Service string `json:"service,omitempty"` + // ExternalAccessConfiguration provides external access configuration for Multi-Cluster. + // +optional + ExternalAccessConfiguration *ExternalAccessConfiguration `json:"externalAccess,omitempty"` + // Amount of members for this MongoDB Replica Set + Members int `json:"members"` + // MemberConfig allows to specify votes, priorities and tags for each of the mongodb process. + // +optional + MemberConfig []automationconfig.MemberOptions `json:"memberConfig,omitempty"` + // +optional + StatefulSetConfiguration *mdbcv1.StatefulSetConfiguration `json:"statefulSet,omitempty"` + // +optional + PodSpec *MongoDbPodSpec `json:"podSpec,omitempty"` +} + +// ClusterSpecItemOverride is almost exact copy of ClusterSpecItem object. +// The object is used in ClusterSpecList in ShardedClusterComponentOverrideSpec in shard overrides. +// The difference lies in some fields being optional, e.g. Members to make it possible to NOT override fields and rely on +// what was set in top level shard configuration. +type ClusterSpecItemOverride struct { + // ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + // name should have a one on one mapping with the service-account created in the central cluster + // to talk to the workload clusters. + ClusterName string `json:"clusterName,omitempty"` + // Amount of members for this MongoDB Replica Set + // +optional + Members *int `json:"members"` + // MemberConfig allows to specify votes, priorities and tags for each of the mongodb process. + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + MemberConfig []automationconfig.MemberOptions `json:"memberConfig,omitempty"` + // +optional + StatefulSetConfiguration *mdbcv1.StatefulSetConfiguration `json:"statefulSet,omitempty"` + // +optional + PodSpec *MongoDbPodSpec `json:"podSpec,omitempty"` +} + +// +kubebuilder:object:generate=false +type DbSpec interface { + Replicas() int + GetClusterDomain() string + GetMongoDBVersion() string + GetSecurityAuthenticationModes() []string + GetResourceType() ResourceType + IsSecurityTLSConfigEnabled() bool + GetFeatureCompatibilityVersion() *string + GetHorizonConfig() []MongoDBHorizonConfig + GetAdditionalMongodConfig() *AdditionalMongodConfig + GetExternalDomain() *string + GetMemberOptions() []automationconfig.MemberOptions + GetAgentConfig() AgentConfig +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type MongoDBList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []MongoDB `json:"items"` +} + +type MongoDBHorizonConfig map[string]string + +type MongoDBConnectivity struct { + // ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + // Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + // [ + // { + // "internal": "my-rs-0.my-internal-domain.com:31843", + // "external": "my-rs-0.my-external-domain.com:21467" + // }, + // { + // "internal": "my-rs-1.my-internal-domain.com:31843", + // "external": "my-rs-1.my-external-domain.com:21467" + // }, + // ... + // ] + // The key of each item in the map is an arbitrary, user-chosen string that + // represents the name of the horizon. The value of the item is the host and, + // optionally, the port that this mongod node will be connected to from. + // +optional + ReplicaSetHorizons []MongoDBHorizonConfig `json:"replicaSetHorizons,omitempty"` +} + +type MongoDbStatus struct { + status.Common `json:",inline"` + BackupStatus *BackupStatus `json:"backup,omitempty"` + status.MongodbShardedClusterSizeConfig `json:",inline"` + SizeStatusInClusters *status.MongodbShardedSizeStatusInClusters `json:"sizeStatusInClusters,omitempty"` + Members int `json:"members,omitempty"` + Version string `json:"version"` + Link string `json:"link,omitempty"` + FeatureCompatibilityVersion string `json:"featureCompatibilityVersion,omitempty"` + Warnings []status.Warning `json:"warnings,omitempty"` +} + +type BackupMode string + +type BackupStatus struct { + StatusName string `json:"statusName"` +} + +type DbCommonSpec struct { + // +kubebuilder:validation:Pattern=^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + // +kubebuilder:validation:Required + Version string `json:"version"` + FeatureCompatibilityVersion *string `json:"featureCompatibilityVersion,omitempty"` + Agent AgentConfig `json:"agent,omitempty"` + // +kubebuilder:validation:Format="hostname" + ClusterDomain string `json:"clusterDomain,omitempty"` + ConnectionSpec `json:",inline"` + + // +kubebuilder:validation:Enum=DEBUG;INFO;WARN;ERROR;FATAL + LogLevel LogLevel `json:"logLevel,omitempty"` + // ExternalAccessConfiguration provides external access configuration. + // +optional + ExternalAccessConfiguration *ExternalAccessConfiguration `json:"externalAccess,omitempty"` + + Persistent *bool `json:"persistent,omitempty"` + // +kubebuilder:validation:Enum=Standalone;ReplicaSet;ShardedCluster + // +kubebuilder:validation:Required + ResourceType ResourceType `json:"type"` + // +optional + Security *Security `json:"security,omitempty"` + Connectivity *MongoDBConnectivity `json:"connectivity,omitempty"` + Backup *Backup `json:"backup,omitempty"` + + // Prometheus configurations. + // +optional + Prometheus *mdbcv1.Prometheus `json:"prometheus,omitempty"` + + // +optional + // StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + // if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + // the global one + StatefulSetConfiguration *mdbcv1.StatefulSetConfiguration `json:"statefulSet,omitempty"` + + // AdditionalMongodConfig is additional configuration that can be passed to + // each data-bearing mongod at runtime. Uses the same structure as the mongod + // configuration file: + // https://docs.mongodb.com/manual/reference/configuration-options/ + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + AdditionalMongodConfig *AdditionalMongodConfig `json:"additionalMongodConfig,omitempty"` + + // In few service mesh options for ex: Istio, by default we would need to duplicate the + // service objects created per pod in all the clusters to enable DNS resolution. Users can + // however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + // enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + // whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + // +optional + DuplicateServiceObjects *bool `json:"duplicateServiceObjects,omitempty"` + + // Topology sets the desired cluster topology of MongoDB resources + // It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + // then clusterSpecList field is mandatory and at least one member cluster has to be specified. + // +kubebuilder:validation:Enum=SingleCluster;MultiCluster + // +optional + Topology string `json:"topology,omitempty"` +} + +type MongoDbSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + DbCommonSpec `json:",inline"` + ShardedClusterSpec `json:",inline"` + status.MongodbShardedClusterSizeConfig `json:",inline"` + + // Amount of members for this MongoDB Replica Set + Members int `json:"members,omitempty"` + PodSpec *MongoDbPodSpec `json:"podSpec,omitempty"` + // DEPRECATED please use `spec.statefulSet.spec.serviceName` to provide a custom service name. + // this is an optional service, it will get the name "-service" in case not provided + Service string `json:"service,omitempty"` + + // MemberConfig allows to specify votes, priorities and tags for each of the mongodb process. + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + MemberConfig []automationconfig.MemberOptions `json:"memberConfig,omitempty"` +} + +func (m *MongoDbSpec) GetExternalDomain() *string { + if m.ExternalAccessConfiguration != nil { + return m.ExternalAccessConfiguration.ExternalDomain + } + return nil +} + +func (m *MongoDbSpec) GetHorizonConfig() []MongoDBHorizonConfig { + return m.Connectivity.ReplicaSetHorizons +} + +func (m *MongoDbSpec) GetMemberOptions() []automationconfig.MemberOptions { + return m.MemberConfig +} + +type SnapshotSchedule struct { + // Number of hours between snapshots. + // +kubebuilder:validation:Enum=6;8;12;24 + // +optional + SnapshotIntervalHours *int `json:"snapshotIntervalHours,omitempty"` + + // Number of days to keep recent snapshots. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=365 + // +optional + SnapshotRetentionDays *int `json:"snapshotRetentionDays,omitempty"` + + // Number of days to retain daily snapshots. Setting 0 will disable this rule. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=365 + // +optional + DailySnapshotRetentionDays *int `json:"dailySnapshotRetentionDays"` + + // Number of weeks to retain weekly snapshots. Setting 0 will disable this rule + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=365 + // +optional + WeeklySnapshotRetentionWeeks *int `json:"weeklySnapshotRetentionWeeks,omitempty"` + // Number of months to retain weekly snapshots. Setting 0 will disable this rule. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=36 + // +optional + MonthlySnapshotRetentionMonths *int `json:"monthlySnapshotRetentionMonths,omitempty"` + // Number of hours in the past for which a point-in-time snapshot can be created. + // +kubebuilder:validation:Enum=1;2;3;4;5;6;7;15;30;60;90;120;180;360 + // +optional + PointInTimeWindowHours *int `json:"pointInTimeWindowHours,omitempty"` + + // Hour of the day to schedule snapshots using a 24-hour clock, in UTC. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=23 + // +optional + ReferenceHourOfDay *int `json:"referenceHourOfDay,omitempty"` + + // Minute of the hour to schedule snapshots, in UTC. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=59 + // +optional + ReferenceMinuteOfHour *int `json:"referenceMinuteOfHour,omitempty"` + + // Day of the week when Ops Manager takes a full snapshot. This ensures a recent complete backup. Ops Manager sets the default value to SUNDAY. + // +kubebuilder:validation:Enum=SUNDAY;MONDAY;TUESDAY;WEDNESDAY;THURSDAY;FRIDAY;SATURDAY + // +optional + FullIncrementalDayOfWeek *string `json:"fullIncrementalDayOfWeek,omitempty"` + + // +kubebuilder:validation:Enum=15;30;60 + ClusterCheckpointIntervalMin *int `json:"clusterCheckpointIntervalMin,omitempty"` +} + +// Backup contains configuration options for configuring +// backup for this MongoDB resource +type Backup struct { + // +kubebuilder:validation:Enum=enabled;disabled;terminated + // +optional + Mode BackupMode `json:"mode"` + + // AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + // when the MongoDB CR is deleted + // +optional + AutoTerminateOnDeletion bool `json:"autoTerminateOnDeletion,omitempty"` + + // +optional + SnapshotSchedule *SnapshotSchedule `json:"snapshotSchedule,omitempty"` + + // Encryption settings + // +optional + Encryption *Encryption `json:"encryption,omitempty"` + + // Assignment Labels set in the Ops Manager + // +optional + AssignmentLabels []string `json:"assignmentLabels,omitempty"` +} + +func (b *Backup) IsKmipEnabled() bool { + if b.Encryption == nil || b.Encryption.Kmip == nil { + return false + } + return true +} + +func (b *Backup) GetKmip() *KmipConfig { + if !b.IsKmipEnabled() { + return nil + } + return b.Encryption.Kmip +} + +// Encryption contains encryption settings +type Encryption struct { + // Kmip corresponds to the KMIP configuration assigned to the Ops Manager Project's configuration. + // +optional + Kmip *KmipConfig `json:"kmip,omitempty"` +} + +// KmipConfig contains Project-level KMIP configuration +type KmipConfig struct { + // KMIP Client configuration + Client v1.KmipClientConfig `json:"client"` +} + +type LogRotateForBackupAndMonitoring struct { + // Maximum size for an individual log file before rotation. + // OM only supports ints + SizeThresholdMB int `json:"sizeThresholdMB,omitempty"` + // Number of hours after which this MongoDB Agent rotates the log file. + TimeThresholdHrs int `json:"timeThresholdHrs,omitempty"` +} + +// AgentLoggingMongodConfig contain settings for the mongodb processes configured by the agent +type AgentLoggingMongodConfig struct { + // +optional + // LogRotate configures log rotation for the mongodb processes + LogRotate *automationconfig.CrdLogRotate `json:"logRotate,omitempty"` + + // LogRotate configures audit log rotation for the mongodb processes + AuditLogRotate *automationconfig.CrdLogRotate `json:"auditlogRotate,omitempty"` + + // +optional + // SystemLog configures system log of mongod + SystemLog *automationconfig.SystemLog `json:"systemLog,omitempty"` +} + +func (a *AgentLoggingMongodConfig) HasLoggingConfigured() bool { + if a.LogRotate != nil || a.AuditLogRotate != nil || a.SystemLog != nil { + return true + } + return false +} + +type BackupAgent struct { + // +optional + // LogRotate configures log rotation for the BackupAgent processes + LogRotate *LogRotateForBackupAndMonitoring `json:"logRotate,omitempty"` +} + +type MonitoringAgent struct { + // +optional + // LogRotate configures log rotation for the BackupAgent processes + LogRotate *LogRotateForBackupAndMonitoring `json:"logRotate,omitempty"` +} + +type AgentConfig struct { + // +optional + BackupAgent BackupAgent `json:"backupAgent,omitempty"` + // +optional + MonitoringAgent MonitoringAgent `json:"monitoringAgent,omitempty"` + // +optional + Mongod AgentLoggingMongodConfig `json:"mongod,omitempty"` + // +optional + ReadinessProbe ReadinessProbe `json:"readinessProbe,omitempty"` + // +optional + StartupParameters StartupParameters `json:"startupOptions"` + // +optional + LogLevel LogLevel `json:"logLevel"` + // +optional + MaxLogFileDurationHours int `json:"maxLogFileDurationHours"` + // DEPRECATED please use mongod.logRotate + // +optional + LogRotate *automationconfig.CrdLogRotate `json:"logRotate,omitempty"` + // DEPRECATED please use mongod.systemLog + // +optional + SystemLog *automationconfig.SystemLog `json:"systemLog,omitempty"` +} + +type MonitoringAgentConfig struct { + StartupParameters StartupParameters `json:"startupOptions"` +} + +type EnvironmentVariables map[string]string + +type ReadinessProbe struct { + EnvironmentVariables `json:"environmentVariables,omitempty"` +} + +// StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains +// log rotation settings as defined here: +type StartupParameters map[string]string + +func (s StartupParameters) ToCommandLineArgs() string { + var keys []string + for k := range s { + keys = append(keys, k) + } + + // order must be preserved to ensure the same set of command line arguments + // results in the same StatefulSet template spec. + sort.SliceStable(keys, func(i, j int) bool { + return keys[i] < keys[j] + }) + + sb := strings.Builder{} + for _, key := range keys { + if value := s[key]; value != "" { + sb.Write([]byte(fmt.Sprintf(" -%s=%s", key, value))) + } else { + sb.Write([]byte(fmt.Sprintf(" -%s", key))) + } + } + return sb.String() +} + +func (m *MongoDB) DesiredReplicas() int { + return m.Spec.Members +} + +func (m *MongoDB) CurrentReplicas() int { + return m.Status.Members +} + +// GetMongoDBVersion returns the version of the MongoDB. +func (m *MongoDbSpec) GetMongoDBVersion() string { + return m.Version +} + +func (m *MongoDbSpec) GetClusterDomain() string { + if m.ClusterDomain != "" { + return m.ClusterDomain + } + return "cluster.local" +} + +func (m *MongoDbSpec) MinimumMajorVersion() uint64 { + if m.FeatureCompatibilityVersion != nil && *m.FeatureCompatibilityVersion != "" { + fcv := *m.FeatureCompatibilityVersion + + // ignore errors here as the format of FCV/version is handled by CRD validation + semverFcv, _ := semver.Make(fmt.Sprintf("%s.0", fcv)) + return semverFcv.Major + } + semverVersion, _ := semver.Make(m.GetMongoDBVersion()) + return semverVersion.Major +} + +// ProjectConfig contains the configuration expected from the `project` (ConfigMap) under Data. +type ProjectConfig struct { + BaseURL string + ProjectName string + OrgID string + Credentials string + UseCustomCA bool + env.SSLProjectConfig +} + +// Credentials contains the configuration expected from the `credentials` (Secret)` attribute in +// `.spec.credentials`. +type Credentials struct { + // +required + PublicAPIKey string + + // +required + PrivateAPIKey string +} + +type ConfigMapRef struct { + Name string `json:"name,omitempty"` +} + +type PrivateCloudConfig struct { + ConfigMapRef ConfigMapRef `json:"configMapRef,omitempty"` +} + +// ConnectionSpec holds fields required to establish an Ops Manager connection +// note, that the fields are marked as 'omitempty' as otherwise they are shown for AppDB +// which is not good +type ConnectionSpec struct { + SharedConnectionSpec `json:",inline"` + // Name of the Secret holding credentials information + // +kubebuilder:validation:Required + Credentials string `json:"credentials"` +} + +type SharedConnectionSpec struct { + // Transient field - the name of the project. By default, is equal to the name of the resource + // though can be overridden if the ConfigMap specifies a different name + ProjectName string `json:"-"` // ignore when marshalling + + // Dev note: don't reference these two fields directly - use the `getProject` method instead + + OpsManagerConfig *PrivateCloudConfig `json:"opsManager,omitempty"` + CloudManagerConfig *PrivateCloudConfig `json:"cloudManager,omitempty"` +} + +type Security struct { + TLSConfig *TLSConfig `json:"tls,omitempty"` + Authentication *Authentication `json:"authentication,omitempty"` + Roles []MongoDbRole `json:"roles,omitempty"` + + // +optional + CertificatesSecretsPrefix string `json:"certsSecretPrefix"` +} + +// MemberCertificateSecretName returns the name of the secret containing the member TLS certs. +func (s *Security) MemberCertificateSecretName(defaultName string) string { + if s.CertificatesSecretsPrefix != "" { + return fmt.Sprintf("%s-%s-cert", s.CertificatesSecretsPrefix, defaultName) + } + + // The default behaviour is to use the `defaultname-cert` format + return fmt.Sprintf("%s-cert", defaultName) +} + +func (d *DbCommonSpec) IsAgentImageOverridden() bool { + if d.StatefulSetConfiguration != nil && isAgentImageOverriden(d.StatefulSetConfiguration.SpecWrapper.Spec.Template.Spec.Containers) { + return true + } + + return false +} + +func (d *DbCommonSpec) GetSecurity() *Security { + if d.Security == nil { + return &Security{} + } + return d.Security +} + +func (d *DbCommonSpec) GetExternalDomain() *string { + if d.ExternalAccessConfiguration != nil { + return d.ExternalAccessConfiguration.ExternalDomain + } + return nil +} + +func (d DbCommonSpec) GetAgentConfig() AgentConfig { + return d.Agent +} + +func (d *DbCommonSpec) GetAdditionalMongodConfig() *AdditionalMongodConfig { + if d == nil || d.AdditionalMongodConfig == nil { + return &AdditionalMongodConfig{} + } + + return d.AdditionalMongodConfig +} + +func (s *Security) IsTLSEnabled() bool { + if s == nil { + return false + } + if s.TLSConfig != nil { + if s.TLSConfig.Enabled { + return true + } + } + return s.CertificatesSecretsPrefix != "" +} + +// GetAgentMechanism returns the authentication mechanism that the agents will be using. +// The agents will use X509 if it is the only mechanism specified, otherwise they will use SCRAM if specified +// and no auth if no mechanisms exist. +func (s *Security) GetAgentMechanism(currentMechanism string) string { + if s == nil || s.Authentication == nil { + return "" + } + auth := s.Authentication + if !s.Authentication.Enabled { + return "" + } + + if currentMechanism == "MONGODB-X509" { + return util.X509 + } + + // If we arrive here, this should + // ALWAYS be true, as we do not allow + // agents.mode to be empty + // if more than one mode in specified in + // spec.authentication.modes + // The check is done in the validation webhook + if len(s.Authentication.Modes) == 1 { + return string(s.Authentication.Modes[0]) + } + return auth.Agents.Mode +} + +// ShouldUseX509 determines if the deployment should have X509 authentication configured +// whether it was configured explicitly or if it required as it would be performing +// an illegal transition otherwise. +func (s *Security) ShouldUseX509(currentAgentAuthMode string) bool { + return s.GetAgentMechanism(currentAgentAuthMode) == util.X509 +} + +// AgentClientCertificateSecretName returns the name of the Secret that holds the agent +// client TLS certificates. +// If no custom name has been defined, it returns the default one. +func (s Security) AgentClientCertificateSecretName(resourceName string) corev1.SecretKeySelector { + secretName := util.AgentSecretName + + if s.CertificatesSecretsPrefix != "" { + secretName = fmt.Sprintf("%s-%s-%s", s.CertificatesSecretsPrefix, resourceName, util.AgentSecretName) + } + if s.ShouldUseClientCertificates() { + secretName = s.Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name + } + + return corev1.SecretKeySelector{ + Key: util.AutomationAgentPemSecretKey, + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + } +} + +// The customer has set ClientCertificateSecretRef. This signals that client certs are required, +// even when no x509 agent-auth has been enabled. +func (s Security) ShouldUseClientCertificates() bool { + return s.Authentication != nil && s.Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name != "" +} + +func (s Security) InternalClusterAuthSecretName(defaultName string) string { + secretName := fmt.Sprintf("%s-clusterfile", defaultName) + if s.CertificatesSecretsPrefix != "" { + secretName = fmt.Sprintf("%s-%s", s.CertificatesSecretsPrefix, secretName) + } + return secretName +} + +// RequiresClientTLSAuthentication checks if client TLS authentication is required, depending +// on a set of defined attributes in the MongoDB resource. This can be explicitly set, setting +// `Authentication.RequiresClientTLSAuthentication` to true or implicitly by setting x509 auth +// as the only auth mechanism. +func (s Security) RequiresClientTLSAuthentication() bool { + if s.Authentication == nil { + return false + } + + if len(s.Authentication.Modes) == 1 && IsAuthPresent(s.Authentication.Modes, util.X509) { + return true + } + + return s.Authentication.RequiresClientTLSAuthentication +} + +func (s *Security) ShouldUseLDAP(currentAgentAuthMode string) bool { + return s.GetAgentMechanism(currentAgentAuthMode) == util.LDAP +} + +func (s *Security) GetInternalClusterAuthenticationMode() string { + if s == nil || s.Authentication == nil { + return "" + } + if s.Authentication.InternalCluster != "" { + return strings.ToUpper(s.Authentication.InternalCluster) + } + return "" +} + +// Authentication holds various authentication related settings that affect +// this MongoDB resource. +type Authentication struct { + Enabled bool `json:"enabled"` + Modes []AuthMode `json:"modes,omitempty"` + InternalCluster string `json:"internalCluster,omitempty"` + // IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + IgnoreUnknownUsers bool `json:"ignoreUnknownUsers,omitempty"` + + // LDAP Configuration + // +optional + Ldap *Ldap `json:"ldap,omitempty"` + + // Agents contains authentication configuration properties for the agents + // +optional + Agents AgentAuthentication `json:"agents,omitempty"` + + // Clients should present valid TLS certificates + RequiresClientTLSAuthentication bool `json:"requireClientTLSAuthentication,omitempty"` +} + +// +kubebuilder:validation:Enum=X509;SCRAM;SCRAM-SHA-1;MONGODB-CR;SCRAM-SHA-256;LDAP +type AuthMode string + +func ConvertAuthModesToStrings(authModes []AuthMode) []string { + stringAuth := make([]string, len(authModes)) + for i, auth := range authModes { + stringAuth[i] = string(auth) + } + return stringAuth +} + +func IsAuthPresent(authModes []AuthMode, auth string) bool { + for _, authMode := range authModes { + if string(authMode) == auth { + return true + } + } + return false +} + +type AuthenticationRestriction struct { + ClientSource []string `json:"clientSource,omitempty"` + ServerAddress []string `json:"serverAddress,omitempty"` +} + +type Resource struct { + // +optional + Db string `json:"db"` + // +optional + Collection string `json:"collection"` + Cluster *bool `json:"cluster,omitempty"` +} + +type Privilege struct { + Actions []string `json:"actions"` + Resource Resource `json:"resource"` +} + +type InheritedRole struct { + Db string `json:"db"` + Role string `json:"role"` +} + +type MongoDbRole struct { + Role string `json:"role"` + AuthenticationRestrictions []AuthenticationRestriction `json:"authenticationRestrictions,omitempty"` + Db string `json:"db"` + // +optional + Privileges []Privilege `json:"privileges"` + Roles []InheritedRole `json:"roles,omitempty"` +} + +type AgentAuthentication struct { + // Mode is the desired Authentication mode that the agents will use + Mode string `json:"mode"` + // +optional + AutomationUserName string `json:"automationUserName"` + // +optional + AutomationPasswordSecretRef corev1.SecretKeySelector `json:"automationPasswordSecretRef"` + // +optional + AutomationLdapGroupDN string `json:"automationLdapGroupDN"` + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + ClientCertificateSecretRefWrap ClientCertificateSecretRefWrapper `json:"clientCertificateSecretRef,omitempty"` +} + +// IsX509Enabled determines if X509 is to be enabled at the project level +// it does not necessarily mean that the agents are using X509 authentication +func (a *Authentication) IsX509Enabled() bool { + return stringutil.Contains(a.GetModes(), util.X509) +} + +// IsLDAPEnabled determines if LDAP is to be enabled at the project level +func (a *Authentication) isLDAPEnabled() bool { + return stringutil.Contains(a.GetModes(), util.LDAP) +} + +// GetModes returns the modes of the Authentication instance of an empty +// list if it is nil +func (a *Authentication) GetModes() []string { + if a == nil { + return []string{} + } + return ConvertAuthModesToStrings(a.Modes) +} + +type Ldap struct { + // +optional + Servers []string `json:"servers"` + + // +kubebuilder:validation:Enum=tls;none + // +optional + TransportSecurity *TransportSecurity `json:"transportSecurity"` + // +optional + ValidateLDAPServerConfig *bool `json:"validateLDAPServerConfig"` + + // Allows to point at a ConfigMap/key with a CA file to mount on the Pod + CAConfigMapRef *corev1.ConfigMapKeySelector `json:"caConfigMapRef,omitempty"` + + // +optional + BindQueryUser string `json:"bindQueryUser"` + // +optional + BindQuerySecretRef SecretRef `json:"bindQueryPasswordSecretRef"` + // +optional + AuthzQueryTemplate string `json:"authzQueryTemplate"` + // +optional + UserToDNMapping string `json:"userToDNMapping"` + // +optional + TimeoutMS int `json:"timeoutMS"` + // +optional + UserCacheInvalidationInterval int `json:"userCacheInvalidationInterval"` +} + +type SecretRef struct { + // +kubebuilder:validation:Required + Name string `json:"name"` +} + +type TLSConfig struct { + // DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + // Enables TLS for this resource. This will make the operator try to mount a + // Secret with a defined name (-cert). + // This is only used when enabling TLS on a MongoDB resource, and not on the + // AppDB, where TLS is configured by setting `secretRef.Name`. + Enabled bool `json:"enabled,omitempty"` + + AdditionalCertificateDomains []string `json:"additionalCertificateDomains,omitempty"` + + // CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + // used to validate the certificates created already. + CA string `json:"ca,omitempty"` +} + +func (m *MongoDbSpec) GetTLSConfig() *TLSConfig { + if m.Security == nil || m.Security.TLSConfig == nil { + return &TLSConfig{} + } + + return m.Security.TLSConfig +} + +// UnmarshalJSON when unmarshalling a MongoDB instance, we don't want to have any nil references +// these are replaced with an empty instance to prevent nil references by calling InitDefaults +func (m *MongoDB) UnmarshalJSON(data []byte) error { + type MongoDBJSON *MongoDB + if err := json.Unmarshal(data, (MongoDBJSON)(m)); err != nil { + return err + } + + m.InitDefaults() + + return nil +} + +// GetLastSpec returns the last spec that has successfully be applied. +func (m *MongoDB) GetLastSpec() (*MongoDbSpec, error) { + lastSpecStr := annotations.GetAnnotation(m, util.LastAchievedSpec) + if lastSpecStr == "" { + return nil, nil + } + + lastSpec := MongoDbSpec{} + if err := json.Unmarshal([]byte(lastSpecStr), &lastSpec); err != nil { + return nil, err + } + + return &lastSpec, nil +} + +func (m *MongoDB) ServiceName() string { + if m.Spec.StatefulSetConfiguration != nil { + svc := m.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName + + if svc != "" { + return svc + } + } + + if m.Spec.Service == "" { + return dns.GetServiceName(m.GetName()) + } + return m.Spec.Service +} + +func (m *MongoDB) ConfigSrvServiceName() string { + return m.Name + "-cs" +} + +func (m *MongoDB) ShardServiceName() string { + return m.Name + "-sh" +} + +func (m *MongoDB) MongosRsName() string { + return m.Name + "-mongos" +} + +func (m *MongoDB) ConfigRsName() string { + return m.Name + "-config" +} + +func (m *MongoDB) ShardRsName(i int) string { + // Unfortunately the pattern used by OM (name_idx) doesn't work as Kubernetes doesn't create the stateful set with an + // exception: "a DNS-1123 subdomain must consist of lower case alphanumeric characters, '-' or '.'" + return fmt.Sprintf("%s-%d", m.Name, i) +} + +func (m *MongoDB) MultiShardRsName(clusterIdx int, shardIdx int) string { + return fmt.Sprintf("%s-%d-%d", m.Name, shardIdx, clusterIdx) +} + +func (m *MongoDB) MultiMongosRsName(clusterIdx int) string { + return fmt.Sprintf("%s-mongos-%d", m.Name, clusterIdx) +} + +func (m *MongoDB) MultiConfigRsName(clusterIdx int) string { + return fmt.Sprintf("%s-config-%d", m.Name, clusterIdx) +} + +func (m *MongoDB) IsLDAPEnabled() bool { + if m.Spec.Security == nil || m.Spec.Security.Authentication == nil { + return false + } + return IsAuthPresent(m.Spec.Security.Authentication.Modes, util.LDAP) +} + +func (m *MongoDB) UpdateStatus(phase status.Phase, statusOptions ...status.Option) { + m.Status.UpdateCommonFields(phase, m.GetGeneration(), statusOptions...) + + if option, exists := status.GetOption(statusOptions, status.BackupStatusOption{}); exists { + if m.Status.BackupStatus == nil { + m.Status.BackupStatus = &BackupStatus{} + } + m.Status.BackupStatus.StatusName = option.(status.BackupStatusOption).Value().(string) + } + + if option, exists := status.GetOption(statusOptions, status.WarningsOption{}); exists { + m.Status.Warnings = append(m.Status.Warnings, option.(status.WarningsOption).Warnings...) + } + if option, exists := status.GetOption(statusOptions, status.BaseUrlOption{}); exists { + m.Status.Link = option.(status.BaseUrlOption).BaseUrl + } + switch m.Spec.ResourceType { + case ReplicaSet: + if option, exists := status.GetOption(statusOptions, status.ReplicaSetMembersOption{}); exists { + m.Status.Members = option.(status.ReplicaSetMembersOption).Members + } + case ShardedCluster: + if option, exists := status.GetOption(statusOptions, status.ShardedClusterSizeConfigOption{}); exists { + if sizeConfig := option.(status.ShardedClusterSizeConfigOption).SizeConfig; sizeConfig != nil { + m.Status.MongodbShardedClusterSizeConfig = *sizeConfig + } + } + if option, exists := status.GetOption(statusOptions, status.ShardedClusterSizeStatusInClustersOption{}); exists { + if sizeConfigInClusters := option.(status.ShardedClusterSizeStatusInClustersOption).SizeConfigInClusters; sizeConfigInClusters != nil { + m.Status.SizeStatusInClusters = sizeConfigInClusters + } + } + } + + if phase == status.PhaseRunning { + m.Status.Version = m.Spec.Version + m.Status.FeatureCompatibilityVersion = m.CalculateFeatureCompatibilityVersion() + m.Status.Message = "" + + switch m.Spec.ResourceType { + case ShardedCluster: + m.Status.ShardCount = m.Spec.ShardCount + } + } +} + +func (m *MongoDB) SetWarnings(warnings []status.Warning, _ ...status.Option) { + m.Status.Warnings = warnings +} + +func (m *MongoDB) AddWarningIfNotExists(warning status.Warning) { + m.Status.Warnings = status.Warnings(m.Status.Warnings).AddIfNotExists(warning) +} + +func (m *MongoDB) GetStatus(...status.Option) interface{} { + return m.Status +} + +func (m *MongoDB) GetCommonStatus(...status.Option) *status.Common { + return &m.Status.Common +} + +func (m *MongoDB) GetPhase() status.Phase { + return m.Status.Phase +} + +func (m *MongoDB) GetStatusPath(...status.Option) string { + return "/status" +} + +// GetProject returns the name of the ConfigMap containing the information about connection to OM/CM, returns empty string if +// neither CloudManager not OpsManager configmap is set +func (c *ConnectionSpec) GetProject() string { + // the contract is that either ops manager or cloud manager must be provided - the controller must validate this + if c.OpsManagerConfig.ConfigMapRef.Name != "" { + return c.OpsManagerConfig.ConfigMapRef.Name + } + if c.CloudManagerConfig.ConfigMapRef.Name != "" { + return c.CloudManagerConfig.ConfigMapRef.Name + } + return "" +} + +// InitDefaults makes sure the MongoDB resource has correct state after initialization: +// - prevents any references from having nil values. +// - makes sure the spec is in correct state +// +// should not be called directly, used in tests and unmarshalling +func (m *MongoDB) InitDefaults() { + // al resources have a pod spec + if m.Spec.PodSpec == nil { + m.Spec.PodSpec = NewMongoDbPodSpec() + } + + if m.Spec.ResourceType == ShardedCluster { + if m.Spec.ConfigSrvPodSpec == nil { + m.Spec.ConfigSrvPodSpec = NewMongoDbPodSpec() + } + if m.Spec.MongosPodSpec == nil { + m.Spec.MongosPodSpec = NewMongoDbPodSpec() + } + if m.Spec.ShardPodSpec == nil { + m.Spec.ShardPodSpec = NewMongoDbPodSpec() + } + if m.Spec.ConfigSrvSpec == nil { + m.Spec.ConfigSrvSpec = &ShardedClusterComponentSpec{} + } + if m.Spec.MongosSpec == nil { + m.Spec.MongosSpec = &ShardedClusterComponentSpec{} + } + if m.Spec.ShardSpec == nil { + m.Spec.ShardSpec = &ShardedClusterComponentSpec{} + } + + } + + if m.Spec.Connectivity == nil { + m.Spec.Connectivity = NewConnectivity() + } + + m.Spec.Security = EnsureSecurity(m.Spec.Security) + + if m.Spec.OpsManagerConfig == nil { + m.Spec.OpsManagerConfig = NewOpsManagerConfig() + } + + if m.Spec.CloudManagerConfig == nil { + m.Spec.CloudManagerConfig = NewOpsManagerConfig() + } + + // ProjectName defaults to the name of the resource + m.Spec.ProjectName = m.Name +} + +func (m *MongoDB) ObjectKey() client.ObjectKey { + return kube.ObjectKey(m.Namespace, m.Name) +} + +func (m *MongoDB) GetLDAP(password, caContents string) *ldap.Ldap { + if !m.IsLDAPEnabled() { + return nil + } + + mdbLdap := m.Spec.Security.Authentication.Ldap + transportSecurity := GetTransportSecurity(mdbLdap) + + validateServerConfig := true + if mdbLdap.ValidateLDAPServerConfig != nil { + validateServerConfig = *mdbLdap.ValidateLDAPServerConfig + } + + return &ldap.Ldap{ + BindQueryUser: mdbLdap.BindQueryUser, + BindQueryPassword: password, + Servers: strings.Join(mdbLdap.Servers, ","), + TransportSecurity: string(transportSecurity), + CaFileContents: caContents, + ValidateLDAPServerConfig: validateServerConfig, + + // Related to LDAP Authorization + AuthzQueryTemplate: mdbLdap.AuthzQueryTemplate, + UserToDnMapping: mdbLdap.UserToDNMapping, + + // TODO: Enable LDAP SASL bind method + BindMethod: "simple", + BindSaslMechanisms: "", + + TimeoutMS: mdbLdap.TimeoutMS, + UserCacheInvalidationInterval: mdbLdap.UserCacheInvalidationInterval, + } +} + +// ExternalAccessConfiguration holds the custom Service override that will be merged into the operator created one. +type ExternalAccessConfiguration struct { + // Provides a way to override the default (NodePort) Service + // +optional + ExternalService ExternalServiceConfiguration `json:"externalService,omitempty"` + + // An external domain that is used for exposing MongoDB to the outside world. + // +optional + ExternalDomain *string `json:"externalDomain,omitempty"` +} + +// ExternalServiceConfiguration is a wrapper for the Service spec object. +type ExternalServiceConfiguration struct { + // A wrapper for the Service spec object. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + SpecWrapper *ServiceSpecWrapper `json:"spec"` + + // A map of annotations that shall be added to the externally available Service. + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +func GetTransportSecurity(mdbLdap *Ldap) TransportSecurity { + transportSecurity := TransportSecurityNone + if mdbLdap.TransportSecurity != nil && strings.ToLower(string(*mdbLdap.TransportSecurity)) != "none" { + transportSecurity = TransportSecurityTLS + } + return transportSecurity +} + +type MongoDbPodSpec struct { + ContainerResourceRequirements `json:"-"` + + // +kubebuilder:pruning:PreserveUnknownFields + PodTemplateWrapper PodTemplateSpecWrapper `json:"podTemplate,omitempty"` + // Note, this field is not serialized in the CRD, it's only present here because of the + // way we currently set defaults for this field in the operator, similar to "ContainerResourceRequirements" + + PodAntiAffinityTopologyKey string `json:"-"` + + // Note, that this field is used by MongoDB resources only, let's keep it here for simplicity + Persistence *Persistence `json:"persistence,omitempty"` +} + +func (m *MongoDbPodSpec) IsAgentImageOverridden() bool { + if m.PodTemplateWrapper.PodTemplate != nil && isAgentImageOverriden(m.PodTemplateWrapper.PodTemplate.Spec.Containers) { + return true + } + return false +} + +type ContainerResourceRequirements struct { + CpuLimit string + CpuRequests string + MemoryLimit string + MemoryRequests string +} + +// This is a struct providing the opportunity to customize the pod created under the hood. +// It naturally delegates to inner object and provides some defaults that can be overriden in each specific case +// TODO remove in favor or 'StatefulSetHelper.setPodSpec(podSpec, defaultPodSpec)' +type PodSpecWrapper struct { + MongoDbPodSpec + // These are the default values, unfortunately Golang doesn't provide the possibility to inline default values into + // structs so use the operator.NewDefaultPodSpec constructor for this + Default MongoDbPodSpec +} + +type Persistence struct { + SingleConfig *PersistenceConfig `json:"single,omitempty"` + MultipleConfig *MultiplePersistenceConfig `json:"multiple,omitempty"` +} + +type MultiplePersistenceConfig struct { + Data *PersistenceConfig `json:"data,omitempty"` + Journal *PersistenceConfig `json:"journal,omitempty"` + Logs *PersistenceConfig `json:"logs,omitempty"` +} + +type PersistenceConfig struct { + Storage string `json:"storage,omitempty"` + StorageClass *string `json:"storageClass,omitempty"` + + // +kubebuilder:pruning:PreserveUnknownFields + LabelSelector *LabelSelectorWrapper `json:"labelSelector,omitempty"` +} + +func (p PodSpecWrapper) GetCpuOrDefault() string { + if p.CpuLimit == "" && p.CpuRequests == "" { + return p.Default.CpuLimit + } + return p.CpuLimit +} + +func (p PodSpecWrapper) GetMemoryOrDefault() string { + // We don't set default if either Memory requests or Memory limits are specified by the User + if p.MemoryLimit == "" && p.MemoryRequests == "" { + return p.Default.MemoryLimit + } + return p.MemoryLimit +} + +func (p PodSpecWrapper) GetCpuRequestsOrDefault() string { + if p.CpuRequests == "" && p.CpuLimit == "" { + return p.Default.CpuRequests + } + return p.CpuRequests +} + +func (p PodSpecWrapper) GetMemoryRequestsOrDefault() string { + // We don't set default if either Memory requests or Memory limits are specified by the User + // otherwise it's possible to get failed Statefulset (e.g. the user specified limits of 200M but we default + // requests to 500M though requests must be less than limits) + if p.MemoryRequests == "" && p.MemoryLimit == "" { + return p.Default.MemoryRequests + } + return p.MemoryRequests +} + +func (p PodSpecWrapper) GetTopologyKeyOrDefault() string { + if p.PodAntiAffinityTopologyKey == "" { + return p.Default.PodAntiAffinityTopologyKey + } + return p.PodAntiAffinityTopologyKey +} + +func (p PodSpecWrapper) SetCpu(cpu string) PodSpecWrapper { + p.CpuLimit = cpu + return p +} + +func (p PodSpecWrapper) SetMemory(memory string) PodSpecWrapper { + p.MemoryLimit = memory + return p +} + +func (p PodSpecWrapper) SetTopology(topology string) PodSpecWrapper { + p.PodAntiAffinityTopologyKey = topology + return p +} + +func GetStorageOrDefault(config *PersistenceConfig, defaultConfig PersistenceConfig) string { + if config == nil || config.Storage == "" { + return defaultConfig.Storage + } + return config.Storage +} + +// Create a MongoDbPodSpec reference without any nil references +// used to initialize any MongoDbPodSpec fields with valid values +// in order to prevent panicking at runtime. +func NewMongoDbPodSpec() *MongoDbPodSpec { + return &MongoDbPodSpec{} +} + +// Replicas returns the number of "user facing" replicas of the MongoDB resource. This method can be used for +// constructing the mongodb URL for example. +// 'Members' would be a more consistent function but go doesn't allow to have the same +func (m *MongoDbSpec) Replicas() int { + var replicasCount int + switch m.ResourceType { + case Standalone: + replicasCount = 1 + case ReplicaSet: + replicasCount = m.Members + case ShardedCluster: + replicasCount = m.MongosCount + default: + panic("Unknown type of resource!") + } + return replicasCount +} + +func (m *MongoDbSpec) GetSecurityAuthenticationModes() []string { + return m.GetSecurity().Authentication.GetModes() +} + +func (m *MongoDbSpec) GetResourceType() ResourceType { + return m.ResourceType +} + +func (d *DbCommonSpec) IsSecurityTLSConfigEnabled() bool { + return d.GetSecurity().IsTLSEnabled() +} + +func (m *MongoDbSpec) GetFeatureCompatibilityVersion() *string { + return m.FeatureCompatibilityVersion +} + +func NewConnectivity() *MongoDBConnectivity { + return &MongoDBConnectivity{} +} + +// PrivateCloudConfig returns and empty `PrivateCloudConfig` object +func NewOpsManagerConfig() *PrivateCloudConfig { + return &PrivateCloudConfig{} +} + +func EnsureSecurity(sec *Security) *Security { + if sec == nil { + sec = newSecurity() + } + if sec.TLSConfig == nil { + sec.TLSConfig = &TLSConfig{} + } + if sec.Roles == nil { + sec.Roles = make([]MongoDbRole, 0) + } + return sec +} + +func newAuthentication() *Authentication { + return &Authentication{Modes: []AuthMode{}} +} + +func newSecurity() *Security { + return &Security{TLSConfig: &TLSConfig{}} +} + +// BuildConnectionString returns a string with a connection string for this resource. +func (m *MongoDB) BuildConnectionString(username, password string, scheme connectionstring.Scheme, connectionParams map[string]string) string { + name := m.Name + if m.Spec.ResourceType == ShardedCluster { + name = m.MongosRsName() + } + builder := connectionstring.Builder(). + SetName(name). + SetNamespace(m.Namespace). + SetUsername(username). + SetPassword(password). + SetReplicas(m.Spec.Replicas()). + SetService(m.ServiceName()). + SetPort(m.Spec.GetAdditionalMongodConfig().GetPortOrDefault()). + SetVersion(m.Spec.GetMongoDBVersion()). + SetAuthenticationModes(m.Spec.GetSecurityAuthenticationModes()). + SetClusterDomain(m.Spec.GetClusterDomain()). + SetExternalDomain(m.Spec.GetExternalDomain()). + SetIsReplicaSet(m.Spec.ResourceType == ReplicaSet). + SetIsTLSEnabled(m.Spec.IsSecurityTLSConfigEnabled()). + SetConnectionParams(connectionParams). + SetScheme(scheme) + + return builder.Build() +} + +func (m *MongoDB) GetAuthenticationModes() []string { + return m.Spec.Security.Authentication.GetModes() +} + +func (m *MongoDB) CalculateFeatureCompatibilityVersion() string { + return fcv.CalculateFeatureCompatibilityVersion(m.Spec.Version, m.Status.FeatureCompatibilityVersion, m.Spec.FeatureCompatibilityVersion) +} + +func (m *MongoDbSpec) IsInChangeVersion(lastSpec *MongoDbSpec) bool { + if lastSpec != nil && (lastSpec.Version != m.Version) { + return true + } + return false +} + +func (m *MongoDbSpec) GetTopology() string { + if m.Topology == "" { + return ClusterTopologySingleCluster + } + return m.Topology +} + +func (m *MongoDbSpec) IsMultiCluster() bool { + return m.GetTopology() == ClusterTopologyMultiCluster +} + +// MongodbCleanUpOptions implements the required interface to be passed +// to the DeleteAllOf function, this cleans up resources of a given type with +// the provided Labels in a specific Namespace. +type MongodbCleanUpOptions struct { + Namespace string + Labels map[string]string +} + +func (m *MongodbCleanUpOptions) ApplyToDeleteAllOf(opts *client.DeleteAllOfOptions) { + opts.Namespace = m.Namespace + opts.LabelSelector = labels.SelectorFromValidatedSet(m.Labels) +} + +func (m ClusterSpecList) GetExternalAccessConfigurationForMemberCluster(clusterName string) *ExternalAccessConfiguration { + for _, csl := range m { + if csl.ClusterName == clusterName { + return csl.ExternalAccessConfiguration + } + } + + return nil +} + +func (m ClusterSpecList) GetExternalDomainForMemberCluster(clusterName string) *string { + if cfg := m.GetExternalAccessConfigurationForMemberCluster(clusterName); cfg != nil { + return cfg.ExternalDomain + } + + return nil +} + +func (m ClusterSpecList) IsExternalDomainSpecifiedInClusterSpecList() bool { + for _, item := range m { + externalAccess := item.ExternalAccessConfiguration + if externalAccess != nil && externalAccess.ExternalDomain != nil { + return true + } + } + + return false +} diff --git a/api/v1/mdb/mongodb_types_test.go b/api/v1/mdb/mongodb_types_test.go new file mode 100644 index 000000000..a953a6177 --- /dev/null +++ b/api/v1/mdb/mongodb_types_test.go @@ -0,0 +1,365 @@ +package mdb + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func TestEnsureSecurity_WithAllNilValues(t *testing.T) { + spec := &MongoDbSpec{ + DbCommonSpec: DbCommonSpec{ + Security: nil, + }, + } + + spec.Security = EnsureSecurity(spec.Security) + assert.NotNil(t, spec.Security) + assert.NotNil(t, spec.Security.TLSConfig) +} + +func TestEnsureSecurity_WithNilTlsConfig(t *testing.T) { + spec := &MongoDbSpec{DbCommonSpec: DbCommonSpec{Security: &Security{TLSConfig: nil, Authentication: &Authentication{}}}} + spec.Security = EnsureSecurity(spec.Security) + assert.NotNil(t, spec.Security) + assert.NotNil(t, spec.Security.TLSConfig) +} + +func TestEnsureSecurity_EmptySpec(t *testing.T) { + spec := &MongoDbSpec{} + spec.Security = EnsureSecurity(spec.Security) + assert.NotNil(t, spec.Security) + assert.NotNil(t, spec.Security.TLSConfig) +} + +func TestGetAgentAuthentication(t *testing.T) { + sec := newSecurity() + sec.Authentication = newAuthentication() + sec.Authentication.Agents.Mode = "SCRAM" + assert.Len(t, sec.Authentication.Modes, 0) + assert.Empty(t, sec.GetAgentMechanism("")) + assert.Equal(t, "", sec.GetAgentMechanism("MONGODB-X509")) + + sec.Authentication.Enabled = true + sec.Authentication.Modes = append(sec.Authentication.Modes, util.X509) + assert.Equal(t, util.X509, sec.GetAgentMechanism("MONGODB-X509"), "if x509 was enabled before, it needs to stay as is") + + sec.Authentication.Modes = append(sec.Authentication.Modes, util.SCRAM) + assert.Equal(t, util.SCRAM, sec.GetAgentMechanism("SCRAM-SHA-256"), "if scram was enabled, scram will be chosen") + + sec.Authentication.Modes = append(sec.Authentication.Modes, util.SCRAMSHA1) + assert.Equal(t, util.SCRAM, sec.GetAgentMechanism("SCRAM-SHA-256"), "Adding SCRAM-SHA-1 doesn't change the default") + + sec.Authentication.Agents.Mode = "X509" + assert.Equal(t, util.X509, sec.GetAgentMechanism("SCRAM-SHA-256"), "transitioning from SCRAM -> X509 is allowed") +} + +func TestMinimumMajorVersion(t *testing.T) { + mdbSpec := MongoDbSpec{ + DbCommonSpec: DbCommonSpec{ + Version: "3.6.0-ent", + FeatureCompatibilityVersion: nil, + }, + } + + assert.Equal(t, mdbSpec.MinimumMajorVersion(), uint64(3)) + + mdbSpec = MongoDbSpec{ + DbCommonSpec: DbCommonSpec{ + Version: "4.0.0-ent", + FeatureCompatibilityVersion: stringutil.Ref("3.6"), + }, + } + + assert.Equal(t, mdbSpec.MinimumMajorVersion(), uint64(3)) + + mdbSpec = MongoDbSpec{ + DbCommonSpec: DbCommonSpec{ + Version: "4.0.0", + FeatureCompatibilityVersion: stringutil.Ref("3.6"), + }, + } + + assert.Equal(t, mdbSpec.MinimumMajorVersion(), uint64(3)) +} + +func TestMongoDB_ConnectionURL_NotSecure(t *testing.T) { + rs := NewReplicaSetBuilder().SetMembers(3).Build() + + var cnx string + cnx = rs.BuildConnectionString("", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017,test-mdb-2.test-mdb-svc.testNS.svc.cluster.local:27017/"+ + "?connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + // Connection parameters. The default one is overridden + cnx = rs.BuildConnectionString("", "", connectionstring.SchemeMongoDB, map[string]string{"connectTimeoutMS": "30000", "readPreference": "secondary"}) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017,test-mdb-2.test-mdb-svc.testNS.svc.cluster.local:27017/"+ + "?connectTimeoutMS=30000&readPreference=secondary&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + // 2 members, custom cluster name + rs = NewReplicaSetBuilder().SetName("paymentsDb").SetMembers(2).SetClusterDomain("company.domain.net").Build() + cnx = rs.BuildConnectionString("", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://paymentsDb-0.paymentsDb-svc.testNS.svc.company.domain.net:27017,"+ + "paymentsDb-1.paymentsDb-svc.testNS.svc.company.domain.net:27017/?connectTimeoutMS=20000&replicaSet=paymentsDb"+ + "&serverSelectionTimeoutMS=20000", + cnx) + + // Sharded cluster + sc := NewClusterBuilder().SetName("contractsDb").SetNamespace("ns").Build() + cnx = sc.BuildConnectionString("", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://contractsDb-mongos-0.contractsDb-svc.ns.svc.cluster.local:27017,"+ + "contractsDb-mongos-1.contractsDb-svc.ns.svc.cluster.local:27017/"+ + "?connectTimeoutMS=20000&serverSelectionTimeoutMS=20000", + cnx) + + // Standalone + st := NewStandaloneBuilder().SetName("foo").Build() + cnx = st.BuildConnectionString("", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://foo-0.foo-svc.testNS.svc.cluster.local:27017/?"+ + "connectTimeoutMS=20000&serverSelectionTimeoutMS=20000", + cnx) +} + +func TestMongoDB_ConnectionURL_Secure(t *testing.T) { + var cnx string + + // Only tls enabled, no auth + rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().Build() + cnx = rs.BuildConnectionString("", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017,test-mdb-2.test-mdb-svc.testNS.svc.cluster.local:27017/?"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000&ssl=true", + cnx) + + // New version of Mongodb -> SCRAM-SHA-256 + rs = NewReplicaSetBuilder().SetMembers(2).SetSecurityTLSEnabled().EnableAuth([]AuthMode{util.SCRAM}).Build() + cnx = rs.BuildConnectionString("the_user", "the_passwd", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://the_user:the_passwd@test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-256&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000&ssl=true", + cnx) + + // Old version of Mongodb -> SCRAM-SHA-1. X509 is a second authentication method - user & password are still appended + rs = NewReplicaSetBuilder().SetMembers(2).SetVersion("3.6.1").EnableAuth([]AuthMode{util.SCRAM, util.X509}).Build() + cnx = rs.BuildConnectionString("the_user", "the_passwd", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://the_user:the_passwd@test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-1&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + // Special symbols in user/password must be encoded + rs = NewReplicaSetBuilder().SetMembers(2).EnableAuth([]AuthMode{util.SCRAM}).Build() + cnx = rs.BuildConnectionString("user/@", "pwd#!@", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://user%2F%40:pwd%23%21%40@test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-256&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + // Caller can override any connection parameters, e.g."authMechanism" + rs = NewReplicaSetBuilder().SetMembers(2).EnableAuth([]AuthMode{util.SCRAM}).Build() + cnx = rs.BuildConnectionString("the_user", "the_passwd", connectionstring.SchemeMongoDB, map[string]string{"authMechanism": "SCRAM-SHA-1"}) + assert.Equal(t, "mongodb://the_user:the_passwd@test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-1&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + // X509 -> no user/password in the url. It's possible to pass user/password in the params though + rs = NewReplicaSetBuilder().SetMembers(2).EnableAuth([]AuthMode{util.X509}).Build() + cnx = rs.BuildConnectionString("the_user", "the_passwd", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?connectTimeoutMS=20000&replicaSet=test-mdb&"+ + "serverSelectionTimeoutMS=20000", cnx) + + // username + password must be provided if scram is enabled + rs = NewReplicaSetBuilder().SetMembers(2).EnableAuth([]AuthMode{util.SCRAM}).Build() + cnx = rs.BuildConnectionString("the_user", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-256&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + cnx = rs.BuildConnectionString("", "the_password", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-256&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) + + cnx = rs.BuildConnectionString("", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.test-mdb-svc.testNS.svc.cluster.local:27017,"+ + "test-mdb-1.test-mdb-svc.testNS.svc.cluster.local:27017/?authMechanism=SCRAM-SHA-256&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) +} + +func TestMongoDBConnectionURLExternalDomainWithAuth(t *testing.T) { + externalDomain := "example.com" + + rs := NewReplicaSetBuilder().SetMembers(2).EnableAuth([]AuthMode{util.SCRAM}).ExposedExternally(nil, nil, &externalDomain).Build() + cnx := rs.BuildConnectionString("the_user", "", connectionstring.SchemeMongoDB, nil) + assert.Equal(t, "mongodb://test-mdb-0.example.com:27017,"+ + "test-mdb-1.example.com:27017/?authMechanism=SCRAM-SHA-256&authSource=admin&"+ + "connectTimeoutMS=20000&replicaSet=test-mdb&serverSelectionTimeoutMS=20000", + cnx) +} + +func TestMongoDB_AddWarningIfNotExists(t *testing.T) { + resource := &MongoDB{} + resource.AddWarningIfNotExists("my test warning") + resource.AddWarningIfNotExists("my test warning") + resource.AddWarningIfNotExists("my other test warning") + assert.Equal(t, []status.Warning{"my test warning;", "my other test warning"}, resource.Status.Warnings) +} + +func TestMongoDB_IsSecurityTLSConfigEnabled(t *testing.T) { + rs := NewReplicaSetBuilder().Build() + tests := []struct { + name string + security *Security + expected bool + }{ + { + name: "TLS is not enabled when Security is nil", + security: nil, + expected: false, + }, + { + name: "TLS is not enabled when TLSConfig is nil", + security: &Security{}, + expected: false, + }, + { + name: "TLS is enabled when CertificatesSecretsPrefix is specified", + security: &Security{ + CertificatesSecretsPrefix: "prefix", + }, + expected: true, + }, + { + name: "TLS is enabled when TLSConfig fields are specified and CertificatesSecretsPrefix is specified", + security: &Security{ + CertificatesSecretsPrefix: "prefix", + TLSConfig: &TLSConfig{ + CA: "issuer-ca", + }, + }, + expected: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rs.Spec.Security = tc.security + assert.Equal(t, tc.expected, rs.GetSecurity().IsTLSEnabled()) + }) + } + rs.GetSpec().IsSecurityTLSConfigEnabled() +} + +func TestMemberCertificateSecretName(t *testing.T) { + rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().Build() + + // If nothing is specified, we return -cert + assert.Equal(t, fmt.Sprintf("%s-cert", rs.Name), rs.GetSecurity().MemberCertificateSecretName(rs.Name)) + + // If the top-level prefix is specified, we use it + rs.Spec.Security.CertificatesSecretsPrefix = "top-level-prefix" + assert.Equal(t, fmt.Sprintf("top-level-prefix-%s-cert", rs.Name), rs.GetSecurity().MemberCertificateSecretName(rs.Name)) +} + +func TestAgentClientCertificateSecretName(t *testing.T) { + rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().EnableAuth([]AuthMode{util.X509}).Build() + + // Default is the hardcoded "agent-certs" + assert.Equal(t, util.AgentSecretName, rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name) + + // If the top-level prefix is there, we use it + rs.Spec.Security.CertificatesSecretsPrefix = "prefix" + assert.Equal(t, fmt.Sprintf("prefix-%s-%s", rs.Name, util.AgentSecretName), rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name) + + // If the name is provided (deprecated) we return it + rs.GetSecurity().Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name = "foo" + assert.Equal(t, "foo", rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name) +} + +func TestInternalClusterAuthSecretName(t *testing.T) { + rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().Build() + + // Default is -clusterfile + assert.Equal(t, fmt.Sprintf("%s-clusterfile", rs.Name), rs.GetSecurity().InternalClusterAuthSecretName(rs.Name)) + + // IF there is a prefix, use it + rs.Spec.Security.CertificatesSecretsPrefix = "prefix" + assert.Equal(t, fmt.Sprintf("prefix-%s-clusterfile", rs.Name), rs.GetSecurity().InternalClusterAuthSecretName(rs.Name)) +} + +func TestGetTransportSecurity(t *testing.T) { + someTLS := TransportSecurity("SomeTLS") + none := TransportSecurity("NONE") + noneUpper := TransportSecurity("none") + + tests := []struct { + name string + mdbLdap *Ldap + want TransportSecurity + }{ + { + name: "enabling transport security", + mdbLdap: &Ldap{ + TransportSecurity: &someTLS, + }, + want: TransportSecurityTLS, + }, + { + name: "no transport set", + mdbLdap: &Ldap{}, + want: TransportSecurityNone, + }, + { + name: "none set", + mdbLdap: &Ldap{ + TransportSecurity: &none, + }, + want: TransportSecurityNone, + }, + { + name: "NONE set", + mdbLdap: &Ldap{ + TransportSecurity: &noneUpper, + }, + want: TransportSecurityNone, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, GetTransportSecurity(tt.mdbLdap), "GetTransportSecurity(%v)", tt.mdbLdap) + }) + } +} + +func TestAdditionalMongodConfigMarshalJSON(t *testing.T) { + mdb := MongoDB{Spec: MongoDbSpec{DbCommonSpec: DbCommonSpec{Version: "4.2.1"}}} + mdb.InitDefaults() + mdb.Spec.AdditionalMongodConfig = &AdditionalMongodConfig{object: map[string]interface{}{"net": map[string]interface{}{"port": "30000"}}} + + marshal, err := json.Marshal(mdb.Spec) + assert.NoError(t, err) + + unmarshalledSpec := MongoDbSpec{} + + err = json.Unmarshal(marshal, &unmarshalledSpec) + assert.NoError(t, err) + + expected := mdb.Spec.GetAdditionalMongodConfig().ToMap() + actual := unmarshalledSpec.AdditionalMongodConfig.ToMap() + assert.Equal(t, expected, actual) +} diff --git a/api/v1/mdb/mongodb_validation.go b/api/v1/mdb/mongodb_validation.go new file mode 100644 index 000000000..9ba8d74df --- /dev/null +++ b/api/v1/mdb/mongodb_validation.go @@ -0,0 +1,354 @@ +package mdb + +import ( + "errors" + "strings" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/strings/slices" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +var _ webhook.Validator = &MongoDB{} + +// ValidateCreate and ValidateUpdate should be the same if we intend to do this +// on every reconciliation as well +func (m *MongoDB) ValidateCreate() (admission.Warnings, error) { + return nil, m.ProcessValidationsOnReconcile(nil) +} + +func (m *MongoDB) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + return nil, m.ProcessValidationsOnReconcile(old.(*MongoDB)) +} + +// ValidateDelete does nothing as we assume validation on deletion is +// unnecessary +func (m *MongoDB) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} + +func replicaSetHorizonsRequireTLS(d DbCommonSpec) v1.ValidationResult { + if len(d.Connectivity.ReplicaSetHorizons) > 0 && !d.IsSecurityTLSConfigEnabled() { + return v1.ValidationError("TLS must be enabled in order to use replica set horizons") + } + return v1.ValidationSuccess() +} + +func horizonsMustEqualMembers(ms MongoDbSpec) v1.ValidationResult { + numHorizonMembers := len(ms.Connectivity.ReplicaSetHorizons) + if numHorizonMembers > 0 && numHorizonMembers != ms.Members { + return v1.ValidationError("Number of horizons must be equal to number of members in replica set") + } + return v1.ValidationSuccess() +} + +func deploymentsMustHaveTLSInX509Env(d DbCommonSpec) v1.ValidationResult { + authSpec := d.Security.Authentication + if authSpec == nil { + return v1.ValidationSuccess() + } + if authSpec.Enabled && authSpec.IsX509Enabled() && !d.GetSecurity().IsTLSEnabled() { + return v1.ValidationError("Cannot have a non-tls deployment when x509 authentication is enabled") + } + return v1.ValidationSuccess() +} + +func deploymentsMustHaveAtLeastOneAuthModeIfAuthIsEnabled(d DbCommonSpec) v1.ValidationResult { + authSpec := d.Security.Authentication + if authSpec == nil { + return v1.ValidationSuccess() + } + if authSpec.Enabled && len(authSpec.Modes) == 0 { + return v1.ValidationError("Cannot enable authentication without modes specified") + } + return v1.ValidationSuccess() +} + +func deploymentsMustHaveAgentModeInAuthModes(d DbCommonSpec) v1.ValidationResult { + authSpec := d.Security.Authentication + if authSpec == nil { + return v1.ValidationSuccess() + } + if !authSpec.Enabled { + return v1.ValidationSuccess() + } + + if authSpec.Agents.Mode != "" && !IsAuthPresent(authSpec.Modes, authSpec.Agents.Mode) { + return v1.ValidationError("Cannot configure an Agent authentication mechanism that is not specified in authentication modes") + } + return v1.ValidationSuccess() +} + +// scramSha1AuthValidation performs the same validation as the Ops Manager does in +// https://github.com/10gen/mms/blob/107304ce6988f6280e8af069d19b7c6226c4f3ce/server/src/main/com/xgen/cloud/atm/publish/_public/svc/AutomationValidationSvc.java +func scramSha1AuthValidation(d DbCommonSpec) v1.ValidationResult { + authSpec := d.Security.Authentication + if authSpec == nil { + return v1.ValidationSuccess() + } + if !authSpec.Enabled { + return v1.ValidationSuccess() + } + + if IsAuthPresent(authSpec.Modes, util.SCRAMSHA1) { + if authSpec.Agents.Mode != util.MONGODBCR { + return v1.ValidationError("Cannot configure SCRAM-SHA-1 without using MONGODB-CR in te Agent Mode") + } + } + return v1.ValidationSuccess() +} + +func ldapAuthRequiresEnterprise(d DbCommonSpec) v1.ValidationResult { + authSpec := d.Security.Authentication + if authSpec != nil && authSpec.isLDAPEnabled() && !strings.HasSuffix(d.Version, "-ent") { + return v1.ValidationError("Cannot enable LDAP authentication with MongoDB Community Builds") + } + return v1.ValidationSuccess() +} + +func additionalMongodConfig(ms MongoDbSpec) v1.ValidationResult { + if ms.ResourceType == ShardedCluster { + if ms.AdditionalMongodConfig != nil && ms.AdditionalMongodConfig.object != nil && len(ms.AdditionalMongodConfig.object) > 0 { + return v1.ValidationError("'spec.additionalMongodConfig' cannot be specified if type of MongoDB is %s", ShardedCluster) + } + return v1.ValidationSuccess() + } + // Standalone or ReplicaSet + if ms.ShardSpec != nil || ms.ConfigSrvSpec != nil || ms.MongosSpec != nil { + return v1.ValidationError("'spec.mongos', 'spec.configSrv', 'spec.shard' cannot be specified if type of MongoDB is %s", ms.ResourceType) + } + return v1.ValidationSuccess() +} + +func replicasetMemberIsSpecified(ms MongoDbSpec) v1.ValidationResult { + if ms.ResourceType == ReplicaSet && ms.Members == 0 { + return v1.ValidationError("'spec.members' must be specified if type of MongoDB is %s", ms.ResourceType) + } + return v1.ValidationSuccess() +} + +func agentModeIsSetIfMoreThanADeploymentAuthModeIsSet(d DbCommonSpec) v1.ValidationResult { + if d.Security == nil || d.Security.Authentication == nil { + return v1.ValidationSuccess() + } + if len(d.Security.Authentication.Modes) > 1 && d.Security.Authentication.Agents.Mode == "" { + return v1.ValidationError("spec.security.authentication.agents.mode must be specified if more than one entry is present in spec.security.authentication.modes") + } + return v1.ValidationSuccess() +} + +func ldapGroupDnIsSetIfLdapAuthzIsEnabledAndAgentsAreExternal(d DbCommonSpec) v1.ValidationResult { + if d.Security == nil || d.Security.Authentication == nil || d.Security.Authentication.Ldap == nil { + return v1.ValidationSuccess() + } + auth := d.Security.Authentication + if auth.Ldap.AuthzQueryTemplate != "" && auth.Agents.AutomationLdapGroupDN == "" && stringutil.Contains([]string{"X509", "LDAP"}, auth.Agents.Mode) { + return v1.ValidationError("automationLdapGroupDN must be specified if LDAP authorization is used and agent auth mode is $external (x509 or LDAP)") + } + return v1.ValidationSuccess() +} + +func resourceTypeImmutable(newObj, oldObj MongoDbSpec) v1.ValidationResult { + if newObj.ResourceType != oldObj.ResourceType { + return v1.ValidationError("'resourceType' cannot be changed once created") + } + return v1.ValidationSuccess() +} + +// This validation blocks topology migrations for any MongoDB resource (Standalone, ReplicaSet, ShardedCluster) +func noTopologyMigration(newObj, oldObj MongoDbSpec) v1.ValidationResult { + if oldObj.GetTopology() != newObj.GetTopology() { + return v1.ValidationError("Automatic Topology Migration (Single/Multi Cluster) is not supported for MongoDB resource") + } + return v1.ValidationSuccess() +} + +// specWithExactlyOneSchema checks that exactly one among "Project/OpsManagerConfig/CloudManagerConfig" +// is configured, doing the "oneOf" validation in the webhook. +func specWithExactlyOneSchema(d DbCommonSpec) v1.ValidationResult { + count := 0 + if *d.OpsManagerConfig != (PrivateCloudConfig{}) { + count += 1 + } + if *d.CloudManagerConfig != (PrivateCloudConfig{}) { + count += 1 + } + + if count != 1 { + return v1.ValidationError("must validate one and only one schema") + } + return v1.ValidationSuccess() +} + +func CommonValidators() []func(d DbCommonSpec) v1.ValidationResult { + return []func(d DbCommonSpec) v1.ValidationResult{ + replicaSetHorizonsRequireTLS, + deploymentsMustHaveTLSInX509Env, + deploymentsMustHaveAtLeastOneAuthModeIfAuthIsEnabled, + deploymentsMustHaveAgentModeInAuthModes, + scramSha1AuthValidation, + ldapAuthRequiresEnterprise, + rolesAttributeisCorrectlyConfigured, + agentModeIsSetIfMoreThanADeploymentAuthModeIsSet, + ldapGroupDnIsSetIfLdapAuthzIsEnabledAndAgentsAreExternal, + specWithExactlyOneSchema, + featureCompatibilityVersionValidation, + } +} + +func featureCompatibilityVersionValidation(d DbCommonSpec) v1.ValidationResult { + fcv := d.FeatureCompatibilityVersion + return ValidateFCV(fcv) +} + +func ValidateFCV(fcv *string) v1.ValidationResult { + if fcv != nil { + f := *fcv + if f == util.AlwaysMatchVersionFCV { + return v1.ValidationSuccess() + } + splitted := strings.Split(f, ".") + if len(splitted) != 2 { + return v1.ValidationError("invalid feature compatibility version: %s, possible values are: '%s' or 'major.minor'", f, util.AlwaysMatchVersionFCV) + } + } + return v1.ValidationResult{} +} + +func (m *MongoDB) RunValidations(old *MongoDB) []v1.ValidationResult { + // The below validators apply to all MongoDB resource (but not MongoDBMulti), regardless of the value of the + // Topology field + mongoDBValidators := []func(m MongoDbSpec) v1.ValidationResult{ + horizonsMustEqualMembers, + additionalMongodConfig, + replicasetMemberIsSpecified, + } + + updateValidators := []func(newObj MongoDbSpec, oldObj MongoDbSpec) v1.ValidationResult{ + resourceTypeImmutable, + noTopologyMigration, + } + + var validationResults []v1.ValidationResult + + for _, validator := range mongoDBValidators { + res := validator(m.Spec) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + for _, validator := range CommonValidators() { + res := validator(m.Spec.DbCommonSpec) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + if m.GetResourceType() == ShardedCluster { + for _, validator := range ShardedClusterCommonValidators() { + res := validator(*m) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + if m.Spec.IsMultiCluster() { + for _, validator := range ShardedClusterMultiValidators() { + results := validator(*m) + for _, res := range results { + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + } + } else { + for _, validator := range ShardedClusterSingleValidators() { + res := validator(*m) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + } + } + + if old == nil { + return validationResults + } + for _, validator := range updateValidators { + res := validator(m.Spec, old.Spec) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + return validationResults +} + +func (m *MongoDB) ProcessValidationsOnReconcile(old *MongoDB) error { + for _, res := range m.RunValidations(old) { + if res.Level == v1.ErrorLevel { + return errors.New(res.Msg) + } + + if res.Level == v1.WarningLevel { + m.AddWarningIfNotExists(status.Warning(res.Msg)) + } + } + + return nil +} + +func ValidateUniqueClusterNames(ms ClusterSpecList) v1.ValidationResult { + present := make(map[string]struct{}) + + for _, e := range ms { + if _, ok := present[e.ClusterName]; ok { + return v1.ValidationError("Multiple clusters with the same name (%s) are not allowed", e.ClusterName) + } + present[e.ClusterName] = struct{}{} + } + return v1.ValidationSuccess() +} + +func ValidateNonEmptyClusterSpecList(ms ClusterSpecList) v1.ValidationResult { + if len(ms) == 0 { + return v1.ValidationError("ClusterSpecList empty is not allowed, please define at least one cluster") + } + return v1.ValidationSuccess() +} + +func ValidateMemberClusterIsSubsetOfKubeConfig(ms ClusterSpecList) v1.ValidationResult { + // read the mounted kubeconfig file and + kubeConfigFile, err := multicluster.NewKubeConfigFile(multicluster.GetKubeConfigPath()) + if err != nil { + // log the error here? + return v1.ValidationSuccess() + } + + kubeConfig, err := kubeConfigFile.LoadKubeConfigFile() + if err != nil { + // log the error here? + return v1.ValidationSuccess() + } + + clusterNames := kubeConfig.GetMemberClusterNames() + notPresentClusters := make([]string, 0) + + for _, e := range ms { + if !slices.Contains(clusterNames, e.ClusterName) { + notPresentClusters = append(notPresentClusters, e.ClusterName) + } + } + if len(notPresentClusters) > 0 { + return v1.ValidationWarning("The following clusters specified in ClusterSpecList is not present in Kubeconfig: %s, instead - the following are: %+v", notPresentClusters, clusterNames) + } + return v1.ValidationSuccess() +} diff --git a/api/v1/mdb/mongodb_validation_test.go b/api/v1/mdb/mongodb_validation_test.go new file mode 100644 index 000000000..dd79841d2 --- /dev/null +++ b/api/v1/mdb/mongodb_validation_test.go @@ -0,0 +1,202 @@ +package mdb + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestMongoDB_ProcessValidations_BadHorizonsMemberCount(t *testing.T) { + replicaSetHorizons := []MongoDBHorizonConfig{ + {"my-horizon": "my-db.com:12345"}, + {"my-horizon": "my-db.com:12346"}, + } + + rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().Build() + rs.Spec.Connectivity = &MongoDBConnectivity{} + rs.Spec.Connectivity.ReplicaSetHorizons = replicaSetHorizons + err := rs.ProcessValidationsOnReconcile(nil) + assert.Contains(t, "Number of horizons must be equal to number of members in replica set", err.Error()) +} + +func TestMongoDB_ProcessValidations_HorizonsWithoutTLS(t *testing.T) { + replicaSetHorizons := []MongoDBHorizonConfig{ + {"my-horizon": "my-db.com:12345"}, + {"my-horizon": "my-db.com:12342"}, + {"my-horizon": "my-db.com:12346"}, + } + + rs := NewReplicaSetBuilder().Build() + rs.Spec.Connectivity = &MongoDBConnectivity{} + rs.Spec.Connectivity.ReplicaSetHorizons = replicaSetHorizons + err := rs.ProcessValidationsOnReconcile(nil) + assert.Equal(t, "TLS must be enabled in order to use replica set horizons", err.Error()) +} + +func TestMongoDB_ProcessValidationsOnReconcile_X509WithoutTls(t *testing.T) { + rs := NewReplicaSetBuilder().Build() + rs.Spec.Security.Authentication = &Authentication{Enabled: true, Modes: []AuthMode{"X509"}} + err := rs.ProcessValidationsOnReconcile(nil) + assert.Equal(t, "Cannot have a non-tls deployment when x509 authentication is enabled", err.Error()) +} + +func TestMongoDB_ValidateCreate_Error(t *testing.T) { + replicaSetHorizons := []MongoDBHorizonConfig{ + {"my-horizon": "my-db.com:12345"}, + {"my-horizon": "my-db.com:12342"}, + {"my-horizon": "my-db.com:12346"}, + } + + rs := NewReplicaSetBuilder().Build() + rs.Spec.Connectivity = &MongoDBConnectivity{} + rs.Spec.Connectivity.ReplicaSetHorizons = replicaSetHorizons + _, err := rs.ValidateCreate() + assert.Equal(t, "TLS must be enabled in order to use replica set horizons", err.Error()) +} + +func TestMongoDB_MultipleAuthsButNoAgentAuth_Error(t *testing.T) { + rs := NewReplicaSetBuilder().SetVersion("4.0.2-ent").Build() + rs.Spec.Security = &Security{ + TLSConfig: &TLSConfig{Enabled: true}, + Authentication: &Authentication{ + Enabled: true, + Modes: []AuthMode{"LDAP", "X509"}, + }, + } + _, err := rs.ValidateCreate() + assert.Errorf(t, err, "spec.security.authentication.agents.mode must be specified if more than one entry is present in spec.security.authentication.modes") +} + +func TestMongoDB_ResourceTypeImmutable(t *testing.T) { + newRs := NewReplicaSetBuilder().Build() + oldRs := NewReplicaSetBuilder().setType(ShardedCluster).Build() + _, err := newRs.ValidateUpdate(oldRs) + assert.Errorf(t, err, "'resourceType' cannot be changed once created") +} + +func TestSpecProjectOnlyOneValue(t *testing.T) { + rs := NewReplicaSetBuilder().Build() + rs.Spec.CloudManagerConfig = &PrivateCloudConfig{ + ConfigMapRef: ConfigMapRef{Name: "cloud-manager"}, + } + _, err := rs.ValidateCreate() + assert.NoError(t, err) +} + +func TestMongoDB_ProcessValidations(t *testing.T) { + rs := NewReplicaSetBuilder().Build() + assert.Error(t, rs.ProcessValidationsOnReconcile(nil), nil) +} + +func TestMongoDB_ValidateAdditionalMongodConfig(t *testing.T) { + t.Run("No sharded cluster additional config for replica set", func(t *testing.T) { + rs := NewReplicaSetBuilder().SetConfigSrvAdditionalConfig(NewAdditionalMongodConfig("systemLog.verbosity", 5)).Build() + _, err := rs.ValidateCreate() + require.Error(t, err) + assert.Equal(t, "'spec.mongos', 'spec.configSrv', 'spec.shard' cannot be specified if type of MongoDB is ReplicaSet", err.Error()) + }) + t.Run("No sharded cluster additional config for standalone", func(t *testing.T) { + rs := NewStandaloneBuilder().SetMongosAdditionalConfig(NewAdditionalMongodConfig("systemLog.verbosity", 5)).Build() + _, err := rs.ValidateCreate() + require.Error(t, err) + assert.Equal(t, "'spec.mongos', 'spec.configSrv', 'spec.shard' cannot be specified if type of MongoDB is Standalone", err.Error()) + }) + t.Run("No replica set additional config for sharded cluster", func(t *testing.T) { + rs := NewClusterBuilder().SetAdditionalConfig(NewAdditionalMongodConfig("systemLog.verbosity", 5)).Build() + _, err := rs.ValidateCreate() + require.Error(t, err) + assert.Equal(t, "'spec.additionalMongodConfig' cannot be specified if type of MongoDB is ShardedCluster", err.Error()) + }) +} + +func TestScramSha1AuthValidation(t *testing.T) { + type TestConfig struct { + MongoDB *MongoDB + ErrorExpected bool + } + tests := map[string]TestConfig{ + "Valid MongoDB with Authentication": { + MongoDB: NewReplicaSetBuilder().EnableAuth([]AuthMode{util.SCRAMSHA1}).Build(), + ErrorExpected: true, + }, + "Valid MongoDB with SCRAM-SHA-1": { + MongoDB: NewReplicaSetBuilder().EnableAuth([]AuthMode{util.SCRAMSHA1, util.MONGODBCR}).EnableAgentAuth(util.MONGODBCR).Build(), + ErrorExpected: false, + }, + } + for testName, testConfig := range tests { + t.Run(testName, func(t *testing.T) { + validationResult := scramSha1AuthValidation(testConfig.MongoDB.Spec.DbCommonSpec) + assert.Equal(t, testConfig.ErrorExpected, v1.ValidationSuccess() != validationResult, "Expected %v, got %v", testConfig.ErrorExpected, validationResult) + }) + } +} + +func TestReplicasetMemberIsSpecified(t *testing.T) { + rs := NewDefaultReplicaSetBuilder().Build() + err := rs.ProcessValidationsOnReconcile(nil) + require.Error(t, err) + assert.Errorf(t, err, "'spec.members' must be specified if type of MongoDB is ReplicaSet") + + rs = NewReplicaSetBuilder().Build() + rs.Spec.CloudManagerConfig = &PrivateCloudConfig{ + ConfigMapRef: ConfigMapRef{Name: "cloud-manager"}, + } + require.NoError(t, rs.ProcessValidationsOnReconcile(nil)) +} + +func TestReplicasetFCV(t *testing.T) { + tests := []struct { + name string + fcv *string + expectError bool + expectedErrorMessage string + }{ + { + name: "Invalid FCV value", + fcv: ptr.To("test"), + expectError: true, + expectedErrorMessage: "invalid feature compatibility version: test, possible values are: 'AlwaysMatchVersion' or 'major.minor'", + }, + { + name: "Valid FCV with specific version", + fcv: ptr.To("4.0"), + expectError: false, + }, + { + name: "Invalid FCV - not major.minor only", + fcv: ptr.To("4.0.0"), + expectError: true, + expectedErrorMessage: "invalid feature compatibility version: 4.0.0, possible values are: 'AlwaysMatchVersion' or 'major.minor'", + }, + { + name: "Valid FCV with AlwaysMatchVersion", + fcv: ptr.To("AlwaysMatchVersion"), + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rs := NewReplicaSetBuilder().Build() + rs.Spec.CloudManagerConfig = &PrivateCloudConfig{ + ConfigMapRef: ConfigMapRef{Name: "cloud-manager"}, + } + rs.Spec.FeatureCompatibilityVersion = tt.fcv + + err := rs.ProcessValidationsOnReconcile(nil) + + if tt.expectError { + require.Error(t, err) + assert.EqualError(t, err, tt.expectedErrorMessage) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/api/v1/mdb/mongodbbuilder.go b/api/v1/mdb/mongodbbuilder.go new file mode 100644 index 000000000..5b0d0118a --- /dev/null +++ b/api/v1/mdb/mongodbbuilder.go @@ -0,0 +1,284 @@ +package mdb + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// TODO must replace all [Standalone|Replicaset|Cluster]Builder classes in 'operator' package +// TODO 2 move this to a separate package 'mongodb' together with 'types.go' and 'podspecbuilder.go' +// Convenience builder for Mongodb object +type MongoDBBuilder struct { + mdb *MongoDB +} + +func NewReplicaSetBuilder() *MongoDBBuilder { + return defaultMongoDB(ReplicaSet).SetMembers(3) +} + +func NewDefaultReplicaSetBuilder() *MongoDBBuilder { + return defaultMongoDB(ReplicaSet) +} + +func NewDefaultShardedClusterBuilder() *MongoDBBuilder { + return defaultMongoDB(ShardedCluster). + SetShardCountSpec(3). + SetMongosCountSpec(1). + SetConfigServerCountSpec(3). + SetMongodsPerShardCountSpec(3). + AddDummyOpsManagerConfig() +} + +func NewDefaultMultiShardedClusterBuilder() *MongoDBBuilder { + return defaultMongoDB(ShardedCluster). + AddDummyOpsManagerConfig(). + SetShardCountSpec(3). + SetMultiClusterTopology(). + SetAllClusterSpecLists( + ClusterSpecList{ + { + ClusterName: "test-cluster-0", + Members: 2, + }, + { + ClusterName: "test-cluster-1", + Members: 3, + }, + }, + ) +} + +func NewStandaloneBuilder() *MongoDBBuilder { + return defaultMongoDB(Standalone) +} + +func NewClusterBuilder() *MongoDBBuilder { + sizeConfig := status.MongodbShardedClusterSizeConfig{ + ShardCount: 2, + MongodsPerShardCount: 3, + ConfigServerCount: 4, + MongosCount: 2, + } + mongodb := defaultMongoDB(ShardedCluster) + mongodb.mdb.Spec.MongodbShardedClusterSizeConfig = sizeConfig + return mongodb +} + +func (b *MongoDBBuilder) ExposedExternally(specOverride *corev1.ServiceSpec, annotationsOverride map[string]string, externalDomain *string) *MongoDBBuilder { + b.mdb.Spec.ExternalAccessConfiguration = &ExternalAccessConfiguration{} + b.mdb.Spec.ExternalAccessConfiguration.ExternalDomain = externalDomain + if specOverride != nil { + b.mdb.Spec.ExternalAccessConfiguration.ExternalService.SpecWrapper = &ServiceSpecWrapper{Spec: *specOverride} + } + if len(annotationsOverride) > 0 { + b.mdb.Spec.ExternalAccessConfiguration.ExternalService.Annotations = annotationsOverride + } + return b +} + +func (b *MongoDBBuilder) SetVersion(version string) *MongoDBBuilder { + b.mdb.Spec.Version = version + return b +} + +func (b *MongoDBBuilder) SetName(name string) *MongoDBBuilder { + b.mdb.Name = name + return b +} + +func (b *MongoDBBuilder) SetNamespace(namespace string) *MongoDBBuilder { + b.mdb.Namespace = namespace + return b +} + +func (b *MongoDBBuilder) SetFCVersion(version string) *MongoDBBuilder { + b.mdb.Spec.FeatureCompatibilityVersion = &version + return b +} + +func (b *MongoDBBuilder) SetMembers(m int) *MongoDBBuilder { + if b.mdb.Spec.ResourceType != ReplicaSet { + panic("Only replicaset can have members configuration") + } + b.mdb.Spec.Members = m + return b +} + +func (b *MongoDBBuilder) SetClusterDomain(m string) *MongoDBBuilder { + b.mdb.Spec.ClusterDomain = m + return b +} + +func (b *MongoDBBuilder) SetAdditionalConfig(c *AdditionalMongodConfig) *MongoDBBuilder { + b.mdb.Spec.AdditionalMongodConfig = c + return b +} + +func (b *MongoDBBuilder) SetMongosAdditionalConfig(c *AdditionalMongodConfig) *MongoDBBuilder { + if b.mdb.Spec.MongosSpec == nil { + b.mdb.Spec.MongosSpec = &ShardedClusterComponentSpec{} + } + b.mdb.Spec.MongosSpec.AdditionalMongodConfig = c + return b +} + +func (b *MongoDBBuilder) SetConfigSrvAdditionalConfig(c *AdditionalMongodConfig) *MongoDBBuilder { + if b.mdb.Spec.ConfigSrvSpec == nil { + b.mdb.Spec.ConfigSrvSpec = &ShardedClusterComponentSpec{} + } + b.mdb.Spec.ConfigSrvSpec.AdditionalMongodConfig = c + return b +} + +func (b *MongoDBBuilder) SetShardAdditionalConfig(c *AdditionalMongodConfig) *MongoDBBuilder { + if b.mdb.Spec.ShardSpec == nil { + b.mdb.Spec.ShardSpec = &ShardedClusterComponentSpec{} + } + b.mdb.Spec.ShardSpec.AdditionalMongodConfig = c + return b +} + +func (b *MongoDBBuilder) SetSecurityTLSEnabled() *MongoDBBuilder { + b.mdb.Spec.Security.TLSConfig.Enabled = true + return b +} + +func (b *MongoDBBuilder) SetLabels(labels map[string]string) *MongoDBBuilder { + b.mdb.Labels = labels + return b +} + +func (b *MongoDBBuilder) SetAnnotations(annotations map[string]string) *MongoDBBuilder { + b.mdb.Annotations = annotations + return b +} + +func (b *MongoDBBuilder) EnableAuth(modes []AuthMode) *MongoDBBuilder { + if b.mdb.Spec.Security.Authentication == nil { + b.mdb.Spec.Security.Authentication = &Authentication{} + } + b.mdb.Spec.Security.Authentication.Enabled = true + b.mdb.Spec.Security.Authentication.Modes = modes + return b +} + +func (b *MongoDBBuilder) EnableAgentAuth(mode string) *MongoDBBuilder { + if b.mdb.Spec.Security.Authentication == nil { + b.mdb.Spec.Security.Authentication = &Authentication{} + } + b.mdb.Spec.Security.Authentication.Agents.Mode = mode + return b +} + +func (b *MongoDBBuilder) SetShardCountSpec(count int) *MongoDBBuilder { + if b.mdb.Spec.ResourceType != ShardedCluster { + panic("Only sharded cluster can have shards configuration") + } + b.mdb.Spec.ShardCount = count + return b +} + +func (b *MongoDBBuilder) SetMongodsPerShardCountSpec(count int) *MongoDBBuilder { + if b.mdb.Spec.ResourceType != ShardedCluster { + panic("Only sharded cluster can have shards configuration") + } + b.mdb.Spec.MongodsPerShardCount = count + return b +} + +func (b *MongoDBBuilder) SetConfigServerCountSpec(count int) *MongoDBBuilder { + if b.mdb.Spec.ResourceType != ShardedCluster { + panic("Only sharded cluster can have config server configuration") + } + b.mdb.Spec.ConfigServerCount = count + return b +} + +func (b *MongoDBBuilder) SetMongosCountSpec(count int) *MongoDBBuilder { + if b.mdb.Spec.ResourceType != ShardedCluster { + panic("Only sharded cluster can have mongos configuration") + } + b.mdb.Spec.MongosCount = count + return b +} + +func (b *MongoDBBuilder) SetAdditionalOptions(config AdditionalMongodConfig) *MongoDBBuilder { + b.mdb.Spec.AdditionalMongodConfig = &config + return b +} + +func (b *MongoDBBuilder) SetBackup(backupSpec Backup) *MongoDBBuilder { + if b.mdb.Spec.ResourceType == Standalone { + panic("Backup is only supported for ReplicaSets and ShardedClusters") + } + b.mdb.Spec.Backup = &backupSpec + return b +} + +func (b *MongoDBBuilder) SetConnectionSpec(spec ConnectionSpec) *MongoDBBuilder { + b.mdb.Spec.ConnectionSpec = spec + return b +} + +func (b *MongoDBBuilder) SetAgentConfig(agentOptions AgentConfig) *MongoDBBuilder { + b.mdb.Spec.Agent = agentOptions + return b +} + +func (b *MongoDBBuilder) SetPersistent(p *bool) *MongoDBBuilder { + b.mdb.Spec.Persistent = p + return b +} + +func (b *MongoDBBuilder) SetPodSpec(podSpec *MongoDbPodSpec) *MongoDBBuilder { + b.mdb.Spec.PodSpec = podSpec + return b +} + +func (b *MongoDBBuilder) SetMultiClusterTopology() *MongoDBBuilder { + b.mdb.Spec.Topology = ClusterTopologyMultiCluster + return b +} + +func (b *MongoDBBuilder) AddDummyOpsManagerConfig() *MongoDBBuilder { + b.mdb.Spec.OpsManagerConfig = &PrivateCloudConfig{ConfigMapRef: ConfigMapRef{Name: "dummy"}} + return b +} + +func (b *MongoDBBuilder) SetAllClusterSpecLists(clusterSpecList ClusterSpecList) *MongoDBBuilder { + b.mdb.Spec.ShardSpec.ClusterSpecList = clusterSpecList + b.mdb.Spec.ConfigSrvSpec.ClusterSpecList = clusterSpecList + b.mdb.Spec.MongosSpec.ClusterSpecList = clusterSpecList + return b +} + +func (b *MongoDBBuilder) SetShardOverrides(shardOverride []ShardOverride) *MongoDBBuilder { + b.mdb.Spec.ShardOverrides = shardOverride + return b +} + +func (b *MongoDBBuilder) Build() *MongoDB { + b.mdb.InitDefaults() + return b.mdb.DeepCopy() +} + +// ************************* Package private methods ********************************************************* + +func defaultMongoDB(resourceType ResourceType) *MongoDBBuilder { + spec := MongoDbSpec{ + DbCommonSpec: DbCommonSpec{ + Version: "4.0.0", + ResourceType: resourceType, + }, + } + mdb := &MongoDB{Spec: spec, ObjectMeta: metav1.ObjectMeta{Name: "test-mdb", Namespace: "testNS"}} + mdb.InitDefaults() + return &MongoDBBuilder{mdb} +} + +func (b *MongoDBBuilder) setType(resourceType ResourceType) *MongoDBBuilder { + b.mdb.Spec.ResourceType = resourceType + return b +} diff --git a/api/v1/mdb/mongodconfig.go b/api/v1/mdb/mongodconfig.go new file mode 100644 index 000000000..1a8f20e16 --- /dev/null +++ b/api/v1/mdb/mongodconfig.go @@ -0,0 +1,117 @@ +package mdb + +import ( + "encoding/json" + "strings" + + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" +) + +// The CRD generator does not support map[string]interface{} +// on the top level and hence we need to work around this with +// a wrapping struct. + +// AdditionalMongodConfig contains a private non exported object with a json tag. +// Because we implement the Json marshal and unmarshal interface, json is still able to convert this object into its write-type. +// Making this field private enables us to make sure we don't directly access this field, making sure it is always initialized. +// The space is on purpose to not generate the comment in the CRD. + +type AdditionalMongodConfig struct { + object map[string]interface{} `json:"-"` +} + +// Note: The MarshalJSON and UnmarshalJSON need to be explicitly implemented in this case as our wrapper type itself cannot be marshalled/unmarshalled by default. Without this custom logic the values provided in the resource definition will not be set in the struct created. +// MarshalJSON defers JSON encoding to the wrapped map +func (amc *AdditionalMongodConfig) MarshalJSON() ([]byte, error) { + return json.Marshal(amc.object) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (amc *AdditionalMongodConfig) UnmarshalJSON(data []byte) error { + if amc.object == nil { + amc.object = map[string]interface{}{} + } + return json.Unmarshal(data, &amc.object) +} + +func NewEmptyAdditionalMongodConfig() *AdditionalMongodConfig { + return &AdditionalMongodConfig{object: make(map[string]interface{})} +} + +func NewAdditionalMongodConfig(key string, value interface{}) *AdditionalMongodConfig { + config := NewEmptyAdditionalMongodConfig() + config.AddOption(key, value) + return config +} + +func (amc *AdditionalMongodConfig) AddOption(key string, value interface{}) *AdditionalMongodConfig { + keys := strings.Split(key, ".") + maputil.SetMapValue(amc.object, value, keys...) + return amc +} + +// ToFlatList returns all mongodb options as a sorted list of string values. +// It performs a recursive traversal of maps and dumps the current config to the final list of configs +func (amc *AdditionalMongodConfig) ToFlatList() []string { + return maputil.ToFlatList(amc.ToMap()) +} + +// GetPortOrDefault returns the port that should be used for the mongo process. +// if no port is specified in the additional mongo args, the default +// port of 27017 will be used +func (amc *AdditionalMongodConfig) GetPortOrDefault() int32 { + if amc == nil || amc.object == nil { + return util.MongoDbDefaultPort + } + + // https://golang.org/pkg/encoding/json/#Unmarshal + // the port will be stored as a float64. + // However, on unit tests, and because of the way the deserialization + // works, this value is returned as an int. That's why we read the + // port as Int which uses the `cast` library to cast both float32 and int + // types into Int. + port := maputil.ReadMapValueAsInt(amc.object, "net", "port") + if port == 0 { + return util.MongoDbDefaultPort + } + + //nolint:gosec // suppressing integer overflow warning for int32(port) + return int32(port) +} + +// DeepCopy is defined manually as codegen utility cannot generate copy methods for 'interface{}' +func (amc *AdditionalMongodConfig) DeepCopy() *AdditionalMongodConfig { + if amc == nil { + return nil + } + out := new(AdditionalMongodConfig) + amc.DeepCopyInto(out) + return out +} + +func (amc *AdditionalMongodConfig) DeepCopyInto(out *AdditionalMongodConfig) { + cp, err := util.MapDeepCopy(amc.object) + if err != nil { + zap.S().Errorf("Failed to copy the map: %s", err) + return + } + config := AdditionalMongodConfig{object: cp} + *out = config +} + +// ToMap creates a copy of the config as a map (Go is quite restrictive to types, and sometimes we need to +// explicitly declare the type as map :( ) +func (amc *AdditionalMongodConfig) ToMap() map[string]interface{} { + if amc == nil || amc.object == nil { + return map[string]interface{}{} + } + cp, err := util.MapDeepCopy(amc.object) + if err != nil { + zap.S().Errorf("Failed to copy the map: %s", err) + return nil + } + return cp +} diff --git a/api/v1/mdb/mongodconfig_test.go b/api/v1/mdb/mongodconfig_test.go new file mode 100644 index 000000000..74a3cfd5e --- /dev/null +++ b/api/v1/mdb/mongodconfig_test.go @@ -0,0 +1,35 @@ +package mdb + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeepCopy(t *testing.T) { + config := NewAdditionalMongodConfig("first.second", "value") + cp := *config.DeepCopy() + + expectedAdditionalConfig := AdditionalMongodConfig{ + object: map[string]interface{}{"first": map[string]interface{}{"second": "value"}}, + } + assert.Equal(t, expectedAdditionalConfig.object, cp.object) + + cp.object["first"].(map[string]interface{})["second"] = "newvalue" + + // The value in the first config hasn't changed + assert.Equal(t, "value", config.object["first"].(map[string]interface{})["second"]) +} + +func TestToFlatList(t *testing.T) { + config := NewAdditionalMongodConfig("one.two.three", "v1") + config.AddOption("one.two.four", 5) + config.AddOption("one.five", true) + config.AddOption("six.seven.eight", "v2") + config.AddOption("six.nine", "v3") + + list := config.ToFlatList() + + expectedStrings := []string{"one.five", "one.two.four", "one.two.three", "six.nine", "six.seven.eight"} + assert.Equal(t, expectedStrings, list) +} diff --git a/api/v1/mdb/podspecbuilder.go b/api/v1/mdb/podspecbuilder.go new file mode 100644 index 000000000..f439b262d --- /dev/null +++ b/api/v1/mdb/podspecbuilder.go @@ -0,0 +1,160 @@ +package mdb + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// TODO remove the wrapper in favor of podSpecBuilder +type PodSpecWrapperBuilder struct { + spec PodSpecWrapper +} + +type PersistenceConfigBuilder struct { + config *PersistenceConfig +} + +// NewPodSpecWrapperBuilder returns the builder with some default values, used in tests mostly +func NewPodSpecWrapperBuilder() *PodSpecWrapperBuilder { + spec := MongoDbPodSpec{ + ContainerResourceRequirements: ContainerResourceRequirements{ + CpuLimit: "1.0", + CpuRequests: "0.5", + MemoryLimit: "500M", + MemoryRequests: "400M", + }, + PodTemplateWrapper: PodTemplateSpecWrapper{&corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{}, + }, + }, + }}, + } + return &PodSpecWrapperBuilder{PodSpecWrapper{ + MongoDbPodSpec: spec, + Default: NewPodSpecWithDefaultValues(), + }} +} + +func NewPodSpecWrapperBuilderFromSpec(spec *MongoDbPodSpec) *PodSpecWrapperBuilder { + if spec == nil { + return &PodSpecWrapperBuilder{PodSpecWrapper{}} + } + return &PodSpecWrapperBuilder{PodSpecWrapper{MongoDbPodSpec: *spec}} +} + +func NewEmptyPodSpecWrapperBuilder() *PodSpecWrapperBuilder { + return &PodSpecWrapperBuilder{spec: PodSpecWrapper{ + MongoDbPodSpec: MongoDbPodSpec{ + Persistence: &Persistence{}, + }, + Default: MongoDbPodSpec{ + Persistence: &Persistence{SingleConfig: &PersistenceConfig{}}, + }, + }} +} + +func (p *PodSpecWrapperBuilder) SetCpuLimit(cpu string) *PodSpecWrapperBuilder { + p.spec.CpuLimit = cpu + return p +} + +func (p *PodSpecWrapperBuilder) SetCpuRequests(cpu string) *PodSpecWrapperBuilder { + p.spec.CpuRequests = cpu + return p +} + +func (p *PodSpecWrapperBuilder) SetMemoryLimit(memory string) *PodSpecWrapperBuilder { + p.spec.MemoryLimit = memory + return p +} + +func (p *PodSpecWrapperBuilder) SetMemoryRequest(memory string) *PodSpecWrapperBuilder { + p.spec.MemoryRequests = memory + return p +} + +func (p *PodSpecWrapperBuilder) SetPodAffinity(affinity corev1.PodAffinity) *PodSpecWrapperBuilder { + p.spec.PodTemplateWrapper.PodTemplate.Spec.Affinity.PodAffinity = &affinity + return p +} + +func (p *PodSpecWrapperBuilder) SetNodeAffinity(affinity corev1.NodeAffinity) *PodSpecWrapperBuilder { + p.spec.PodTemplateWrapper.PodTemplate.Spec.Affinity.NodeAffinity = &affinity + return p +} + +func (p *PodSpecWrapperBuilder) SetPodAntiAffinityTopologyKey(topologyKey string) *PodSpecWrapperBuilder { + p.spec.PodAntiAffinityTopologyKey = topologyKey + return p +} + +func (p *PodSpecWrapperBuilder) SetPodTemplate(template *corev1.PodTemplateSpec) *PodSpecWrapperBuilder { + p.spec.PodTemplateWrapper.PodTemplate = template + return p +} + +func (p *PodSpecWrapperBuilder) SetSinglePersistence(builder *PersistenceConfigBuilder) *PodSpecWrapperBuilder { + if p.spec.Persistence == nil { + p.spec.Persistence = &Persistence{} + } + p.spec.Persistence.SingleConfig = builder.config + return p +} + +func (p *PodSpecWrapperBuilder) SetMultiplePersistence(dataBuilder, journalBuilder, logsBuilder *PersistenceConfigBuilder) *PodSpecWrapperBuilder { + if p.spec.Persistence == nil { + p.spec.Persistence = &Persistence{} + } + p.spec.Persistence.MultipleConfig = &MultiplePersistenceConfig{} + if dataBuilder != nil { + p.spec.Persistence.MultipleConfig.Data = dataBuilder.config + } + if journalBuilder != nil { + p.spec.Persistence.MultipleConfig.Journal = journalBuilder.config + } + if logsBuilder != nil { + p.spec.Persistence.MultipleConfig.Logs = logsBuilder.config + } + return p +} + +func (p *PodSpecWrapperBuilder) SetDefault(builder *PodSpecWrapperBuilder) *PodSpecWrapperBuilder { + p.spec.Default = builder.Build().MongoDbPodSpec + return p +} + +func (p *PodSpecWrapperBuilder) Build() *PodSpecWrapper { + return p.spec.DeepCopy() +} + +func NewPersistenceBuilder(size string) *PersistenceConfigBuilder { + return &PersistenceConfigBuilder{config: &PersistenceConfig{Storage: size}} +} + +func (p *PersistenceConfigBuilder) SetStorageClass(class string) *PersistenceConfigBuilder { + p.config.StorageClass = &class + return p +} + +func (p *PersistenceConfigBuilder) SetLabelSelector(labels map[string]string) *PersistenceConfigBuilder { + p.config.LabelSelector = &LabelSelectorWrapper{metav1.LabelSelector{MatchLabels: labels}} + return p +} + +func NewPodSpecWithDefaultValues() MongoDbPodSpec { + defaultPodSpec := MongoDbPodSpec{PodAntiAffinityTopologyKey: "kubernetes.io/hostname"} + defaultPodSpec.Persistence = &Persistence{ + SingleConfig: &PersistenceConfig{Storage: "30G"}, + MultipleConfig: &MultiplePersistenceConfig{ + Data: &PersistenceConfig{Storage: util.DefaultMongodStorageSize}, + Journal: &PersistenceConfig{Storage: util.DefaultJournalStorageSize}, + Logs: &PersistenceConfig{Storage: util.DefaultLogsStorageSize}, + }, + } + defaultPodSpec.PodTemplateWrapper = NewMongoDbPodSpec().PodTemplateWrapper + return defaultPodSpec +} diff --git a/api/v1/mdb/sharded_cluster_validation.go b/api/v1/mdb/sharded_cluster_validation.go new file mode 100644 index 000000000..fad8d7b9e --- /dev/null +++ b/api/v1/mdb/sharded_cluster_validation.go @@ -0,0 +1,376 @@ +package mdb + +import ( + "fmt" + "regexp" + "strconv" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" +) + +var MemberConfigErrorMessage = "there must be at least as many entries in MemberConfig as specified in the 'members' field" + +func ShardedClusterCommonValidators() []func(m MongoDB) v1.ValidationResult { + return []func(m MongoDB) v1.ValidationResult{ + shardOverridesShardNamesNotEmpty, + shardOverridesShardNamesUnique, + shardOverridesShardNamesCorrectValues, + shardOverridesClusterSpecListsCorrect, + shardCountSpecified, + } +} + +func ShardedClusterSingleValidators() []func(m MongoDB) v1.ValidationResult { + return []func(m MongoDB) v1.ValidationResult{ + emptyClusterSpecLists, + duplicateServiceObjectsIsIgnoredInSingleCluster, + mandatorySingleClusterFieldsAreSpecified, + } +} + +func ShardedClusterMultiValidators() []func(m MongoDB) []v1.ValidationResult { + return []func(m MongoDB) []v1.ValidationResult{ + noIgnoredFieldUsed, + func(m MongoDB) []v1.ValidationResult { + return []v1.ValidationResult{hasClusterSpecListsDefined(m)} + }, + func(m MongoDB) []v1.ValidationResult { + return []v1.ValidationResult{validClusterSpecLists(m)} + }, + func(m MongoDB) []v1.ValidationResult { + return []v1.ValidationResult{validateMemberClusterIsSubsetOfKubeConfig(m)} + }, + } +} + +// This applies to any topology +func shardCountSpecified(m MongoDB) v1.ValidationResult { + if m.Spec.ShardCount == 0 { + return v1.ValidationError("shardCount must be specified") + } + return v1.ValidationSuccess() +} + +func mandatorySingleClusterFieldsAreSpecified(m MongoDB) v1.ValidationResult { + if m.Spec.MongodsPerShardCount == 0 || + m.Spec.MongosCount == 0 || + m.Spec.ConfigServerCount == 0 { + return v1.ValidationError("The following fields must be specified in single cluster topology: mongodsPerShardCount, mongosCount, configServerCount") + } + return v1.ValidationSuccess() +} + +func hasClusterSpecListsDefined(m MongoDB) v1.ValidationResult { + msg := "cluster spec list in %s must be defined in Multi Cluster topology" + if !hasClusterSpecList(m.Spec.ShardSpec.ClusterSpecList) { + return v1.ValidationError(msg, "spec.shardSpec") + } + if !hasClusterSpecList(m.Spec.ConfigSrvSpec.ClusterSpecList) { + return v1.ValidationError(msg, "spec.configSrvSpec") + } + if !hasClusterSpecList(m.Spec.MongosSpec.ClusterSpecList) { + return v1.ValidationError(msg, "spec.mongosSpec") + } + return v1.ValidationSuccess() +} + +func emptyClusterSpecLists(m MongoDB) v1.ValidationResult { + msg := "cluster spec list in %s must be empty in Single Cluster topology" + if hasClusterSpecList(m.Spec.ShardSpec.ClusterSpecList) { + return v1.ValidationError(msg, "spec.shardSpec") + } + if hasClusterSpecList(m.Spec.ConfigSrvSpec.ClusterSpecList) { + return v1.ValidationError(msg, "spec.configSrvSpec") + } + if hasClusterSpecList(m.Spec.MongosSpec.ClusterSpecList) { + return v1.ValidationError(msg, "spec.mongosSpec") + } + for _, shardOverride := range m.Spec.ShardOverrides { + if len(shardOverride.ClusterSpecList) > 0 { + return v1.ValidationError(msg, "spec.shardOverrides") + } + } + return v1.ValidationSuccess() +} + +func hasClusterSpecList(clusterSpecList ClusterSpecList) bool { + return len(clusterSpecList) > 0 +} + +// Validate clusterSpecList field, the validation for shard overrides clusterSpecList require different rules +func validClusterSpecLists(m MongoDB) v1.ValidationResult { + clusterSpecs := []struct { + list ClusterSpecList + specName string + }{ + {m.Spec.ShardSpec.ClusterSpecList, "spec.shardSpec"}, + {m.Spec.ConfigSrvSpec.ClusterSpecList, "spec.configSrvSpec"}, + {m.Spec.MongosSpec.ClusterSpecList, "spec.mongosSpec"}, + } + for _, spec := range clusterSpecs { + if result := isValidClusterSpecList(spec.list, spec.specName); result != v1.ValidationSuccess() { + return result + } + } + // MemberConfig and Members fields are ignored at top level for MC Sharded + if len(m.Spec.MemberConfig) > 0 && len(m.Spec.MemberConfig) < m.Spec.Members { + return v1.ValidationError("Invalid clusterSpecList: %s", MemberConfigErrorMessage) + } + return v1.ValidationSuccess() +} + +func isValidClusterSpecList(clusterSpecList ClusterSpecList, specName string) v1.ValidationResult { + for _, clusterSpecItem := range clusterSpecList { + if clusterSpecItem.ClusterName == "" { + return v1.ValidationError("All clusters specified in %s.clusterSpecList require clusterName and members fields", specName) + } + if len(clusterSpecItem.MemberConfig) > 0 && len(clusterSpecItem.MemberConfig) < clusterSpecItem.Members { + return v1.ValidationError("Invalid member configuration in %s.clusterSpecList: %s", specName, MemberConfigErrorMessage) + } + } + return v1.ValidationSuccess() +} + +func validateShardOverrideClusterSpecList(clusterSpecList []ClusterSpecItemOverride, shardNames []string) (bool, v1.ValidationResult) { + if len(clusterSpecList) == 0 { + return true, v1.ValidationError("shard override for shards %+v has an empty clusterSpecList", shardNames) + } + for _, clusterSpec := range clusterSpecList { + // Note that it is okay for a shard override clusterSpecList to have Members = 0 + if clusterSpec.ClusterName == "" { + return true, v1.ValidationError("shard override for shards %+v has an empty clusterName in clusterSpecList, this field must be specified", shardNames) + } + // This check is performed for overrides cluster spec lists as well + if len(clusterSpec.MemberConfig) > 0 && clusterSpec.Members != nil && + len(clusterSpec.MemberConfig) < *clusterSpec.Members { + return true, v1.ValidationError("shard override for shards %+v is incorrect: %s", shardNames, MemberConfigErrorMessage) + } + } + return false, v1.ValidationSuccess() +} + +func shardOverridesShardNamesNotEmpty(m MongoDB) v1.ValidationResult { + for idx, shardOverride := range m.Spec.ShardOverrides { + if len(shardOverride.ShardNames) == 0 { + return v1.ValidationError("spec.shardOverride[*].shardNames cannot be empty, shardOverride with index %d is invalid", idx) + } + } + return v1.ValidationSuccess() +} + +func shardOverridesShardNamesUnique(m MongoDB) v1.ValidationResult { + idSet := make(map[string]bool) + for _, shardOverride := range m.Spec.ShardOverrides { + for _, shardName := range shardOverride.ShardNames { + if idSet[shardName] && shardName != "" { + return v1.ValidationError("spec.shardOverride[*].shardNames elements must be unique in shardOverrides, shardName %s is a duplicate", shardName) + } + idSet[shardName] = true + } + } + return v1.ValidationSuccess() +} + +func shardOverridesShardNamesCorrectValues(m MongoDB) v1.ValidationResult { + for _, shardOverride := range m.Spec.ShardOverrides { + for _, shardName := range shardOverride.ShardNames { + if !validateShardName(shardName, m.Spec.ShardCount, m.Name) { + return v1.ValidationError("name %s is incorrect, it must follow the following format: %s-{shard index} with shardIndex < %d (shardCount)", shardName, m.Name, m.Spec.ShardCount) + } + } + } + return v1.ValidationSuccess() +} + +func shardOverridesClusterSpecListsCorrect(m MongoDB) v1.ValidationResult { + for _, shardOverride := range m.Spec.ShardOverrides { + if shardOverride.ClusterSpecList != nil { + if hasError, result := validateShardOverrideClusterSpecList(shardOverride.ClusterSpecList, shardOverride.ShardNames); hasError { + return result + } + } + } + return v1.ValidationSuccess() + // Note that shardOverride.Members and shardOverride.MemberConfig should not be checked as they are ignored, + // shardOverride.ClusterSpecList.Members and shardOverride.ClusterSpecList.MemberConfig are used instead +} + +// If the MDB resource name is foo, and we have n shards, we verify that shard names ∈ {foo-0 , foo-1 ..., foo-(n-1)} +func validateShardName(shardName string, shardCount int, resourceName string) bool { + // The shard number should not have leading zeros except for 0 itself + pattern := fmt.Sprintf(`^%s-(0|[1-9][0-9]*)$`, resourceName) + + re := regexp.MustCompile(pattern) + if !re.MatchString(shardName) { + return false + } + + // Extract the shard number from the matched part + parts := re.FindStringSubmatch(shardName) + shardNumber, err := strconv.Atoi(parts[1]) + if err != nil { + return false + } + + if shardNumber < 0 || shardNumber >= shardCount { + return false + } + return true +} + +func noIgnoredFieldUsed(m MongoDB) []v1.ValidationResult { + var warnings []v1.ValidationResult + var errors []v1.ValidationResult + + if m.Spec.MongodsPerShardCount != 0 { + appendValidationError(&errors, "spec.mongodsPerShardCount", "spec.shard.clusterSpecList.members") + } + + if m.Spec.MongosCount != 0 { + appendValidationError(&errors, "spec.mongosCount", "spec.mongos.clusterSpecList.members") + } + + if m.Spec.ConfigServerCount != 0 { + appendValidationError(&errors, "spec.configServerCount", "spec.configSrv.clusterSpecList.members") + } + + if m.Spec.Members != 0 { + appendValidationWarning(&warnings, "spec.members", "spec.[...].clusterSpecList.members") + } + + if m.Spec.MemberConfig != nil { + appendValidationWarning(&warnings, "spec.memberConfig", "spec.[...].clusterSpecList.memberConfig") + } + + for _, clusterSpec := range m.Spec.ShardSpec.ClusterSpecList { + if clusterSpec.PodSpec != nil && clusterSpec.PodSpec.PodTemplateWrapper.PodTemplate != nil { + appendValidationWarning(&warnings, "spec.shard.clusterSpecList.podSpec.podTemplate", "spec.shard.clusterSpecList.statefulSetConfiguration") + } + } + + for _, clusterSpec := range m.Spec.ConfigSrvSpec.ClusterSpecList { + if clusterSpec.PodSpec != nil && clusterSpec.PodSpec.PodTemplateWrapper.PodTemplate != nil { + appendValidationWarning(&warnings, "spec.configSrv.clusterSpecList.podSpec.podTemplate", "spec.configSrv.clusterSpecList.statefulSetConfiguration") + } + } + + for _, clusterSpec := range m.Spec.MongosSpec.ClusterSpecList { + if clusterSpec.PodSpec != nil && clusterSpec.PodSpec.PodTemplateWrapper.PodTemplate != nil { + appendValidationWarning(&warnings, "spec.mongos.clusterSpecList.podSpec.podTemplate", "spec.mongos.clusterSpecList.statefulSetConfiguration") + } + } + + for _, shardOverride := range m.Spec.ShardOverrides { + if shardOverride.MemberConfig != nil { + appendValidationWarning(&warnings, "spec.shardOverrides.memberConfig", "spec.shardOverrides.clusterSpecList.memberConfig") + } + + if shardOverride.Members != nil { + appendValidationWarning(&warnings, "spec.shardOverrides.members", "spec.shardOverrides.clusterSpecList.members") + } + + if shardOverride.PodSpec != nil && shardOverride.PodSpec.PodTemplateWrapper.PodTemplate != nil { + appendValidationWarning(&warnings, "spec.shardOverrides.podSpec.podTemplate", "spec.shardOverrides.statefulSetConfiguration") + } + + for _, clusterSpec := range shardOverride.ClusterSpecList { + if clusterSpec.PodSpec != nil && clusterSpec.PodSpec.PodTemplateWrapper.PodTemplate != nil { + appendValidationWarning(&warnings, "spec.shardOverrides.clusterSpecList.podSpec.podTemplate", "spec.shardOverrides.clusterSpecList.statefulSetConfiguration") + } + } + } + + if len(errors) > 0 { + return errors + } + + if len(warnings) > 0 { + return warnings + } + + return []v1.ValidationResult{v1.ValidationSuccess()} +} + +func appendValidationWarning(warnings *[]v1.ValidationResult, ignoredField string, preferredField string) { + *warnings = append(*warnings, v1.ValidationWarning("%s is ignored in Multi Cluster topology. "+ + "Use instead: %s", ignoredField, preferredField)) +} + +func appendValidationError(errors *[]v1.ValidationResult, ignoredField string, preferredField string) { + *errors = append(*errors, v1.ValidationError("%s must not be set in Multi Cluster topology. "+ + "The member count will depend on: %s", ignoredField, preferredField)) +} + +func duplicateServiceObjectsIsIgnoredInSingleCluster(m MongoDB) v1.ValidationResult { + if m.Spec.DuplicateServiceObjects != nil { + return v1.ValidationWarning("In Single Cluster topology, spec.duplicateServiceObjects field is ignored") + } + return v1.ValidationSuccess() +} + +// This is used to validate all kind of cluster spec in the same way, whether it is an override or not +func convertOverrideToClusterSpec(override ClusterSpecItemOverride) ClusterSpecItem { + var overrideMembers int + if override.Members != nil { + overrideMembers = *override.Members + } else { + overrideMembers = 0 + } + return ClusterSpecItem{ + ClusterName: override.ClusterName, + Service: "", // Field doesn't exist in override + ExternalAccessConfiguration: nil, // Field doesn't exist in override + Members: overrideMembers, + MemberConfig: override.MemberConfig, + StatefulSetConfiguration: override.StatefulSetConfiguration, + PodSpec: override.PodSpec, + } +} + +func validateMemberClusterIsSubsetOfKubeConfig(m MongoDB) v1.ValidationResult { + // We first extract every cluster spec lists from the resource (from Shard, ConfigServer, Mongos and ShardOverrides) + // And we put them in a single flat structure, to be able to run all validations in a single for loop + + // Slice of structs to hold name and ClusterSpecList + var clusterSpecLists []struct { + name string + list ClusterSpecList + } + + // Helper function to append a ClusterSpecList to the slice + appendClusterSpec := func(name string, list ClusterSpecList) { + clusterSpecLists = append(clusterSpecLists, struct { + name string + list ClusterSpecList + }{ + name: name, + list: list, + }) + } + + // Convert ClusterSpecItemOverride to ClusterSpecItem + for _, override := range m.Spec.ShardOverrides { + var convertedList ClusterSpecList + for _, overrideItem := range override.ClusterSpecList { + convertedList = append(convertedList, convertOverrideToClusterSpec(overrideItem)) + } + appendClusterSpec(fmt.Sprintf("shard %+v override", override.ShardNames), convertedList) + } + + // Append other ClusterSpecLists + appendClusterSpec("spec.shardSpec", m.Spec.ShardSpec.ClusterSpecList) + appendClusterSpec("spec.configSrvSpec", m.Spec.ConfigSrvSpec.ClusterSpecList) + appendClusterSpec("spec.mongosSpec", m.Spec.MongosSpec.ClusterSpecList) + + // Validate each ClusterSpecList + for _, specList := range clusterSpecLists { + validationResult := ValidateMemberClusterIsSubsetOfKubeConfig(specList.list) + if validationResult.Level == v1.WarningLevel { + return v1.ValidationWarning("Warning when validating %s ClusterSpecList: %s", specList.name, validationResult.Msg) + } else if validationResult.Level == v1.ErrorLevel { + return v1.ValidationError("Error when validating %s ClusterSpecList: %s", specList.name, validationResult.Msg) + } + } + + return v1.ValidationSuccess() +} diff --git a/api/v1/mdb/sharded_cluster_validation_test.go b/api/v1/mdb/sharded_cluster_validation_test.go new file mode 100644 index 000000000..9aa763ec1 --- /dev/null +++ b/api/v1/mdb/sharded_cluster_validation_test.go @@ -0,0 +1,671 @@ +package mdb + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + corev1 "k8s.io/api/core/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +func makeMemberConfig(members int) []automationconfig.MemberOptions { + return make([]automationconfig.MemberOptions, members) +} + +var defaultMemberConfig = makeMemberConfig(1) + +func TestShardCountIsSpecified(t *testing.T) { + errString := "shardCount must be specified" + scSingle := NewDefaultShardedClusterBuilder().Build() + scSingle.Spec.ShardCount = 0 + _, err := scSingle.ValidateCreate() + require.Error(t, err) + assert.Equal(t, errString, err.Error()) + + scMulti := NewDefaultMultiShardedClusterBuilder().Build() + scMulti.Spec.ShardCount = 0 + _, err = scMulti.ValidateCreate() + require.Error(t, err) + assert.Equal(t, errString, err.Error()) +} + +func TestMandatorySingleClusterFieldsAreSpecified(t *testing.T) { + scSingle := NewDefaultShardedClusterBuilder().Build() + scSingle.Spec.MongosCount = 0 + _, err := scSingle.ValidateCreate() + require.Error(t, err) + assert.Equal(t, "The following fields must be specified in single cluster topology: mongodsPerShardCount, mongosCount, configServerCount", err.Error()) +} + +func TestShardOverridesAreCorrect(t *testing.T) { + intPointer := ptr.To(3) + resourceName := "foo" + tests := []struct { + name string + isMultiCluster bool + shardCount int + shardOverrides []ShardOverride + expectError bool + errorMessage string + expectWarning bool + expectedWarningMessage string + }{ + { + name: "Validate correct shard overrides for SingleCluster topology", + isMultiCluster: false, + shardCount: 3, + shardOverrides: []ShardOverride{{ShardNames: []string{"foo-1"}}, {ShardNames: []string{"foo-0", "foo-2"}}}, + }, + { + name: "Validate incorrect shard overrides for SingleCluster topology", + isMultiCluster: false, + shardCount: 3, + shardOverrides: []ShardOverride{{ShardNames: []string{"foo-100"}}, {ShardNames: []string{"foo-3"}}}, + expectError: true, + errorMessage: "name foo-100 is incorrect, it must follow the following format: foo-{shard index} with shardIndex < 3 (shardCount)", + }, + { + name: "No error for correct shard overrides", + isMultiCluster: true, + shardCount: 4, + shardOverrides: []ShardOverride{{ShardNames: []string{"foo-2"}}, {ShardNames: []string{"foo-0", "foo-3"}}}, + }, + { + name: "Error for incorrect shard name", + isMultiCluster: true, + shardCount: 3, + shardOverrides: []ShardOverride{{ShardNames: []string{"foo-4"}}, {ShardNames: []string{"foo-0", "foo-1"}}}, + expectError: true, + errorMessage: "name foo-4 is incorrect, it must follow the following format: foo-{shard index} with shardIndex < 3 (shardCount)", + }, + { + name: "Error for incorrect shard name with leading zeros", + isMultiCluster: true, + shardCount: 3, + shardOverrides: []ShardOverride{{ShardNames: []string{"foo-000"}}, {ShardNames: []string{"foo-0", "foo-1"}}}, + expectError: true, + errorMessage: "name foo-000 is incorrect, it must follow the following format: foo-{shard index} with shardIndex < 3 (shardCount)", + }, + { + name: "Error for duplicate shard names", + isMultiCluster: true, + shardCount: 3, + shardOverrides: []ShardOverride{{ShardNames: []string{"foo-2"}}, {ShardNames: []string{"foo-0", "foo-2"}}}, + expectError: true, + errorMessage: "spec.shardOverride[*].shardNames elements must be unique in shardOverrides, shardName foo-2 is a duplicate", + }, + { + name: "Error for empty shard names slice", + isMultiCluster: true, + shardCount: 3, + shardOverrides: []ShardOverride{{ShardNames: []string{}}}, + expectError: true, + errorMessage: "spec.shardOverride[*].shardNames cannot be empty, shardOverride with index 0 is invalid", + }, + { + name: "Error when ClusterName is empty in ClusterSpecList", + isMultiCluster: true, + shardCount: 2, + shardOverrides: []ShardOverride{ + { + ShardNames: []string{"foo-0"}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ClusterName: "", Members: intPointer}}, + }, + }, + }, + expectError: true, + errorMessage: "shard override for shards [foo-0] has an empty clusterName in clusterSpecList, this field must be specified", + }, + { + name: "Error when ClusterSpecList is empty in ShardOverrides", + isMultiCluster: true, + shardCount: 5, + shardOverrides: []ShardOverride{ + { + ShardNames: []string{"foo-0", "foo-1", "foo-4"}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{}, + }, + }, + }, + expectError: true, + errorMessage: "shard override for shards [foo-0 foo-1 foo-4] has an empty clusterSpecList", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var sc *MongoDB + if tt.isMultiCluster { + sc = NewDefaultMultiShardedClusterBuilder().SetName(resourceName).Build() + } else { + sc = NewDefaultShardedClusterBuilder().SetName(resourceName).Build() + } + sc.Spec.ShardCount = tt.shardCount + sc.Spec.ShardOverrides = tt.shardOverrides + + _, err := sc.ValidateCreate() + + if tt.expectError { + require.Error(t, err) + assert.Equal(t, tt.errorMessage, err.Error()) + } else { + assert.NoError(t, err) + } + + if tt.expectWarning { + assertWarningExists(t, sc.Status.Warnings, status.Warning(tt.expectedWarningMessage)) + } + }) + } +} + +func TestValidClusterSpecLists(t *testing.T) { + tests := []struct { + name string + shardSpec ClusterSpecItem + configSrvSpec ClusterSpecItem + mongosSpec ClusterSpecItem + members int + memberConfig int + expectError bool + errorMessage string + }{ + { + name: "Error when Members and MemberConfig mismatch", + shardSpec: ClusterSpecItem{ClusterName: "shard-cluster", Members: 3}, + configSrvSpec: ClusterSpecItem{ClusterName: "config-cluster", Members: 1}, + mongosSpec: ClusterSpecItem{ClusterName: "mongos-cluster", Members: 1}, + members: 3, + memberConfig: 2, + expectError: true, + errorMessage: "Invalid clusterSpecList: " + MemberConfigErrorMessage, + }, + { + name: "No error when ClusterSpecLists are valid", + shardSpec: ClusterSpecItem{ClusterName: "shard-cluster", Members: 3}, + configSrvSpec: ClusterSpecItem{ClusterName: "config-cluster", Members: 1}, + mongosSpec: ClusterSpecItem{ClusterName: "mongos-cluster", Members: 1}, + members: 3, + memberConfig: 3, + expectError: false, + }, + { + name: "No error when ClusterSpecLists members is 0", + shardSpec: ClusterSpecItem{ClusterName: "shard-cluster", Members: 0}, + configSrvSpec: ClusterSpecItem{ClusterName: "config-cluster", Members: 0}, + mongosSpec: ClusterSpecItem{ClusterName: "mongos-cluster", Members: 0}, + members: 3, + memberConfig: 3, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := NewDefaultMultiShardedClusterBuilder().Build() + sc.Spec.ShardSpec.ClusterSpecList = ClusterSpecList{tt.shardSpec} + sc.Spec.ConfigSrvSpec.ClusterSpecList = ClusterSpecList{tt.configSrvSpec} + sc.Spec.MongosSpec.ClusterSpecList = ClusterSpecList{tt.mongosSpec} + sc.Spec.Members = tt.members + sc.Spec.MemberConfig = make([]automationconfig.MemberOptions, tt.memberConfig) + + _, err := sc.ValidateCreate() + + if tt.expectError { + require.Error(t, err) + assert.Equal(t, tt.errorMessage, err.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestNoIgnoredFieldUsed(t *testing.T) { + podSpecWithTemplate := &MongoDbPodSpec{ + PodTemplateWrapper: PodTemplateSpecWrapper{PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{}, + }}, + } + tests := []struct { + name string + isMultiCluster bool + mongodsPerShard int + configServerCount int + mongosCount int + members int + memberConfig []automationconfig.MemberOptions + shardOverrides []ShardOverride + expectWarning bool + expectError bool + errorMessage string + expectedWarnings []status.Warning + }{ + { + name: "No warning when fields are set in SingleCluster topology", + isMultiCluster: false, + mongodsPerShard: 3, + configServerCount: 2, + mongosCount: 2, + shardOverrides: []ShardOverride{ + {ShardNames: []string{"foo-0"}, MemberConfig: defaultMemberConfig}, + {ShardNames: []string{"foo-1"}, Members: ptr.To(2)}, + {ShardNames: []string{"foo-2"}, StatefulSetConfiguration: &v1.StatefulSetConfiguration{}}, + }, + expectWarning: false, + expectedWarnings: []status.Warning{}, + }, + { + name: "No warning when no ignored fields are used in MultiCluster topology", + isMultiCluster: true, + }, + { + name: "Error when MongodsPerShardCount is set in MultiCluster topology", + isMultiCluster: true, + mongodsPerShard: 3, + expectError: true, + errorMessage: "spec.mongodsPerShardCount must not be set in Multi Cluster topology. The member count will depend on: spec.shard.clusterSpecList.members", + }, + { + name: "Error when ConfigServerCount is set in MultiCluster topology", + isMultiCluster: true, + configServerCount: 2, + expectError: true, + errorMessage: "spec.configServerCount must not be set in Multi Cluster topology. The member count will depend on: spec.configSrv.clusterSpecList.members", + }, + { + name: "Error when MongosCount is set in MultiCluster topology", + isMultiCluster: true, + mongosCount: 2, + expectError: true, + errorMessage: "spec.mongosCount must not be set in Multi Cluster topology. The member count will depend on: spec.mongos.clusterSpecList.members", + }, + { + name: "Warning when MemberConfig is set in ShardOverrides in MultiCluster topology", + isMultiCluster: true, + shardOverrides: []ShardOverride{ + {ShardNames: []string{"foo-0"}, MemberConfig: defaultMemberConfig}, + }, + expectWarning: true, + expectedWarnings: []status.Warning{ + "spec.shardOverrides.memberConfig is ignored in Multi Cluster topology. Use instead: spec.shardOverrides.clusterSpecList.memberConfig", + }, + }, + { + name: "Warning when Members is set in ShardOverrides in MultiCluster topology", + isMultiCluster: true, + shardOverrides: []ShardOverride{ + {ShardNames: []string{"foo-0"}, Members: ptr.To(2)}, + }, + expectWarning: true, + expectedWarnings: []status.Warning{ + "spec.shardOverrides.members is ignored in Multi Cluster topology. Use instead: spec.shardOverrides.clusterSpecList.members", + }, + }, + { + name: "Warnings when Members and MemberConfig are set at top level", + isMultiCluster: true, + members: 1, + memberConfig: []automationconfig.MemberOptions{{ + Votes: ptr.To(1), + Priority: ptr.To("3"), + }}, + expectWarning: true, + expectedWarnings: []status.Warning{ + "spec.members is ignored in Multi Cluster topology. Use instead: spec.[...].clusterSpecList.members", + "spec.memberConfig is ignored in Multi Cluster topology. Use instead: spec.[...].clusterSpecList.memberConfig", + }, + }, + { + name: "Multiple warnings when several ignored fields are set in MultiCluster topology", + isMultiCluster: true, + shardOverrides: []ShardOverride{ + {ShardNames: []string{"foo-0"}, MemberConfig: defaultMemberConfig}, + {ShardNames: []string{"foo-1"}, Members: ptr.To(2)}, + {ShardNames: []string{"foo-2"}, StatefulSetConfiguration: &v1.StatefulSetConfiguration{}}, + { + ShardNames: []string{"foo-3"}, + PodSpec: podSpecWithTemplate, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ + ClusterName: "cluster-0", + PodSpec: podSpecWithTemplate, + }}, + }, + }, + }, + expectWarning: true, + expectedWarnings: []status.Warning{ + "spec.shardOverrides.memberConfig is ignored in Multi Cluster topology. Use instead: spec.shardOverrides.clusterSpecList.memberConfig", + "spec.shardOverrides.members is ignored in Multi Cluster topology. Use instead: spec.shardOverrides.clusterSpecList.members", + "spec.shardOverrides.podSpec.podTemplate is ignored in Multi Cluster topology. Use instead: spec.shardOverrides.statefulSetConfiguration", + "spec.shardOverrides.clusterSpecList.podSpec.podTemplate is ignored in Multi Cluster topology. Use instead: spec.shardOverrides.clusterSpecList.statefulSetConfiguration", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var sc *MongoDB + if tt.isMultiCluster { + sc = NewDefaultMultiShardedClusterBuilder().SetName("foo").Build() + } else { + sc = NewDefaultShardedClusterBuilder().SetName("foo").Build() + } + + sc.Spec.MongodsPerShardCount = tt.mongodsPerShard + sc.Spec.ConfigServerCount = tt.configServerCount + sc.Spec.MongosCount = tt.mongosCount + sc.Spec.ShardOverrides = tt.shardOverrides + sc.Spec.Members = tt.members + sc.Spec.MemberConfig = tt.memberConfig + // To avoid validation errors + if len(tt.shardOverrides) > 0 { + sc.Spec.ShardCount = len(tt.shardOverrides) + } else { + sc.Spec.ShardCount = 3 + } + + _, err := sc.ValidateCreate() + // In case there is a validation error, we cannot expect warnings as well, the validation will stop with + // the error + if tt.expectError { + require.Error(t, err) + assert.Equal(t, tt.errorMessage, err.Error()) + } else { + assert.NoError(t, err) + } + if tt.expectWarning { + require.NotEmpty(t, sc.Status.Warnings) + for _, expectedWarning := range tt.expectedWarnings { + assertWarningExists(t, sc.Status.Warnings, expectedWarning) + } + } else { + assert.Empty(t, sc.Status.Warnings) + } + }) + } +} + +func TestPodSpecTemplatesWarnings(t *testing.T) { + sc := NewDefaultMultiShardedClusterBuilder().Build() + mongoPodSpec := &MongoDbPodSpec{PodTemplateWrapper: PodTemplateSpecWrapper{PodTemplate: &corev1.PodTemplateSpec{}}} + sc.Spec.ShardSpec.ClusterSpecList[0].PodSpec = mongoPodSpec + sc.Spec.ConfigSrvSpec.ClusterSpecList[0].PodSpec = mongoPodSpec + sc.Spec.MongosSpec.ClusterSpecList[0].PodSpec = mongoPodSpec + _, err := sc.ValidateCreate() + assert.NoError(t, err) + expectedWarnings := []status.Warning{ + "spec.shard.clusterSpecList.podSpec.podTemplate is ignored in Multi Cluster topology. Use instead: spec.shard.clusterSpecList.statefulSetConfiguration", + "spec.configSrv.clusterSpecList.podSpec.podTemplate is ignored in Multi Cluster topology. Use instead: spec.configSrv.clusterSpecList.statefulSetConfiguration", + "spec.mongos.clusterSpecList.podSpec.podTemplate is ignored in Multi Cluster topology. Use instead: spec.mongos.clusterSpecList.statefulSetConfiguration", + } + for _, expectedWarning := range expectedWarnings { + assertWarningExists(t, sc.Status.Warnings, expectedWarning) + } +} + +func TestDuplicateServiceObjectsIsIgnoredInSingleCluster(t *testing.T) { + sc := NewDefaultShardedClusterBuilder().Build() + truePointer := ptr.To(true) + sc.Spec.DuplicateServiceObjects = truePointer + _, err := sc.ValidateCreate() + assert.NoError(t, err) + assertWarningExists(t, sc.Status.Warnings, "In Single Cluster topology, spec.duplicateServiceObjects field is ignored") +} + +func TestEmptyClusterSpecListInOverrides(t *testing.T) { + sc := NewDefaultShardedClusterBuilder().SetShardCountSpec(1).Build() + sc.Spec.ShardOverrides = []ShardOverride{ + { + ShardNames: []string{fmt.Sprintf("%s-0", sc.Name)}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ClusterName: "test-cluster"}}, + }, + }, + } + _, err := sc.ValidateCreate() + require.Error(t, err) + assert.Equal(t, "cluster spec list in spec.shardOverrides must be empty in Single Cluster topology", err.Error()) +} + +func assertWarningExists(t *testing.T, warnings status.Warnings, expectedWarning status.Warning) { + assert.NotEmpty(t, warnings) + + // Check if the expected warning exists in the warnings, either with or without a semicolon + var found bool + for _, warning := range warnings { + if warning == expectedWarning || warning == expectedWarning+status.SEP { + found = true + break + } + } + + // If not found, print the list of warnings and fail the test + if !found { + assert.Fail(t, "Expected warning not found", "Expected warning: %q, but it was not found in warnings: %v", expectedWarning, warnings) + } +} + +func TestValidateShardName(t *testing.T) { + // Example usage + resourceName := "foo" + shardCount := 5 + + tests := []struct { + shardName string + expect bool + }{ + { + shardName: "foo-0", + expect: true, + }, + { + shardName: "foo-3", + expect: true, + }, + { + shardName: "foo-5", + expect: false, + }, + { + shardName: "foo-01", + expect: false, + }, + { + shardName: "foo2", + expect: false, + }, + { + shardName: "bar-2", + expect: false, + }, + { + shardName: "foo-a", + expect: false, + }, + { + shardName: "foo-2-2", + expect: false, + }, + { + shardName: "", + expect: false, + }, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("name %s", tt.shardName), func(t *testing.T) { + assert.Equal(t, tt.expect, validateShardName(tt.shardName, shardCount, resourceName)) + }) + } +} + +func TestNoTopologyMigration(t *testing.T) { + scSingle := NewDefaultShardedClusterBuilder().Build() + scMulti := NewDefaultShardedClusterBuilder().SetMultiClusterTopology().Build() + _, err := scSingle.ValidateUpdate(scMulti) + require.Error(t, err) + assert.Equal(t, "Automatic Topology Migration (Single/Multi Cluster) is not supported for MongoDB resource", err.Error()) +} + +func TestValidateMemberClusterIsSubsetOfKubeConfig(t *testing.T) { + testCases := []struct { + name string + clusterSpec ClusterSpecList + shardOverrides []ShardOverride + expectedWarning bool + expectedMsg string + }{ + { + name: "Failure due to mismatched clusters", + clusterSpec: ClusterSpecList{ + {ClusterName: "hello", Members: 1}, + {ClusterName: "hi", Members: 2}, + }, + expectedWarning: true, + expectedMsg: "Warning when validating spec.shardSpec ClusterSpecList: The following clusters specified in ClusterSpecList is not present in Kubeconfig: [hello hi], instead - the following are: [foo bar]", + }, + { + name: "Success when clusters match", + clusterSpec: ClusterSpecList{ + {ClusterName: "foo", Members: 1}, + }, + expectedWarning: false, + }, + { + name: "Failure with partial mismatch of clusters", + clusterSpec: ClusterSpecList{ + {ClusterName: "foo", Members: 1}, + {ClusterName: "unknown", Members: 2}, + }, + expectedWarning: true, + expectedMsg: "Warning when validating spec.shardSpec ClusterSpecList: The following clusters specified in ClusterSpecList is not present in Kubeconfig: [unknown], instead - the following are: [foo bar]", + }, + { + name: "Success with multiple clusters in KubeConfig", + clusterSpec: ClusterSpecList{ + {ClusterName: "foo", Members: 1}, + {ClusterName: "bar", Members: 2}, + }, + expectedWarning: false, + }, + { + name: "Success with multiple clusters in shard overrides", + clusterSpec: ClusterSpecList{ + {ClusterName: "foo", Members: 1}, + {ClusterName: "bar", Members: 2}, + }, + shardOverrides: []ShardOverride{ + { + ShardNames: []string{"foo-0"}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ClusterName: "foo"}, {ClusterName: "bar"}}, + }, + }, + { + ShardNames: []string{"foo-1", "foo-2"}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ClusterName: "foo"}}, + }, + }, + }, + expectedWarning: false, + }, + { + name: "Error with incorrect clusters in shard overrides", + clusterSpec: ClusterSpecList{ + {ClusterName: "foo", Members: 1}, + {ClusterName: "bar", Members: 2}, + }, + shardOverrides: []ShardOverride{ + { + ShardNames: []string{"foo-0"}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ClusterName: "foo"}, {ClusterName: "unknown"}}, + }, + }, + { + ShardNames: []string{"foo-1", "foo-2"}, + ShardedClusterComponentOverrideSpec: ShardedClusterComponentOverrideSpec{ + ClusterSpecList: []ClusterSpecItemOverride{{ClusterName: "foo"}}, + }, + }, + }, + expectedWarning: true, + expectedMsg: "Warning when validating shard [foo-0] override ClusterSpecList: The following clusters specified in ClusterSpecList is not present in Kubeconfig: [unknown], instead - the following are: [foo bar]", + }, + } + + // Run each test case + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + // The below function will create a temporary file and set the correct environment variable, so that + // the validation checking if clusters belong to the KubeConfig find this file + file := createTestKubeConfigAndSetEnvMultipleClusters(t) + defer os.Remove(file.Name()) + + sc := NewDefaultMultiShardedClusterBuilder(). + SetName("foo"). + SetShardCountSpec(3). + SetAllClusterSpecLists(tt.clusterSpec). + SetShardOverrides(tt.shardOverrides). + Build() + _, err := sc.ValidateCreate() + + if tt.expectedWarning { + require.NoError(t, err) + assert.Contains(t, sc.Status.Warnings, status.Warning(tt.expectedMsg)) + } else { + require.NoError(t, err) + } + }) + } +} + +// TODO: partially duplicated from mongodbmulti_validation_test.go, consider moving to another file +// Helper function to create a KubeConfig with multiple clusters +func createTestKubeConfigAndSetEnvMultipleClusters(t *testing.T) *os.File { + //nolint + testKubeConfig := fmt.Sprintf(` +apiVersion: v1 +contexts: +- context: + cluster: foo + namespace: a-1661872869-pq35wlt3zzz + user: foo + name: foo +- context: + cluster: bar + namespace: b-1661872869-pq35wlt3yyy + user: bar + name: bar +kind: Config +users: +- name: foo + user: + token: eyJhbGciOi +- name: bar + user: + token: eyJhbGciOi +`) + + file, err := os.CreateTemp("", "kubeconfig") + assert.NoError(t, err) + _, err = file.WriteString(testKubeConfig) + assert.NoError(t, err) + t.Setenv(multicluster.KubeConfigPathEnv, file.Name()) + return file +} diff --git a/api/v1/mdb/shardedcluster.go b/api/v1/mdb/shardedcluster.go new file mode 100644 index 000000000..4b3e0c404 --- /dev/null +++ b/api/v1/mdb/shardedcluster.go @@ -0,0 +1,103 @@ +package mdb + +import ( + "fmt" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" +) + +// ShardedClusterSpec is the spec consisting of configuration specific for sharded cluster only. +type ShardedClusterSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + ConfigSrvSpec *ShardedClusterComponentSpec `json:"configSrv,omitempty"` + // +kubebuilder:pruning:PreserveUnknownFields + MongosSpec *ShardedClusterComponentSpec `json:"mongos,omitempty"` + // +kubebuilder:pruning:PreserveUnknownFields + ShardSpec *ShardedClusterComponentSpec `json:"shard,omitempty"` + // ShardOverrides allow for overriding the configuration of a specific shard. + // It replaces deprecated spec.shard.shardSpecificPodSpec field. When spec.shard.shardSpecificPodSpec is still defined then + // spec.shard.shardSpecificPodSpec is applied first to the particular shard and then spec.shardOverrides is applied on top + // of that (if defined for the same shard). + // +kubebuilder:pruning:PreserverUnknownFields + // +optional + ShardOverrides []ShardOverride `json:"shardOverrides,omitempty"` + + ConfigSrvPodSpec *MongoDbPodSpec `json:"configSrvPodSpec,omitempty"` + MongosPodSpec *MongoDbPodSpec `json:"mongosPodSpec,omitempty"` + ShardPodSpec *MongoDbPodSpec `json:"shardPodSpec,omitempty"` + // ShardSpecificPodSpec allows you to provide a Statefulset override per shard. + // DEPRECATED please use spec.shard.shardOverrides instead + // +optional + ShardSpecificPodSpec []MongoDbPodSpec `json:"shardSpecificPodSpec,omitempty"` +} + +type ShardedClusterComponentSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + AdditionalMongodConfig *AdditionalMongodConfig `json:"additionalMongodConfig,omitempty"` + // Configuring logRotation is not allowed under this section. + // Please use the most top level resource fields for this; spec.Agent + Agent AgentConfig `json:"agent,omitempty"` + ClusterSpecList ClusterSpecList `json:"clusterSpecList,omitempty"` +} + +type ShardedClusterComponentOverrideSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + AdditionalMongodConfig *AdditionalMongodConfig `json:"additionalMongodConfig,omitempty"` + Agent *AgentConfig `json:"agent,omitempty"` + ClusterSpecList []ClusterSpecItemOverride `json:"clusterSpecList,omitempty"` +} + +type ShardOverride struct { + // +kubebuilder:validation:MinItems=1 + ShardNames []string `json:"shardNames"` + + ShardedClusterComponentOverrideSpec `json:",inline"` + + // The following override fields work for SingleCluster only. For MultiCluster - fields from specific clusters are used. + // +optional + PodSpec *MongoDbPodSpec `json:"podSpec,omitempty"` + + // Number of member nodes in this shard. Used only in SingleCluster. For MultiCluster the number of members is specified in ShardOverride.ClusterSpecList. + // +optional + Members *int `json:"members"` + // Process configuration override for this shard. Used in SingleCluster only. The number of items specified must be >= spec.mongodsPerShardCount or spec.shardOverride.members. + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + MemberConfig []automationconfig.MemberOptions `json:"memberConfig,omitempty"` + // Statefulset override for this particular shard. + // +optional + StatefulSetConfiguration *mdbcv1.StatefulSetConfiguration `json:"statefulSet,omitempty"` +} + +func (s *ShardedClusterComponentSpec) GetAdditionalMongodConfig() *AdditionalMongodConfig { + if s == nil { + return &AdditionalMongodConfig{} + } + + if s.AdditionalMongodConfig == nil { + return &AdditionalMongodConfig{} + } + + return s.AdditionalMongodConfig +} + +func (s *ShardedClusterComponentSpec) GetAgentConfig() *AgentConfig { + if s == nil { + return &AgentConfig{ + StartupParameters: StartupParameters{}, + } + } + return &s.Agent +} + +func (s *ShardedClusterComponentSpec) GetClusterSpecItem(clusterName string) ClusterSpecItem { + for i := range s.ClusterSpecList { + if s.ClusterSpecList[i].ClusterName == clusterName { + return s.ClusterSpecList[i] + } + } + // it should never occur - we preprocess all clusterSpecLists + panic(fmt.Errorf("clusterName %s not found in clusterSpecList", clusterName)) +} diff --git a/api/v1/mdb/wrap.go b/api/v1/mdb/wrap.go new file mode 100644 index 000000000..6dfe26af2 --- /dev/null +++ b/api/v1/mdb/wrap.go @@ -0,0 +1,152 @@ +// Contains the wrapped types which are needed for generating +// CRD yamls using kubebuilder. They prevent each of the fields showing up in CRD yaml thereby +// resulting in a relatively smaller file. +package mdb + +import ( + "encoding/json" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ClientCertificateSecretRefWrapper struct { + ClientCertificateSecretRef corev1.SecretKeySelector `json:"-"` +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (c *ClientCertificateSecretRefWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(c.ClientCertificateSecretRef) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (c *ClientCertificateSecretRefWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &c.ClientCertificateSecretRef) +} + +func (c *ClientCertificateSecretRefWrapper) DeepCopy() *ClientCertificateSecretRefWrapper { + return &ClientCertificateSecretRefWrapper{ + ClientCertificateSecretRef: c.ClientCertificateSecretRef, + } +} + +type PodTemplateSpecWrapper struct { + PodTemplate *corev1.PodTemplateSpec `json:"-"` +} + +type LabelSelectorWrapper struct { + LabelSelector metav1.LabelSelector `json:"-"` +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (m *PodTemplateSpecWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(m.PodTemplate) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (m *PodTemplateSpecWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &m.PodTemplate) +} + +func (m *PodTemplateSpecWrapper) DeepCopy() *PodTemplateSpecWrapper { + return &PodTemplateSpecWrapper{ + PodTemplate: m.PodTemplate, + } +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (m *LabelSelectorWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(m.LabelSelector) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (m *LabelSelectorWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &m.LabelSelector) +} + +func (m *LabelSelectorWrapper) DeepCopy() *LabelSelectorWrapper { + return &LabelSelectorWrapper{ + LabelSelector: m.LabelSelector, + } +} + +type PodAffinityWrapper struct { + PodAffinity *corev1.PodAffinity `json:"-"` +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (m *PodAffinityWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(m.PodAffinity) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (m *PodAffinityWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &m.PodAffinity) +} + +func (m *PodAffinityWrapper) DeepCopy() *PodAffinityWrapper { + return &PodAffinityWrapper{ + PodAffinity: m.PodAffinity, + } +} + +type NodeAffinityWrapper struct { + NodeAffinity *corev1.NodeAffinity `json:"-"` +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (m *NodeAffinityWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(m.NodeAffinity) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (m *NodeAffinityWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &m.NodeAffinity) +} + +func (m *NodeAffinityWrapper) DeepCopy() *NodeAffinityWrapper { + return &NodeAffinityWrapper{ + NodeAffinity: m.NodeAffinity, + } +} + +type StatefulSetSpecWrapper struct { + Spec appsv1.StatefulSetSpec `json:"-"` +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (s *StatefulSetSpecWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(s.Spec) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (s *StatefulSetSpecWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &s.Spec) +} + +func (s *StatefulSetSpecWrapper) DeepCopy() *StatefulSetSpecWrapper { + return &StatefulSetSpecWrapper{ + Spec: s.Spec, + } +} + +type ServiceSpecWrapper struct { + Spec corev1.ServiceSpec `json:"-"` +} + +// MarshalJSON defers JSON encoding to the wrapped map +func (s *ServiceSpecWrapper) MarshalJSON() ([]byte, error) { + return json.Marshal(s.Spec) +} + +// UnmarshalJSON will decode the data into the wrapped map +func (s *ServiceSpecWrapper) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &s.Spec) +} + +func (s *ServiceSpecWrapper) DeepCopy() *ServiceSpecWrapper { + return &ServiceSpecWrapper{ + Spec: s.Spec, + } +} diff --git a/api/v1/mdb/zz_generated.deepcopy.go b/api/v1/mdb/zz_generated.deepcopy.go new file mode 100644 index 000000000..5b2d3ca53 --- /dev/null +++ b/api/v1/mdb/zz_generated.deepcopy.go @@ -0,0 +1,1537 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package mdb + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentAuthentication) DeepCopyInto(out *AgentAuthentication) { + *out = *in + in.AutomationPasswordSecretRef.DeepCopyInto(&out.AutomationPasswordSecretRef) + in.ClientCertificateSecretRefWrap.DeepCopyInto(&out.ClientCertificateSecretRefWrap) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentAuthentication. +func (in *AgentAuthentication) DeepCopy() *AgentAuthentication { + if in == nil { + return nil + } + out := new(AgentAuthentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentConfig) DeepCopyInto(out *AgentConfig) { + *out = *in + in.BackupAgent.DeepCopyInto(&out.BackupAgent) + in.MonitoringAgent.DeepCopyInto(&out.MonitoringAgent) + in.Mongod.DeepCopyInto(&out.Mongod) + in.ReadinessProbe.DeepCopyInto(&out.ReadinessProbe) + if in.StartupParameters != nil { + in, out := &in.StartupParameters, &out.StartupParameters + *out = make(StartupParameters, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.LogRotate != nil { + in, out := &in.LogRotate, &out.LogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } + if in.SystemLog != nil { + in, out := &in.SystemLog, &out.SystemLog + *out = new(automationconfig.SystemLog) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentConfig. +func (in *AgentConfig) DeepCopy() *AgentConfig { + if in == nil { + return nil + } + out := new(AgentConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentLoggingMongodConfig) DeepCopyInto(out *AgentLoggingMongodConfig) { + *out = *in + if in.LogRotate != nil { + in, out := &in.LogRotate, &out.LogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } + if in.AuditLogRotate != nil { + in, out := &in.AuditLogRotate, &out.AuditLogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } + if in.SystemLog != nil { + in, out := &in.SystemLog, &out.SystemLog + *out = new(automationconfig.SystemLog) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentLoggingMongodConfig. +func (in *AgentLoggingMongodConfig) DeepCopy() *AgentLoggingMongodConfig { + if in == nil { + return nil + } + out := new(AgentLoggingMongodConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Authentication) DeepCopyInto(out *Authentication) { + *out = *in + if in.Modes != nil { + in, out := &in.Modes, &out.Modes + *out = make([]AuthMode, len(*in)) + copy(*out, *in) + } + if in.Ldap != nil { + in, out := &in.Ldap, &out.Ldap + *out = new(Ldap) + (*in).DeepCopyInto(*out) + } + in.Agents.DeepCopyInto(&out.Agents) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Authentication. +func (in *Authentication) DeepCopy() *Authentication { + if in == nil { + return nil + } + out := new(Authentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthenticationRestriction) DeepCopyInto(out *AuthenticationRestriction) { + *out = *in + if in.ClientSource != nil { + in, out := &in.ClientSource, &out.ClientSource + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ServerAddress != nil { + in, out := &in.ServerAddress, &out.ServerAddress + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthenticationRestriction. +func (in *AuthenticationRestriction) DeepCopy() *AuthenticationRestriction { + if in == nil { + return nil + } + out := new(AuthenticationRestriction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Backup) DeepCopyInto(out *Backup) { + *out = *in + if in.SnapshotSchedule != nil { + in, out := &in.SnapshotSchedule, &out.SnapshotSchedule + *out = new(SnapshotSchedule) + (*in).DeepCopyInto(*out) + } + if in.Encryption != nil { + in, out := &in.Encryption, &out.Encryption + *out = new(Encryption) + (*in).DeepCopyInto(*out) + } + if in.AssignmentLabels != nil { + in, out := &in.AssignmentLabels, &out.AssignmentLabels + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Backup. +func (in *Backup) DeepCopy() *Backup { + if in == nil { + return nil + } + out := new(Backup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupAgent) DeepCopyInto(out *BackupAgent) { + *out = *in + if in.LogRotate != nil { + in, out := &in.LogRotate, &out.LogRotate + *out = new(LogRotateForBackupAndMonitoring) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupAgent. +func (in *BackupAgent) DeepCopy() *BackupAgent { + if in == nil { + return nil + } + out := new(BackupAgent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus. +func (in *BackupStatus) DeepCopy() *BackupStatus { + if in == nil { + return nil + } + out := new(BackupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientCertificateSecretRefWrapper) DeepCopyInto(out *ClientCertificateSecretRefWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSpecItem) DeepCopyInto(out *ClusterSpecItem) { + *out = *in + if in.ExternalAccessConfiguration != nil { + in, out := &in.ExternalAccessConfiguration, &out.ExternalAccessConfiguration + *out = new(ExternalAccessConfiguration) + (*in).DeepCopyInto(*out) + } + if in.MemberConfig != nil { + in, out := &in.MemberConfig, &out.MemberConfig + *out = make([]automationconfig.MemberOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpecItem. +func (in *ClusterSpecItem) DeepCopy() *ClusterSpecItem { + if in == nil { + return nil + } + out := new(ClusterSpecItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSpecItemOverride) DeepCopyInto(out *ClusterSpecItemOverride) { + *out = *in + if in.Members != nil { + in, out := &in.Members, &out.Members + *out = new(int) + **out = **in + } + if in.MemberConfig != nil { + in, out := &in.MemberConfig, &out.MemberConfig + *out = make([]automationconfig.MemberOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpecItemOverride. +func (in *ClusterSpecItemOverride) DeepCopy() *ClusterSpecItemOverride { + if in == nil { + return nil + } + out := new(ClusterSpecItemOverride) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ClusterSpecList) DeepCopyInto(out *ClusterSpecList) { + { + in := &in + *out = make(ClusterSpecList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpecList. +func (in ClusterSpecList) DeepCopy() ClusterSpecList { + if in == nil { + return nil + } + out := new(ClusterSpecList) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapRef) DeepCopyInto(out *ConfigMapRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapRef. +func (in *ConfigMapRef) DeepCopy() *ConfigMapRef { + if in == nil { + return nil + } + out := new(ConfigMapRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionSpec) DeepCopyInto(out *ConnectionSpec) { + *out = *in + in.SharedConnectionSpec.DeepCopyInto(&out.SharedConnectionSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionSpec. +func (in *ConnectionSpec) DeepCopy() *ConnectionSpec { + if in == nil { + return nil + } + out := new(ConnectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerResourceRequirements) DeepCopyInto(out *ContainerResourceRequirements) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerResourceRequirements. +func (in *ContainerResourceRequirements) DeepCopy() *ContainerResourceRequirements { + if in == nil { + return nil + } + out := new(ContainerResourceRequirements) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Credentials) DeepCopyInto(out *Credentials) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Credentials. +func (in *Credentials) DeepCopy() *Credentials { + if in == nil { + return nil + } + out := new(Credentials) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DbCommonSpec) DeepCopyInto(out *DbCommonSpec) { + *out = *in + if in.FeatureCompatibilityVersion != nil { + in, out := &in.FeatureCompatibilityVersion, &out.FeatureCompatibilityVersion + *out = new(string) + **out = **in + } + in.Agent.DeepCopyInto(&out.Agent) + in.ConnectionSpec.DeepCopyInto(&out.ConnectionSpec) + if in.ExternalAccessConfiguration != nil { + in, out := &in.ExternalAccessConfiguration, &out.ExternalAccessConfiguration + *out = new(ExternalAccessConfiguration) + (*in).DeepCopyInto(*out) + } + if in.Persistent != nil { + in, out := &in.Persistent, &out.Persistent + *out = new(bool) + **out = **in + } + if in.Security != nil { + in, out := &in.Security, &out.Security + *out = new(Security) + (*in).DeepCopyInto(*out) + } + if in.Connectivity != nil { + in, out := &in.Connectivity, &out.Connectivity + *out = new(MongoDBConnectivity) + (*in).DeepCopyInto(*out) + } + if in.Backup != nil { + in, out := &in.Backup, &out.Backup + *out = new(Backup) + (*in).DeepCopyInto(*out) + } + if in.Prometheus != nil { + in, out := &in.Prometheus, &out.Prometheus + *out = new(v1.Prometheus) + **out = **in + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } + if in.AdditionalMongodConfig != nil { + in, out := &in.AdditionalMongodConfig, &out.AdditionalMongodConfig + *out = (*in).DeepCopy() + } + if in.DuplicateServiceObjects != nil { + in, out := &in.DuplicateServiceObjects, &out.DuplicateServiceObjects + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DbCommonSpec. +func (in *DbCommonSpec) DeepCopy() *DbCommonSpec { + if in == nil { + return nil + } + out := new(DbCommonSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Encryption) DeepCopyInto(out *Encryption) { + *out = *in + if in.Kmip != nil { + in, out := &in.Kmip, &out.Kmip + *out = new(KmipConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Encryption. +func (in *Encryption) DeepCopy() *Encryption { + if in == nil { + return nil + } + out := new(Encryption) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in EnvironmentVariables) DeepCopyInto(out *EnvironmentVariables) { + { + in := &in + *out = make(EnvironmentVariables, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvironmentVariables. +func (in EnvironmentVariables) DeepCopy() EnvironmentVariables { + if in == nil { + return nil + } + out := new(EnvironmentVariables) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalAccessConfiguration) DeepCopyInto(out *ExternalAccessConfiguration) { + *out = *in + in.ExternalService.DeepCopyInto(&out.ExternalService) + if in.ExternalDomain != nil { + in, out := &in.ExternalDomain, &out.ExternalDomain + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalAccessConfiguration. +func (in *ExternalAccessConfiguration) DeepCopy() *ExternalAccessConfiguration { + if in == nil { + return nil + } + out := new(ExternalAccessConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalServiceConfiguration) DeepCopyInto(out *ExternalServiceConfiguration) { + *out = *in + if in.SpecWrapper != nil { + in, out := &in.SpecWrapper, &out.SpecWrapper + *out = (*in).DeepCopy() + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalServiceConfiguration. +func (in *ExternalServiceConfiguration) DeepCopy() *ExternalServiceConfiguration { + if in == nil { + return nil + } + out := new(ExternalServiceConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InheritedRole) DeepCopyInto(out *InheritedRole) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InheritedRole. +func (in *InheritedRole) DeepCopy() *InheritedRole { + if in == nil { + return nil + } + out := new(InheritedRole) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KmipConfig) DeepCopyInto(out *KmipConfig) { + *out = *in + out.Client = in.Client +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KmipConfig. +func (in *KmipConfig) DeepCopy() *KmipConfig { + if in == nil { + return nil + } + out := new(KmipConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LabelSelectorWrapper) DeepCopyInto(out *LabelSelectorWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Ldap) DeepCopyInto(out *Ldap) { + *out = *in + if in.Servers != nil { + in, out := &in.Servers, &out.Servers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TransportSecurity != nil { + in, out := &in.TransportSecurity, &out.TransportSecurity + *out = new(TransportSecurity) + **out = **in + } + if in.ValidateLDAPServerConfig != nil { + in, out := &in.ValidateLDAPServerConfig, &out.ValidateLDAPServerConfig + *out = new(bool) + **out = **in + } + if in.CAConfigMapRef != nil { + in, out := &in.CAConfigMapRef, &out.CAConfigMapRef + *out = new(corev1.ConfigMapKeySelector) + (*in).DeepCopyInto(*out) + } + out.BindQuerySecretRef = in.BindQuerySecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ldap. +func (in *Ldap) DeepCopy() *Ldap { + if in == nil { + return nil + } + out := new(Ldap) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogRotateForBackupAndMonitoring) DeepCopyInto(out *LogRotateForBackupAndMonitoring) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogRotateForBackupAndMonitoring. +func (in *LogRotateForBackupAndMonitoring) DeepCopy() *LogRotateForBackupAndMonitoring { + if in == nil { + return nil + } + out := new(LogRotateForBackupAndMonitoring) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDB) DeepCopyInto(out *MongoDB) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDB. +func (in *MongoDB) DeepCopy() *MongoDB { + if in == nil { + return nil + } + out := new(MongoDB) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDB) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBBuilder) DeepCopyInto(out *MongoDBBuilder) { + *out = *in + if in.mdb != nil { + in, out := &in.mdb, &out.mdb + *out = new(MongoDB) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBBuilder. +func (in *MongoDBBuilder) DeepCopy() *MongoDBBuilder { + if in == nil { + return nil + } + out := new(MongoDBBuilder) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBConnectivity) DeepCopyInto(out *MongoDBConnectivity) { + *out = *in + if in.ReplicaSetHorizons != nil { + in, out := &in.ReplicaSetHorizons, &out.ReplicaSetHorizons + *out = make([]MongoDBHorizonConfig, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = make(MongoDBHorizonConfig, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBConnectivity. +func (in *MongoDBConnectivity) DeepCopy() *MongoDBConnectivity { + if in == nil { + return nil + } + out := new(MongoDBConnectivity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in MongoDBHorizonConfig) DeepCopyInto(out *MongoDBHorizonConfig) { + { + in := &in + *out = make(MongoDBHorizonConfig, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBHorizonConfig. +func (in MongoDBHorizonConfig) DeepCopy() MongoDBHorizonConfig { + if in == nil { + return nil + } + out := new(MongoDBHorizonConfig) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBList) DeepCopyInto(out *MongoDBList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MongoDB, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBList. +func (in *MongoDBList) DeepCopy() *MongoDBList { + if in == nil { + return nil + } + out := new(MongoDBList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDbPodSpec) DeepCopyInto(out *MongoDbPodSpec) { + *out = *in + out.ContainerResourceRequirements = in.ContainerResourceRequirements + in.PodTemplateWrapper.DeepCopyInto(&out.PodTemplateWrapper) + if in.Persistence != nil { + in, out := &in.Persistence, &out.Persistence + *out = new(Persistence) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDbPodSpec. +func (in *MongoDbPodSpec) DeepCopy() *MongoDbPodSpec { + if in == nil { + return nil + } + out := new(MongoDbPodSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDbRole) DeepCopyInto(out *MongoDbRole) { + *out = *in + if in.AuthenticationRestrictions != nil { + in, out := &in.AuthenticationRestrictions, &out.AuthenticationRestrictions + *out = make([]AuthenticationRestriction, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Privileges != nil { + in, out := &in.Privileges, &out.Privileges + *out = make([]Privilege, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]InheritedRole, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDbRole. +func (in *MongoDbRole) DeepCopy() *MongoDbRole { + if in == nil { + return nil + } + out := new(MongoDbRole) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDbSpec) DeepCopyInto(out *MongoDbSpec) { + *out = *in + in.DbCommonSpec.DeepCopyInto(&out.DbCommonSpec) + in.ShardedClusterSpec.DeepCopyInto(&out.ShardedClusterSpec) + out.MongodbShardedClusterSizeConfig = in.MongodbShardedClusterSizeConfig + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } + if in.MemberConfig != nil { + in, out := &in.MemberConfig, &out.MemberConfig + *out = make([]automationconfig.MemberOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDbSpec. +func (in *MongoDbSpec) DeepCopy() *MongoDbSpec { + if in == nil { + return nil + } + out := new(MongoDbSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDbStatus) DeepCopyInto(out *MongoDbStatus) { + *out = *in + in.Common.DeepCopyInto(&out.Common) + if in.BackupStatus != nil { + in, out := &in.BackupStatus, &out.BackupStatus + *out = new(BackupStatus) + **out = **in + } + out.MongodbShardedClusterSizeConfig = in.MongodbShardedClusterSizeConfig + if in.SizeStatusInClusters != nil { + in, out := &in.SizeStatusInClusters, &out.SizeStatusInClusters + *out = new(status.MongodbShardedSizeStatusInClusters) + (*in).DeepCopyInto(*out) + } + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]status.Warning, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDbStatus. +func (in *MongoDbStatus) DeepCopy() *MongoDbStatus { + if in == nil { + return nil + } + out := new(MongoDbStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongodbCleanUpOptions) DeepCopyInto(out *MongodbCleanUpOptions) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongodbCleanUpOptions. +func (in *MongodbCleanUpOptions) DeepCopy() *MongodbCleanUpOptions { + if in == nil { + return nil + } + out := new(MongodbCleanUpOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MonitoringAgent) DeepCopyInto(out *MonitoringAgent) { + *out = *in + if in.LogRotate != nil { + in, out := &in.LogRotate, &out.LogRotate + *out = new(LogRotateForBackupAndMonitoring) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MonitoringAgent. +func (in *MonitoringAgent) DeepCopy() *MonitoringAgent { + if in == nil { + return nil + } + out := new(MonitoringAgent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MonitoringAgentConfig) DeepCopyInto(out *MonitoringAgentConfig) { + *out = *in + if in.StartupParameters != nil { + in, out := &in.StartupParameters, &out.StartupParameters + *out = make(StartupParameters, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MonitoringAgentConfig. +func (in *MonitoringAgentConfig) DeepCopy() *MonitoringAgentConfig { + if in == nil { + return nil + } + out := new(MonitoringAgentConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiplePersistenceConfig) DeepCopyInto(out *MultiplePersistenceConfig) { + *out = *in + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = new(PersistenceConfig) + (*in).DeepCopyInto(*out) + } + if in.Journal != nil { + in, out := &in.Journal, &out.Journal + *out = new(PersistenceConfig) + (*in).DeepCopyInto(*out) + } + if in.Logs != nil { + in, out := &in.Logs, &out.Logs + *out = new(PersistenceConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiplePersistenceConfig. +func (in *MultiplePersistenceConfig) DeepCopy() *MultiplePersistenceConfig { + if in == nil { + return nil + } + out := new(MultiplePersistenceConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeAffinityWrapper) DeepCopyInto(out *NodeAffinityWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Persistence) DeepCopyInto(out *Persistence) { + *out = *in + if in.SingleConfig != nil { + in, out := &in.SingleConfig, &out.SingleConfig + *out = new(PersistenceConfig) + (*in).DeepCopyInto(*out) + } + if in.MultipleConfig != nil { + in, out := &in.MultipleConfig, &out.MultipleConfig + *out = new(MultiplePersistenceConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Persistence. +func (in *Persistence) DeepCopy() *Persistence { + if in == nil { + return nil + } + out := new(Persistence) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PersistenceConfig) DeepCopyInto(out *PersistenceConfig) { + *out = *in + if in.StorageClass != nil { + in, out := &in.StorageClass, &out.StorageClass + *out = new(string) + **out = **in + } + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PersistenceConfig. +func (in *PersistenceConfig) DeepCopy() *PersistenceConfig { + if in == nil { + return nil + } + out := new(PersistenceConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PersistenceConfigBuilder) DeepCopyInto(out *PersistenceConfigBuilder) { + *out = *in + if in.config != nil { + in, out := &in.config, &out.config + *out = new(PersistenceConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PersistenceConfigBuilder. +func (in *PersistenceConfigBuilder) DeepCopy() *PersistenceConfigBuilder { + if in == nil { + return nil + } + out := new(PersistenceConfigBuilder) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodAffinityWrapper) DeepCopyInto(out *PodAffinityWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodSpecWrapper) DeepCopyInto(out *PodSpecWrapper) { + *out = *in + in.MongoDbPodSpec.DeepCopyInto(&out.MongoDbPodSpec) + in.Default.DeepCopyInto(&out.Default) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpecWrapper. +func (in *PodSpecWrapper) DeepCopy() *PodSpecWrapper { + if in == nil { + return nil + } + out := new(PodSpecWrapper) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodSpecWrapperBuilder) DeepCopyInto(out *PodSpecWrapperBuilder) { + *out = *in + in.spec.DeepCopyInto(&out.spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpecWrapperBuilder. +func (in *PodSpecWrapperBuilder) DeepCopy() *PodSpecWrapperBuilder { + if in == nil { + return nil + } + out := new(PodSpecWrapperBuilder) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodTemplateSpecWrapper) DeepCopyInto(out *PodTemplateSpecWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrivateCloudConfig) DeepCopyInto(out *PrivateCloudConfig) { + *out = *in + out.ConfigMapRef = in.ConfigMapRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrivateCloudConfig. +func (in *PrivateCloudConfig) DeepCopy() *PrivateCloudConfig { + if in == nil { + return nil + } + out := new(PrivateCloudConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Privilege) DeepCopyInto(out *Privilege) { + *out = *in + if in.Actions != nil { + in, out := &in.Actions, &out.Actions + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Resource.DeepCopyInto(&out.Resource) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Privilege. +func (in *Privilege) DeepCopy() *Privilege { + if in == nil { + return nil + } + out := new(Privilege) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProjectConfig) DeepCopyInto(out *ProjectConfig) { + *out = *in + out.SSLProjectConfig = in.SSLProjectConfig +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectConfig. +func (in *ProjectConfig) DeepCopy() *ProjectConfig { + if in == nil { + return nil + } + out := new(ProjectConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReadinessProbe) DeepCopyInto(out *ReadinessProbe) { + *out = *in + if in.EnvironmentVariables != nil { + in, out := &in.EnvironmentVariables, &out.EnvironmentVariables + *out = make(EnvironmentVariables, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReadinessProbe. +func (in *ReadinessProbe) DeepCopy() *ReadinessProbe { + if in == nil { + return nil + } + out := new(ReadinessProbe) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Resource) DeepCopyInto(out *Resource) { + *out = *in + if in.Cluster != nil { + in, out := &in.Cluster, &out.Cluster + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resource. +func (in *Resource) DeepCopy() *Resource { + if in == nil { + return nil + } + out := new(Resource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretRef) DeepCopyInto(out *SecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretRef. +func (in *SecretRef) DeepCopy() *SecretRef { + if in == nil { + return nil + } + out := new(SecretRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Security) DeepCopyInto(out *Security) { + *out = *in + if in.TLSConfig != nil { + in, out := &in.TLSConfig, &out.TLSConfig + *out = new(TLSConfig) + (*in).DeepCopyInto(*out) + } + if in.Authentication != nil { + in, out := &in.Authentication, &out.Authentication + *out = new(Authentication) + (*in).DeepCopyInto(*out) + } + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]MongoDbRole, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Security. +func (in *Security) DeepCopy() *Security { + if in == nil { + return nil + } + out := new(Security) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceSpecWrapper) DeepCopyInto(out *ServiceSpecWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardOverride) DeepCopyInto(out *ShardOverride) { + *out = *in + if in.ShardNames != nil { + in, out := &in.ShardNames, &out.ShardNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.ShardedClusterComponentOverrideSpec.DeepCopyInto(&out.ShardedClusterComponentOverrideSpec) + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } + if in.Members != nil { + in, out := &in.Members, &out.Members + *out = new(int) + **out = **in + } + if in.MemberConfig != nil { + in, out := &in.MemberConfig, &out.MemberConfig + *out = make([]automationconfig.MemberOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardOverride. +func (in *ShardOverride) DeepCopy() *ShardOverride { + if in == nil { + return nil + } + out := new(ShardOverride) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedClusterComponentOverrideSpec) DeepCopyInto(out *ShardedClusterComponentOverrideSpec) { + *out = *in + if in.AdditionalMongodConfig != nil { + in, out := &in.AdditionalMongodConfig, &out.AdditionalMongodConfig + *out = (*in).DeepCopy() + } + if in.Agent != nil { + in, out := &in.Agent, &out.Agent + *out = new(AgentConfig) + (*in).DeepCopyInto(*out) + } + if in.ClusterSpecList != nil { + in, out := &in.ClusterSpecList, &out.ClusterSpecList + *out = make([]ClusterSpecItemOverride, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedClusterComponentOverrideSpec. +func (in *ShardedClusterComponentOverrideSpec) DeepCopy() *ShardedClusterComponentOverrideSpec { + if in == nil { + return nil + } + out := new(ShardedClusterComponentOverrideSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedClusterComponentSpec) DeepCopyInto(out *ShardedClusterComponentSpec) { + *out = *in + if in.AdditionalMongodConfig != nil { + in, out := &in.AdditionalMongodConfig, &out.AdditionalMongodConfig + *out = (*in).DeepCopy() + } + in.Agent.DeepCopyInto(&out.Agent) + if in.ClusterSpecList != nil { + in, out := &in.ClusterSpecList, &out.ClusterSpecList + *out = make(ClusterSpecList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedClusterComponentSpec. +func (in *ShardedClusterComponentSpec) DeepCopy() *ShardedClusterComponentSpec { + if in == nil { + return nil + } + out := new(ShardedClusterComponentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedClusterSpec) DeepCopyInto(out *ShardedClusterSpec) { + *out = *in + if in.ConfigSrvSpec != nil { + in, out := &in.ConfigSrvSpec, &out.ConfigSrvSpec + *out = new(ShardedClusterComponentSpec) + (*in).DeepCopyInto(*out) + } + if in.MongosSpec != nil { + in, out := &in.MongosSpec, &out.MongosSpec + *out = new(ShardedClusterComponentSpec) + (*in).DeepCopyInto(*out) + } + if in.ShardSpec != nil { + in, out := &in.ShardSpec, &out.ShardSpec + *out = new(ShardedClusterComponentSpec) + (*in).DeepCopyInto(*out) + } + if in.ShardOverrides != nil { + in, out := &in.ShardOverrides, &out.ShardOverrides + *out = make([]ShardOverride, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ConfigSrvPodSpec != nil { + in, out := &in.ConfigSrvPodSpec, &out.ConfigSrvPodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } + if in.MongosPodSpec != nil { + in, out := &in.MongosPodSpec, &out.MongosPodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } + if in.ShardPodSpec != nil { + in, out := &in.ShardPodSpec, &out.ShardPodSpec + *out = new(MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } + if in.ShardSpecificPodSpec != nil { + in, out := &in.ShardSpecificPodSpec, &out.ShardSpecificPodSpec + *out = make([]MongoDbPodSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedClusterSpec. +func (in *ShardedClusterSpec) DeepCopy() *ShardedClusterSpec { + if in == nil { + return nil + } + out := new(ShardedClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SharedConnectionSpec) DeepCopyInto(out *SharedConnectionSpec) { + *out = *in + if in.OpsManagerConfig != nil { + in, out := &in.OpsManagerConfig, &out.OpsManagerConfig + *out = new(PrivateCloudConfig) + **out = **in + } + if in.CloudManagerConfig != nil { + in, out := &in.CloudManagerConfig, &out.CloudManagerConfig + *out = new(PrivateCloudConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SharedConnectionSpec. +func (in *SharedConnectionSpec) DeepCopy() *SharedConnectionSpec { + if in == nil { + return nil + } + out := new(SharedConnectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SnapshotSchedule) DeepCopyInto(out *SnapshotSchedule) { + *out = *in + if in.SnapshotIntervalHours != nil { + in, out := &in.SnapshotIntervalHours, &out.SnapshotIntervalHours + *out = new(int) + **out = **in + } + if in.SnapshotRetentionDays != nil { + in, out := &in.SnapshotRetentionDays, &out.SnapshotRetentionDays + *out = new(int) + **out = **in + } + if in.DailySnapshotRetentionDays != nil { + in, out := &in.DailySnapshotRetentionDays, &out.DailySnapshotRetentionDays + *out = new(int) + **out = **in + } + if in.WeeklySnapshotRetentionWeeks != nil { + in, out := &in.WeeklySnapshotRetentionWeeks, &out.WeeklySnapshotRetentionWeeks + *out = new(int) + **out = **in + } + if in.MonthlySnapshotRetentionMonths != nil { + in, out := &in.MonthlySnapshotRetentionMonths, &out.MonthlySnapshotRetentionMonths + *out = new(int) + **out = **in + } + if in.PointInTimeWindowHours != nil { + in, out := &in.PointInTimeWindowHours, &out.PointInTimeWindowHours + *out = new(int) + **out = **in + } + if in.ReferenceHourOfDay != nil { + in, out := &in.ReferenceHourOfDay, &out.ReferenceHourOfDay + *out = new(int) + **out = **in + } + if in.ReferenceMinuteOfHour != nil { + in, out := &in.ReferenceMinuteOfHour, &out.ReferenceMinuteOfHour + *out = new(int) + **out = **in + } + if in.FullIncrementalDayOfWeek != nil { + in, out := &in.FullIncrementalDayOfWeek, &out.FullIncrementalDayOfWeek + *out = new(string) + **out = **in + } + if in.ClusterCheckpointIntervalMin != nil { + in, out := &in.ClusterCheckpointIntervalMin, &out.ClusterCheckpointIntervalMin + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotSchedule. +func (in *SnapshotSchedule) DeepCopy() *SnapshotSchedule { + if in == nil { + return nil + } + out := new(SnapshotSchedule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in StartupParameters) DeepCopyInto(out *StartupParameters) { + { + in := &in + *out = make(StartupParameters, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StartupParameters. +func (in StartupParameters) DeepCopy() StartupParameters { + if in == nil { + return nil + } + out := new(StartupParameters) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetSpecWrapper) DeepCopyInto(out *StatefulSetSpecWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { + *out = *in + if in.AdditionalCertificateDomains != nil { + in, out := &in.AdditionalCertificateDomains, &out.AdditionalCertificateDomains + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { + if in == nil { + return nil + } + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1/mdbmulti/doc.go b/api/v1/mdbmulti/doc.go new file mode 100644 index 000000000..d6a165e6c --- /dev/null +++ b/api/v1/mdbmulti/doc.go @@ -0,0 +1,4 @@ +package mdbmulti + +// +k8s:deepcopy-gen=package +// +versionName=v1 diff --git a/api/v1/mdbmulti/groupversion_info.go b/api/v1/mdbmulti/groupversion_info.go new file mode 100644 index 000000000..00ca5c87e --- /dev/null +++ b/api/v1/mdbmulti/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1 contains API Schema definitions for the mongodb v1 API group +// +kubebuilder:object:generate=true +// +groupName=mongodb.com +package mdbmulti + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "mongodb.com", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1/mdbmulti/mongodb_multi_types.go b/api/v1/mdbmulti/mongodb_multi_types.go new file mode 100644 index 000000000..8ac231d54 --- /dev/null +++ b/api/v1/mdbmulti/mongodb_multi_types.go @@ -0,0 +1,667 @@ +package mdbmulti + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/blang/semver" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbc "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/fcv" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster/failedcluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + intp "github.com/10gen/ops-manager-kubernetes/pkg/util/int" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func init() { + v1.SchemeBuilder.Register(&MongoDBMultiCluster{}, &MongoDBMultiClusterList{}) +} + +type TransportSecurity string + +const ( + LastClusterNumMapping = "mongodb.com/v1.lastClusterNumMapping" + TransportSecurityNone TransportSecurity = "none" + TransportSecurityTLS TransportSecurity = "tls" +) + +// The MongoDBMultiCluster resource allows users to create MongoDB deployment spread over +// multiple clusters + +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:path= mongodbmulticluster,scope=Namespaced,shortName=mdbmc +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Current state of the MongoDB deployment." +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="The time since the MongoDBMultiCluster resource was created." +type MongoDBMultiCluster struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + // +optional + Status MongoDBMultiStatus `json:"status"` + Spec MongoDBMultiSpec `json:"spec"` +} + +func (m *MongoDBMultiCluster) GetProjectConfigMapNamespace() string { + return m.Namespace +} + +func (m *MongoDBMultiCluster) GetCredentialsSecretNamespace() string { + return m.Namespace +} + +func (m *MongoDBMultiCluster) GetProjectConfigMapName() string { + return m.Spec.OpsManagerConfig.ConfigMapRef.Name +} + +func (m *MongoDBMultiCluster) GetCredentialsSecretName() string { + return m.Spec.Credentials +} + +func (m *MongoDBMultiCluster) GetMultiClusterAgentHostnames() ([]string, error) { + hostnames := make([]string, 0) + + clusterSpecList, err := m.GetClusterSpecItems() + if err != nil { + return nil, err + } + + for _, spec := range clusterSpecList { + hostnames = append(hostnames, dns.GetMultiClusterProcessHostnames(m.Name, m.Namespace, m.ClusterNum(spec.ClusterName), spec.Members, m.Spec.GetClusterDomain(), nil)...) + } + return hostnames, nil +} + +func (m *MongoDBMultiCluster) MultiStatefulsetName(clusterNum int) string { + return dns.GetMultiStatefulSetName(m.Name, clusterNum) +} + +func (m *MongoDBMultiCluster) MultiHeadlessServiceName(clusterNum int) string { + return fmt.Sprintf("%s-svc", m.MultiStatefulsetName(clusterNum)) +} + +func (m *MongoDBMultiCluster) GetBackupSpec() *mdbv1.Backup { + return m.Spec.Backup +} + +func (m *MongoDBMultiCluster) GetResourceType() mdbv1.ResourceType { + return m.Spec.ResourceType +} + +func (m *MongoDBMultiCluster) GetResourceName() string { + return m.Name +} + +func (m *MongoDBMultiCluster) GetSecurity() *mdbv1.Security { + return m.Spec.Security +} + +func (m *MongoDBMultiCluster) GetConnectionSpec() *mdbv1.ConnectionSpec { + return &m.Spec.ConnectionSpec +} + +func (m *MongoDBMultiCluster) GetPrometheus() *mdbc.Prometheus { + return m.Spec.Prometheus +} + +func (m *MongoDBMultiCluster) IsLDAPEnabled() bool { + if m.Spec.Security == nil || m.Spec.Security.Authentication == nil { + return false + } + return stringutil.Contains(m.Spec.GetSecurityAuthenticationModes(), util.LDAP) +} + +func (m *MongoDBMultiCluster) GetLDAP(password, caContents string) *ldap.Ldap { + if !m.IsLDAPEnabled() { + return nil + } + mdbLdap := m.Spec.Security.Authentication.Ldap + transportSecurity := mdbv1.GetTransportSecurity(mdbLdap) + + validateServerConfig := true + if mdbLdap.ValidateLDAPServerConfig != nil { + validateServerConfig = *mdbLdap.ValidateLDAPServerConfig + } + + return &ldap.Ldap{ + BindQueryUser: mdbLdap.BindQueryUser, + BindQueryPassword: password, + Servers: strings.Join(mdbLdap.Servers, ","), + TransportSecurity: string(transportSecurity), + CaFileContents: caContents, + ValidateLDAPServerConfig: validateServerConfig, + + // Related to LDAP Authorization + AuthzQueryTemplate: mdbLdap.AuthzQueryTemplate, + UserToDnMapping: mdbLdap.UserToDNMapping, + + // TODO: Enable LDAP SASL bind method + BindMethod: "simple", + BindSaslMechanisms: "", + + TimeoutMS: mdbLdap.TimeoutMS, + UserCacheInvalidationInterval: mdbLdap.UserCacheInvalidationInterval, + } +} + +func (m *MongoDBMultiCluster) GetHostNameOverrideConfigmapName() string { + return fmt.Sprintf("%s-hostname-override", m.Name) +} + +func (m *MongoDBMultiCluster) ObjectKey() client.ObjectKey { + return kube.ObjectKey(m.Namespace, m.Name) +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type MongoDBMultiClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []MongoDBMultiCluster `json:"items"` +} + +func (m *MongoDBMultiCluster) GetClusterSpecByName(clusterName string) *mdbv1.ClusterSpecItem { + for _, csi := range m.Spec.ClusterSpecList { + if csi.ClusterName == clusterName { + return &csi + } + } + return nil +} + +// ClusterStatusList holds a list of clusterStatuses corresponding to each cluster +type ClusterStatusList struct { + ClusterStatuses []ClusterStatusItem `json:"clusterStatuses,omitempty"` +} + +// ClusterStatusItem is the mongodb multi-cluster spec that is specific to a +// particular Kubernetes cluster, this maps to the statefulset created in each cluster +type ClusterStatusItem struct { + // ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + // name should have a one on one mapping with the service-account created in the central cluster + // to talk to the workload clusters. + ClusterName string `json:"clusterName,omitempty"` + status.Common `json:",inline"` + Members int `json:"members,omitempty"` + Warnings []status.Warning `json:"warnings,omitempty"` +} + +type MongoDBMultiStatus struct { + status.Common `json:",inline"` + ClusterStatusList ClusterStatusList `json:"clusterStatusList,omitempty"` + BackupStatus *mdbv1.BackupStatus `json:"backup,omitempty"` + Version string `json:"version"` + Link string `json:"link,omitempty"` + FeatureCompatibilityVersion string `json:"featureCompatibilityVersion,omitempty"` + Warnings []status.Warning `json:"warnings,omitempty"` +} + +type MongoDBMultiSpec struct { + // +kubebuilder:pruning:PreserveUnknownFields + mdbv1.DbCommonSpec `json:",inline"` + + ClusterSpecList mdbv1.ClusterSpecList `json:"clusterSpecList,omitempty"` + + // Mapping stores the deterministic index for a given cluster-name. + Mapping map[string]int `json:"-"` +} + +func (m *MongoDBMultiSpec) GetAgentConfig() mdbv1.AgentConfig { + return m.Agent +} + +func (m *MongoDBMultiCluster) GetStatus(...status.Option) interface{} { + return m.Status +} + +func (m *MongoDBMultiCluster) GetCommonStatus(...status.Option) *status.Common { + return &m.Status.Common +} + +func (m *MongoDBMultiCluster) GetStatusPath(...status.Option) string { + return "/status" +} + +func (m *MongoDBMultiCluster) SetWarnings(warnings []status.Warning, _ ...status.Option) { + m.Status.Warnings = warnings +} + +func (m *MongoDBMultiCluster) UpdateStatus(phase status.Phase, statusOptions ...status.Option) { + m.Status.UpdateCommonFields(phase, m.GetGeneration(), statusOptions...) + + if option, exists := status.GetOption(statusOptions, status.BackupStatusOption{}); exists { + if m.Status.BackupStatus == nil { + m.Status.BackupStatus = &mdbv1.BackupStatus{} + } + m.Status.BackupStatus.StatusName = option.(status.BackupStatusOption).Value().(string) + } + + if phase == status.PhaseRunning { + m.Status.FeatureCompatibilityVersion = m.CalculateFeatureCompatibilityVersion() + } +} + +// GetClusterSpecItems returns the cluster spec items that should be used for reconciliation. +// These may not be the values specified in the spec directly, this takes into account the following three conditions: +// 1. Adding/Removing cluster from the clusterSpecList. +// 2. Scaling the number of nodes of each cluster. +// 3. When there is a cluster outage, there is an annotation put in the CR to orchestrate workloads out +// of the impacted cluster to the remaining clusters. +// The return value should be used in the reconciliation loop when determining which processes +// should be added to the automation config and which services need to be created and how many replicas +// each StatefulSet should have. +// This function should always be used instead of accessing the struct fields directly in the Reconcile function. +func (m *MongoDBMultiCluster) GetClusterSpecItems() (mdbv1.ClusterSpecList, error) { + desiredSpecList := m.GetDesiredSpecList() + prevSpec, err := m.ReadLastAchievedSpec() + if err != nil { + return nil, err + } + + if prevSpec == nil { + return desiredSpecList, nil + } + + prevSpecs := prevSpec.GetClusterSpecList() + + desiredSpecMap := clusterSpecItemListToMap(desiredSpecList) + prevSpecsMap := clusterSpecItemListToMap(prevSpecs) + + var specsForThisReconciliation mdbv1.ClusterSpecList + + // We only care about the members of the previous reconcile, the rest should be reflecting the CRD definition. + for _, spec := range prevSpecs { + if desiredSpec, ok := desiredSpecMap[spec.ClusterName]; ok { + prevMembers := spec.Members + spec = desiredSpec + spec.Members = prevMembers + } + specsForThisReconciliation = append(specsForThisReconciliation, spec) + } + + // When we remove a cluster, this means that there will be an entry in the resource annotation (the previous spec) + // but not in the current spec. In order to make scaling work, we add an entry for the removed cluster that has + // 0 members. This allows the following scaling down logic to handle the transition from n -> 0 members, with a + // decrementing value of one with each reconciliation. After this, we delete the StatefulSet if the spec item + // was removed. + + // E.g. + // Reconciliation 1: + // 3 clusters all with 3 members + // Reconciliation 2: + // 2 clusters with 3 members (we removed the last cluster). + // The spec has 2 members, but we add a third with 0 members. + // This "dummy" item will be handled the same as another spec item. + // This is only relevant for the first reconciliation after removal since this cluster spec will be saved + // in an annotation, and the regular scaling logic will happen in subsequent reconciliations. + // We go from members 3-3-3 to 3-3-2 + // Reconciliation 3: + // We go from 3-3-2 to 3-3-1 + // Reconciliation 4: + // We go from 3-3-1 to 3-3-0 (and then delete the StatefulSet in this final reconciliation) + + for _, previousItem := range prevSpecs { + if _, ok := desiredSpecMap[previousItem.ClusterName]; !ok { + previousItem.Members = 0 + desiredSpecList = append(desiredSpecList, previousItem) + } + } + + for _, item := range desiredSpecList { + // If a spec item exists but was not there previously, we add it with a single member. + // This allows subsequent reconciliations to go from 1-> n one member at a time as usual. + // It will never be possible to add a new member at the maximum members since scaling can only ever be done + // one at a time. Adding the item with 1 member allows the regular logic to handle scaling one a time until + // we reach the desired member count. + prevItem, ok := prevSpecsMap[item.ClusterName] + if !ok { + if item.Members > 1 { + item.Members = 1 + } + return append(specsForThisReconciliation, item), nil + } + // can only scale one member at a time, so we return early on each increment. + if item.Members > prevItem.Members { + specsForThisReconciliation[m.ClusterNum(item.ClusterName)].Members += 1 + return specsForThisReconciliation, nil + } + if item.Members < prevItem.Members { + specsForThisReconciliation[m.ClusterNum(item.ClusterName)].Members -= 1 + return specsForThisReconciliation, nil + } + } + + return specsForThisReconciliation, nil +} + +// HasClustersToFailOver checks if the MongoDBMultiCluster CR has ""clusterSpecOverride" annotation which is put when one or more clusters +// are not reachable. +func HasClustersToFailOver(annotations map[string]string) (string, bool) { + if annotations == nil { + return "", false + } + val, ok := annotations[failedcluster.ClusterSpecOverrideAnnotation] + return val, ok +} + +// GetFailedClusters returns the current set of failed clusters for the MongoDBMultiCluster CR. +func (m *MongoDBMultiCluster) GetFailedClusters() ([]failedcluster.FailedCluster, error) { + if m.Annotations == nil { + return nil, nil + } + failedClusterBytes, ok := m.Annotations[failedcluster.FailedClusterAnnotation] + if !ok { + return []failedcluster.FailedCluster{}, nil + } + var failedClusters []failedcluster.FailedCluster + err := json.Unmarshal([]byte(failedClusterBytes), &failedClusters) + if err != nil { + return nil, err + } + return failedClusters, err +} + +// GetFailedClusterNames returns the current set of failed cluster names for the MongoDBMultiCluster CR. +func (m *MongoDBMultiCluster) GetFailedClusterNames() ([]string, error) { + failedClusters, err := m.GetFailedClusters() + if err != nil { + return nil, err + } + clusterNames := []string{} + for _, c := range failedClusters { + clusterNames = append(clusterNames, c.ClusterName) + } + return clusterNames, nil +} + +// clusterSpecItemListToMap converts a slice of cluster spec items into a map using the name as the key. +func clusterSpecItemListToMap(clusterSpecItems mdbv1.ClusterSpecList) map[string]mdbv1.ClusterSpecItem { + m := map[string]mdbv1.ClusterSpecItem{} + for _, c := range clusterSpecItems { + m[c.ClusterName] = c + } + return m +} + +// ReadLastAchievedSpec fetches the previously achieved spec. +func (m *MongoDBMultiCluster) ReadLastAchievedSpec() (*MongoDBMultiSpec, error) { + if m.Annotations == nil { + return nil, nil + } + specBytes, ok := m.Annotations[util.LastAchievedSpec] + if !ok { + return nil, nil + } + + prevSpec := &MongoDBMultiSpec{} + if err := json.Unmarshal([]byte(specBytes), &prevSpec); err != nil { + return nil, err + } + return prevSpec, nil +} + +func (m *MongoDBMultiCluster) GetLastAdditionalMongodConfig() map[string]interface{} { + lastSpec, err := m.ReadLastAchievedSpec() + if lastSpec == nil || err != nil { + return map[string]interface{}{} + } + return lastSpec.GetAdditionalMongodConfig().ToMap() +} + +// when unmarshalling a MongoDBMultiCluster instance, we don't want to have any nil references +// these are replaced with an empty instance to prevent nil references +func (m *MongoDBMultiCluster) UnmarshalJSON(data []byte) error { + type MongoDBJSON *MongoDBMultiCluster + if err := json.Unmarshal(data, (MongoDBJSON)(m)); err != nil { + return err + } + + m.InitDefaults() + return nil +} + +// InitDefaults makes sure the MongoDBMultiCluster resource has correct state after initialization: +// - prevents any references from having nil values. +// - makes sure the spec is in correct state +// +// should not be called directly, used in tests and unmarshalling +func (m *MongoDBMultiCluster) InitDefaults() { + m.Spec.Security = mdbv1.EnsureSecurity(m.Spec.Security) + + // TODO: add more default if need be + // ProjectName defaults to the name of the resource + if m.Spec.ProjectName == "" { + m.Spec.ProjectName = m.Name + } + + if m.Spec.Agent.StartupParameters == nil { + m.Spec.Agent.StartupParameters = map[string]string{} + } + + if m.Spec.AdditionalMongodConfig == nil || m.Spec.AdditionalMongodConfig.ToMap() == nil { + m.Spec.AdditionalMongodConfig = &mdbv1.AdditionalMongodConfig{} + } + + if m.Spec.CloudManagerConfig == nil { + m.Spec.CloudManagerConfig = mdbv1.NewOpsManagerConfig() + } + + if m.Spec.OpsManagerConfig == nil { + m.Spec.OpsManagerConfig = mdbv1.NewOpsManagerConfig() + } + + if m.Spec.Connectivity == nil { + m.Spec.Connectivity = mdbv1.NewConnectivity() + } + + m.Spec.Security = mdbv1.EnsureSecurity(m.Spec.Security) +} + +// Replicas returns the total number of MongoDB members running across all the clusters +func (m *MongoDBMultiSpec) Replicas() int { + num := 0 + for _, e := range m.ClusterSpecList { + num += e.Members + } + return num +} + +func (m *MongoDBMultiSpec) GetClusterDomain() string { + if m.ClusterDomain != "" { + return m.ClusterDomain + } + return "cluster.local" +} + +func (m *MongoDBMultiSpec) GetMongoDBVersion() string { + return m.Version +} + +func (m *MongoDBMultiSpec) GetSecurityAuthenticationModes() []string { + return m.GetSecurity().Authentication.GetModes() +} + +func (m *MongoDBMultiSpec) GetResourceType() mdbv1.ResourceType { + return m.ResourceType +} + +func (m *MongoDBMultiSpec) IsSecurityTLSConfigEnabled() bool { + return m.GetSecurity().IsTLSEnabled() +} + +func (m *MongoDBMultiSpec) GetFeatureCompatibilityVersion() *string { + return m.FeatureCompatibilityVersion +} + +func (m *MongoDBMultiSpec) GetHorizonConfig() []mdbv1.MongoDBHorizonConfig { + return m.Connectivity.ReplicaSetHorizons +} + +func (m *MongoDBMultiSpec) GetMemberOptions() []automationconfig.MemberOptions { + specList := m.GetClusterSpecList() + var options []automationconfig.MemberOptions + for _, item := range specList { + options = append(options, item.MemberConfig...) + } + return options +} + +func (m *MongoDBMultiSpec) MinimumMajorVersion() uint64 { + if m.FeatureCompatibilityVersion != nil && *m.FeatureCompatibilityVersion != "" { + fcv := *m.FeatureCompatibilityVersion + + // ignore errors here as the format of FCV/version is handled by CRD validation + semverFcv, _ := semver.Make(fmt.Sprintf("%s.0", fcv)) + return semverFcv.Major + } + semverVersion, _ := semver.Make(m.GetMongoDBVersion()) + return semverVersion.Major +} + +func (m *MongoDBMultiSpec) GetPersistence() bool { + if m.Persistent == nil { + return true + } + return *m.Persistent +} + +// GetClusterSpecList returns the cluster spec items. +// This method should ideally be not used in the reconciler. Always, prefer to +// use the GetHealthyMemberClusters() method from the reconciler. +func (m *MongoDBMultiSpec) GetClusterSpecList() mdbv1.ClusterSpecList { + return m.ClusterSpecList +} + +func (m *MongoDBMultiSpec) GetExternalAccessConfigurationForMemberCluster(clusterName string) *mdbv1.ExternalAccessConfiguration { + for _, csl := range m.ClusterSpecList { + if csl.ClusterName == clusterName && csl.ExternalAccessConfiguration != nil { + return csl.ExternalAccessConfiguration + } + } + + return m.ExternalAccessConfiguration +} + +func (m *MongoDBMultiSpec) GetExternalDomainForMemberCluster(clusterName string) *string { + if cfg := m.GetExternalAccessConfigurationForMemberCluster(clusterName); cfg != nil { + if externalDomain := cfg.ExternalDomain; externalDomain != nil { + return externalDomain + } + } + + return m.GetExternalDomain() +} + +// GetDesiredSpecList returns the desired cluster spec list for a given reconcile operation. +// Returns the failerOver annotation if present else reads the cluster spec list from the CR. +func (m *MongoDBMultiCluster) GetDesiredSpecList() mdbv1.ClusterSpecList { + clusterSpecList := m.Spec.ClusterSpecList + + if val, ok := HasClustersToFailOver(m.GetAnnotations()); ok { + var clusterSpecOverride mdbv1.ClusterSpecList + + err := json.Unmarshal([]byte(val), &clusterSpecOverride) + if err != nil { + return clusterSpecList + } + clusterSpecList = clusterSpecOverride + } + return clusterSpecList +} + +// ClusterNum returns the index associated with a given clusterName, it assigns a unique id to each +// clustername taking into account addition and removal of clusters. We don't reuse cluster indexes since +// the clusters can be removed and then added back. +func (m *MongoDBMultiCluster) ClusterNum(clusterName string) int { + if m.Spec.Mapping == nil { + m.Spec.Mapping = make(map[string]int) + } + // first check if the entry exists in local map before making any API call + if val, ok := m.Spec.Mapping[clusterName]; ok { + return val + } + + // next check if the clusterName is present in the annotations + if bytes, ok := m.Annotations[LastClusterNumMapping]; ok { + _ = json.Unmarshal([]byte(bytes), &m.Spec.Mapping) + + if val, ok := m.Spec.Mapping[clusterName]; ok { + return val + } + } + + index := getNextIndex(m.Spec.Mapping) + m.Spec.Mapping[clusterName] = index + return index +} + +// BuildConnectionString for a MultiCluster user. +// +// Not yet functional, because m.Service() is not defined. Waiting for CLOUDP-105817 +// to complete. +func (m *MongoDBMultiCluster) BuildConnectionString(username, password string, scheme connectionstring.Scheme, connectionParams map[string]string) string { + hostnames := make([]string, 0) + for _, spec := range m.Spec.GetClusterSpecList() { + hostnames = append(hostnames, dns.GetMultiClusterProcessHostnames(m.Name, m.Namespace, m.ClusterNum(spec.ClusterName), spec.Members, m.Spec.GetClusterDomain(), nil)...) + } + builder := connectionstring.Builder(). + SetName(m.Name). + SetNamespace(m.Namespace). + SetUsername(username). + SetPassword(password). + SetReplicas(m.Spec.Replicas()). + SetService(m.Name + "-svc"). + SetPort(m.Spec.GetAdditionalMongodConfig().GetPortOrDefault()). + SetVersion(m.Spec.GetMongoDBVersion()). + SetAuthenticationModes(m.Spec.GetSecurityAuthenticationModes()). + SetClusterDomain(m.Spec.GetClusterDomain()). + SetExternalDomain(m.Spec.GetExternalDomain()). + SetIsReplicaSet(true). + SetIsTLSEnabled(m.Spec.IsSecurityTLSConfigEnabled()). + SetMultiClusterHosts(hostnames). + SetScheme(scheme) + + return builder.Build() +} + +func (m *MongoDBMultiCluster) GetAuthenticationModes() []string { + return m.Spec.Security.Authentication.GetModes() +} + +// getNextIndex returns the next higher index from the current cluster indexes +func getNextIndex(m map[string]int) int { + maxi := -1 + + for _, val := range m { + maxi = intp.Max(maxi, val) + } + return maxi + 1 +} + +func (m *MongoDBMultiCluster) IsInChangeVersion() bool { + spec, err := m.ReadLastAchievedSpec() + if err != nil { + return false + } + if spec != nil && (spec.Version != m.Spec.Version) { + return true + } + return false +} + +func (m *MongoDBMultiCluster) CalculateFeatureCompatibilityVersion() string { + return fcv.CalculateFeatureCompatibilityVersion(m.Spec.Version, m.Status.FeatureCompatibilityVersion, m.Spec.FeatureCompatibilityVersion) +} diff --git a/api/v1/mdbmulti/mongodb_multi_types_test.go b/api/v1/mdbmulti/mongodb_multi_types_test.go new file mode 100644 index 000000000..cf8b18d0b --- /dev/null +++ b/api/v1/mdbmulti/mongodb_multi_types_test.go @@ -0,0 +1,40 @@ +package mdbmulti + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +func TestMongoDBMultiSpecMinimumMajorVersion(t *testing.T) { + tests := []struct { + name string + DbCommonSpec mdb.DbCommonSpec + want uint64 + }{ + { + name: "non ent", + DbCommonSpec: mdb.DbCommonSpec{ + Version: "7.1.0", + }, + want: 7, + }, + { + name: "ent", + DbCommonSpec: mdb.DbCommonSpec{ + Version: "7.0.2-ent", + }, + want: 7, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := &MongoDBMultiSpec{ + DbCommonSpec: tt.DbCommonSpec, + } + assert.Equalf(t, tt.want, m.MinimumMajorVersion(), "MinimumMajorVersion()") + }) + } +} diff --git a/api/v1/mdbmulti/mongodbmulti_validation.go b/api/v1/mdbmulti/mongodbmulti_validation.go new file mode 100644 index 000000000..ef08c95f6 --- /dev/null +++ b/api/v1/mdbmulti/mongodbmulti_validation.go @@ -0,0 +1,116 @@ +package mdbmulti + +import ( + "errors" + + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + runtime "k8s.io/apimachinery/pkg/runtime" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +var _ webhook.Validator = &MongoDBMultiCluster{} + +func (m *MongoDBMultiCluster) ValidateCreate() (admission.Warnings, error) { + return nil, m.ProcessValidationsOnReconcile(nil) +} + +func (m *MongoDBMultiCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + return nil, m.ProcessValidationsOnReconcile(old.(*MongoDBMultiCluster)) +} + +func (m *MongoDBMultiCluster) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} + +func (m *MongoDBMultiCluster) ProcessValidationsOnReconcile(old *MongoDBMultiCluster) error { + for _, res := range m.RunValidations(old) { + if res.Level == v1.ErrorLevel { + return errors.New(res.Msg) + } + if res.Level == v1.WarningLevel { + m.AddWarningIfNotExists(status.Warning(res.Msg)) + } + } + return nil +} + +func (m *MongoDBMultiCluster) RunValidations(old *MongoDBMultiCluster) []v1.ValidationResult { + multiClusterValidators := []func(ms MongoDBMultiSpec) v1.ValidationResult{ + validateUniqueExternalDomains, + } + + // shared validators between MongoDBMulti and AppDB + multiClusterAppDBSharedClusterValidators := []func(ms mdbv1.ClusterSpecList) v1.ValidationResult{ + mdbv1.ValidateUniqueClusterNames, + mdbv1.ValidateNonEmptyClusterSpecList, + mdbv1.ValidateMemberClusterIsSubsetOfKubeConfig, + } + + var validationResults []v1.ValidationResult + + for _, validator := range mdbv1.CommonValidators() { + res := validator(m.Spec.DbCommonSpec) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + for _, validator := range multiClusterValidators { + res := validator(m.Spec) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + for _, validator := range multiClusterAppDBSharedClusterValidators { + res := validator(m.Spec.ClusterSpecList) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + return validationResults +} + +// validateUniqueExternalDomains validates uniqueness of the domains if they are provided. +// External domain might be specified at the top level in spec.externalAccess.externalDomain or in every member cluster. +// We make sure that if external domains are used, every member cluster has unique external domain defined. +func validateUniqueExternalDomains(ms MongoDBMultiSpec) v1.ValidationResult { + externalDomains := make(map[string]string) + + for _, e := range ms.ClusterSpecList { + if externalDomain := ms.GetExternalDomainForMemberCluster(e.ClusterName); externalDomain != nil { + externalDomains[e.ClusterName] = *externalDomain + } + } + + // We don't need to validate external domains if there aren't any specified. + // We don't have any flag that enables usage of external domains. We use them if they are provided. + if len(externalDomains) == 0 { + return v1.ValidationSuccess() + } + + present := map[string]struct{}{} + for _, e := range ms.ClusterSpecList { + externalDomain, ok := externalDomains[e.ClusterName] + if !ok { + return v1.ValidationError("The externalDomain is not set for member cluster: %s", e.ClusterName) + } + + if _, ok := present[externalDomain]; ok { + return v1.ValidationError("Multiple member clusters with the same externalDomain (%s) are not allowed. "+ + "Check if all spec.clusterSpecList[*].externalAccess.externalDomain fields are defined and are unique.", externalDomain) + } + present[externalDomain] = struct{}{} + } + return v1.ValidationSuccess() +} + +func (m *MongoDBMultiCluster) AddWarningIfNotExists(warning status.Warning) { + m.Status.Warnings = status.Warnings(m.Status.Warnings).AddIfNotExists(warning) +} diff --git a/api/v1/mdbmulti/mongodbmulti_validation_test.go b/api/v1/mdbmulti/mongodbmulti_validation_test.go new file mode 100644 index 000000000..0eb2c5c29 --- /dev/null +++ b/api/v1/mdbmulti/mongodbmulti_validation_test.go @@ -0,0 +1,151 @@ +package mdbmulti + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +func TestUniqueClusterNames(t *testing.T) { + mrs := DefaultMultiReplicaSetBuilder().Build() + mrs.Spec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: "abc", + Members: 2, + }, + { + ClusterName: "def", + Members: 1, + }, + { + ClusterName: "abc", + Members: 1, + }, + } + + _, err := mrs.ValidateCreate() + assert.ErrorContains(t, err, "Multiple clusters with the same name (abc) are not allowed") +} + +func TestUniqueExternalDomains(t *testing.T) { + mrs := DefaultMultiReplicaSetBuilder().Build() + mrs.Spec.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{} + mrs.Spec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: "1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("test")}, + }, + { + ClusterName: "2", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("test")}, + }, + { + ClusterName: "3", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("test")}, + }, + } + + _, err := mrs.ValidateCreate() + assert.ErrorContains(t, err, "Multiple member clusters with the same externalDomain (test) are not allowed") +} + +func TestAllExternalDomainsSet(t *testing.T) { + mrs := DefaultMultiReplicaSetBuilder().Build() + mrs.Spec.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{} + mrs.Spec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: "1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("test")}, + }, + { + ClusterName: "2", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: nil}, + }, + { + ClusterName: "3", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("test")}, + }, + } + + _, err := mrs.ValidateCreate() + assert.ErrorContains(t, err, "The externalDomain is not set for member cluster: 2") +} + +func TestMongoDBMultiValidattionHorzonsWithoutTLS(t *testing.T) { + replicaSetHorizons := []mdbv1.MongoDBHorizonConfig{ + {"my-horizon": "my-db.com:12345"}, + {"my-horizon": "my-db.com:12342"}, + {"my-horizon": "my-db.com:12346"}, + } + + mrs := DefaultMultiReplicaSetBuilder().Build() + mrs.Spec.Connectivity = &mdbv1.MongoDBConnectivity{ + ReplicaSetHorizons: replicaSetHorizons, + } + mrs.Spec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: "foo", + }, + } + + _, err := mrs.ValidateCreate() + assert.ErrorContains(t, err, "TLS must be enabled in order to use replica set horizons") +} + +func TestSpecProjectOnlyOneValue(t *testing.T) { + file := createTestKubeConfigAndSetEnv(t) + defer os.Remove(file.Name()) + + mrs := DefaultMultiReplicaSetBuilder().Build() + mrs.Spec.OpsManagerConfig = &mdbv1.PrivateCloudConfig{ + ConfigMapRef: mdbv1.ConfigMapRef{Name: "cloud-manager"}, + } + mrs.Spec.ClusterSpecList = mdbv1.ClusterSpecList{{ + ClusterName: "foo", + }} + + _, err := mrs.ValidateCreate() + assert.NoError(t, err) +} + +func createTestKubeConfigAndSetEnv(t *testing.T) *os.File { + //lint:ignore S1039 I avoid to modify this string to not ruin the format + //nolint + testKubeConfig := fmt.Sprintf( + ` +apiVersion: v1 +contexts: +- context: + cluster: foo + namespace: a-1661872869-pq35wlt3zzz + user: foo + name: foo +kind: Config +users: +- name: foo + user: + token: eyJhbGciOi +`) + + file, err := os.CreateTemp("", "kubeconfig") + assert.NoError(t, err) + + _, err = file.WriteString(testKubeConfig) + assert.NoError(t, err) + + t.Setenv(multicluster.KubeConfigPathEnv, file.Name()) + + return file +} diff --git a/api/v1/mdbmulti/mongodbmultibuilder.go b/api/v1/mdbmulti/mongodbmultibuilder.go new file mode 100644 index 000000000..5e11adaaa --- /dev/null +++ b/api/v1/mdbmulti/mongodbmultibuilder.go @@ -0,0 +1,134 @@ +package mdbmulti + +import ( + "crypto/rand" + "fmt" + "math/big" + + v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type MultiReplicaSetBuilder struct { + *MongoDBMultiCluster +} + +const ( + TestProjectConfigMapName = om.TestGroupName + TestCredentialsSecretName = "my-credentials" + TestNamespace = "my-namespace" +) + +func DefaultMultiReplicaSetBuilder() *MultiReplicaSetBuilder { + spec := MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + Connectivity: &mdbv1.MongoDBConnectivity{}, + Version: "7.0.0", + Persistent: util.BooleanRef(false), + ConnectionSpec: mdbv1.ConnectionSpec{ + SharedConnectionSpec: mdbv1.SharedConnectionSpec{ + OpsManagerConfig: &mdbv1.PrivateCloudConfig{ + ConfigMapRef: mdbv1.ConfigMapRef{ + Name: TestProjectConfigMapName, + }, + }, + }, + Credentials: TestCredentialsSecretName, + }, + ResourceType: mdbv1.ReplicaSet, + Security: &mdbv1.Security{ + TLSConfig: &mdbv1.TLSConfig{}, + Authentication: &mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{}, + }, + Roles: []mdbv1.MongoDbRole{}, + }, + DuplicateServiceObjects: util.BooleanRef(false), + }, + } + + mrs := &MongoDBMultiCluster{Spec: spec, ObjectMeta: metav1.ObjectMeta{Name: "temple", Namespace: TestNamespace}} + return &MultiReplicaSetBuilder{mrs} +} + +func (m *MultiReplicaSetBuilder) Build() *MongoDBMultiCluster { + // initialize defaults + res := m.MongoDBMultiCluster.DeepCopy() + res.InitDefaults() + return res +} + +func (m *MultiReplicaSetBuilder) SetVersion(version string) *MultiReplicaSetBuilder { + m.Spec.Version = version + return m +} + +func (m *MultiReplicaSetBuilder) SetSecurity(s *mdbv1.Security) *MultiReplicaSetBuilder { + m.Spec.Security = s + return m +} + +func (m *MultiReplicaSetBuilder) SetClusterSpecList(clusters []string) *MultiReplicaSetBuilder { + randFive, err := rand.Int(rand.Reader, big.NewInt(5)) + if err != nil { + panic(err) + } + + randFiveAsInt := int(randFive.Int64()) + + for _, e := range clusters { + m.Spec.ClusterSpecList = append(m.Spec.ClusterSpecList, mdbv1.ClusterSpecItem{ + ClusterName: e, + Members: randFiveAsInt + 1, // number of cluster members b/w 1 to 5 + }) + } + return m +} + +func (m *MultiReplicaSetBuilder) SetExternalAccess(configuration mdbv1.ExternalAccessConfiguration, externalDomainTemplate *string) *MultiReplicaSetBuilder { + m.Spec.ExternalAccessConfiguration = &configuration + + for i := range m.Spec.ClusterSpecList { + if externalDomainTemplate != nil { + s := fmt.Sprintf(*externalDomainTemplate, i) + m.Spec.ClusterSpecList[i].ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: &s, + } + } + } + + return m +} + +func (m *MultiReplicaSetBuilder) SetConnectionSpec(spec mdbv1.ConnectionSpec) *MultiReplicaSetBuilder { + m.Spec.ConnectionSpec = spec + return m +} + +func (m *MultiReplicaSetBuilder) SetBackup(backupSpec mdbv1.Backup) *MultiReplicaSetBuilder { + m.Spec.Backup = &backupSpec + return m +} + +func (m *MultiReplicaSetBuilder) SetPodSpecTemplate(spec corev1.PodTemplateSpec) *MultiReplicaSetBuilder { + if m.Spec.StatefulSetConfiguration == nil { + m.Spec.StatefulSetConfiguration = &v1.StatefulSetConfiguration{} + } + m.Spec.StatefulSetConfiguration.SpecWrapper.Spec.Template = spec + return m +} + +func (m *MultiReplicaSetBuilder) SetName(name string) *MultiReplicaSetBuilder { + m.Name = name + return m +} + +func (m *MultiReplicaSetBuilder) SetOpsManagerConfigMapName(configMapName string) *MultiReplicaSetBuilder { + m.Spec.OpsManagerConfig.ConfigMapRef.Name = configMapName + return m +} diff --git a/api/v1/mdbmulti/zz_generated.deepcopy.go b/api/v1/mdbmulti/zz_generated.deepcopy.go new file mode 100644 index 000000000..f4583ec40 --- /dev/null +++ b/api/v1/mdbmulti/zz_generated.deepcopy.go @@ -0,0 +1,206 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package mdbmulti + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterStatusItem) DeepCopyInto(out *ClusterStatusItem) { + *out = *in + in.Common.DeepCopyInto(&out.Common) + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]status.Warning, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatusItem. +func (in *ClusterStatusItem) DeepCopy() *ClusterStatusItem { + if in == nil { + return nil + } + out := new(ClusterStatusItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterStatusList) DeepCopyInto(out *ClusterStatusList) { + *out = *in + if in.ClusterStatuses != nil { + in, out := &in.ClusterStatuses, &out.ClusterStatuses + *out = make([]ClusterStatusItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatusList. +func (in *ClusterStatusList) DeepCopy() *ClusterStatusList { + if in == nil { + return nil + } + out := new(ClusterStatusList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBMultiCluster) DeepCopyInto(out *MongoDBMultiCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBMultiCluster. +func (in *MongoDBMultiCluster) DeepCopy() *MongoDBMultiCluster { + if in == nil { + return nil + } + out := new(MongoDBMultiCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBMultiCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBMultiClusterList) DeepCopyInto(out *MongoDBMultiClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MongoDBMultiCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBMultiClusterList. +func (in *MongoDBMultiClusterList) DeepCopy() *MongoDBMultiClusterList { + if in == nil { + return nil + } + out := new(MongoDBMultiClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBMultiClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBMultiSpec) DeepCopyInto(out *MongoDBMultiSpec) { + *out = *in + in.DbCommonSpec.DeepCopyInto(&out.DbCommonSpec) + if in.ClusterSpecList != nil { + in, out := &in.ClusterSpecList, &out.ClusterSpecList + *out = make(mdb.ClusterSpecList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Mapping != nil { + in, out := &in.Mapping, &out.Mapping + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBMultiSpec. +func (in *MongoDBMultiSpec) DeepCopy() *MongoDBMultiSpec { + if in == nil { + return nil + } + out := new(MongoDBMultiSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBMultiStatus) DeepCopyInto(out *MongoDBMultiStatus) { + *out = *in + in.Common.DeepCopyInto(&out.Common) + in.ClusterStatusList.DeepCopyInto(&out.ClusterStatusList) + if in.BackupStatus != nil { + in, out := &in.BackupStatus, &out.BackupStatus + *out = new(mdb.BackupStatus) + **out = **in + } + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]status.Warning, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBMultiStatus. +func (in *MongoDBMultiStatus) DeepCopy() *MongoDBMultiStatus { + if in == nil { + return nil + } + out := new(MongoDBMultiStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiReplicaSetBuilder) DeepCopyInto(out *MultiReplicaSetBuilder) { + *out = *in + if in.MongoDBMultiCluster != nil { + in, out := &in.MongoDBMultiCluster, &out.MongoDBMultiCluster + *out = new(MongoDBMultiCluster) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiReplicaSetBuilder. +func (in *MultiReplicaSetBuilder) DeepCopy() *MultiReplicaSetBuilder { + if in == nil { + return nil + } + out := new(MultiReplicaSetBuilder) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1/om/appdb_types.go b/api/v1/om/appdb_types.go new file mode 100644 index 000000000..5403e189a --- /dev/null +++ b/api/v1/om/appdb_types.go @@ -0,0 +1,611 @@ +package om + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/authtypes" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/constants" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +const ( + appDBKeyfilePath = "/var/lib/mongodb-mms-automation/authentication/keyfile" + ClusterTopologyMultiCluster = "MultiCluster" +) + +type AppDBSpec struct { + // +kubebuilder:validation:Pattern=^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + Version string `json:"version"` + // Amount of members for this MongoDB Replica Set + // +kubebuilder:validation:Maximum=50 + // +kubebuilder:validation:Minimum=3 + Members int `json:"members,omitempty"` + PodSpec *mdbv1.MongoDbPodSpec `json:"podSpec,omitempty"` + FeatureCompatibilityVersion *string `json:"featureCompatibilityVersion,omitempty"` + + // +optional + Security *mdbv1.Security `json:"security,omitempty"` + ClusterDomain string `json:"clusterDomain,omitempty"` + // +kubebuilder:validation:Enum=Standalone;ReplicaSet;ShardedCluster + ResourceType mdbv1.ResourceType `json:"type,omitempty"` + + Connectivity *mdbv1.MongoDBConnectivity `json:"connectivity,omitempty"` + + // ExternalAccessConfiguration provides external access configuration. + // +optional + ExternalAccessConfiguration *mdbv1.ExternalAccessConfiguration `json:"externalAccess,omitempty"` + + // AdditionalMongodConfig are additional configurations that can be passed to + // each data-bearing mongod at runtime. Uses the same structure as the mongod + // configuration file: + // https://docs.mongodb.com/manual/reference/configuration-options/ + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + AdditionalMongodConfig *mdbv1.AdditionalMongodConfig `json:"additionalMongodConfig,omitempty"` + + // specify configuration like startup flags and automation config settings for the AutomationAgent and MonitoringAgent + AutomationAgent mdbv1.AgentConfig `json:"agent,omitempty"` + + // Specify configuration like startup flags just for the MonitoringAgent. + // These take precedence over + // the flags set in AutomationAgent + MonitoringAgent mdbv1.MonitoringAgentConfig `json:"monitoringAgent,omitempty"` + ConnectionSpec `json:",inline"` + + // PasswordSecretKeyRef contains a reference to the secret which contains the password + // for the mongodb-ops-manager SCRAM-SHA user + PasswordSecretKeyRef *userv1.SecretKeyRef `json:"passwordSecretKeyRef,omitempty"` + + // Enables Prometheus integration on the AppDB. + Prometheus *mdbcv1.Prometheus `json:"prometheus,omitempty"` + + // Transient fields. + // These fields are cleaned before serialization, see 'MarshalJSON()' + // note, that we cannot include the 'OpsManager' instance here as this creates circular dependency and problems with + // 'DeepCopy' + + OpsManagerName string `json:"-"` + Namespace string `json:"-"` + // this is an optional service, it will get the name "-svc" in case not provided + Service string `json:"service,omitempty"` + + // AutomationConfigOverride holds any fields that will be merged on top of the Automation Config + // that the operator creates for the AppDB. Currently only the process.disabled and logRotate field is recognized. + AutomationConfigOverride *mdbcv1.AutomationConfigOverride `json:"automationConfig,omitempty"` + + UpdateStrategyType appsv1.StatefulSetUpdateStrategyType `json:"-"` + + // MemberConfig allows to specify votes, priorities and tags for each of the mongodb process. + // +optional + MemberConfig []automationconfig.MemberOptions `json:"memberConfig,omitempty"` + + // +kubebuilder:validation:Enum=SingleCluster;MultiCluster + // +optional + Topology string `json:"topology,omitempty"` + // +optional + ClusterSpecList mdbv1.ClusterSpecList `json:"clusterSpecList,omitempty"` +} + +func (m *AppDBSpec) GetAgentConfig() mdbv1.AgentConfig { + return m.AutomationAgent +} + +func (m *AppDBSpec) GetAgentLogLevel() mdbcv1.LogLevel { + agentLogLevel := mdbcv1.LogLevelInfo + if m.AutomationAgent.LogLevel != "" { + agentLogLevel = mdbcv1.LogLevel(m.AutomationAgent.LogLevel) + } + return agentLogLevel +} + +func (m *AppDBSpec) GetAgentMaxLogFileDurationHours() int { + agentMaxLogFileDurationHours := automationconfig.DefaultAgentMaxLogFileDurationHours + if m.AutomationAgent.MaxLogFileDurationHours != 0 { + agentMaxLogFileDurationHours = m.AutomationAgent.MaxLogFileDurationHours + } + return agentMaxLogFileDurationHours +} + +// ObjectKey returns the client.ObjectKey with m.OpsManagerName because the name is used to identify the object to enqueue and reconcile. +func (m *AppDBSpec) ObjectKey() client.ObjectKey { + return kube.ObjectKey(m.Namespace, m.OpsManagerName) +} + +// GetConnectionSpec returns nil because no connection spec for appDB is implemented for the watcher setup +func (m *AppDBSpec) GetConnectionSpec() *mdbv1.ConnectionSpec { + return nil +} + +func (m *AppDBSpec) GetMongodConfiguration() mdbcv1.MongodConfiguration { + mongodConfig := mdbcv1.NewMongodConfiguration() + if m.GetAdditionalMongodConfig() == nil || m.AdditionalMongodConfig.ToMap() == nil { + return mongodConfig + } + for k, v := range m.AdditionalMongodConfig.ToMap() { + mongodConfig.SetOption(k, v) + } + return mongodConfig +} + +func (m *AppDBSpec) GetHorizonConfig() []mdbv1.MongoDBHorizonConfig { + return nil // no horizon support for AppDB currently +} + +func (m *AppDBSpec) GetAdditionalMongodConfig() *mdbv1.AdditionalMongodConfig { + if m.AdditionalMongodConfig != nil { + return m.AdditionalMongodConfig + } + return &mdbv1.AdditionalMongodConfig{} +} + +func (m *AppDBSpec) GetMemberOptions() []automationconfig.MemberOptions { + return m.MemberConfig +} + +// GetAgentPasswordSecretNamespacedName returns the NamespacedName for the secret +// which contains the Automation Agent's password. +func (m *AppDBSpec) GetAgentPasswordSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{ + Namespace: m.Namespace, + Name: m.Name() + "-agent-password", + } +} + +// GetAgentKeyfileSecretNamespacedName returns the NamespacedName for the secret +// which contains the keyfile. +func (m *AppDBSpec) GetAgentKeyfileSecretNamespacedName() types.NamespacedName { + return types.NamespacedName{ + Namespace: m.Namespace, + Name: m.Name() + "-keyfile", + } +} + +// GetAuthOptions returns a set of Options which is used to configure Scram Sha authentication +// in the AppDB. +func (m *AppDBSpec) GetAuthOptions() authtypes.Options { + return authtypes.Options{ + AuthoritativeSet: false, + KeyFile: appDBKeyfilePath, + AuthMechanisms: []string{ + constants.Sha256, + constants.Sha1, + }, + AgentName: util.AutomationAgentName, + AutoAuthMechanism: constants.Sha1, + } +} + +// GetAuthUsers returns a list of all scram users for this deployment. +// in this case it is just the Ops Manager user for the AppDB. +func (m *AppDBSpec) GetAuthUsers() []authtypes.User { + passwordSecretName := m.GetOpsManagerUserPasswordSecretName() + if m.PasswordSecretKeyRef != nil && m.PasswordSecretKeyRef.Name != "" { + passwordSecretName = m.PasswordSecretKeyRef.Name + } + return []authtypes.User{ + { + Username: util.OpsManagerMongoDBUserName, + Database: util.DefaultUserDatabase, + // required roles for the AppDB user are outlined in the documentation + // https://docs.opsmanager.mongodb.com/current/tutorial/prepare-backing-mongodb-instances/#replica-set-security + Roles: []authtypes.Role{ + { + Name: "readWriteAnyDatabase", + Database: "admin", + }, + { + Name: "dbAdminAnyDatabase", + Database: "admin", + }, + { + Name: "clusterMonitor", + Database: "admin", + }, + // Enables backup and restoration roles + // https://docs.mongodb.com/manual/reference/built-in-roles/#backup-and-restoration-roles + { + Name: "backup", + Database: "admin", + }, + { + Name: "restore", + Database: "admin", + }, + // Allows user to do db.fsyncLock required by CLOUDP-78890 + // https://docs.mongodb.com/manual/reference/built-in-roles/#hostManager + { + Name: "hostManager", + Database: "admin", + }, + }, + PasswordSecretKey: m.GetOpsManagerUserPasswordSecretKey(), + PasswordSecretName: passwordSecretName, + ScramCredentialsSecretName: m.OpsManagerUserScramCredentialsName(), + }, + } +} + +// used in AppDBConfigurable to implement scram.Configurable +func (m *AppDBSpec) NamespacedName() types.NamespacedName { + return types.NamespacedName{Name: m.Name(), Namespace: m.Namespace} +} + +// GetOpsManagerUserPasswordSecretName returns the name of the secret +// that will store the Ops Manager user's password. +func (m *AppDBSpec) GetOpsManagerUserPasswordSecretName() string { + return m.Name() + "-om-password" +} + +// GetOpsManagerUserPasswordSecretKey returns the key that should be used to map to the Ops Manager user's +// password in the secret. +func (m *AppDBSpec) GetOpsManagerUserPasswordSecretKey() string { + if m.PasswordSecretKeyRef != nil && m.PasswordSecretKeyRef.Key != "" { + return m.PasswordSecretKeyRef.Key + } + return util.DefaultAppDbPasswordKey +} + +// OpsManagerUserScramCredentialsName returns the name of the Secret +// which will store the Ops Manager MongoDB user's scram credentials. +func (m *AppDBSpec) OpsManagerUserScramCredentialsName() string { + return m.Name() + "-om-user-scram-credentials" +} + +type ConnectionSpec struct { + mdbv1.SharedConnectionSpec `json:",inline"` + + // Credentials differ to mdbv1.ConnectionSpec because they are optional here. + + // Name of the Secret holding credentials information + Credentials string `json:"credentials,omitempty"` +} + +type AppDbBuilder struct { + appDb *AppDBSpec +} + +// GetMongoDBVersion returns the version of the MongoDB. +// For AppDB we directly rely on the version field which can +// contain -ent or not for enterprise and static containers. +func (m *AppDBSpec) GetMongoDBVersion() string { + return m.Version +} + +func (m *AppDBSpec) GetClusterDomain() string { + if m.ClusterDomain != "" { + return m.ClusterDomain + } + return "cluster.local" +} + +// Replicas returns the number of "user facing" replicas of the MongoDB resource. This method can be used for +// constructing the mongodb URL for example. +// 'Members' would be a more consistent function but go doesn't allow to have the same +// For AppDB there is a validation that number of members is in the range [3, 50] +func (m *AppDBSpec) Replicas() int { + return m.Members +} + +func (m *AppDBSpec) GetSecurityAuthenticationModes() []string { + return m.GetSecurity().Authentication.GetModes() +} + +func (m *AppDBSpec) GetResourceType() mdbv1.ResourceType { + return m.ResourceType +} + +func (m *AppDBSpec) IsSecurityTLSConfigEnabled() bool { + return m.GetSecurity().IsTLSEnabled() +} + +func (m *AppDBSpec) GetFeatureCompatibilityVersion() *string { + return m.FeatureCompatibilityVersion +} + +func (m *AppDBSpec) GetSecurity() *mdbv1.Security { + if m.Security == nil { + return &mdbv1.Security{} + } + return m.Security +} + +func (m *AppDBSpec) GetTLSConfig() *mdbv1.TLSConfig { + if m.Security == nil || m.Security.TLSConfig == nil { + return &mdbv1.TLSConfig{} + } + + return m.Security.TLSConfig +} + +func DefaultAppDbBuilder() *AppDbBuilder { + appDb := &AppDBSpec{ + Version: "4.2.0", + Members: 3, + PodSpec: &mdbv1.MongoDbPodSpec{}, + PasswordSecretKeyRef: &userv1.SecretKeyRef{}, + } + return &AppDbBuilder{appDb: appDb} +} + +func (b *AppDbBuilder) Build() *AppDBSpec { + return b.appDb.DeepCopy() +} + +func (m *AppDBSpec) GetSecretName() string { + return m.Name() + "-password" +} + +func (m *AppDBSpec) UnmarshalJSON(data []byte) error { + type MongoDBJSON *AppDBSpec + if err := json.Unmarshal(data, (MongoDBJSON)(m)); err != nil { + return err + } + + // if a reference is specified without a key, we will default to "password" + if m.PasswordSecretKeyRef != nil && m.PasswordSecretKeyRef.Key == "" { + m.PasswordSecretKeyRef.Key = util.DefaultAppDbPasswordKey + } + + m.Credentials = "" + m.CloudManagerConfig = nil + m.OpsManagerConfig = nil + + // all resources have a pod spec + if m.PodSpec == nil { + m.PodSpec = mdbv1.NewMongoDbPodSpec() + } + return nil +} + +// Name returns the name of the StatefulSet for the AppDB +func (m *AppDBSpec) Name() string { + return m.OpsManagerName + "-db" +} + +func (m *AppDBSpec) ProjectIDConfigMapName() string { + return m.Name() + "-project-id" +} + +func (m *AppDBSpec) ClusterMappingConfigMapName() string { + return m.Name() + "-cluster-mapping" +} + +func (m *AppDBSpec) LastAppliedMemberSpecConfigMapName() string { + return m.Name() + "-member-spec" +} + +func (m *AppDBSpec) ServiceName() string { + if m.Service == "" { + return m.Name() + "-svc" + } + return m.Service +} + +func (m *AppDBSpec) AutomationConfigSecretName() string { + return m.Name() + "-config" +} + +func (m *AppDBSpec) MonitoringAutomationConfigSecretName() string { + return m.Name() + "-monitoring-config" +} + +func (m *AppDBSpec) GetAgentLogFile() string { + return automationconfig.DefaultAgentLogFile +} + +// This function is used in community to determine whether we need to create a single +// volume for data+logs or two separate ones +// unless spec.PodSpec.Persistence.MultipleConfig is set, a single volume will be created +func (m *AppDBSpec) HasSeparateDataAndLogsVolumes() bool { + p := m.PodSpec.Persistence + return p != nil && (p.MultipleConfig != nil && p.SingleConfig == nil) +} + +func (m *AppDBSpec) GetUpdateStrategyType() appsv1.StatefulSetUpdateStrategyType { + return m.UpdateStrategyType +} + +// GetCAConfigMapName returns the name of the ConfigMap which contains +// the CA which will recognize the certificates used to connect to the AppDB +// deployment +func (m *AppDBSpec) GetCAConfigMapName() string { + security := m.Security + if security != nil && security.TLSConfig != nil { + return security.TLSConfig.CA + } + return "" +} + +// GetTlsCertificatesSecretName returns the name of the secret +// which holds the certificates used to connect to the AppDB +func (m *AppDBSpec) GetTlsCertificatesSecretName() string { + return m.GetSecurity().MemberCertificateSecretName(m.Name()) +} + +func (m *AppDBSpec) GetName() string { + return m.Name() +} + +func (m *AppDBSpec) GetNamespace() string { + return m.Namespace +} + +func (m *AppDBSpec) DataVolumeName() string { + return "data" +} + +func (m *AppDBSpec) LogsVolumeName() string { + return "logs" +} + +func (m *AppDBSpec) NeedsAutomationConfigVolume() bool { + return !vault.IsVaultSecretBackend() +} + +func (m *AppDBSpec) AutomationConfigConfigMapName() string { + return fmt.Sprintf("%s-automation-config-version", m.Name()) +} + +func (m *AppDBSpec) MonitoringAutomationConfigConfigMapName() string { + return fmt.Sprintf("%s-monitoring-automation-config-version", m.Name()) +} + +// GetSecretsMountedIntoPod returns the list of strings mounted into the pod that we need to watch. +func (m *AppDBSpec) GetSecretsMountedIntoPod() []string { + secrets := []string{} + if m.PasswordSecretKeyRef != nil { + secrets = append(secrets, m.PasswordSecretKeyRef.Name) + } + + if m.Security.IsTLSEnabled() { + secrets = append(secrets, m.GetTlsCertificatesSecretName()) + } + return secrets +} + +func (m *AppDBSpec) GetMemberClusterSpecByName(memberClusterName string) mdbv1.ClusterSpecItem { + for _, clusterSpec := range m.GetClusterSpecList() { + if clusterSpec.ClusterName == memberClusterName { + return clusterSpec + } + } + + // In case the member cluster is not found in the cluster spec list, we return an empty ClusterSpecItem + // with 0 members to handle the case of removing a cluster from the spec list without a panic. + return mdbv1.ClusterSpecItem{ + ClusterName: memberClusterName, + Members: 0, + } +} + +func (m *AppDBSpec) BuildConnectionURL(username, password string, scheme connectionstring.Scheme, connectionParams map[string]string, multiClusterHostnames []string) string { + builder := connectionstring.Builder(). + SetName(m.Name()). + SetNamespace(m.Namespace). + SetUsername(username). + SetPassword(password). + SetReplicas(m.Replicas()). + SetService(m.ServiceName()). + SetVersion(m.GetMongoDBVersion()). + SetAuthenticationModes(m.GetSecurityAuthenticationModes()). + SetClusterDomain(m.GetClusterDomain()). + SetExternalDomain(m.GetExternalDomain()). + SetIsReplicaSet(true). + SetIsTLSEnabled(m.IsSecurityTLSConfigEnabled()). + SetConnectionParams(connectionParams). + SetScheme(scheme) + + if m.IsMultiCluster() { + builder.SetReplicas(len(multiClusterHostnames)) + builder.SetMultiClusterHosts(multiClusterHostnames) + } + + return builder.Build() +} + +func (m *AppDBSpec) GetClusterSpecList() mdbv1.ClusterSpecList { + if m.IsMultiCluster() { + return m.ClusterSpecList + } else { + return mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: m.Members, + MemberConfig: m.GetMemberOptions(), + }, + } + } +} + +func (m *AppDBSpec) IsMultiCluster() bool { + return m.Topology == ClusterTopologyMultiCluster +} + +func (m *AppDBSpec) NameForCluster(memberClusterNum int) string { + if !m.IsMultiCluster() { + return m.GetName() + } + + return fmt.Sprintf("%s-%d", m.GetName(), memberClusterNum) +} + +func (m *AppDBSpec) HeadlessServiceSelectorAppLabel(memberClusterNum int) string { + return m.HeadlessServiceNameForCluster(memberClusterNum) +} + +func (m *AppDBSpec) HeadlessServiceNameForCluster(memberClusterNum int) string { + if !m.IsMultiCluster() { + return m.ServiceName() + } + + if m.Service == "" { + return dns.GetMultiHeadlessServiceName(m.GetName(), memberClusterNum) + } + + return fmt.Sprintf("%s-%d", m.Service, memberClusterNum) +} + +func GetAppDBCaPemPath() string { + return util.AppDBMmsCaFileDirInContainer + "ca-pem" +} + +func (m *AppDBSpec) GetPodName(clusterIdx int, podIdx int) string { + if m.IsMultiCluster() { + return dns.GetMultiPodName(m.Name(), clusterIdx, podIdx) + } + return dns.GetPodName(m.Name(), podIdx) +} + +func (m *AppDBSpec) GetExternalServiceName(clusterIdx int, podIdx int) string { + if m.IsMultiCluster() { + return dns.GetMultiExternalServiceName(m.GetName(), clusterIdx, podIdx) + } + return dns.GetExternalServiceName(m.Name(), podIdx) +} + +func (m *AppDBSpec) GetExternalAccessConfiguration() *mdbv1.ExternalAccessConfiguration { + return m.ExternalAccessConfiguration +} + +func (m *AppDBSpec) GetExternalDomain() *string { + if m.ExternalAccessConfiguration != nil { + return m.ExternalAccessConfiguration.ExternalDomain + } + return nil +} + +func (m *AppDBSpec) GetExternalAccessConfigurationForMemberCluster(clusterName string) *mdbv1.ExternalAccessConfiguration { + for _, csl := range m.ClusterSpecList { + if csl.ClusterName == clusterName && csl.ExternalAccessConfiguration != nil { + return csl.ExternalAccessConfiguration + } + } + + return m.ExternalAccessConfiguration +} + +func (m *AppDBSpec) GetExternalDomainForMemberCluster(clusterName string) *string { + if cfg := m.GetExternalAccessConfigurationForMemberCluster(clusterName); cfg != nil { + if externalDomain := cfg.ExternalDomain; externalDomain != nil { + return externalDomain + } + } + + return m.GetExternalDomain() +} diff --git a/api/v1/om/doc.go b/api/v1/om/doc.go new file mode 100644 index 000000000..13d6e428a --- /dev/null +++ b/api/v1/om/doc.go @@ -0,0 +1,4 @@ +package om + +// +k8s:deepcopy-gen=package +// +versionName=v1 diff --git a/api/v1/om/groupversion_info.go b/api/v1/om/groupversion_info.go new file mode 100644 index 000000000..0097c2ffb --- /dev/null +++ b/api/v1/om/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1 contains API Schema definitions for the mongodb v1 API group +// +kubebuilder:object:generate=true +// +groupName=mongodb.com +package om + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "mongodb.com", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1/om/opsmanager_types.go b/api/v1/om/opsmanager_types.go new file mode 100644 index 000000000..db8ad0655 --- /dev/null +++ b/api/v1/om/opsmanager_types.go @@ -0,0 +1,1108 @@ +package om + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + + mdbc "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/fcv" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func init() { + v1.SchemeBuilder.Register(&MongoDBOpsManager{}, &MongoDBOpsManagerList{}) +} + +const ( + queryableBackupConfigPath string = "brs.queryable.proxyPort" + debuggingPortConfigPath string = "mms.k8s.debuggingPort" + queryableBackupDefaultPort int32 = 25999 +) + +// The MongoDBOpsManager resource allows you to deploy Ops Manager within your Kubernetes cluster +// +k8s:deepcopy-gen=true +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=opsmanagers,scope=Namespaced,shortName=om,singular=opsmanager +// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".spec.replicas",description="The number of replicas of MongoDBOpsManager." +// +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.version",description="The version of MongoDBOpsManager." +// +kubebuilder:printcolumn:name="State (OpsManager)",type="string",JSONPath=".status.opsManager.phase",description="The current state of the MongoDBOpsManager." +// +kubebuilder:printcolumn:name="State (AppDB)",type="string",JSONPath=".status.applicationDatabase.phase",description="The current state of the MongoDBOpsManager Application Database." +// +kubebuilder:printcolumn:name="State (Backup)",type="string",JSONPath=".status.backup.phase",description="The current state of the MongoDBOpsManager Backup Daemon." +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="The time since the MongoDBOpsManager resource was created." +// +kubebuilder:printcolumn:name="Warnings",type="string",JSONPath=".status.warnings",description="Warnings." +type MongoDBOpsManager struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec MongoDBOpsManagerSpec `json:"spec"` + // +optional + Status MongoDBOpsManagerStatus `json:"status"` +} + +func (om *MongoDBOpsManager) GetAppDBProjectConfig(ctx context.Context, secretClient secrets.SecretClient, client kubernetesClient.Client) (mdbv1.ProjectConfig, error) { + var operatorVaultSecretPath string + if secretClient.VaultClient != nil { + operatorVaultSecretPath = secretClient.VaultClient.OperatorSecretPath() + } + secretName, err := om.APIKeySecretName(ctx, secretClient, operatorVaultSecretPath) + if err != nil { + return mdbv1.ProjectConfig{}, err + } + + if om.IsTLSEnabled() { + opsManagerCA := om.Spec.GetOpsManagerCA() + cm, err := client.GetConfigMap(ctx, kube.ObjectKey(om.Namespace, opsManagerCA)) + if err != nil { + return mdbv1.ProjectConfig{}, err + } + ca := cm.Data["mms-ca.crt"] + return mdbv1.ProjectConfig{ + BaseURL: om.CentralURL(), + ProjectName: om.Spec.AppDB.Name(), + Credentials: secretName, + UseCustomCA: true, + SSLProjectConfig: env.SSLProjectConfig{ + SSLRequireValidMMSServerCertificates: true, + SSLMMSCAConfigMap: opsManagerCA, + SSLMMSCAConfigMapContents: ca, + }, + }, nil + } + + return mdbv1.ProjectConfig{ + BaseURL: om.CentralURL(), + ProjectName: om.Spec.AppDB.Name(), + Credentials: secretName, + }, nil +} + +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type MongoDBOpsManagerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []MongoDBOpsManager `json:"items"` +} + +type MongoDBOpsManagerSpec struct { + // The configuration properties passed to Ops Manager/Backup Daemon + // +optional + Configuration map[string]string `json:"configuration,omitempty"` + + Version string `json:"version"` + // +optional + // +kubebuilder:validation:Minimum=1 + Replicas int `json:"replicas"` + // Deprecated: This has been replaced by the ClusterDomain which should be + // used instead + // +kubebuilder:validation:Format="hostname" + ClusterName string `json:"clusterName,omitempty"` + // +optional + // +kubebuilder:validation:Format="hostname" + ClusterDomain string `json:"clusterDomain,omitempty"` + + // AdminSecret is the secret for the first admin user to create + // has the fields: "Username", "Password", "FirstName", "LastName" + AdminSecret string `json:"adminCredentials,omitempty"` + AppDB AppDBSpec `json:"applicationDatabase"` + + Logging *Logging `json:"logging,omitempty"` + // Custom JVM parameters passed to the Ops Manager JVM + // +optional + JVMParams []string `json:"jvmParameters,omitempty"` + + // Backup + // +optional + Backup *MongoDBOpsManagerBackup `json:"backup,omitempty"` + + // InternalConnectivity if set allows for overriding the settings of the default service + // used for internal connectivity to the OpsManager servers. + // +optional + InternalConnectivity *MongoDBOpsManagerServiceDefinition `json:"internalConnectivity,omitempty"` + + // MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + // accessing this Ops Manager resource from outside the Kubernetes cluster. + // +optional + MongoDBOpsManagerExternalConnectivity *MongoDBOpsManagerServiceDefinition `json:"externalConnectivity,omitempty"` + + // Configure HTTPS. + // +optional + Security *MongoDBOpsManagerSecurity `json:"security,omitempty"` + + // Configure custom StatefulSet configuration + // +optional + StatefulSetConfiguration *mdbc.StatefulSetConfiguration `json:"statefulSet,omitempty"` + + // Topology sets the desired cluster topology of Ops Manager deployment. + // It defaults (and if not set) to SingleCluster. If MultiCluster specified, + // then clusterSpecList field is mandatory and at least one member cluster has to be specified. + // +kubebuilder:validation:Enum=SingleCluster;MultiCluster + // +optional + Topology string `json:"topology,omitempty"` + + // +optional + ClusterSpecList []ClusterSpecOMItem `json:"clusterSpecList,omitempty"` + + // OpsManagerURL specified the URL with which the operator and AppDB monitoring agent should access Ops Manager instance (or instances). + // When not set, the operator is using FQDN of Ops Manager's headless service `{name}-svc.{namespace}.svc.cluster.local` to connect to the instance. If that URL cannot be used, then URL in this field should be provided for the operator to connect to Ops Manager instances. + // +optional + OpsManagerURL string `json:"opsManagerURL,omitempty"` +} + +type Logging struct { + // LogBackAccessRef points at a ConfigMap/key with the logback access configuration file to mount on the Pod + LogBackAccessRef *mdbv1.ConfigMapRef `json:"LogBackAccessRef,omitempty"` + + // LogBackRef points at a ConfigMap/key with the logback configuration file to mount on the Pod + LogBackRef *mdbv1.ConfigMapRef `json:"LogBackRef,omitempty"` +} + +// ClusterSpecOMItem defines members cluster details for Ops Manager multi-cluster deployment. +type ClusterSpecOMItem struct { + // ClusterName is name of the cluster where the Ops Manager Statefulset will be scheduled. + // The operator is using ClusterName to find API credentials in `mongodb-enterprise-operator-member-list` config map to use for this member cluster. + // If the credentials are not found, then the member cluster is considered unreachable and ignored in the reconcile process. + // +kubebuilder:validation:Required + ClusterName string `json:"clusterName,omitempty"` + + // +kubebuilder:validation:Required + // Number of Ops Manager instances in this member cluster. + Members int `json:"members"` + + // Cluster domain to override the default *.svc.cluster.local if the default cluster domain has been changed on a cluster level. + // +optional + // +kubebuilder:validation:Format="hostname" + ClusterDomain string `json:"clusterDomain,omitempty"` + + // The configuration properties passed to Ops Manager and Backup Daemon in this cluster. + // If specified (not empty) then this field overrides `spec.configuration` field entirely. + // If not specified, then `spec.configuration` field is used for the Ops Manager and Backup Daemon instances in this cluster. + // +optional + Configuration map[string]string `json:"configuration,omitempty"` + + // JVM parameters to pass to Ops Manager and Backup Daemon instances in this member cluster. + // If specified (not empty) then this field overrides `spec.jvmParameters` field entirely. + // If not specified, then `spec.jvmParameters` field is used for the Ops Manager and Backup Daemon instances in this cluster. + // +optional + JVMParams []string `json:"jvmParameters,omitempty"` + + // MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + // accessing Ops Manager instances in this member cluster from outside the Kubernetes cluster. + // If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + // If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + // +optional + MongoDBOpsManagerExternalConnectivity *MongoDBOpsManagerServiceDefinition `json:"externalConnectivity,omitempty"` + + // Configure custom StatefulSet configuration to override in Ops Manager's statefulset in this member cluster. + // If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + // If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + // +optional + StatefulSetConfiguration *mdbc.StatefulSetConfiguration `json:"statefulSet,omitempty"` + + // Backup contains settings to override from top-level `spec.backup` for this member cluster. + // If the value is not set here, then the value is taken from `spec.backup`. + // +optional + Backup *MongoDBOpsManagerBackupClusterSpecItem `json:"backup,omitempty"` + + // Legacy if true switches to using legacy, single-cluster naming convention for that cluster. + // Define to true if this cluster contains existing OM deployment that needs to be migrated to multi-cluster topology. + Legacy bool `json:"-"` +} + +func (ms *ClusterSpecOMItem) GetStatefulSetSpecOverride() *appsv1.StatefulSetSpec { + if ms != nil && ms.StatefulSetConfiguration != nil { + return ms.StatefulSetConfiguration.SpecWrapper.Spec.DeepCopy() + } + return nil +} + +func (ms *ClusterSpecOMItem) GetBackupStatefulSetSpecOverride() *appsv1.StatefulSetSpec { + if ms != nil && ms.Backup != nil && ms.Backup.StatefulSetConfiguration != nil { + return ms.Backup.StatefulSetConfiguration.SpecWrapper.Spec.DeepCopy() + } + return nil +} + +type MongoDBOpsManagerSecurity struct { + // +optional + TLS MongoDBOpsManagerTLS `json:"tls"` + + // +optional + CertificatesSecretsPrefix string `json:"certsSecretPrefix"` +} + +type MongoDBOpsManagerTLS struct { + // +optional + SecretRef TLSSecretRef `json:"secretRef"` + // +optional + CA string `json:"ca"` +} + +type TLSSecretRef struct { + // +kubebuilder:validation:Required + Name string `json:"name"` +} + +func (ms MongoDBOpsManagerSpec) IsKmipEnabled() bool { + if ms.Backup == nil || !ms.Backup.Enabled || ms.Backup.Encryption == nil || ms.Backup.Encryption.Kmip == nil { + return false + } + return true +} + +func (ms MongoDBOpsManagerSpec) GetClusterDomain() string { + if ms.ClusterDomain != "" { + return ms.ClusterDomain + } + if ms.ClusterName != "" { + return ms.ClusterName + } + return "cluster.local" +} + +func (ms MongoDBOpsManagerSpec) GetOpsManagerCA() string { + if ms.Security != nil { + return ms.Security.TLS.CA + } + return "" +} + +func (ms MongoDBOpsManagerSpec) GetAppDbCA() string { + if ms.AppDB.Security != nil && ms.AppDB.Security.TLSConfig != nil { + return ms.AppDB.Security.TLSConfig.CA + } + return "" +} + +func (ms *MongoDBOpsManagerSpec) IsMultiCluster() bool { + return ms.Topology == ClusterTopologyMultiCluster +} + +func (ms *MongoDBOpsManagerSpec) GetClusterStatusList() []status.OMClusterStatusItem { + clusterStatuses := make([]status.OMClusterStatusItem, 0) + for _, item := range ms.ClusterSpecList { + clusterStatuses = append(clusterStatuses, status.OMClusterStatusItem{ClusterName: item.ClusterName, Replicas: item.Members}) + } + return clusterStatuses +} + +func (ms *MongoDBOpsManagerSpec) GetBackupClusterStatusList() []status.OMClusterStatusItem { + clusterStatuses := make([]status.OMClusterStatusItem, 0) + for _, item := range ms.ClusterSpecList { + if item.Backup != nil { + clusterStatuses = append(clusterStatuses, status.OMClusterStatusItem{ClusterName: item.ClusterName, Replicas: item.Backup.Members}) + } + } + return clusterStatuses +} + +// GetTotalReplicas gets the number of OpsManager replicas, taking into account all the member cluster in the case of a multicluster deployment. +func (ms *MongoDBOpsManagerSpec) GetTotalReplicas() int { + if ms.IsMultiCluster() { + replicas := 0 + for _, item := range ms.ClusterSpecList { + replicas += item.Members + } + return replicas + } + return ms.Replicas +} + +func (om *MongoDBOpsManager) ObjectKey() client.ObjectKey { + return kube.ObjectKey(om.Namespace, om.Name) +} + +func (om *MongoDBOpsManager) AppDBStatefulSetObjectKey(memberClusterNum int) client.ObjectKey { + return kube.ObjectKey(om.Namespace, om.Spec.AppDB.NameForCluster(memberClusterNum)) +} + +// MongoDBOpsManagerServiceDefinition struct that defines the mechanism by which this Ops Manager resource +// is exposed, via a Service, to the outside of the Kubernetes Cluster. +type MongoDBOpsManagerServiceDefinition struct { + // Type of the `Service` to be created. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=LoadBalancer;NodePort;ClusterIP + Type corev1.ServiceType `json:"type"` + + // Port in which this `Service` will listen to, this applies to `NodePort`. + Port int32 `json:"port,omitempty"` + + // LoadBalancerIP IP that will be assigned to this LoadBalancer. + LoadBalancerIP string `json:"loadBalancerIP,omitempty"` + + // ClusterIP IP that will be assigned to this Service when creating a ClusterIP type Service + // +optional + ClusterIP *string `json:"clusterIP,omitempty"` + + // ExternalTrafficPolicy mechanism to preserve the client source IP. + // Only supported on GCE and Google Kubernetes Engine. + // +kubebuilder:validation:Enum=Cluster;Local + ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty"` + + // Annotations is a list of annotations to be directly passed to the Service object. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// MongoDBOpsManagerBackup backup structure for Ops Manager resources +type MongoDBOpsManagerBackup struct { + // Enabled indicates if Backups will be enabled for this Ops Manager. + Enabled bool `json:"enabled"` + ExternalServiceEnabled *bool `json:"externalServiceEnabled,omitempty"` + + // Members indicate the number of backup daemon pods to create. + // +optional + // +kubebuilder:validation:Minimum=1 + Members int `json:"members,omitempty"` + + // Assignment Labels set in the Ops Manager + // +optional + AssignmentLabels []string `json:"assignmentLabels,omitempty"` + + // HeadDB specifies configuration options for the HeadDB + HeadDB *mdbv1.PersistenceConfig `json:"headDB,omitempty"` + JVMParams []string `json:"jvmParameters,omitempty"` + + // S3OplogStoreConfigs describes the list of s3 oplog store configs used for backup. + S3OplogStoreConfigs []S3Config `json:"s3OpLogStores,omitempty"` + + // OplogStoreConfigs describes the list of oplog store configs used for backup + OplogStoreConfigs []DataStoreConfig `json:"opLogStores,omitempty"` + BlockStoreConfigs []DataStoreConfig `json:"blockStores,omitempty"` + S3Configs []S3Config `json:"s3Stores,omitempty"` + FileSystemStoreConfigs []FileSystemStoreConfig `json:"fileSystemStores,omitempty"` + StatefulSetConfiguration *mdbc.StatefulSetConfiguration `json:"statefulSet,omitempty"` + + // QueryableBackupSecretRef references the secret which contains the pem file which is used + // for queryable backup. This will be mounted into the Ops Manager pod. + // +optional + QueryableBackupSecretRef SecretRef `json:"queryableBackupSecretRef,omitempty"` + + // Encryption settings + // +optional + Encryption *Encryption `json:"encryption,omitempty"` + + Logging *Logging `json:"logging,omitempty"` +} + +// MongoDBOpsManagerBackupClusterSpecItem backup structure for overriding top-level backup definition in Ops Manager's clusterSpecList. +type MongoDBOpsManagerBackupClusterSpecItem struct { + // Members indicate the number of backup daemon pods to create. + // +required + // +kubebuilder:validation:Minimum=0 + Members int `json:"members,omitempty"` + + // Assignment Labels set in the Ops Manager + // +optional + AssignmentLabels []string `json:"assignmentLabels,omitempty"` + + // HeadDB specifies configuration options for the HeadDB + HeadDB *mdbv1.PersistenceConfig `json:"headDB,omitempty"` + JVMParams []string `json:"jvmParameters,omitempty"` + + // StatefulSetConfiguration specified optional overrides for backup datemon statefulset. + // +optional + StatefulSetConfiguration *mdbc.StatefulSetConfiguration `json:"statefulSet,omitempty"` +} + +// Encryption contains encryption settings +type Encryption struct { + // Kmip corresponds to the KMIP configuration assigned to the Ops Manager Project's configuration. + // +optional + Kmip *KmipConfig `json:"kmip,omitempty"` +} + +// KmipConfig contains Project-level KMIP configuration +type KmipConfig struct { + // KMIP Server configuration + Server v1.KmipServerConfig `json:"server"` +} + +type MongoDBOpsManagerStatus struct { + OpsManagerStatus OpsManagerStatus `json:"opsManager,omitempty"` + AppDbStatus AppDbStatus `json:"applicationDatabase,omitempty"` + BackupStatus BackupStatus `json:"backup,omitempty"` +} + +type OpsManagerStatus struct { + status.Common `json:",inline"` + Replicas int `json:"replicas,omitempty"` + Version string `json:"version,omitempty"` + Url string `json:"url,omitempty"` + Warnings []status.Warning `json:"warnings,omitempty"` + ClusterStatusList []status.OMClusterStatusItem `json:"clusterStatusList,omitempty"` +} + +type AgentVersion struct { + AgentVersion string `json:"agent_version"` +} + +type OpsManagerVersion string + +type OpsManagerVersionMapping struct { + OpsManager map[OpsManagerVersion]AgentVersion `json:"ops_manager"` + CloudManager string `json:"cloud_manager"` +} + +type AppDbStatus struct { + mdbv1.MongoDbStatus `json:",inline"` + ClusterStatusList []status.ClusterStatusItem `json:"clusterStatusList,omitempty"` +} + +type BackupStatus struct { + status.Common `json:",inline"` + Version string `json:"version,omitempty"` + Warnings []status.Warning `json:"warnings,omitempty"` + ClusterStatusList []status.OMClusterStatusItem `json:"clusterStatusList,omitempty"` +} + +type FileSystemStoreConfig struct { + Name string `json:"name"` +} + +// DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores +// Optionally references the user if the Mongodb is configured with authentication +type DataStoreConfig struct { + Name string `json:"name"` + MongoDBResourceRef userv1.MongoDBResourceRef `json:"mongodbResourceRef"` + MongoDBUserRef *MongoDBUserRef `json:"mongodbUserRef,omitempty"` + // Assignment Labels set in the Ops Manager + // +optional + AssignmentLabels []string `json:"assignmentLabels,omitempty"` +} + +func (f DataStoreConfig) Identifier() interface{} { + return f.Name +} + +type SecretRef struct { + Name string `json:"name"` +} + +type S3Config struct { + MongoDBResourceRef *userv1.MongoDBResourceRef `json:"mongodbResourceRef,omitempty"` + MongoDBUserRef *MongoDBUserRef `json:"mongodbUserRef,omitempty"` + // S3SecretRef is the secret that contains the AWS credentials used to access S3 + // It is optional because the credentials can be provided via AWS IRSA + // +optional + S3SecretRef *SecretRef `json:"s3SecretRef,omitempty"` + Name string `json:"name"` + PathStyleAccessEnabled bool `json:"pathStyleAccessEnabled"` + S3BucketEndpoint string `json:"s3BucketEndpoint"` + S3BucketName string `json:"s3BucketName"` + // +optional + S3RegionOverride string `json:"s3RegionOverride"` + // Set this to "true" to use the appDBCa as a CA to access S3. + // Deprecated: This has been replaced by CustomCertificateSecretRefs, + // In the future all custom certificates, which includes the appDBCa + // for s3Config should be configured in CustomCertificateSecretRefs instead. + // +optional + CustomCertificate bool `json:"customCertificate"` + // This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + // S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + // +optional + IRSAEnabled bool `json:"irsaEnabled"` + // Assignment Labels set in the Ops Manager + // +optional + AssignmentLabels []string `json:"assignmentLabels"` + // CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + // that apply to the associated S3 bucket. + // +optional + CustomCertificateSecretRefs []corev1.SecretKeySelector `json:"customCertificateSecretRefs"` +} + +func (s S3Config) Identifier() interface{} { + return s.Name +} + +// MongodbResourceObjectKey returns the "name-namespace" object key. Uses the AppDB name if the mongodb resource is not +// specified +func (s S3Config) MongodbResourceObjectKey(opsManager *MongoDBOpsManager) client.ObjectKey { + ns := opsManager.Namespace + if s.MongoDBResourceRef == nil { + return client.ObjectKey{} + } + if s.MongoDBResourceRef.Namespace != "" { + ns = s.MongoDBResourceRef.Namespace + } + return client.ObjectKey{Name: s.MongoDBResourceRef.Name, Namespace: ns} +} + +func (s S3Config) MongodbUserObjectKey(defaultNamespace string) client.ObjectKey { + ns := defaultNamespace + if s.MongoDBResourceRef == nil { + return client.ObjectKey{} + } + if s.MongoDBResourceRef.Namespace != "" { + ns = s.MongoDBResourceRef.Namespace + } + return client.ObjectKey{Name: s.MongoDBUserRef.Name, Namespace: ns} +} + +// MongodbResourceObjectKey returns the object key for the mongodb resource referenced by the dataStoreConfig. +// It uses the "parent" object namespace if it is not overriden by 'MongoDBResourceRef.namespace' +func (f *DataStoreConfig) MongodbResourceObjectKey(defaultNamespace string) client.ObjectKey { + ns := defaultNamespace + if f.MongoDBResourceRef.Namespace != "" { + ns = f.MongoDBResourceRef.Namespace + } + return client.ObjectKey{Name: f.MongoDBResourceRef.Name, Namespace: ns} +} + +func (f *DataStoreConfig) MongodbUserObjectKey(defaultNamespace string) client.ObjectKey { + ns := defaultNamespace + if f.MongoDBResourceRef.Namespace != "" { + ns = f.MongoDBResourceRef.Namespace + } + return client.ObjectKey{Name: f.MongoDBUserRef.Name, Namespace: ns} +} + +type MongoDBUserRef struct { + Name string `json:"name"` +} + +func (om *MongoDBOpsManager) UnmarshalJSON(data []byte) error { + type MongoDBJSON *MongoDBOpsManager + if err := json.Unmarshal(data, (MongoDBJSON)(om)); err != nil { + return err + } + om.InitDefaultFields() + + return nil +} + +func (om *MongoDBOpsManager) InitDefaultFields() { + // providing backward compatibility for the deployments which didn't specify the 'replicas' before Operator 1.3.1 + // This doesn't update the object in Api server so the real spec won't change + // All newly created resources will pass through the normal validation so 'replicas' will never be 0 + if om.Spec.Replicas == 0 { + om.Spec.Replicas = 1 + } + + if om.Spec.Backup == nil { + om.Spec.Backup = newBackup() + } + + if om.Spec.Backup.Members == 0 { + om.Spec.Backup.Members = 1 + } + + om.Spec.AppDB.Security = ensureSecurityWithSCRAM(om.Spec.AppDB.Security) + + // setting ops manager name, namespace and ClusterDomain for the appdb (transient fields) + om.Spec.AppDB.OpsManagerName = om.Name + om.Spec.AppDB.Namespace = om.Namespace + om.Spec.AppDB.ClusterDomain = om.Spec.GetClusterDomain() + om.Spec.AppDB.ResourceType = mdbv1.ReplicaSet +} + +func ensureSecurityWithSCRAM(specSecurity *mdbv1.Security) *mdbv1.Security { + if specSecurity == nil { + specSecurity = &mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{}} + } + // the only allowed authentication is SCRAM - it's implicit to the user and hidden from him + specSecurity.Authentication = &mdbv1.Authentication{Modes: []mdbv1.AuthMode{util.SCRAM}} + return specSecurity +} + +func (om *MongoDBOpsManager) SvcName() string { + return om.Name + "-svc" +} + +func (om *MongoDBOpsManager) ExternalSvcName() string { + return om.SvcName() + "-ext" +} + +func (om *MongoDBOpsManager) AppDBMongoConnectionStringSecretName() string { + return om.Spec.AppDB.Name() + "-connection-string" +} + +func (om *MongoDBOpsManager) BackupDaemonServiceName() string { + return om.BackupDaemonStatefulSetName() + "-svc" +} + +func (om *MongoDBOpsManager) BackupDaemonHeadlessServiceNameForClusterIndex(clusterIndex int) string { + return fmt.Sprintf("%s-svc", om.BackupDaemonStatefulSetNameForClusterIndex(clusterIndex)) +} + +func (ms MongoDBOpsManagerSpec) BackupDaemonSvcPort() (int32, error) { + if port, ok := ms.Configuration[queryableBackupConfigPath]; ok { + val, err := strconv.ParseInt(port, 10, 32) + if err != nil { + return -1, err + } + return int32(val), nil + } + return queryableBackupDefaultPort, nil +} + +func (ms MongoDBOpsManagerSpec) DebugPort() (int32, error) { + if port, ok := ms.Configuration[debuggingPortConfigPath]; ok { + val, err := strconv.ParseInt(port, 10, 32) + if err != nil { + return 0, fmt.Errorf("failed to parse debugging port %s: %w", port, err) + } + return int32(val), nil + } + return 0, nil +} + +func (om *MongoDBOpsManager) AddConfigIfDoesntExist(key, value string) bool { + if om.Spec.Configuration == nil { + om.Spec.Configuration = make(map[string]string) + } + if _, ok := om.Spec.Configuration[key]; !ok { + om.Spec.Configuration[key] = value + return true + } + return false +} + +func (om *MongoDBOpsManager) UpdateStatus(phase status.Phase, statusOptions ...status.Option) { + var statusPart status.Part + if option, exists := status.GetOption(statusOptions, status.OMPartOption{}); exists { + statusPart = option.(status.OMPartOption).StatusPart + } + + switch statusPart { + case status.AppDb: + om.updateStatusAppDb(phase, statusOptions...) + case status.OpsManager: + om.updateStatusOpsManager(phase, statusOptions...) + case status.Backup: + om.updateStatusBackup(phase, statusOptions...) + case status.None: + + } +} + +func (om *MongoDBOpsManager) updateStatusAppDb(phase status.Phase, statusOptions ...status.Option) { + om.Status.AppDbStatus.UpdateCommonFields(phase, om.GetGeneration(), statusOptions...) + + if option, exists := status.GetOption(statusOptions, status.ReplicaSetMembersOption{}); exists { + om.Status.AppDbStatus.Members = option.(status.ReplicaSetMembersOption).Members + } + + if option, exists := status.GetOption(statusOptions, status.MultiReplicaSetMemberOption{}); exists { + om.Status.AppDbStatus.Members = option.(status.MultiReplicaSetMemberOption).Members + om.Status.AppDbStatus.ClusterStatusList = option.(status.MultiReplicaSetMemberOption).ClusterStatusList + } + + if option, exists := status.GetOption(statusOptions, status.WarningsOption{}); exists { + om.Status.AppDbStatus.Warnings = append(om.Status.AppDbStatus.Warnings, option.(status.WarningsOption).Warnings...) + } + + if phase == status.PhaseRunning { + spec := om.Spec.AppDB + om.Status.AppDbStatus.FeatureCompatibilityVersion = om.CalculateFeatureCompatibilityVersion() + om.Status.AppDbStatus.Version = spec.GetMongoDBVersion() + om.Status.AppDbStatus.Message = "" + } +} + +func (om *MongoDBOpsManager) updateStatusOpsManager(phase status.Phase, statusOptions ...status.Option) { + om.Status.OpsManagerStatus.UpdateCommonFields(phase, om.GetGeneration(), statusOptions...) + + if option, exists := status.GetOption(statusOptions, status.BaseUrlOption{}); exists { + om.Status.OpsManagerStatus.Url = option.(status.BaseUrlOption).BaseUrl + } + + if option, exists := status.GetOption(statusOptions, status.WarningsOption{}); exists { + om.Status.OpsManagerStatus.Warnings = append(om.Status.OpsManagerStatus.Warnings, option.(status.WarningsOption).Warnings...) + } + + if phase == status.PhaseRunning { + om.Status.OpsManagerStatus.Replicas = om.Spec.GetTotalReplicas() + om.Status.OpsManagerStatus.ClusterStatusList = om.Spec.GetClusterStatusList() + om.Status.OpsManagerStatus.Version = om.Spec.Version + om.Status.OpsManagerStatus.Message = "" + } +} + +func (om *MongoDBOpsManager) updateStatusBackup(phase status.Phase, statusOptions ...status.Option) { + om.Status.BackupStatus.UpdateCommonFields(phase, om.GetGeneration(), statusOptions...) + + if option, exists := status.GetOption(statusOptions, status.WarningsOption{}); exists { + om.Status.BackupStatus.Warnings = append(om.Status.BackupStatus.Warnings, option.(status.WarningsOption).Warnings...) + } + if phase == status.PhaseRunning { + om.Status.BackupStatus.Message = "" + om.Status.BackupStatus.Version = om.Spec.Version + om.Status.BackupStatus.ClusterStatusList = om.Spec.GetBackupClusterStatusList() + } +} + +func (om *MongoDBOpsManager) SetWarnings(warnings []status.Warning, options ...status.Option) { + for _, part := range getPartsFromStatusOptions(options...) { + switch part { + case status.OpsManager: + om.Status.OpsManagerStatus.Warnings = warnings + case status.Backup: + om.Status.BackupStatus.Warnings = warnings + case status.AppDb: + om.Status.AppDbStatus.Warnings = warnings + default: + } + } +} + +func (om *MongoDBOpsManager) AddOpsManagerWarningIfNotExists(warning status.Warning) { + om.Status.OpsManagerStatus.Warnings = status.Warnings(om.Status.OpsManagerStatus.Warnings).AddIfNotExists(warning) +} + +func (om *MongoDBOpsManager) AddAppDBWarningIfNotExists(warning status.Warning) { + om.Status.AppDbStatus.Warnings = status.Warnings(om.Status.AppDbStatus.Warnings).AddIfNotExists(warning) +} + +func (om *MongoDBOpsManager) AddBackupWarningIfNotExists(warning status.Warning) { + om.Status.BackupStatus.Warnings = status.Warnings(om.Status.BackupStatus.Warnings).AddIfNotExists(warning) +} + +func (om *MongoDBOpsManager) GetStatus(options ...status.Option) interface{} { + if part, exists := status.GetOption(options, status.OMPartOption{}); exists { + switch part.Value().(status.Part) { + case status.OpsManager: + return om.Status.OpsManagerStatus + case status.AppDb: + return om.Status.AppDbStatus + case status.Backup: + return om.Status.BackupStatus + default: + } + } + return om.Status +} + +func (om *MongoDBOpsManager) GetStatusWarnings(part status.Part) []status.Warning { + switch part { + case status.OpsManager: + return om.Status.OpsManagerStatus.Warnings + case status.AppDb: + return om.Status.AppDbStatus.Warnings + case status.Backup: + return om.Status.BackupStatus.Warnings + default: + return []status.Warning{} + } +} + +func (om *MongoDBOpsManager) GetCommonStatus(options ...status.Option) *status.Common { + if part, exists := status.GetOption(options, status.OMPartOption{}); exists { + switch part.Value().(status.Part) { + case status.OpsManager: + return &om.Status.OpsManagerStatus.Common + case status.AppDb: + return &om.Status.AppDbStatus.Common + case status.Backup: + return &om.Status.BackupStatus.Common + default: + } + } + return nil +} + +func (om *MongoDBOpsManager) GetPhase() status.Phase { + return om.Status.OpsManagerStatus.Phase +} + +func (om *MongoDBOpsManager) GetStatusPath(options ...status.Option) string { + if part, exists := status.GetOption(options, status.OMPartOption{}); exists { + switch part.Value().(status.Part) { + case status.OpsManager: + return "/status/opsManager" + case status.AppDb: + return "/status/applicationDatabase" + case status.Backup: + return "/status/backup" + default: + } + } + // we should never actually reach this + return "/status" +} + +// APIKeySecretName returns the secret object name to store the API key to communicate to ops-manager. +// To ensure backward compatibility, it checks if a secret key is present with the old format name({$ops-manager-name}-admin-key), +// if not it returns the new name format ({$ops-manager-namespace}-${ops-manager-name}-admin-key), to have multiple om deployments +// with the same name. +func (om *MongoDBOpsManager) APIKeySecretName(ctx context.Context, client secrets.SecretClientInterface, operatorSecretPath string) (string, error) { + oldAPISecretName := fmt.Sprintf("%s-admin-key", om.Name) + operatorNamespace := env.ReadOrPanic(util.CurrentNamespace) // nolint:forbidigo + oldAPIKeySecretNamespacedName := types.NamespacedName{Name: oldAPISecretName, Namespace: operatorNamespace} + + _, err := client.ReadSecret(ctx, oldAPIKeySecretNamespacedName, fmt.Sprintf("%s/%s/%s", operatorSecretPath, operatorNamespace, oldAPISecretName)) + if err != nil { + if secrets.SecretNotExist(err) { + return fmt.Sprintf("%s-%s-admin-key", om.Namespace, om.Name), nil + } + + return "", err + } + return oldAPISecretName, nil +} + +func (om *MongoDBOpsManager) GetSecurity() MongoDBOpsManagerSecurity { + if om.Spec.Security == nil { + return MongoDBOpsManagerSecurity{} + } + return *om.Spec.Security +} + +func (om *MongoDBOpsManager) OpsManagerStatefulSetName() string { + return om.GetName() +} + +func (om *MongoDBOpsManager) BackupDaemonStatefulSetName() string { + return fmt.Sprintf("%s-backup-daemon", om.GetName()) +} + +func (om *MongoDBOpsManager) BackupDaemonStatefulSetNameForClusterIndex(clusterIndex int) string { + return fmt.Sprintf("%s-%d-backup-daemon", om.GetName(), clusterIndex) +} + +func (om *MongoDBOpsManager) GetSchemePort() (corev1.URIScheme, int32) { + if om.IsTLSEnabled() { + return SchemePortFromAnnotation("https") + } + return SchemePortFromAnnotation("http") +} + +func (om *MongoDBOpsManager) IsTLSEnabled() bool { + return om.Spec.Security != nil && (om.Spec.Security.TLS.SecretRef.Name != "" || om.Spec.Security.CertificatesSecretsPrefix != "") +} + +func (om *MongoDBOpsManager) TLSCertificateSecretName() string { + // The old field has the precedence + if om.GetSecurity().TLS.SecretRef.Name != "" { + return om.GetSecurity().TLS.SecretRef.Name + } + if om.GetSecurity().CertificatesSecretsPrefix != "" { + return fmt.Sprintf("%s-%s-cert", om.GetSecurity().CertificatesSecretsPrefix, om.Name) + } + return "" +} + +func (om *MongoDBOpsManager) CentralURL() string { + if om.Spec.OpsManagerURL != "" { + return om.Spec.OpsManagerURL + } + + fqdn := dns.GetServiceFQDN(om.SvcName(), om.Namespace, om.Spec.GetClusterDomain()) + scheme, port := om.GetSchemePort() + + centralURL := url.URL{ + Scheme: string(scheme), + Host: fmt.Sprintf("%s:%d", fqdn, port), + } + return strings.ToLower(centralURL.String()) +} + +func (om *MongoDBOpsManager) BackupDaemonFQDNs() []string { + hostnames, _ := dns.GetDNSNames(om.BackupDaemonStatefulSetName(), om.BackupDaemonServiceName(), om.Namespace, om.Spec.GetClusterDomain(), om.Spec.Backup.Members, nil) + return hostnames +} + +// VersionedImplForMemberCluster is a proxy type for implementing community's annotations.Versioned. +// Originally it was implemented directly in MongoDBOpsManager, but we need to have different implementations +// returning name of stateful set in different member clusters. +// +k8s:deepcopy-gen=false +type VersionedImplForMemberCluster struct { + client.Object + memberClusterNum int + opsManager *MongoDBOpsManager +} + +func (v VersionedImplForMemberCluster) NamespacedName() types.NamespacedName { + return types.NamespacedName{Name: v.opsManager.Spec.AppDB.NameForCluster(v.memberClusterNum), Namespace: v.opsManager.Namespace} +} + +func (v VersionedImplForMemberCluster) GetMongoDBVersionForAnnotation() string { + return v.opsManager.Spec.AppDB.Version +} + +func (v VersionedImplForMemberCluster) IsChangingVersion() bool { + return v.opsManager.IsChangingVersion() +} + +func (om *MongoDBOpsManager) GetVersionedImplForMemberCluster(memberClusterNum int) *VersionedImplForMemberCluster { + return &VersionedImplForMemberCluster{ + Object: om, + memberClusterNum: memberClusterNum, + opsManager: om, + } +} + +func (om *MongoDBOpsManager) IsChangingVersion() bool { + prevVersion := om.GetPreviousVersion() + return prevVersion != "" && prevVersion != om.Spec.AppDB.Version +} + +func (om *MongoDBOpsManager) GetPreviousVersion() string { + return annotations.GetAnnotation(om, annotations.LastAppliedMongoDBVersion) +} + +func (om *MongoDBOpsManager) CalculateFeatureCompatibilityVersion() string { + return fcv.CalculateFeatureCompatibilityVersion(om.Spec.AppDB.Version, om.Status.AppDbStatus.FeatureCompatibilityVersion, om.Spec.AppDB.FeatureCompatibilityVersion) +} + +// GetSecretsMountedIntoPod returns the list of strings mounted into the pod that we need to watch. +func (om *MongoDBOpsManager) GetSecretsMountedIntoPod() []string { + var secretNames []string + tls := om.TLSCertificateSecretName() + if tls != "" { + secretNames = append(secretNames, tls) + } + + if om.Spec.AdminSecret != "" { + secretNames = append(secretNames, om.Spec.AdminSecret) + } + + if om.Spec.Backup != nil { + for _, config := range om.Spec.Backup.S3Configs { + if config.S3SecretRef != nil && config.S3SecretRef.Name != "" { + secretNames = append(secretNames, config.S3SecretRef.Name) + } + } + } + + return secretNames +} + +func (om *MongoDBOpsManager) GetClusterSpecList() []ClusterSpecOMItem { + if om.Spec.IsMultiCluster() { + return om.Spec.ClusterSpecList + } else { + return []ClusterSpecOMItem{om.getLegacyClusterSpecOMItem()} + } +} + +func (om *MongoDBOpsManager) getLegacyClusterSpecOMItem() ClusterSpecOMItem { + legacyClusterSpecOMItem := ClusterSpecOMItem{ + ClusterName: multicluster.LegacyCentralClusterName, + Members: om.Spec.Replicas, + Legacy: true, + StatefulSetConfiguration: om.Spec.StatefulSetConfiguration, + } + if om.Spec.Backup != nil { + legacyClusterSpecOMItem.Backup = &MongoDBOpsManagerBackupClusterSpecItem{ + Members: om.Spec.Backup.Members, + AssignmentLabels: om.Spec.Backup.AssignmentLabels, + HeadDB: om.Spec.Backup.HeadDB, + JVMParams: om.Spec.Backup.JVMParams, + StatefulSetConfiguration: om.Spec.Backup.StatefulSetConfiguration, + } + } + return legacyClusterSpecOMItem +} + +func (om *MongoDBOpsManager) GetMemberClusterSpecByName(memberClusterName string) ClusterSpecOMItem { + for _, clusterSpec := range om.GetClusterSpecList() { + if clusterSpec.ClusterName == memberClusterName { + return clusterSpec + } + } + return ClusterSpecOMItem{ + ClusterName: memberClusterName, + Members: 0, + } +} + +func (om *MongoDBOpsManager) GetMemberClusterBackupAssignmentLabels(memberClusterName string) []string { + clusterSpecItem := om.GetMemberClusterSpecByName(memberClusterName) + if clusterSpecItem.Backup != nil && clusterSpecItem.Backup.AssignmentLabels != nil { + return clusterSpecItem.Backup.AssignmentLabels + } + return om.Spec.Backup.AssignmentLabels +} + +func (om *MongoDBOpsManager) ClusterMappingConfigMapName() string { + return om.Name + "-cluster-mapping" +} + +func (om *MongoDBOpsManager) GetExternalConnectivityConfigurationForMemberCluster(clusterName string) *MongoDBOpsManagerServiceDefinition { + for _, csl := range om.Spec.ClusterSpecList { + if csl.ClusterName == clusterName && csl.MongoDBOpsManagerExternalConnectivity != nil { + return csl.MongoDBOpsManagerExternalConnectivity + } + } + + return om.Spec.MongoDBOpsManagerExternalConnectivity +} + +// newBackup returns an empty backup object +func newBackup() *MongoDBOpsManagerBackup { + return &MongoDBOpsManagerBackup{Enabled: true} +} + +// ConvertToEnvVarFormat takes a property in the form of +// mms.mail.transport, and converts it into the expected env var format of +// OM_PROP_mms_mail_transport +func ConvertNameToEnvVarFormat(propertyFormat string) string { + withPrefix := fmt.Sprintf("%s%s", util.OmPropertyPrefix, propertyFormat) + return strings.ReplaceAll(withPrefix, ".", "_") +} + +func SchemePortFromAnnotation(annotation string) (corev1.URIScheme, int32) { + scheme := corev1.URISchemeHTTP + port := int32(util.OpsManagerDefaultPortHTTP) + if strings.ToUpper(annotation) == "HTTPS" { + scheme = corev1.URISchemeHTTPS + port = int32(util.OpsManagerDefaultPortHTTPS) + } + + return scheme, port +} + +func getPartsFromStatusOptions(options ...status.Option) []status.Part { + var parts []status.Part + for _, part := range options { + if omPart, ok := part.(status.OMPartOption); ok { + statusPart := omPart.Value().(status.Part) + parts = append(parts, statusPart) + } + } + return parts +} + +// AppDBConfigurable holds information needed to enable SCRAM-SHA +// and combines AppDBSpec (includes SCRAM configuration) +// and MongoDBOpsManager instance (used as the owner reference for the SCRAM related resources) +type AppDBConfigurable struct { + AppDBSpec + OpsManager MongoDBOpsManager +} + +// GetOwnerReferences returns the OwnerReferences pointing to the MongoDBOpsManager instance and used by SCRAM related resources. +func (m *AppDBConfigurable) GetOwnerReferences() []metav1.OwnerReference { + groupVersionKind := schema.GroupVersionKind{ + Group: GroupVersion.Group, + Version: GroupVersion.Version, + Kind: m.OpsManager.Kind, + } + ownerReference := *metav1.NewControllerRef(&m.OpsManager, groupVersionKind) + return []metav1.OwnerReference{ownerReference} +} diff --git a/api/v1/om/opsmanager_types_test.go b/api/v1/om/opsmanager_types_test.go new file mode 100644 index 000000000..aa59d6ce9 --- /dev/null +++ b/api/v1/om/opsmanager_types_test.go @@ -0,0 +1,173 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +func TestMongoDBOpsManager_AddWarningIfNotExists(t *testing.T) { + resource := &MongoDBOpsManager{} + resource.AddOpsManagerWarningIfNotExists("my test warning") + resource.AddOpsManagerWarningIfNotExists("my test warning") + resource.AddOpsManagerWarningIfNotExists("my other test warning") + assert.Equal(t, []status.Warning{"my test warning;", "my other test warning"}, resource.Status.OpsManagerStatus.Warnings) + assert.Empty(t, resource.Status.AppDbStatus.Warnings) + assert.Empty(t, resource.Status.BackupStatus.Warnings) +} + +func TestAppDB_AddWarningIfNotExists(t *testing.T) { + resource := &MongoDBOpsManager{} + resource.AddAppDBWarningIfNotExists("my test warning") + resource.AddAppDBWarningIfNotExists("my test warning") + resource.AddAppDBWarningIfNotExists("my other test warning") + assert.Equal(t, []status.Warning{"my test warning;", "my other test warning"}, resource.Status.AppDbStatus.Warnings) + assert.Empty(t, resource.Status.BackupStatus.Warnings) + assert.Empty(t, resource.Status.OpsManagerStatus.Warnings) +} + +func TestBackup_AddWarningIfNotExists(t *testing.T) { + resource := &MongoDBOpsManager{} + resource.AddBackupWarningIfNotExists("my test warning") + resource.AddBackupWarningIfNotExists("my test warning") + resource.AddBackupWarningIfNotExists("my other test warning") + assert.Equal(t, []status.Warning{"my test warning;", "my other test warning"}, resource.Status.BackupStatus.Warnings) + assert.Empty(t, resource.Status.AppDbStatus.Warnings) + assert.Empty(t, resource.Status.OpsManagerStatus.Warnings) +} + +func TestGetPartsFromStatusOptions(t *testing.T) { + t.Run("Empty list returns nil slice", func(t *testing.T) { + assert.Nil(t, getPartsFromStatusOptions()) + }) + + t.Run("Ops Manager parts are extracted correctly", func(t *testing.T) { + statusOptions := []status.Option{ + status.NewBackupStatusOption("some-status"), + status.NewOMPartOption(status.OpsManager), + status.NewOMPartOption(status.Backup), + status.NewOMPartOption(status.AppDb), + status.NewBaseUrlOption("base-url"), + } + res := getPartsFromStatusOptions(statusOptions...) + assert.Len(t, res, 3) + assert.Equal(t, status.OpsManager, res[0]) + assert.Equal(t, status.Backup, res[1]) + assert.Equal(t, status.AppDb, res[2]) + }) +} + +func TestTLSCertificateSecretName(t *testing.T) { + om := NewOpsManagerBuilderDefault().Build() + om.SetName("new-manager") + tests := []struct { + name string + security MongoDBOpsManagerSecurity + expected string + }{ + { + name: "TLS Certificate Secret name empty", + security: MongoDBOpsManagerSecurity{}, + expected: "", + }, + { + name: "TLS Certificate Secret name from TLS.SecretRef.Name", + security: MongoDBOpsManagerSecurity{ + TLS: MongoDBOpsManagerTLS{ + SecretRef: TLSSecretRef{ + Name: "ops-manager-cert", + }, + }, + }, + expected: "ops-manager-cert", + }, + { + name: "TLS Certificate Secret name from Security.CertificatesSecretPrefix", + security: MongoDBOpsManagerSecurity{ + CertificatesSecretsPrefix: "om", + }, + expected: "om-new-manager-cert", + }, + { + name: "TLS Certificate Secret name from TLS.SecretRef.Name has priority", + security: MongoDBOpsManagerSecurity{ + TLS: MongoDBOpsManagerTLS{ + SecretRef: TLSSecretRef{ + Name: "ops-manager-cert", + }, + }, + CertificatesSecretsPrefix: "prefix", + }, + expected: "ops-manager-cert", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + om.Spec.Security = &tc.security + assert.Equal(t, tc.expected, om.TLSCertificateSecretName()) + }) + } +} + +func TestIsTLSEnabled(t *testing.T) { + om := NewOpsManagerBuilderDefault().Build() + tests := []struct { + name string + security *MongoDBOpsManagerSecurity + expected bool + }{ + { + name: "TLS is not enabled when security is not specified", + security: nil, + expected: false, + }, + { + name: "TLS is not enabled when TLS.SecretRef.Name is not specified", + security: &MongoDBOpsManagerSecurity{ + TLS: MongoDBOpsManagerTLS{ + SecretRef: TLSSecretRef{}, + }, + }, + expected: false, + }, + { + name: "TLS is enabled when TLS.SecretRef.Name is specified", + security: &MongoDBOpsManagerSecurity{ + TLS: MongoDBOpsManagerTLS{ + SecretRef: TLSSecretRef{ + Name: "ops-manager-cert", + }, + }, + }, + expected: true, + }, + { + name: "TLS is enabled when CertificatesSecretsPrefix is specified", + security: &MongoDBOpsManagerSecurity{ + CertificatesSecretsPrefix: "prefix", + }, + expected: true, + }, + { + name: "TLS is enabled when both sources of cert secret name are specified", + security: &MongoDBOpsManagerSecurity{ + TLS: MongoDBOpsManagerTLS{ + SecretRef: TLSSecretRef{ + Name: "ops-manager-cert", + }, + }, + CertificatesSecretsPrefix: "prefix", + }, + expected: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + om.Spec.Security = tc.security + assert.Equal(t, tc.expected, om.IsTLSEnabled()) + }) + } +} diff --git a/api/v1/om/opsmanager_validation.go b/api/v1/om/opsmanager_validation.go new file mode 100644 index 000000000..8440c689d --- /dev/null +++ b/api/v1/om/opsmanager_validation.go @@ -0,0 +1,332 @@ +package om + +import ( + "errors" + "fmt" + "net" + + "github.com/blang/semver" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +// IMPORTANT: this package is intended to contain only "simple" validation—in +// other words, validation that is based only on the properties in the Ops Manager +// resource. More complex validation, such as validation that needs to observe +// the state of the cluster, belongs somewhere else. + +var _ webhook.Validator = &MongoDBOpsManager{} + +// ValidateCreate and ValidateUpdate should be the same if we intend to do this +// on every reconciliation as well +func (om *MongoDBOpsManager) ValidateCreate() (admission.Warnings, error) { + return nil, om.ProcessValidationsWebhook() +} + +func (om *MongoDBOpsManager) ValidateUpdate(_ runtime.Object) (admission.Warnings, error) { + return nil, om.ProcessValidationsWebhook() +} + +// ValidateDelete does nothing as we assume validation on deletion is +// unnecessary +func (om *MongoDBOpsManager) ValidateDelete() (admission.Warnings, error) { + return nil, nil +} + +func errorNotConfigurableForAppDB(field string) v1.ValidationResult { + return v1.OpsManagerResourceValidationError(fmt.Sprintf("%s field is not configurable for application databases", field), status.AppDb) +} + +func validOmVersion(os MongoDBOpsManagerSpec) v1.ValidationResult { + _, err := versionutil.StringToSemverVersion(os.Version) + if err != nil { + return v1.OpsManagerResourceValidationError(fmt.Sprintf("'%s' is an invalid value for spec.version: %s", os.Version, err), status.OpsManager) + } + return v1.ValidationSuccess() +} + +func validAppDBVersion(os MongoDBOpsManagerSpec) v1.ValidationResult { + version := os.AppDB.GetMongoDBVersion() + v, err := semver.Make(version) + if err != nil { + return v1.OpsManagerResourceValidationError(fmt.Sprintf("'%s' is an invalid value for spec.applicationDatabase.version: %s", version, err), status.AppDb) + } + fourZero, _ := semver.Make("4.0.0") + if v.LT(fourZero) { + return v1.OpsManagerResourceValidationError("the version of Application Database must be >= 4.0", status.AppDb) + } + + return v1.ValidationSuccess() +} + +func connectivityIsNotConfigurable(os MongoDBOpsManagerSpec) v1.ValidationResult { + if os.AppDB.Connectivity != nil { + return errorNotConfigurableForAppDB("connectivity") + } + return v1.ValidationSuccess() +} + +// ConnectionSpec fields +func credentialsIsNotConfigurable(os MongoDBOpsManagerSpec) v1.ValidationResult { + if os.AppDB.Credentials != "" { + return errorNotConfigurableForAppDB("credentials") + } + return v1.ValidationSuccess() +} + +func opsManagerConfigIsNotConfigurable(os MongoDBOpsManagerSpec) v1.ValidationResult { + if os.AppDB.OpsManagerConfig != nil { + return errorNotConfigurableForAppDB("opsManager") + } + return v1.ValidationSuccess() +} + +func cloudManagerConfigIsNotConfigurable(os MongoDBOpsManagerSpec) v1.ValidationResult { + if os.AppDB.CloudManagerConfig != nil { + return errorNotConfigurableForAppDB("cloudManager") + } + return v1.ValidationSuccess() +} + +// onlyFileSystemStoreIsEnabled checks if only FileSystemSnapshotStore is configured and not S3Store/Blockstore +func onlyFileSystemStoreIsEnabled(bp MongoDBOpsManagerBackup) bool { + if len(bp.BlockStoreConfigs) == 0 && len(bp.S3Configs) == 0 && len(bp.FileSystemStoreConfigs) > 0 { + return true + } + return false +} + +// s3StoreMongodbUserSpecifiedNoMongoResource checks that 'mongodbResourceRef' is provided if 'mongodbUserRef' is configured +func s3StoreMongodbUserSpecifiedNoMongoResource(os MongoDBOpsManagerSpec) v1.ValidationResult { + if !os.Backup.Enabled || len(os.Backup.S3Configs) == 0 { + return v1.ValidationSuccess() + } + + if onlyFileSystemStoreIsEnabled(*os.Backup) { + return v1.ValidationSuccess() + } + + for _, config := range os.Backup.S3Configs { + if config.MongoDBUserRef != nil && config.MongoDBResourceRef == nil { + return v1.OpsManagerResourceValidationError( + fmt.Sprintf("'mongodbResourceRef' must be specified if 'mongodbUserRef' is configured (S3 Store: %s)", config.Name), status.OpsManager, + ) + } + } + return v1.ValidationSuccess() +} + +func kmipValidation(os MongoDBOpsManagerSpec) v1.ValidationResult { + if os.Backup == nil || !os.Backup.Enabled || os.Backup.Encryption == nil || os.Backup.Encryption.Kmip == nil { + return v1.ValidationSuccess() + } + + if _, _, err := net.SplitHostPort(os.Backup.Encryption.Kmip.Server.URL); err != nil { + return v1.OpsManagerResourceValidationError(fmt.Sprintf("kmip url can not be splitted into host and port, see %v", err), status.OpsManager) + } + + if len(os.Backup.Encryption.Kmip.Server.CA) == 0 { + return v1.OpsManagerResourceValidationError("kmip CA ConfigMap name can not be empty", status.OpsManager) + } + + return v1.ValidationSuccess() +} + +func validateEmptyClusterSpecListSingleCluster(os MongoDBOpsManagerSpec) v1.ValidationResult { + if !os.AppDB.IsMultiCluster() { + if len(os.AppDB.ClusterSpecList) > 0 { + return v1.OpsManagerResourceValidationError("Single cluster AppDB deployment should have empty clusterSpecList", status.OpsManager) + } + } + return v1.ValidationSuccess() +} + +func validateTopologyIsSpecified(os MongoDBOpsManagerSpec) v1.ValidationResult { + if len(os.ClusterSpecList) > 0 { + if !os.IsMultiCluster() { + return v1.OpsManagerResourceValidationError("Topology 'MultiCluster' must be specified while setting a not empty spec.clusterSpecList", status.OpsManager) + } + } + return v1.ValidationSuccess() +} + +func featureCompatibilityVersionValidation(os MongoDBOpsManagerSpec) v1.ValidationResult { + fcv := os.AppDB.FeatureCompatibilityVersion + return mdb.ValidateFCV(fcv) +} + +func validateClusterSpecList(os MongoDBOpsManagerSpec) v1.ValidationResult { + if os.IsMultiCluster() { + if len(os.ClusterSpecList) == 0 { + return v1.OpsManagerResourceValidationError("At least one ClusterSpecList entry must be specified for MultiCluster mode OM", status.OpsManager) + } + if os.Backup != nil && os.Backup.Enabled { + backupMembersConfigured := false + for _, clusterSpec := range os.ClusterSpecList { + if clusterSpec.Backup != nil && clusterSpec.Backup.Members > 0 { + backupMembersConfigured = true + break + } + } + if !backupMembersConfigured { + return v1.OpsManagerResourceValidationError("At least one ClusterSpecList item must have backup members configured", status.OpsManager) + } + } + } + if !os.IsMultiCluster() { + if len(os.ClusterSpecList) > 0 { + return v1.OpsManagerResourceValidationError("ClusterSpecList cannot be specified for SingleCluster mode OM", status.OpsManager) + } + } + return v1.ValidationSuccess() +} + +// validateAppDBUniqueExternalDomains validates uniqueness of the domains if they are provided. +// External domain might be specified at the top level in spec.externalAccess.externalDomain or in every member cluster. +// We make sure that if external domains are used, every member cluster has unique external domain defined. +func validateAppDBUniqueExternalDomains(os MongoDBOpsManagerSpec) v1.ValidationResult { + appDBSpec := os.AppDB + + externalDomains := make(map[string]string) + for _, e := range appDBSpec.ClusterSpecList { + if externalDomain := appDBSpec.GetExternalDomainForMemberCluster(e.ClusterName); externalDomain != nil { + externalDomains[e.ClusterName] = *externalDomain + } + } + + // We don't need to validate external domains if there aren't any specified. + // We don't have any flag that enables usage of external domains. We use them if they are provided. + if len(externalDomains) == 0 { + return v1.ValidationSuccess() + } + + present := map[string]struct{}{} + for _, e := range appDBSpec.ClusterSpecList { + externalDomain, ok := externalDomains[e.ClusterName] + if !ok { + return v1.ValidationError("The externalDomain is not set for member cluster: %s", e.ClusterName) + } + + if _, ok := present[externalDomain]; ok { + return v1.ValidationError("Multiple member clusters with the same externalDomain (%s) are not allowed. "+ + "Check if all spec.applicationDatabase.clusterSpecList[*].externalAccess.externalDomain fields are defined and are unique.", externalDomain) + } + present[externalDomain] = struct{}{} + } + return v1.ValidationSuccess() +} + +func validateBackupS3Stores(os MongoDBOpsManagerSpec) v1.ValidationResult { + backup := os.Backup + if backup == nil || !backup.Enabled { + return v1.ValidationSuccess() + } + + if len(backup.S3Configs) > 0 { + for _, config := range backup.S3Configs { + if config.IRSAEnabled { + if config.S3SecretRef != nil { + return v1.OpsManagerResourceValidationWarning("'s3SecretRef' must not be specified if using IRSA (S3 Store: %s)", status.OpsManager, config.Name) + } + } else if config.S3SecretRef == nil || config.S3SecretRef.Name == "" { + return v1.OpsManagerResourceValidationError("'s3SecretRef' must be specified if not using IRSA (S3 Store: %s)", status.OpsManager, config.Name) + } + } + } + + if len(backup.S3OplogStoreConfigs) > 0 { + for _, oplogStoreConfig := range backup.S3OplogStoreConfigs { + if oplogStoreConfig.IRSAEnabled { + if oplogStoreConfig.S3SecretRef != nil { + return v1.OpsManagerResourceValidationWarning("'s3SecretRef' must not be specified if using IRSA (S3 OpLog Store: %s)", status.OpsManager, oplogStoreConfig.Name) + } + } else if oplogStoreConfig.S3SecretRef == nil || oplogStoreConfig.S3SecretRef.Name == "" { + return v1.OpsManagerResourceValidationError("'s3SecretRef' must be specified if not using IRSA (S3 OpLog Store: %s)", status.OpsManager, oplogStoreConfig.Name) + } + } + } + + return v1.ValidationSuccess() +} + +func (om *MongoDBOpsManager) RunValidations() []v1.ValidationResult { + validators := []func(m MongoDBOpsManagerSpec) v1.ValidationResult{ + validOmVersion, + validAppDBVersion, + connectivityIsNotConfigurable, + cloudManagerConfigIsNotConfigurable, + opsManagerConfigIsNotConfigurable, + credentialsIsNotConfigurable, + s3StoreMongodbUserSpecifiedNoMongoResource, + kmipValidation, + validateEmptyClusterSpecListSingleCluster, + validateTopologyIsSpecified, + validateClusterSpecList, + validateBackupS3Stores, + featureCompatibilityVersionValidation, + validateAppDBUniqueExternalDomains, + } + + multiClusterAppDBSharedClusterValidators := []func(ms mdb.ClusterSpecList) v1.ValidationResult{ + mdb.ValidateUniqueClusterNames, + mdb.ValidateNonEmptyClusterSpecList, + mdb.ValidateMemberClusterIsSubsetOfKubeConfig, + } + + var validationResults []v1.ValidationResult + + for _, validator := range validators { + res := validator(om.Spec) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + + // Explicit tests for AppDB multi-cluster + if om.Spec.AppDB.IsMultiCluster() { + for _, validator := range multiClusterAppDBSharedClusterValidators { + res := validator(om.Spec.AppDB.ClusterSpecList) + if res.Level > 0 { + validationResults = append(validationResults, res) + } + } + } + + return validationResults +} + +func (om *MongoDBOpsManager) ProcessValidationsWebhook() error { + for _, res := range om.RunValidations() { + if res.Level == v1.ErrorLevel { + return errors.New(res.Msg) + } + } + return nil +} + +func (om *MongoDBOpsManager) ProcessValidationsOnReconcile() (status.Part, error) { + for _, res := range om.RunValidations() { + if res.Level == v1.ErrorLevel { + return res.OmStatusPart, errors.New(res.Msg) + } + + if res.Level == v1.WarningLevel { + switch res.OmStatusPart { + case status.OpsManager: + om.AddOpsManagerWarningIfNotExists(status.Warning(res.Msg)) + case status.AppDb: + om.AddAppDBWarningIfNotExists(status.Warning(res.Msg)) + case status.Backup: + om.AddBackupWarningIfNotExists(status.Warning(res.Msg)) + } + } + } + + return status.None, nil +} diff --git a/api/v1/om/opsmanager_validation_test.go b/api/v1/om/opsmanager_validation_test.go new file mode 100644 index 000000000..04871feb7 --- /dev/null +++ b/api/v1/om/opsmanager_validation_test.go @@ -0,0 +1,382 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +func TestOpsManagerValidation(t *testing.T) { + type args struct { + testedOm *MongoDBOpsManager + expectedPart status.Part + expectedErrorMessage string + expectedWarningMessage status.Warning + } + tests := map[string]args{ + "Valid KMIP configuration": { + testedOm: NewOpsManagerBuilderDefault().SetBackup(MongoDBOpsManagerBackup{ + Enabled: true, + Encryption: &Encryption{ + Kmip: &KmipConfig{ + Server: v1.KmipServerConfig{ + CA: "kmip-ca", + URL: "kmip.mongodb.com:5696", + }, + }, + }, + }).Build(), + expectedPart: status.None, + }, + "Valid disabled KMIP configuration": { + testedOm: NewOpsManagerBuilderDefault().SetBackup(MongoDBOpsManagerBackup{ + Enabled: false, + Encryption: &Encryption{ + Kmip: &KmipConfig{ + Server: v1.KmipServerConfig{ + URL: "::this::is::a::wrong::address", + }, + }, + }, + }).Build(), + expectedPart: status.None, + }, + "Invalid KMIP configuration with wrong url": { + testedOm: NewOpsManagerBuilderDefault().SetBackup(MongoDBOpsManagerBackup{ + Enabled: true, + Encryption: &Encryption{ + Kmip: &KmipConfig{ + Server: v1.KmipServerConfig{ + CA: "kmip-ca", + URL: "wrong:::url:::123", + }, + }, + }, + }).Build(), + expectedErrorMessage: "kmip url can not be splitted into host and port, see address wrong:::url:::123: too many colons in address", + expectedPart: status.OpsManager, + }, + "Invalid KMIP configuration url and no port": { + testedOm: NewOpsManagerBuilderDefault().SetBackup(MongoDBOpsManagerBackup{ + Enabled: true, + Encryption: &Encryption{ + Kmip: &KmipConfig{ + Server: v1.KmipServerConfig{ + CA: "kmip-ca", + URL: "localhost", + }, + }, + }, + }).Build(), + expectedErrorMessage: "kmip url can not be splitted into host and port, see address localhost: missing port in address", + expectedPart: status.OpsManager, + }, + "Invalid KMIP configuration without CA": { + testedOm: NewOpsManagerBuilderDefault().SetBackup(MongoDBOpsManagerBackup{ + Enabled: true, + Encryption: &Encryption{ + Kmip: &KmipConfig{ + Server: v1.KmipServerConfig{ + URL: "kmip.mongodb.com:5696", + }, + }, + }, + }).Build(), + expectedErrorMessage: "kmip CA ConfigMap name can not be empty", + expectedPart: status.OpsManager, + }, + "Valid default OpsManager": { + testedOm: NewOpsManagerBuilderDefault().Build(), + expectedPart: status.None, + }, + "Invalid AppDB connectivity spec": { + testedOm: NewOpsManagerBuilderDefault(). + SetAppDbConnectivity(mdbv1.MongoDBConnectivity{ReplicaSetHorizons: []mdbv1.MongoDBHorizonConfig{}}). + Build(), + expectedErrorMessage: "connectivity field is not configurable for application databases", + expectedPart: status.AppDb, + }, + "Invalid AppDB credentials": { + testedOm: NewOpsManagerBuilderDefault(). + SetAppDbCredentials("invalid"). + Build(), + expectedErrorMessage: "credentials field is not configurable for application databases", + expectedPart: status.AppDb, + }, + "Invalid AppDB OpsManager config": { + testedOm: NewOpsManagerBuilderDefault(). + SetOpsManagerConfig(mdbv1.PrivateCloudConfig{}). + Build(), + expectedErrorMessage: "opsManager field is not configurable for application databases", + expectedPart: status.AppDb, + }, + "Invalid AppDB CloudManager config": { + testedOm: NewOpsManagerBuilderDefault(). + SetCloudManagerConfig(mdbv1.PrivateCloudConfig{}). + Build(), + expectedErrorMessage: "cloudManager field is not configurable for application databases", + expectedPart: status.AppDb, + }, + "Invalid S3 Store config": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3SnapshotStore(S3Config{Name: "test", MongoDBUserRef: &MongoDBUserRef{Name: "foo"}}). + Build(), + expectedErrorMessage: "'mongodbResourceRef' must be specified if 'mongodbUserRef' is configured (S3 Store: test)", + expectedPart: status.OpsManager, + }, + "Invalid S3 Store config - missing s3SecretRef": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3SnapshotStore(S3Config{Name: "test", S3SecretRef: nil}). + Build(), + expectedErrorMessage: "'s3SecretRef' must be specified if not using IRSA (S3 Store: test)", + expectedPart: status.OpsManager, + }, + "Invalid S3 Store config - missing s3SecretRef.Name": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3SnapshotStore(S3Config{Name: "test", S3SecretRef: &SecretRef{}}). + Build(), + expectedErrorMessage: "'s3SecretRef' must be specified if not using IRSA (S3 Store: test)", + expectedPart: status.OpsManager, + }, + "Valid S3 Store config - no s3SecretRef if irsaEnabled": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3SnapshotStore(S3Config{Name: "test", IRSAEnabled: true}). + Build(), + expectedPart: status.None, + }, + "Valid S3 Store config with warning - s3SecretRef present when irsaEnabled": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3SnapshotStore(S3Config{Name: "test", S3SecretRef: &SecretRef{}, IRSAEnabled: true}). + Build(), + expectedWarningMessage: "'s3SecretRef' must not be specified if using IRSA (S3 Store: test)", + expectedPart: status.OpsManager, + }, + "Invalid S3 OpLog Store config - missing s3SecretRef": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3OplogStoreConfig(S3Config{Name: "test", S3SecretRef: nil}). + Build(), + expectedErrorMessage: "'s3SecretRef' must be specified if not using IRSA (S3 OpLog Store: test)", + expectedPart: status.OpsManager, + }, + "Invalid S3 OpLog Store config - missing s3SecretRef.Name": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3OplogStoreConfig(S3Config{Name: "test", S3SecretRef: &SecretRef{}}). + Build(), + expectedErrorMessage: "'s3SecretRef' must be specified if not using IRSA (S3 OpLog Store: test)", + expectedPart: status.OpsManager, + }, + "Valid S3 OpLog Store config - no s3SecretRef if irsaEnabled": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3OplogStoreConfig(S3Config{Name: "test", IRSAEnabled: true}). + Build(), + expectedPart: status.None, + }, + "Valid S3 OpLog Store config with warning - s3SecretRef present when irsaEnabled": { + testedOm: NewOpsManagerBuilderDefault(). + AddS3OplogStoreConfig(S3Config{Name: "test", S3SecretRef: &SecretRef{}, IRSAEnabled: true}). + Build(), + expectedWarningMessage: "'s3SecretRef' must not be specified if using IRSA (S3 OpLog Store: test)", + expectedPart: status.OpsManager, + }, + "Invalid OpsManager version": { + testedOm: NewOpsManagerBuilderDefault(). + SetVersion("4.4"). + Build(), + expectedErrorMessage: "'4.4' is an invalid value for spec.version: Ops Manager Status spec.version 4.4 is invalid", + expectedPart: status.OpsManager, + }, + "Invalid foo OpsManager version": { + testedOm: NewOpsManagerBuilderDefault(). + SetVersion("foo"). + Build(), + expectedErrorMessage: "'foo' is an invalid value for spec.version: Ops Manager Status spec.version foo is invalid", + expectedPart: status.OpsManager, + }, + "Invalid 4_4.4.0 OpsManager version": { + testedOm: NewOpsManagerBuilderDefault(). + SetVersion("4_4.4.0"). + Build(), + expectedErrorMessage: "'4_4.4.0' is an invalid value for spec.version: Ops Manager Status spec.version 4_4.4.0 is invalid", + expectedPart: status.OpsManager, + }, + "Invalid 4.4_4.0 OpsManager version": { + testedOm: NewOpsManagerBuilderDefault(). + SetVersion("4.4_4.0"). + Build(), + expectedErrorMessage: "'4.4_4.0' is an invalid value for spec.version: Ops Manager Status spec.version 4.4_4.0 is invalid", + expectedPart: status.OpsManager, + }, + "Invalid 4.4.0_0 OpsManager version": { + testedOm: NewOpsManagerBuilderDefault(). + SetVersion("4.4.0_0"). + Build(), + expectedErrorMessage: "'4.4.0_0' is an invalid value for spec.version: Ops Manager Status spec.version 4.4.0_0 is invalid", + expectedPart: status.OpsManager, + }, + "Too low AppDB version": { + testedOm: NewOpsManagerBuilderDefault(). + SetAppDbVersion("3.6.12"). + Build(), + expectedErrorMessage: "the version of Application Database must be >= 4.0", + expectedPart: status.AppDb, + }, + "Valid 4.0.0 OpsManager version": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.0.0").Build(), + expectedPart: status.None, + }, + "Valid 4.2.0-rc1 OpsManager version": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.2.0-rc1").Build(), + expectedPart: status.None, + }, + "Valid 4.5.0-ent OpsManager version": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.5.0-ent").Build(), + expectedPart: status.None, + }, + "Single cluster AppDB deployment should have empty clusterSpecList": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.5.0-ent"). + SetOpsManagerTopology(mdbv1.ClusterTopologySingleCluster). + SetOpsManagerClusterSpecList([]ClusterSpecOMItem{{ClusterName: "test"}}). + SetAppDBClusterSpecList([]mdbv1.ClusterSpecItem{{ClusterName: "test"}}). + Build(), + expectedPart: status.OpsManager, + expectedErrorMessage: "Single cluster AppDB deployment should have empty clusterSpecList", + }, + "Topology 'MultiCluster' must be specified while setting a not empty spec.clusterSpecList": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.5.0-ent"). + SetOpsManagerTopology(mdbv1.ClusterTopologySingleCluster). + SetOpsManagerClusterSpecList([]ClusterSpecOMItem{{ClusterName: "test"}}). + Build(), + expectedPart: status.OpsManager, + expectedErrorMessage: "Topology 'MultiCluster' must be specified while setting a not empty spec.clusterSpecList", + }, + "Uniform externalDomain can be overwritten multi cluster AppDB": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.5.0-ent"). + SetAppDBTopology(ClusterTopologyMultiCluster). + SetAppDbExternalAccess(mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test"), + }). + SetAppDBClusterSpecList([]mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test1"), + }, + }, + { + ClusterName: "cluster2", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test2"), + }, + }, + { + ClusterName: "cluster3", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test3"), + }, + }, + }). + Build(), + expectedPart: status.None, + expectedErrorMessage: "", + }, + "Uniform externalDomain is not allowed for multi cluster AppDB": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.5.0-ent"). + SetAppDBTopology(ClusterTopologyMultiCluster). + SetAppDbExternalAccess(mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test"), + }). + SetAppDBClusterSpecList([]mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 1, + }, + { + ClusterName: "cluster3", + Members: 1, + }, + }). + Build(), + expectedPart: status.AppDb, + expectedErrorMessage: "Multiple member clusters with the same externalDomain (test) are not allowed. " + + "Check if all spec.applicationDatabase.clusterSpecList[*].externalAccess.externalDomain fields are defined and are unique.", + }, + "Multiple member clusters with the same externalDomain are not allowed": { + testedOm: NewOpsManagerBuilderDefault().SetVersion("4.5.0-ent"). + SetAppDBTopology(ClusterTopologyMultiCluster). + SetAppDBClusterSpecList([]mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test"), + }, + }, + { + ClusterName: "cluster2", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test"), + }, + }, + { + ClusterName: "cluster3", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("test"), + }, + }, + }). + Build(), + expectedPart: status.AppDb, + expectedErrorMessage: "Multiple member clusters with the same externalDomain (test) are not allowed. " + + "Check if all spec.applicationDatabase.clusterSpecList[*].externalAccess.externalDomain fields are defined and are unique.", + }, + } + + for testName := range tests { + t.Run(testName, func(t *testing.T) { + testConfig := tests[testName] + part, err := testConfig.testedOm.ProcessValidationsOnReconcile() + + if testConfig.expectedErrorMessage != "" { + assert.NotNil(t, err) + assert.Equal(t, testConfig.expectedPart, part) + assert.Equal(t, testConfig.expectedErrorMessage, err.Error()) + } else { + assert.Nil(t, err) + assert.Equal(t, status.None, part) + } + + if testConfig.expectedWarningMessage != "" { + warnings := testConfig.testedOm.GetStatusWarnings(testConfig.expectedPart) + assert.Contains(t, warnings, testConfig.expectedWarningMessage) + } + }) + } +} + +func TestOpsManager_RunValidations_InvalidPreRelease(t *testing.T) { + om := NewOpsManagerBuilder().SetVersion("3.5.0-1193-x86_64").SetAppDbVersion("4.4.4-ent").Build() + version, err := versionutil.StringToSemverVersion(om.Spec.Version) + assert.NoError(t, err) + + part, err := om.ProcessValidationsOnReconcile() + assert.Equal(t, status.None, part) + assert.NoError(t, err) + assert.Equal(t, uint64(3), version.Major) + assert.Equal(t, uint64(5), version.Minor) + assert.Equal(t, uint64(0), version.Patch) +} diff --git a/api/v1/om/opsmanagerbuilder.go b/api/v1/om/opsmanagerbuilder.go new file mode 100644 index 000000000..5204d8443 --- /dev/null +++ b/api/v1/om/opsmanagerbuilder.go @@ -0,0 +1,280 @@ +package om + +import ( + "k8s.io/apimachinery/pkg/types" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbc "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" +) + +type OpsManagerBuilder struct { + om MongoDBOpsManager +} + +func NewOpsManagerBuilder() *OpsManagerBuilder { + return &OpsManagerBuilder{} +} + +func NewOpsManagerBuilderDefault() *OpsManagerBuilder { + return NewOpsManagerBuilder().SetName("default-om").SetVersion("4.4.1").SetAppDbMembers(3).SetAppDbPodSpec(*DefaultAppDbBuilder().Build().PodSpec).SetAppDbVersion("4.2.20") +} + +func NewOpsManagerBuilderFromResource(resource MongoDBOpsManager) *OpsManagerBuilder { + return &OpsManagerBuilder{om: resource} +} + +func (b *OpsManagerBuilder) SetVersion(version string) *OpsManagerBuilder { + b.om.Spec.Version = version + return b +} + +func (b *OpsManagerBuilder) SetAppDbVersion(version string) *OpsManagerBuilder { + b.om.Spec.AppDB.Version = version + return b +} + +func (b *OpsManagerBuilder) SetAppDbPodSpec(podSpec mdbv1.MongoDbPodSpec) *OpsManagerBuilder { + b.om.Spec.AppDB.PodSpec = &podSpec + return b +} + +func (b *OpsManagerBuilder) SetOpsManagerConfig(config mdbv1.PrivateCloudConfig) *OpsManagerBuilder { + b.om.Spec.AppDB.OpsManagerConfig = &config + return b +} + +func (b *OpsManagerBuilder) SetCloudManagerConfig(config mdbv1.PrivateCloudConfig) *OpsManagerBuilder { + b.om.Spec.AppDB.CloudManagerConfig = &config + return b +} + +func (b *OpsManagerBuilder) SetAppDbConnectivity(connectivitySpec mdbv1.MongoDBConnectivity) *OpsManagerBuilder { + b.om.Spec.AppDB.Connectivity = &connectivitySpec + return b +} + +func (b *OpsManagerBuilder) SetAppDbExternalAccess(externalAccessConfiguration mdbv1.ExternalAccessConfiguration) *OpsManagerBuilder { + b.om.Spec.AppDB.ExternalAccessConfiguration = &externalAccessConfiguration + return b +} + +func (b *OpsManagerBuilder) SetAppDBTLSConfig(config mdbv1.TLSConfig) *OpsManagerBuilder { + if b.om.Spec.AppDB.Security == nil { + b.om.Spec.AppDB.Security = &mdbv1.Security{} + } + + b.om.Spec.AppDB.Security.TLSConfig = &config + return b +} + +func (b *OpsManagerBuilder) SetTLSConfig(config MongoDBOpsManagerTLS) *OpsManagerBuilder { + if b.om.Spec.Security == nil { + b.om.Spec.Security = &MongoDBOpsManagerSecurity{} + } + + b.om.Spec.Security.TLS = config + return b +} + +func (b *OpsManagerBuilder) AddS3Config(s3ConfigName, credentialsName string) *OpsManagerBuilder { + if b.om.Spec.Backup == nil { + b.om.Spec.Backup = &MongoDBOpsManagerBackup{Enabled: true} + } + if b.om.Spec.Backup.S3Configs == nil { + b.om.Spec.Backup.S3Configs = []S3Config{} + } + b.om.Spec.Backup.S3Configs = append(b.om.Spec.Backup.S3Configs, S3Config{ + S3SecretRef: &SecretRef{ + Name: credentialsName, + }, + Name: s3ConfigName, + }) + return b +} + +func (b *OpsManagerBuilder) AddOplogStoreConfig(oplogStoreName, userName string, mdbNsName types.NamespacedName) *OpsManagerBuilder { + if b.om.Spec.Backup == nil { + b.om.Spec.Backup = &MongoDBOpsManagerBackup{Enabled: true} + } + if b.om.Spec.Backup.OplogStoreConfigs == nil { + b.om.Spec.Backup.OplogStoreConfigs = []DataStoreConfig{} + } + b.om.Spec.Backup.OplogStoreConfigs = append(b.om.Spec.Backup.OplogStoreConfigs, DataStoreConfig{ + Name: oplogStoreName, + MongoDBResourceRef: userv1.MongoDBResourceRef{ + Name: mdbNsName.Name, + Namespace: mdbNsName.Namespace, + }, + MongoDBUserRef: &MongoDBUserRef{ + Name: userName, + }, + }) + return b +} + +func (b *OpsManagerBuilder) AddS3OplogStoreConfig(s3Config S3Config) *OpsManagerBuilder { + if b.om.Spec.Backup == nil { + b.om.Spec.Backup = &MongoDBOpsManagerBackup{Enabled: true} + } + if b.om.Spec.Backup.S3OplogStoreConfigs == nil { + b.om.Spec.Backup.S3OplogStoreConfigs = []S3Config{} + } + b.om.Spec.Backup.S3OplogStoreConfigs = append(b.om.Spec.Backup.S3OplogStoreConfigs, s3Config) + return b +} + +func (b *OpsManagerBuilder) AddBlockStoreConfig(blockStoreName, userName string, mdbNsName types.NamespacedName) *OpsManagerBuilder { + if b.om.Spec.Backup == nil { + b.om.Spec.Backup = &MongoDBOpsManagerBackup{Enabled: true} + } + if b.om.Spec.Backup.BlockStoreConfigs == nil { + b.om.Spec.Backup.BlockStoreConfigs = []DataStoreConfig{} + } + b.om.Spec.Backup.BlockStoreConfigs = append(b.om.Spec.Backup.BlockStoreConfigs, DataStoreConfig{ + Name: blockStoreName, + MongoDBResourceRef: userv1.MongoDBResourceRef{ + Name: mdbNsName.Name, + Namespace: mdbNsName.Namespace, + }, + MongoDBUserRef: &MongoDBUserRef{ + Name: userName, + }, + }) + return b +} + +func (b *OpsManagerBuilder) SetClusterDomain(clusterDomain string) *OpsManagerBuilder { + b.om.Spec.ClusterDomain = clusterDomain + return b +} + +func (b *OpsManagerBuilder) SetName(name string) *OpsManagerBuilder { + b.om.Name = name + return b +} + +func (b *OpsManagerBuilder) SetNamespace(namespace string) *OpsManagerBuilder { + b.om.Namespace = namespace + return b +} + +func (b *OpsManagerBuilder) SetAppDbMembers(members int) *OpsManagerBuilder { + b.om.Spec.AppDB.Members = members + return b +} + +func (b *OpsManagerBuilder) SetAppDbCredentials(credentials string) *OpsManagerBuilder { + b.om.Spec.AppDB.Credentials = credentials + return b +} + +func (b *OpsManagerBuilder) SetBackupMembers(members int) *OpsManagerBuilder { + if b.om.Spec.Backup == nil { + b.om.Spec.Backup = &MongoDBOpsManagerBackup{Enabled: true} + } + b.om.Spec.Backup.Members = members + return b +} + +func (b *OpsManagerBuilder) SetAdditionalMongodbConfig(config *mdbv1.AdditionalMongodConfig) *OpsManagerBuilder { + b.om.Spec.AppDB.AdditionalMongodConfig = config + return b +} + +func (b *OpsManagerBuilder) SetAppDbFeatureCompatibility(version string) *OpsManagerBuilder { + b.om.Spec.AppDB.FeatureCompatibilityVersion = &version + return b +} + +func (b *OpsManagerBuilder) SetStatefulSetSpec(customSpec appsv1.StatefulSetSpec) *OpsManagerBuilder { + b.om.Spec.StatefulSetConfiguration = &mdbc.StatefulSetConfiguration{SpecWrapper: mdbc.StatefulSetSpecWrapper{Spec: customSpec}} + return b +} + +func (b *OpsManagerBuilder) SetLogRotate(logRotate *automationconfig.CrdLogRotate) *OpsManagerBuilder { + b.om.Spec.AppDB.AutomationAgent.LogRotate = logRotate + return b +} + +func (b *OpsManagerBuilder) SetSystemLog(systemLog *automationconfig.SystemLog) *OpsManagerBuilder { + b.om.Spec.AppDB.AutomationAgent.SystemLog = systemLog + return b +} + +func (b *OpsManagerBuilder) SetAppDBPassword(secretName, key string) *OpsManagerBuilder { + b.om.Spec.AppDB.PasswordSecretKeyRef = &userv1.SecretKeyRef{Name: secretName, Key: key} + return b +} + +func (b *OpsManagerBuilder) SetAppDBAutomationConfigOverride(acOverride mdbc.AutomationConfigOverride) *OpsManagerBuilder { + b.om.Spec.AppDB.AutomationConfigOverride = &acOverride + return b +} + +func (b *OpsManagerBuilder) SetBackup(backup MongoDBOpsManagerBackup) *OpsManagerBuilder { + b.om.Spec.Backup = &backup + return b +} + +func (b *OpsManagerBuilder) AddConfiguration(key, value string) *OpsManagerBuilder { + b.om.AddConfigIfDoesntExist(key, value) + return b +} + +func (b *OpsManagerBuilder) AddS3SnapshotStore(config S3Config) *OpsManagerBuilder { + if b.om.Spec.Backup == nil { + b.om.Spec.Backup = newBackup() + } + if b.om.Spec.Backup.S3Configs == nil { + b.om.Spec.Backup.S3Configs = []S3Config{} + } + b.om.Spec.Backup.S3Configs = append(b.om.Spec.Backup.S3Configs, config) + return b +} + +func (b *OpsManagerBuilder) SetOMStatusVersion(version string) *OpsManagerBuilder { + b.om.Status.OpsManagerStatus.Version = version + return b +} + +func (b *OpsManagerBuilder) SetInternalConnectivity(internalConnectivity MongoDBOpsManagerServiceDefinition) *OpsManagerBuilder { + b.om.Spec.InternalConnectivity = &internalConnectivity + return b +} + +func (b *OpsManagerBuilder) SetExternalConnectivity(externalConnectivity MongoDBOpsManagerServiceDefinition) *OpsManagerBuilder { + b.om.Spec.MongoDBOpsManagerExternalConnectivity = &externalConnectivity + return b +} + +func (b *OpsManagerBuilder) SetAppDBTopology(topology string) *OpsManagerBuilder { + b.om.Spec.AppDB.Topology = topology + return b +} + +func (b *OpsManagerBuilder) SetOpsManagerTopology(topology string) *OpsManagerBuilder { + b.om.Spec.Topology = topology + return b +} + +func (b *OpsManagerBuilder) SetAppDBClusterSpecList(clusterSpecItems mdbv1.ClusterSpecList) *OpsManagerBuilder { + b.om.Spec.AppDB.ClusterSpecList = append(b.om.Spec.AppDB.ClusterSpecList, clusterSpecItems...) + return b +} + +func (b *OpsManagerBuilder) SetOpsManagerClusterSpecList(clusterSpecItems []ClusterSpecOMItem) *OpsManagerBuilder { + b.om.Spec.ClusterSpecList = append(b.om.Spec.ClusterSpecList, clusterSpecItems...) + return b +} + +func (b *OpsManagerBuilder) Build() *MongoDBOpsManager { + b.om.InitDefaultFields() + return b.om.DeepCopy() +} + +// ************************* Private methods ************************************ diff --git a/api/v1/om/zz_generated.deepcopy.go b/api/v1/om/zz_generated.deepcopy.go new file mode 100644 index 000000000..9b6fcc201 --- /dev/null +++ b/api/v1/om/zz_generated.deepcopy.go @@ -0,0 +1,837 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package om + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentVersion) DeepCopyInto(out *AgentVersion) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentVersion. +func (in *AgentVersion) DeepCopy() *AgentVersion { + if in == nil { + return nil + } + out := new(AgentVersion) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AppDBConfigurable) DeepCopyInto(out *AppDBConfigurable) { + *out = *in + in.AppDBSpec.DeepCopyInto(&out.AppDBSpec) + in.OpsManager.DeepCopyInto(&out.OpsManager) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppDBConfigurable. +func (in *AppDBConfigurable) DeepCopy() *AppDBConfigurable { + if in == nil { + return nil + } + out := new(AppDBConfigurable) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AppDBSpec) DeepCopyInto(out *AppDBSpec) { + *out = *in + if in.PodSpec != nil { + in, out := &in.PodSpec, &out.PodSpec + *out = new(mdb.MongoDbPodSpec) + (*in).DeepCopyInto(*out) + } + if in.FeatureCompatibilityVersion != nil { + in, out := &in.FeatureCompatibilityVersion, &out.FeatureCompatibilityVersion + *out = new(string) + **out = **in + } + if in.Security != nil { + in, out := &in.Security, &out.Security + *out = new(mdb.Security) + (*in).DeepCopyInto(*out) + } + if in.Connectivity != nil { + in, out := &in.Connectivity, &out.Connectivity + *out = new(mdb.MongoDBConnectivity) + (*in).DeepCopyInto(*out) + } + if in.ExternalAccessConfiguration != nil { + in, out := &in.ExternalAccessConfiguration, &out.ExternalAccessConfiguration + *out = new(mdb.ExternalAccessConfiguration) + (*in).DeepCopyInto(*out) + } + if in.AdditionalMongodConfig != nil { + in, out := &in.AdditionalMongodConfig, &out.AdditionalMongodConfig + *out = (*in).DeepCopy() + } + in.AutomationAgent.DeepCopyInto(&out.AutomationAgent) + in.MonitoringAgent.DeepCopyInto(&out.MonitoringAgent) + in.ConnectionSpec.DeepCopyInto(&out.ConnectionSpec) + if in.PasswordSecretKeyRef != nil { + in, out := &in.PasswordSecretKeyRef, &out.PasswordSecretKeyRef + *out = new(user.SecretKeyRef) + **out = **in + } + if in.Prometheus != nil { + in, out := &in.Prometheus, &out.Prometheus + *out = new(v1.Prometheus) + **out = **in + } + if in.AutomationConfigOverride != nil { + in, out := &in.AutomationConfigOverride, &out.AutomationConfigOverride + *out = new(v1.AutomationConfigOverride) + (*in).DeepCopyInto(*out) + } + if in.MemberConfig != nil { + in, out := &in.MemberConfig, &out.MemberConfig + *out = make([]automationconfig.MemberOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ClusterSpecList != nil { + in, out := &in.ClusterSpecList, &out.ClusterSpecList + *out = make(mdb.ClusterSpecList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppDBSpec. +func (in *AppDBSpec) DeepCopy() *AppDBSpec { + if in == nil { + return nil + } + out := new(AppDBSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AppDbBuilder) DeepCopyInto(out *AppDbBuilder) { + *out = *in + if in.appDb != nil { + in, out := &in.appDb, &out.appDb + *out = new(AppDBSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppDbBuilder. +func (in *AppDbBuilder) DeepCopy() *AppDbBuilder { + if in == nil { + return nil + } + out := new(AppDbBuilder) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AppDbStatus) DeepCopyInto(out *AppDbStatus) { + *out = *in + in.MongoDbStatus.DeepCopyInto(&out.MongoDbStatus) + if in.ClusterStatusList != nil { + in, out := &in.ClusterStatusList, &out.ClusterStatusList + *out = make([]status.ClusterStatusItem, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AppDbStatus. +func (in *AppDbStatus) DeepCopy() *AppDbStatus { + if in == nil { + return nil + } + out := new(AppDbStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { + *out = *in + in.Common.DeepCopyInto(&out.Common) + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]status.Warning, len(*in)) + copy(*out, *in) + } + if in.ClusterStatusList != nil { + in, out := &in.ClusterStatusList, &out.ClusterStatusList + *out = make([]status.OMClusterStatusItem, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus. +func (in *BackupStatus) DeepCopy() *BackupStatus { + if in == nil { + return nil + } + out := new(BackupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSpecOMItem) DeepCopyInto(out *ClusterSpecOMItem) { + *out = *in + if in.Configuration != nil { + in, out := &in.Configuration, &out.Configuration + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.JVMParams != nil { + in, out := &in.JVMParams, &out.JVMParams + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MongoDBOpsManagerExternalConnectivity != nil { + in, out := &in.MongoDBOpsManagerExternalConnectivity, &out.MongoDBOpsManagerExternalConnectivity + *out = new(MongoDBOpsManagerServiceDefinition) + (*in).DeepCopyInto(*out) + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } + if in.Backup != nil { + in, out := &in.Backup, &out.Backup + *out = new(MongoDBOpsManagerBackupClusterSpecItem) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpecOMItem. +func (in *ClusterSpecOMItem) DeepCopy() *ClusterSpecOMItem { + if in == nil { + return nil + } + out := new(ClusterSpecOMItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionSpec) DeepCopyInto(out *ConnectionSpec) { + *out = *in + in.SharedConnectionSpec.DeepCopyInto(&out.SharedConnectionSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionSpec. +func (in *ConnectionSpec) DeepCopy() *ConnectionSpec { + if in == nil { + return nil + } + out := new(ConnectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataStoreConfig) DeepCopyInto(out *DataStoreConfig) { + *out = *in + out.MongoDBResourceRef = in.MongoDBResourceRef + if in.MongoDBUserRef != nil { + in, out := &in.MongoDBUserRef, &out.MongoDBUserRef + *out = new(MongoDBUserRef) + **out = **in + } + if in.AssignmentLabels != nil { + in, out := &in.AssignmentLabels, &out.AssignmentLabels + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreConfig. +func (in *DataStoreConfig) DeepCopy() *DataStoreConfig { + if in == nil { + return nil + } + out := new(DataStoreConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Encryption) DeepCopyInto(out *Encryption) { + *out = *in + if in.Kmip != nil { + in, out := &in.Kmip, &out.Kmip + *out = new(KmipConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Encryption. +func (in *Encryption) DeepCopy() *Encryption { + if in == nil { + return nil + } + out := new(Encryption) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FileSystemStoreConfig) DeepCopyInto(out *FileSystemStoreConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileSystemStoreConfig. +func (in *FileSystemStoreConfig) DeepCopy() *FileSystemStoreConfig { + if in == nil { + return nil + } + out := new(FileSystemStoreConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KmipConfig) DeepCopyInto(out *KmipConfig) { + *out = *in + out.Server = in.Server +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KmipConfig. +func (in *KmipConfig) DeepCopy() *KmipConfig { + if in == nil { + return nil + } + out := new(KmipConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Logging) DeepCopyInto(out *Logging) { + *out = *in + if in.LogBackAccessRef != nil { + in, out := &in.LogBackAccessRef, &out.LogBackAccessRef + *out = new(mdb.ConfigMapRef) + **out = **in + } + if in.LogBackRef != nil { + in, out := &in.LogBackRef, &out.LogBackRef + *out = new(mdb.ConfigMapRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Logging. +func (in *Logging) DeepCopy() *Logging { + if in == nil { + return nil + } + out := new(Logging) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManager) DeepCopyInto(out *MongoDBOpsManager) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManager. +func (in *MongoDBOpsManager) DeepCopy() *MongoDBOpsManager { + if in == nil { + return nil + } + out := new(MongoDBOpsManager) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBOpsManager) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerBackup) DeepCopyInto(out *MongoDBOpsManagerBackup) { + *out = *in + if in.ExternalServiceEnabled != nil { + in, out := &in.ExternalServiceEnabled, &out.ExternalServiceEnabled + *out = new(bool) + **out = **in + } + if in.AssignmentLabels != nil { + in, out := &in.AssignmentLabels, &out.AssignmentLabels + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.HeadDB != nil { + in, out := &in.HeadDB, &out.HeadDB + *out = new(mdb.PersistenceConfig) + (*in).DeepCopyInto(*out) + } + if in.JVMParams != nil { + in, out := &in.JVMParams, &out.JVMParams + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.S3OplogStoreConfigs != nil { + in, out := &in.S3OplogStoreConfigs, &out.S3OplogStoreConfigs + *out = make([]S3Config, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.OplogStoreConfigs != nil { + in, out := &in.OplogStoreConfigs, &out.OplogStoreConfigs + *out = make([]DataStoreConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.BlockStoreConfigs != nil { + in, out := &in.BlockStoreConfigs, &out.BlockStoreConfigs + *out = make([]DataStoreConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.S3Configs != nil { + in, out := &in.S3Configs, &out.S3Configs + *out = make([]S3Config, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.FileSystemStoreConfigs != nil { + in, out := &in.FileSystemStoreConfigs, &out.FileSystemStoreConfigs + *out = make([]FileSystemStoreConfig, len(*in)) + copy(*out, *in) + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } + out.QueryableBackupSecretRef = in.QueryableBackupSecretRef + if in.Encryption != nil { + in, out := &in.Encryption, &out.Encryption + *out = new(Encryption) + (*in).DeepCopyInto(*out) + } + if in.Logging != nil { + in, out := &in.Logging, &out.Logging + *out = new(Logging) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerBackup. +func (in *MongoDBOpsManagerBackup) DeepCopy() *MongoDBOpsManagerBackup { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerBackupClusterSpecItem) DeepCopyInto(out *MongoDBOpsManagerBackupClusterSpecItem) { + *out = *in + if in.AssignmentLabels != nil { + in, out := &in.AssignmentLabels, &out.AssignmentLabels + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.HeadDB != nil { + in, out := &in.HeadDB, &out.HeadDB + *out = new(mdb.PersistenceConfig) + (*in).DeepCopyInto(*out) + } + if in.JVMParams != nil { + in, out := &in.JVMParams, &out.JVMParams + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerBackupClusterSpecItem. +func (in *MongoDBOpsManagerBackupClusterSpecItem) DeepCopy() *MongoDBOpsManagerBackupClusterSpecItem { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerBackupClusterSpecItem) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerList) DeepCopyInto(out *MongoDBOpsManagerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MongoDBOpsManager, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerList. +func (in *MongoDBOpsManagerList) DeepCopy() *MongoDBOpsManagerList { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBOpsManagerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerSecurity) DeepCopyInto(out *MongoDBOpsManagerSecurity) { + *out = *in + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerSecurity. +func (in *MongoDBOpsManagerSecurity) DeepCopy() *MongoDBOpsManagerSecurity { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerSecurity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerServiceDefinition) DeepCopyInto(out *MongoDBOpsManagerServiceDefinition) { + *out = *in + if in.ClusterIP != nil { + in, out := &in.ClusterIP, &out.ClusterIP + *out = new(string) + **out = **in + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerServiceDefinition. +func (in *MongoDBOpsManagerServiceDefinition) DeepCopy() *MongoDBOpsManagerServiceDefinition { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerServiceDefinition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerSpec) DeepCopyInto(out *MongoDBOpsManagerSpec) { + *out = *in + if in.Configuration != nil { + in, out := &in.Configuration, &out.Configuration + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.AppDB.DeepCopyInto(&out.AppDB) + if in.Logging != nil { + in, out := &in.Logging, &out.Logging + *out = new(Logging) + (*in).DeepCopyInto(*out) + } + if in.JVMParams != nil { + in, out := &in.JVMParams, &out.JVMParams + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Backup != nil { + in, out := &in.Backup, &out.Backup + *out = new(MongoDBOpsManagerBackup) + (*in).DeepCopyInto(*out) + } + if in.InternalConnectivity != nil { + in, out := &in.InternalConnectivity, &out.InternalConnectivity + *out = new(MongoDBOpsManagerServiceDefinition) + (*in).DeepCopyInto(*out) + } + if in.MongoDBOpsManagerExternalConnectivity != nil { + in, out := &in.MongoDBOpsManagerExternalConnectivity, &out.MongoDBOpsManagerExternalConnectivity + *out = new(MongoDBOpsManagerServiceDefinition) + (*in).DeepCopyInto(*out) + } + if in.Security != nil { + in, out := &in.Security, &out.Security + *out = new(MongoDBOpsManagerSecurity) + **out = **in + } + if in.StatefulSetConfiguration != nil { + in, out := &in.StatefulSetConfiguration, &out.StatefulSetConfiguration + *out = new(v1.StatefulSetConfiguration) + (*in).DeepCopyInto(*out) + } + if in.ClusterSpecList != nil { + in, out := &in.ClusterSpecList, &out.ClusterSpecList + *out = make([]ClusterSpecOMItem, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerSpec. +func (in *MongoDBOpsManagerSpec) DeepCopy() *MongoDBOpsManagerSpec { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerStatus) DeepCopyInto(out *MongoDBOpsManagerStatus) { + *out = *in + in.OpsManagerStatus.DeepCopyInto(&out.OpsManagerStatus) + in.AppDbStatus.DeepCopyInto(&out.AppDbStatus) + in.BackupStatus.DeepCopyInto(&out.BackupStatus) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerStatus. +func (in *MongoDBOpsManagerStatus) DeepCopy() *MongoDBOpsManagerStatus { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBOpsManagerTLS) DeepCopyInto(out *MongoDBOpsManagerTLS) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBOpsManagerTLS. +func (in *MongoDBOpsManagerTLS) DeepCopy() *MongoDBOpsManagerTLS { + if in == nil { + return nil + } + out := new(MongoDBOpsManagerTLS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBUserRef) DeepCopyInto(out *MongoDBUserRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUserRef. +func (in *MongoDBUserRef) DeepCopy() *MongoDBUserRef { + if in == nil { + return nil + } + out := new(MongoDBUserRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpsManagerBuilder) DeepCopyInto(out *OpsManagerBuilder) { + *out = *in + in.om.DeepCopyInto(&out.om) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpsManagerBuilder. +func (in *OpsManagerBuilder) DeepCopy() *OpsManagerBuilder { + if in == nil { + return nil + } + out := new(OpsManagerBuilder) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpsManagerStatus) DeepCopyInto(out *OpsManagerStatus) { + *out = *in + in.Common.DeepCopyInto(&out.Common) + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]status.Warning, len(*in)) + copy(*out, *in) + } + if in.ClusterStatusList != nil { + in, out := &in.ClusterStatusList, &out.ClusterStatusList + *out = make([]status.OMClusterStatusItem, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpsManagerStatus. +func (in *OpsManagerStatus) DeepCopy() *OpsManagerStatus { + if in == nil { + return nil + } + out := new(OpsManagerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpsManagerVersionMapping) DeepCopyInto(out *OpsManagerVersionMapping) { + *out = *in + if in.OpsManager != nil { + in, out := &in.OpsManager, &out.OpsManager + *out = make(map[OpsManagerVersion]AgentVersion, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpsManagerVersionMapping. +func (in *OpsManagerVersionMapping) DeepCopy() *OpsManagerVersionMapping { + if in == nil { + return nil + } + out := new(OpsManagerVersionMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3Config) DeepCopyInto(out *S3Config) { + *out = *in + if in.MongoDBResourceRef != nil { + in, out := &in.MongoDBResourceRef, &out.MongoDBResourceRef + *out = new(user.MongoDBResourceRef) + **out = **in + } + if in.MongoDBUserRef != nil { + in, out := &in.MongoDBUserRef, &out.MongoDBUserRef + *out = new(MongoDBUserRef) + **out = **in + } + if in.S3SecretRef != nil { + in, out := &in.S3SecretRef, &out.S3SecretRef + *out = new(SecretRef) + **out = **in + } + if in.AssignmentLabels != nil { + in, out := &in.AssignmentLabels, &out.AssignmentLabels + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CustomCertificateSecretRefs != nil { + in, out := &in.CustomCertificateSecretRefs, &out.CustomCertificateSecretRefs + *out = make([]corev1.SecretKeySelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Config. +func (in *S3Config) DeepCopy() *S3Config { + if in == nil { + return nil + } + out := new(S3Config) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretRef) DeepCopyInto(out *SecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretRef. +func (in *SecretRef) DeepCopy() *SecretRef { + if in == nil { + return nil + } + out := new(SecretRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSSecretRef) DeepCopyInto(out *TLSSecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSSecretRef. +func (in *TLSSecretRef) DeepCopy() *TLSSecretRef { + if in == nil { + return nil + } + out := new(TLSSecretRef) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1/register.go b/api/v1/register.go new file mode 100644 index 000000000..2efd9821b --- /dev/null +++ b/api/v1/register.go @@ -0,0 +1,19 @@ +// NOTE: Boilerplate only. Ignore this file. + +// Package v1 contains API Schema definitions for the mongodb v1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=mongodb.com +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "mongodb.com", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +) diff --git a/api/v1/status/doc.go b/api/v1/status/doc.go new file mode 100644 index 000000000..6c12ae38b --- /dev/null +++ b/api/v1/status/doc.go @@ -0,0 +1 @@ +package status diff --git a/api/v1/status/option.go b/api/v1/status/option.go new file mode 100644 index 000000000..85ac32b82 --- /dev/null +++ b/api/v1/status/option.go @@ -0,0 +1,123 @@ +package status + +import ( + "reflect" +) + +type Option interface { + Value() interface{} +} + +type noOption struct{} + +func (o noOption) Value() interface{} { + return nil +} + +// MessageOption describes the status message +type MessageOption struct { + Message string +} + +func NewMessageOption(message string) MessageOption { + return MessageOption{Message: message} +} + +func (o MessageOption) Value() interface{} { + return o.Message +} + +// WarningsOption describes the status warnings +type WarningsOption struct { + Warnings []Warning +} + +func NewWarningsOption(warnings []Warning) WarningsOption { + return WarningsOption{Warnings: warnings} +} + +func (o WarningsOption) Value() interface{} { + return o.Warnings +} + +// BaseUrlOption describes the Ops Manager base URL. +type BaseUrlOption struct { + BaseUrl string +} + +func NewBaseUrlOption(baseUrl string) BaseUrlOption { + return BaseUrlOption{BaseUrl: baseUrl} +} + +func (o BaseUrlOption) Value() interface{} { + return o.BaseUrl +} + +// OMPartOption describes the part of Ops Manager resource status to be updated +type OMPartOption struct { + StatusPart Part +} + +func NewOMPartOption(statusPart Part) OMPartOption { + return OMPartOption{StatusPart: statusPart} +} + +func (o OMPartOption) Value() interface{} { + return o.StatusPart +} + +// ResourcesNotReadyOption describes the resources dependent on the resource which are not ready +type ResourcesNotReadyOption struct { + ResourcesNotReady []ResourceNotReady +} + +func NewResourcesNotReadyOption(resourceNotReady []ResourceNotReady) ResourcesNotReadyOption { + return ResourcesNotReadyOption{ResourcesNotReady: resourceNotReady} +} + +func (o ResourcesNotReadyOption) Value() interface{} { + return o.ResourcesNotReady +} + +type BackupStatusOption struct { + statusName string +} + +func NewBackupStatusOption(statusName string) BackupStatusOption { + return BackupStatusOption{ + statusName: statusName, + } +} + +func (o BackupStatusOption) Value() interface{} { + return o.statusName +} + +func GetOption(statusOptions []Option, targetOption Option) (Option, bool) { + for _, s := range statusOptions { + if reflect.TypeOf(s) == reflect.TypeOf(targetOption) { + return s, true + } + } + return noOption{}, false +} + +// PVCStatusOption describes the resources pvc statuses +type PVCStatusOption struct { + PVC *PVC +} + +func NewPVCsStatusOption(pvc *PVC) PVCStatusOption { + return PVCStatusOption{PVC: pvc} +} + +func (o PVCStatusOption) Value() interface{} { + return o.PVC +} + +// NewPVCsStatusOptionEmptyStatus sets a nil status; such that later in r.updateStatus(), commonUpdate sets the field +// explicitly to nil to remove that field. +// Otherwise, that field will forever be in the status field. +func NewPVCsStatusOptionEmptyStatus() PVCStatusOption { + return PVCStatusOption{PVC: nil} +} diff --git a/api/v1/status/part.go b/api/v1/status/part.go new file mode 100644 index 000000000..c71722cd5 --- /dev/null +++ b/api/v1/status/part.go @@ -0,0 +1,11 @@ +package status + +// Part is the logical constant for specific field in status in the MongoDBOpsManager +type Part int + +const ( + AppDb Part = iota + OpsManager + Backup + None +) diff --git a/api/v1/status/phase.go b/api/v1/status/phase.go new file mode 100644 index 000000000..064577d3c --- /dev/null +++ b/api/v1/status/phase.go @@ -0,0 +1,36 @@ +package status + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Phase string + +const ( + // PhasePending means the reconciliation has finished but the resource is neither in Error nor Running state - + // most of all waiting for some event to happen (CSRs approved, shard rebalanced etc) + PhasePending Phase = "Pending" + + // PhaseRunning means the Mongodb Resource is in a running state + PhaseRunning Phase = "Running" + + // PhaseFailed means the Mongodb Resource is in a failed state + PhaseFailed Phase = "Failed" + + // PhaseDisabled means that the resource is not enabled + PhaseDisabled Phase = "Disabled" + + // PhaseUpdated means a MongoDBUser was successfully updated + PhaseUpdated Phase = "Updated" + + // PhaseUnsupported means a resource is not supported by the current Operator version + PhaseUnsupported Phase = "Unsupported" +) + +type Updater interface { + GetStatusPath(options ...Option) string + GetStatus(options ...Option) interface{} + UpdateStatus(phase Phase, statusOptions ...Option) + GetCommonStatus(options ...Option) *Common + client.Object +} diff --git a/api/v1/status/pvc/phase.go b/api/v1/status/pvc/phase.go new file mode 100644 index 000000000..853a01c86 --- /dev/null +++ b/api/v1/status/pvc/phase.go @@ -0,0 +1,9 @@ +package pvc + +type Phase string + +const ( + PhaseNoAction Phase = "PVC Resize - NoAction" + PhasePVCResize Phase = "PVC Resize - PVC Is Resizing" + PhaseSTSOrphaned Phase = "PVC Resize - STS has been orphaned" +) diff --git a/api/v1/status/resourcenotready.go b/api/v1/status/resourcenotready.go new file mode 100644 index 000000000..ff470c018 --- /dev/null +++ b/api/v1/status/resourcenotready.go @@ -0,0 +1,23 @@ +package status + +// ResourceKind specifies a kind of a Kubernetes resource. Used in status of a Custom Resource +type ResourceKind string + +const ( + StatefulsetKind ResourceKind = "StatefulSet" +) + +// ResourceNotReady describes the dependent resource which is not ready yet +// +k8s:deepcopy-gen=true +type ResourceNotReady struct { + Kind ResourceKind `json:"kind"` + Name string `json:"name"` + Errors []ResourceError `json:"errors,omitempty"` + Message string `json:"message,omitempty"` +} + +// +k8s:deepcopy-gen=true +type ResourceError struct { + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` +} diff --git a/api/v1/status/scaling_status.go b/api/v1/status/scaling_status.go new file mode 100644 index 000000000..004782cb1 --- /dev/null +++ b/api/v1/status/scaling_status.go @@ -0,0 +1,152 @@ +package status + +import ( + "fmt" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers/interfaces" +) + +func MembersOption(replicaSetscaler scale.ReplicaSetScaler) Option { + return ReplicaSetMembersOption{Members: scale.ReplicasThisReconciliation(replicaSetscaler)} +} + +func AppDBMemberOptions(appDBScalers ...interfaces.MultiClusterReplicaSetScaler) Option { + members := 0 + clusterStatusList := []ClusterStatusItem{} + for _, scaler := range appDBScalers { + members += scale.ReplicasThisReconciliation(scaler) + clusterStatusList = append(clusterStatusList, ClusterStatusItem{ + Members: scale.ReplicasThisReconciliation(scaler), + ClusterName: scaler.MemberClusterName(), + }) + } + return MultiReplicaSetMemberOption{Members: members, ClusterStatusList: clusterStatusList} +} + +// ReplicaSetMembersOption is required in order to ensure that the status of a resource +// is only updated one member at a time. The logic which scales incrementally relies +// on the current status of the resource to be accurate. +type ReplicaSetMembersOption struct { + Members int +} + +func (o ReplicaSetMembersOption) Value() interface{} { + return o.Members +} + +type OMClusterStatusItem struct { + ClusterName string `json:"clusterName,omitempty"` + Replicas int `json:"replicas,omitempty"` +} + +type ClusterStatusItem struct { + ClusterName string `json:"clusterName,omitempty"` + Members int `json:"members,omitempty"` +} + +type ClusterStatusList struct { + ClusterStatuses []ClusterStatusItem `json:"clusterStatuses,omitempty"` +} + +type MultiReplicaSetMemberOption struct { + Members int + ClusterStatusList []ClusterStatusItem +} + +func (o MultiReplicaSetMemberOption) Value() interface{} { + return struct { + Members int + ClusterStatusList []ClusterStatusItem + }{ + o.Members, + o.ClusterStatusList, + } +} + +type ShardedClusterMongodsPerShardCountOption struct { + Members int +} + +func (o ShardedClusterMongodsPerShardCountOption) Value() interface{} { + return o.Members +} + +type ShardedClusterConfigServerOption struct { + Members int +} + +func (o ShardedClusterConfigServerOption) Value() interface{} { + return o.Members +} + +type ShardedClusterMongosOption struct { + Members int +} + +func (o ShardedClusterMongosOption) Value() interface{} { + return o.Members +} + +type ShardedClusterSizeConfigOption struct { + SizeConfig *MongodbShardedClusterSizeConfig +} + +func (o ShardedClusterSizeConfigOption) Value() interface{} { + return o.SizeConfig +} + +type ShardedClusterSizeStatusInClustersOption struct { + SizeConfigInClusters *MongodbShardedSizeStatusInClusters +} + +func (o ShardedClusterSizeStatusInClustersOption) Value() interface{} { + return o.SizeConfigInClusters +} + +// MongodbShardedClusterSizeConfig describes the numbers and sizes of replica sets inside sharded cluster +// +k8s:deepcopy-gen=true +type MongodbShardedClusterSizeConfig struct { + ShardCount int `json:"shardCount,omitempty"` + MongodsPerShardCount int `json:"mongodsPerShardCount,omitempty"` + MongosCount int `json:"mongosCount,omitempty"` + ConfigServerCount int `json:"configServerCount,omitempty"` +} + +func (m *MongodbShardedClusterSizeConfig) String() string { + return fmt.Sprintf("%+v", *m) +} + +// MongodbShardedSizeStatusInClusters describes the number and sizes of replica sets members deployed across member clusters +// +k8s:deepcopy-gen=true +type MongodbShardedSizeStatusInClusters struct { + ShardMongodsInClusters map[string]int `json:"shardMongodsInClusters,omitempty"` + ShardOverridesInClusters map[string]map[string]int `json:"shardOverridesInClusters,omitempty"` + MongosCountInClusters map[string]int `json:"mongosCountInClusters,omitempty"` + ConfigServerMongodsInClusters map[string]int `json:"configServerMongodsInClusters,omitempty"` +} + +func String(m *MongodbShardedSizeStatusInClusters) string { + return fmt.Sprintf("%+v", *m) +} + +func sumMap(m map[string]int) int { + sum := 0 + for _, v := range m { + sum += v + } + return sum +} + +func (s *MongodbShardedSizeStatusInClusters) TotalShardMongodsInClusters() int { + return sumMap(s.ShardMongodsInClusters) +} + +func (s *MongodbShardedSizeStatusInClusters) TotalConfigServerMongodsInClusters() int { + return sumMap(s.ConfigServerMongodsInClusters) +} + +func (s *MongodbShardedSizeStatusInClusters) TotalMongosCountInClusters() int { + return sumMap(s.MongosCountInClusters) +} diff --git a/api/v1/status/status.go b/api/v1/status/status.go new file mode 100644 index 000000000..582d683e1 --- /dev/null +++ b/api/v1/status/status.go @@ -0,0 +1,100 @@ +package status + +import ( + "reflect" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status/pvc" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/timeutil" +) + +type Writer interface { + // UpdateStatus updates the status of the object + UpdateStatus(phase Phase, statusOptions ...Option) + + // SetWarnings sets the warnings for the object, the list of Options + // provided indicate which Status subresource should be updated in the case + // of AppDB, OpsManager and Backup + SetWarnings([]Warning, ...Option) + + // GetStatusPath should return the path that should be used + // to patch the Status object + GetStatusPath(options ...Option) string +} + +type Reader interface { + // GetStatus returns the status of the object. The list of Options + // provided indicates which subresource will be returned. AppDB, OM or Backup + GetStatus(options ...Option) interface{} + GetCommonStatus(options ...Option) *Common +} + +// Common is the struct shared by all statuses in existing Custom Resources. +// +kubebuilder:object:generate:=true +type Common struct { + Phase Phase `json:"phase"` + Message string `json:"message,omitempty"` + LastTransition string `json:"lastTransition,omitempty"` + ResourcesNotReady []ResourceNotReady `json:"resourcesNotReady,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + PVCs PVCS `json:"pvc,omitempty"` +} + +type PVCS []PVC + +func (p *PVCS) Merge(pvc2 PVC) PVCS { + if p == nil { + return nil + } + + found := false + for i := range *p { + if (*p)[i].StatefulsetName == pvc2.StatefulsetName { + found = true + (*p)[i].Phase = pvc2.Phase + } + } + + if !found { + *p = append(*p, pvc2) + } + + return *p +} + +type PVC struct { + Phase pvc.Phase `json:"phase"` + StatefulsetName string `json:"statefulsetName"` +} + +func (p *PVC) GetPhase() pvc.Phase { + if p == nil || p.StatefulsetName == "" || p.Phase == "" { + return pvc.PhaseNoAction + } + return p.Phase +} + +// UpdateCommonFields is the update function to update common fields used in statuses of all managed CRs +func (s *Common) UpdateCommonFields(phase Phase, generation int64, statusOptions ...Option) { + previousStatus := s.DeepCopy() + s.Phase = phase + s.ObservedGeneration = generation + if option, exists := GetOption(statusOptions, MessageOption{}); exists { + s.Message = stringutil.UpperCaseFirstChar(option.(MessageOption).Message) + } + if option, exists := GetOption(statusOptions, ResourcesNotReadyOption{}); exists { + s.ResourcesNotReady = option.(ResourcesNotReadyOption).ResourcesNotReady + } + if option, exists := GetOption(statusOptions, PVCStatusOption{}); exists { + p := option.(PVCStatusOption).PVC + if p == nil { + s.PVCs = nil + } else { + s.PVCs.Merge(*p) + } + } + // We update the time only if the status really changed. Otherwise, we'd like to preserve the old one. + if !reflect.DeepEqual(previousStatus, s) { + s.LastTransition = timeutil.Now() + } +} diff --git a/api/v1/status/status_test.go b/api/v1/status/status_test.go new file mode 100644 index 000000000..d36732b96 --- /dev/null +++ b/api/v1/status/status_test.go @@ -0,0 +1,45 @@ +package status + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_NotRefreshingLastTransitionTime(t *testing.T) { + // given + testTime := "test" + + status := &Common{ + Phase: PhaseFailed, + Message: "test", + LastTransition: testTime, + ObservedGeneration: 1, + } + + // when + status.UpdateCommonFields(PhaseFailed, 1) + timeAfterTheTest := status.LastTransition + + // then + assert.Equal(t, testTime, timeAfterTheTest) +} + +func Test_RefreshingLastTransitionTime(t *testing.T) { + // given + testTime := "test" + + status := &Common{ + Phase: PhaseFailed, + Message: "test", + LastTransition: testTime, + ObservedGeneration: 1, + } + + // when + status.UpdateCommonFields(PhaseRunning, 2) + timeAfterTheTest := status.LastTransition + + // then + assert.NotEqual(t, testTime, timeAfterTheTest) +} diff --git a/api/v1/status/warnings.go b/api/v1/status/warnings.go new file mode 100644 index 000000000..0a472051e --- /dev/null +++ b/api/v1/status/warnings.go @@ -0,0 +1,27 @@ +package status + +type Warning string + +type Warnings []Warning + +const ( + SEP Warning = ";" +) + +func (m Warnings) AddIfNotExists(warning Warning) Warnings { + for _, existingWarning := range m { + if existingWarning == warning || existingWarning == warning+SEP { + return m + } + } + + // separate warnings by a ; + for i := 0; i < len(m); i++ { + existingWarning := m[i] + if existingWarning[len(existingWarning)-1:] != SEP { + m[i] += SEP + } + } + + return append(m, warning) +} diff --git a/api/v1/status/zz_generated.deepcopy.go b/api/v1/status/zz_generated.deepcopy.go new file mode 100644 index 000000000..3636e83fb --- /dev/null +++ b/api/v1/status/zz_generated.deepcopy.go @@ -0,0 +1,154 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package status + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Common) DeepCopyInto(out *Common) { + *out = *in + if in.ResourcesNotReady != nil { + in, out := &in.ResourcesNotReady, &out.ResourcesNotReady + *out = make([]ResourceNotReady, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PVCs != nil { + in, out := &in.PVCs, &out.PVCs + *out = make(PVCS, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Common. +func (in *Common) DeepCopy() *Common { + if in == nil { + return nil + } + out := new(Common) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongodbShardedClusterSizeConfig) DeepCopyInto(out *MongodbShardedClusterSizeConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongodbShardedClusterSizeConfig. +func (in *MongodbShardedClusterSizeConfig) DeepCopy() *MongodbShardedClusterSizeConfig { + if in == nil { + return nil + } + out := new(MongodbShardedClusterSizeConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongodbShardedSizeStatusInClusters) DeepCopyInto(out *MongodbShardedSizeStatusInClusters) { + *out = *in + if in.ShardMongodsInClusters != nil { + in, out := &in.ShardMongodsInClusters, &out.ShardMongodsInClusters + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ShardOverridesInClusters != nil { + in, out := &in.ShardOverridesInClusters, &out.ShardOverridesInClusters + *out = make(map[string]map[string]int, len(*in)) + for key, val := range *in { + var outVal map[string]int + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.MongosCountInClusters != nil { + in, out := &in.MongosCountInClusters, &out.MongosCountInClusters + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ConfigServerMongodsInClusters != nil { + in, out := &in.ConfigServerMongodsInClusters, &out.ConfigServerMongodsInClusters + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongodbShardedSizeStatusInClusters. +func (in *MongodbShardedSizeStatusInClusters) DeepCopy() *MongodbShardedSizeStatusInClusters { + if in == nil { + return nil + } + out := new(MongodbShardedSizeStatusInClusters) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceError) DeepCopyInto(out *ResourceError) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceError. +func (in *ResourceError) DeepCopy() *ResourceError { + if in == nil { + return nil + } + out := new(ResourceError) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceNotReady) DeepCopyInto(out *ResourceNotReady) { + *out = *in + if in.Errors != nil { + in, out := &in.Errors, &out.Errors + *out = make([]ResourceError, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceNotReady. +func (in *ResourceNotReady) DeepCopy() *ResourceNotReady { + if in == nil { + return nil + } + out := new(ResourceNotReady) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1/user/doc.go b/api/v1/user/doc.go new file mode 100644 index 000000000..bb014d2e2 --- /dev/null +++ b/api/v1/user/doc.go @@ -0,0 +1,4 @@ +package user + +// +k8s:deepcopy-gen=package +// +versionName=v1 diff --git a/api/v1/user/groupversion_info.go b/api/v1/user/groupversion_info.go new file mode 100644 index 000000000..38b73e558 --- /dev/null +++ b/api/v1/user/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1 contains API Schema definitions for the mongodb v1 API group +// +kubebuilder:object:generate=true +// +groupName=mongodb.com +package user + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "mongodb.com", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1/user/mongodbuser_types.go b/api/v1/user/mongodbuser_types.go new file mode 100644 index 000000000..56e07938d --- /dev/null +++ b/api/v1/user/mongodbuser_types.go @@ -0,0 +1,200 @@ +package user + +import ( + "context" + "fmt" + "regexp" + "strings" + + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/client" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func init() { + v1.SchemeBuilder.Register(&MongoDBUser{}, &MongoDBUserList{}) +} + +// The MongoDBUser resource allows you to create, deletion and configure +// users for your MongoDB deployments + +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +kubebuilder:resource:shortName=mdbu +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="The current state of the MongoDB User." +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="The time since the MongoDB User resource was created." +type MongoDBUser struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + // +optional + Status MongoDBUserStatus `json:"status"` + Spec MongoDBUserSpec `json:"spec"` +} + +func (u *MongoDBUser) GetCommonStatus(options ...status.Option) *status.Common { + return &u.Status.Common +} + +// GetPassword returns the password of the user as stored in the referenced +// secret. If the password secret reference is unset then a blank password and +// a nil error will be returned. +func (u MongoDBUser) GetPassword(ctx context.Context, secretClient secrets.SecretClient) (string, error) { + if u.Spec.PasswordSecretKeyRef.Name == "" { + return "", nil + } + + nsName := client.ObjectKey{ + Namespace: u.Namespace, + Name: u.Spec.PasswordSecretKeyRef.Name, + } + var databaseSecretPath string + if vault.IsVaultSecretBackend() { + databaseSecretPath = secretClient.VaultClient.DatabaseSecretPath() + } + secretData, err := secretClient.ReadSecret(ctx, nsName, databaseSecretPath) + if err != nil { + return "", xerrors.Errorf("could not retrieve user password secret: %w", err) + } + + passwordBytes, passwordIsSet := secretData[u.Spec.PasswordSecretKeyRef.Key] + if !passwordIsSet { + return "", xerrors.Errorf("password is not set in password secret") + } + + return passwordBytes, nil +} + +// SecretKeyRef is a reference to a value in a given secret in the same +// namespace. Based on: +// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core +type SecretKeyRef struct { + Name string `json:"name"` + Key string `json:"key,omitempty"` +} + +type MongoDBResourceRef struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` +} + +type MongoDBUserSpec struct { + Roles []Role `json:"roles,omitempty"` + Username string `json:"username"` + Database string `json:"db"` + // +optional + MongoDBResourceRef MongoDBResourceRef `json:"mongodbResourceRef"` + // +optional + PasswordSecretKeyRef SecretKeyRef `json:"passwordSecretKeyRef"` + // +optional + ConnectionStringSecretName string `json:"connectionStringSecretName"` +} + +type MongoDBUserStatus struct { + status.Common `json:",inline"` + Roles []Role `json:"roles,omitempty"` + Username string `json:"username"` + Database string `json:"db"` + Project string `json:"project"` + Warnings []status.Warning `json:"warnings,omitempty"` +} + +type Role struct { + RoleName string `json:"name"` + Database string `json:"db"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type MongoDBUserList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + Items []MongoDBUser `json:"items"` +} + +// ChangedIdentifier determines if the user has changed a value that is used in +// uniquely identifying them. Either username or db. This function relies on the status +// of the resource and is required in order to remove the old user before +// adding a new one to avoid leaving stale state in Ops Manger. +func (u *MongoDBUser) ChangedIdentifier() bool { + if u.Status.Username == "" || u.Status.Database == "" { + return false + } + return u.Status.Username != u.Spec.Username || u.Status.Database != u.Spec.Database +} + +func (u *MongoDBUser) UpdateStatus(phase status.Phase, statusOptions ...status.Option) { + u.Status.UpdateCommonFields(phase, u.GetGeneration(), statusOptions...) + if option, exists := status.GetOption(statusOptions, status.WarningsOption{}); exists { + u.Status.Warnings = append(u.Status.Warnings, option.(status.WarningsOption).Warnings...) + } + + if phase == status.PhaseRunning { + u.Status.Phase = status.PhaseUpdated + u.Status.Roles = u.Spec.Roles + u.Status.Database = u.Spec.Database + u.Status.Username = u.Spec.Username + } +} + +func (u MongoDBUser) GetConnectionStringSecretName() string { + if u.Spec.ConnectionStringSecretName != "" { + return u.Spec.ConnectionStringSecretName + } + var resourceRef string + if u.Spec.MongoDBResourceRef.Name != "" { + resourceRef = u.Spec.MongoDBResourceRef.Name + "-" + } + + database := u.Spec.Database + if database == "$external" { + database = strings.TrimPrefix(database, "$") + } + + return normalizeName(fmt.Sprintf("%s%s-%s", resourceRef, u.Name, database)) +} + +// normalizeName returns a string that conforms to RFC-1123. +// This logic is duplicated in the community operator in https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/api/v1/mongodbcommunity_types.go. +// The logic should be reused if/when we unify the user types or observe that the logic needs to be changed for business logic reasons, to avoid modifying it +// in separate places in the future. +func normalizeName(name string) string { + errors := validation.IsDNS1123Subdomain(name) + if len(errors) == 0 { + return name + } + + // convert name to lowercase and replace invalid characters with '-' + name = strings.ToLower(name) + re := regexp.MustCompile("[^a-z0-9-]+") + name = re.ReplaceAllString(name, "-") + + // Remove duplicate `-` resulting from contiguous non-allowed chars. + re = regexp.MustCompile(`\-+`) + name = re.ReplaceAllString(name, "-") + + name = strings.Trim(name, "-") + + if len(name) > validation.DNS1123SubdomainMaxLength { + name = name[0:validation.DNS1123SubdomainMaxLength] + } + return name +} + +func (u *MongoDBUser) SetWarnings(warnings []status.Warning, _ ...status.Option) { + u.Status.Warnings = warnings +} + +func (u *MongoDBUser) GetStatus(...status.Option) interface{} { + return u.Status +} + +func (u MongoDBUser) GetStatusPath(...status.Option) string { + return "/status" +} diff --git a/api/v1/user/mongodbuser_types_test.go b/api/v1/user/mongodbuser_types_test.go new file mode 100644 index 000000000..ba9eefdc6 --- /dev/null +++ b/api/v1/user/mongodbuser_types_test.go @@ -0,0 +1,42 @@ +package user + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMongoDBUser_ChangedIdentifier(t *testing.T) { + before := MongoDBUser{ + Spec: MongoDBUserSpec{ + Username: "before-name", + Database: "before-db", + }, + } + + after := MongoDBUser{ + Spec: MongoDBUserSpec{ + Username: "after-name", + Database: "after-db", + }, + Status: MongoDBUserStatus{ + Username: "before-name", + Database: "before-db", + }, + } + + assert.False(t, before.ChangedIdentifier(), "Status has not be initialized yet so the identifier should not have changed") + assert.True(t, after.ChangedIdentifier(), "Status differs from Spec, so identifier should have changed") + + before = MongoDBUser{ + Spec: MongoDBUserSpec{ + Username: "before-name", + Database: "before-db", + }, + Status: MongoDBUserStatus{ + Username: "before-name", + Database: "before-db", + }, + } + assert.False(t, before.ChangedIdentifier(), "Identifier before and after are the same, identifier should not have changed") +} diff --git a/api/v1/user/zz_generated.deepcopy.go b/api/v1/user/zz_generated.deepcopy.go new file mode 100644 index 000000000..1593d2a96 --- /dev/null +++ b/api/v1/user/zz_generated.deepcopy.go @@ -0,0 +1,178 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package user + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBResourceRef) DeepCopyInto(out *MongoDBResourceRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBResourceRef. +func (in *MongoDBResourceRef) DeepCopy() *MongoDBResourceRef { + if in == nil { + return nil + } + out := new(MongoDBResourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBUser) DeepCopyInto(out *MongoDBUser) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUser. +func (in *MongoDBUser) DeepCopy() *MongoDBUser { + if in == nil { + return nil + } + out := new(MongoDBUser) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBUser) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBUserList) DeepCopyInto(out *MongoDBUserList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MongoDBUser, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUserList. +func (in *MongoDBUserList) DeepCopy() *MongoDBUserList { + if in == nil { + return nil + } + out := new(MongoDBUserList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBUserList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBUserSpec) DeepCopyInto(out *MongoDBUserSpec) { + *out = *in + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]Role, len(*in)) + copy(*out, *in) + } + out.MongoDBResourceRef = in.MongoDBResourceRef + out.PasswordSecretKeyRef = in.PasswordSecretKeyRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUserSpec. +func (in *MongoDBUserSpec) DeepCopy() *MongoDBUserSpec { + if in == nil { + return nil + } + out := new(MongoDBUserSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBUserStatus) DeepCopyInto(out *MongoDBUserStatus) { + *out = *in + in.Common.DeepCopyInto(&out.Common) + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]Role, len(*in)) + copy(*out, *in) + } + if in.Warnings != nil { + in, out := &in.Warnings, &out.Warnings + *out = make([]status.Warning, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUserStatus. +func (in *MongoDBUserStatus) DeepCopy() *MongoDBUserStatus { + if in == nil { + return nil + } + out := new(MongoDBUserStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Role) DeepCopyInto(out *Role) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Role. +func (in *Role) DeepCopy() *Role { + if in == nil { + return nil + } + out := new(Role) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyRef. +func (in *SecretKeyRef) DeepCopy() *SecretKeyRef { + if in == nil { + return nil + } + out := new(SecretKeyRef) + in.DeepCopyInto(out) + return out +} diff --git a/api/v1/validation.go b/api/v1/validation.go new file mode 100644 index 000000000..3878924fc --- /dev/null +++ b/api/v1/validation.go @@ -0,0 +1,43 @@ +package v1 + +import ( + "fmt" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +type validationLevel int + +const ( + SuccessLevel validationLevel = iota + WarningLevel + ErrorLevel +) + +type ValidationResult struct { + Msg string + Level validationLevel + // OmStatusPart indicates which Warnings array this ValidationResult + // should correspond to. Either OpsManager, AppDB or Backup + OmStatusPart status.Part +} + +func ValidationSuccess() ValidationResult { + return ValidationResult{Level: SuccessLevel} +} + +func ValidationWarning(msg string, params ...interface{}) ValidationResult { + return ValidationResult{Msg: fmt.Sprintf(msg, params...), Level: WarningLevel} +} + +func ValidationError(msg string, params ...interface{}) ValidationResult { + return ValidationResult{Msg: fmt.Sprintf(msg, params...), Level: ErrorLevel} +} + +func OpsManagerResourceValidationWarning(msg string, part status.Part, params ...interface{}) ValidationResult { + return ValidationResult{Msg: fmt.Sprintf(msg, params...), Level: WarningLevel, OmStatusPart: part} +} + +func OpsManagerResourceValidationError(msg string, part status.Part, params ...interface{}) ValidationResult { + return ValidationResult{Msg: fmt.Sprintf(msg, params...), Level: ErrorLevel, OmStatusPart: part} +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index df22b4876..ab950d890 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -20,565 +20,49 @@ limitations under the License. package v1 -import ( - "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" -) +import () // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AgentConfiguration) DeepCopyInto(out *AgentConfiguration) { +func (in *KmipClientConfig) DeepCopyInto(out *KmipClientConfig) { *out = *in - if in.LogRotate != nil { - in, out := &in.LogRotate, &out.LogRotate - *out = new(automationconfig.CrdLogRotate) - **out = **in - } - if in.AuditLogRotate != nil { - in, out := &in.AuditLogRotate, &out.AuditLogRotate - *out = new(automationconfig.CrdLogRotate) - **out = **in - } - if in.SystemLog != nil { - in, out := &in.SystemLog, &out.SystemLog - *out = new(automationconfig.SystemLog) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentConfiguration. -func (in *AgentConfiguration) DeepCopy() *AgentConfiguration { - if in == nil { - return nil - } - out := new(AgentConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Authentication) DeepCopyInto(out *Authentication) { - *out = *in - if in.Modes != nil { - in, out := &in.Modes, &out.Modes - *out = make([]AuthMode, len(*in)) - copy(*out, *in) - } - if in.AgentCertificateSecret != nil { - in, out := &in.AgentCertificateSecret, &out.AgentCertificateSecret - *out = new(corev1.LocalObjectReference) - **out = **in - } - if in.IgnoreUnknownUsers != nil { - in, out := &in.IgnoreUnknownUsers, &out.IgnoreUnknownUsers - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Authentication. -func (in *Authentication) DeepCopy() *Authentication { - if in == nil { - return nil - } - out := new(Authentication) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AuthenticationRestriction) DeepCopyInto(out *AuthenticationRestriction) { - *out = *in - if in.ClientSource != nil { - in, out := &in.ClientSource, &out.ClientSource - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.ServerAddress != nil { - in, out := &in.ServerAddress, &out.ServerAddress - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthenticationRestriction. -func (in *AuthenticationRestriction) DeepCopy() *AuthenticationRestriction { - if in == nil { - return nil - } - out := new(AuthenticationRestriction) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AutomationConfigOverride) DeepCopyInto(out *AutomationConfigOverride) { - *out = *in - if in.Processes != nil { - in, out := &in.Processes, &out.Processes - *out = make([]OverrideProcess, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.ReplicaSet.DeepCopyInto(&out.ReplicaSet) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutomationConfigOverride. -func (in *AutomationConfigOverride) DeepCopy() *AutomationConfigOverride { - if in == nil { - return nil - } - out := new(AutomationConfigOverride) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CustomRole) DeepCopyInto(out *CustomRole) { - *out = *in - if in.Privileges != nil { - in, out := &in.Privileges, &out.Privileges - *out = make([]Privilege, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - if in.AuthenticationRestrictions != nil { - in, out := &in.AuthenticationRestrictions, &out.AuthenticationRestrictions - *out = make([]AuthenticationRestriction, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomRole. -func (in *CustomRole) DeepCopy() *CustomRole { - if in == nil { - return nil - } - out := new(CustomRole) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MapWrapper) DeepCopyInto(out *MapWrapper) { - clone := in.DeepCopy() - *out = *clone -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MongoDBCommunity) DeepCopyInto(out *MongoDBCommunity) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunity. -func (in *MongoDBCommunity) DeepCopy() *MongoDBCommunity { - if in == nil { - return nil - } - out := new(MongoDBCommunity) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MongoDBCommunity) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MongoDBCommunityList) DeepCopyInto(out *MongoDBCommunityList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]MongoDBCommunity, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunityList. -func (in *MongoDBCommunityList) DeepCopy() *MongoDBCommunityList { - if in == nil { - return nil - } - out := new(MongoDBCommunityList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MongoDBCommunityList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MongoDBCommunitySpec) DeepCopyInto(out *MongoDBCommunitySpec) { - *out = *in - if in.ReplicaSetHorizons != nil { - in, out := &in.ReplicaSetHorizons, &out.ReplicaSetHorizons - *out = make(ReplicaSetHorizonConfiguration, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = make(automationconfig.ReplicaSetHorizons, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - } - } - in.Security.DeepCopyInto(&out.Security) - if in.Users != nil { - in, out := &in.Users, &out.Users - *out = make([]MongoDBUser, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.StatefulSetConfiguration.DeepCopyInto(&out.StatefulSetConfiguration) - in.AgentConfiguration.DeepCopyInto(&out.AgentConfiguration) - in.AdditionalMongodConfig.DeepCopyInto(&out.AdditionalMongodConfig) - if in.AutomationConfigOverride != nil { - in, out := &in.AutomationConfigOverride, &out.AutomationConfigOverride - *out = new(AutomationConfigOverride) - (*in).DeepCopyInto(*out) - } - if in.Prometheus != nil { - in, out := &in.Prometheus, &out.Prometheus - *out = new(Prometheus) - **out = **in - } - in.AdditionalConnectionStringConfig.DeepCopyInto(&out.AdditionalConnectionStringConfig) - if in.MemberConfig != nil { - in, out := &in.MemberConfig, &out.MemberConfig - *out = make([]automationconfig.MemberOptions, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunitySpec. -func (in *MongoDBCommunitySpec) DeepCopy() *MongoDBCommunitySpec { - if in == nil { - return nil - } - out := new(MongoDBCommunitySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MongoDBCommunityStatus) DeepCopyInto(out *MongoDBCommunityStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunityStatus. -func (in *MongoDBCommunityStatus) DeepCopy() *MongoDBCommunityStatus { - if in == nil { - return nil - } - out := new(MongoDBCommunityStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MongoDBUser) DeepCopyInto(out *MongoDBUser) { - *out = *in - out.PasswordSecretRef = in.PasswordSecretRef - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]Role, len(*in)) - copy(*out, *in) - } - in.AdditionalConnectionStringConfig.DeepCopyInto(&out.AdditionalConnectionStringConfig) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUser. -func (in *MongoDBUser) DeepCopy() *MongoDBUser { - if in == nil { - return nil - } - out := new(MongoDBUser) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MongodConfiguration) DeepCopyInto(out *MongodConfiguration) { - *out = *in - in.MapWrapper.DeepCopyInto(&out.MapWrapper) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongodConfiguration. -func (in *MongodConfiguration) DeepCopy() *MongodConfiguration { - if in == nil { - return nil - } - out := new(MongodConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OverrideProcess) DeepCopyInto(out *OverrideProcess) { - *out = *in - if in.LogRotate != nil { - in, out := &in.LogRotate, &out.LogRotate - *out = new(automationconfig.CrdLogRotate) - **out = **in - } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverrideProcess. -func (in *OverrideProcess) DeepCopy() *OverrideProcess { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KmipClientConfig. +func (in *KmipClientConfig) DeepCopy() *KmipClientConfig { if in == nil { return nil } - out := new(OverrideProcess) + out := new(KmipClientConfig) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OverrideReplicaSet) DeepCopyInto(out *OverrideReplicaSet) { +func (in *KmipServerConfig) DeepCopyInto(out *KmipServerConfig) { *out = *in - if in.Id != nil { - in, out := &in.Id, &out.Id - *out = new(string) - **out = **in - } - in.Settings.DeepCopyInto(&out.Settings) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverrideReplicaSet. -func (in *OverrideReplicaSet) DeepCopy() *OverrideReplicaSet { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KmipServerConfig. +func (in *KmipServerConfig) DeepCopy() *KmipServerConfig { if in == nil { return nil } - out := new(OverrideReplicaSet) + out := new(KmipServerConfig) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Privilege) DeepCopyInto(out *Privilege) { +func (in *ValidationResult) DeepCopyInto(out *ValidationResult) { *out = *in - in.Resource.DeepCopyInto(&out.Resource) - if in.Actions != nil { - in, out := &in.Actions, &out.Actions - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Privilege. -func (in *Privilege) DeepCopy() *Privilege { - if in == nil { - return nil - } - out := new(Privilege) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Prometheus) DeepCopyInto(out *Prometheus) { - *out = *in - out.PasswordSecretRef = in.PasswordSecretRef - out.TLSSecretRef = in.TLSSecretRef -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Prometheus. -func (in *Prometheus) DeepCopy() *Prometheus { - if in == nil { - return nil - } - out := new(Prometheus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ReplicaSetHorizonConfiguration) DeepCopyInto(out *ReplicaSetHorizonConfiguration) { - { - in := &in - *out = make(ReplicaSetHorizonConfiguration, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = make(automationconfig.ReplicaSetHorizons, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaSetHorizonConfiguration. -func (in ReplicaSetHorizonConfiguration) DeepCopy() ReplicaSetHorizonConfiguration { - if in == nil { - return nil - } - out := new(ReplicaSetHorizonConfiguration) - in.DeepCopyInto(out) - return *out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Resource) DeepCopyInto(out *Resource) { - *out = *in - if in.DB != nil { - in, out := &in.DB, &out.DB - *out = new(string) - **out = **in - } - if in.Collection != nil { - in, out := &in.Collection, &out.Collection - *out = new(string) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resource. -func (in *Resource) DeepCopy() *Resource { - if in == nil { - return nil - } - out := new(Resource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Role) DeepCopyInto(out *Role) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Role. -func (in *Role) DeepCopy() *Role { - if in == nil { - return nil - } - out := new(Role) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SecretKeyReference) DeepCopyInto(out *SecretKeyReference) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyReference. -func (in *SecretKeyReference) DeepCopy() *SecretKeyReference { - if in == nil { - return nil - } - out := new(SecretKeyReference) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Security) DeepCopyInto(out *Security) { - *out = *in - in.Authentication.DeepCopyInto(&out.Authentication) - in.TLS.DeepCopyInto(&out.TLS) - if in.Roles != nil { - in, out := &in.Roles, &out.Roles - *out = make([]CustomRole, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Security. -func (in *Security) DeepCopy() *Security { - if in == nil { - return nil - } - out := new(Security) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StatefulSetConfiguration) DeepCopyInto(out *StatefulSetConfiguration) { - *out = *in - in.SpecWrapper.DeepCopyInto(&out.SpecWrapper) - in.MetadataWrapper.DeepCopyInto(&out.MetadataWrapper) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetConfiguration. -func (in *StatefulSetConfiguration) DeepCopy() *StatefulSetConfiguration { - if in == nil { - return nil - } - out := new(StatefulSetConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StatefulSetMetadataWrapper) DeepCopyInto(out *StatefulSetMetadataWrapper) { - clone := in.DeepCopy() - *out = *clone -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StatefulSetSpecWrapper) DeepCopyInto(out *StatefulSetSpecWrapper) { - clone := in.DeepCopy() - *out = *clone -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TLS) DeepCopyInto(out *TLS) { - *out = *in - out.CertificateKeySecret = in.CertificateKeySecret - if in.CaCertificateSecret != nil { - in, out := &in.CaCertificateSecret, &out.CaCertificateSecret - *out = new(corev1.LocalObjectReference) - **out = **in - } - if in.CaConfigMap != nil { - in, out := &in.CaConfigMap, &out.CaConfigMap - *out = new(corev1.LocalObjectReference) - **out = **in - } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLS. -func (in *TLS) DeepCopy() *TLS { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValidationResult. +func (in *ValidationResult) DeepCopy() *ValidationResult { if in == nil { return nil } - out := new(TLS) + out := new(ValidationResult) in.DeepCopyInto(out) return out } diff --git a/config/crd/bases/mongodb.com_mongodb.yaml b/config/crd/bases/mongodb.com_mongodb.yaml new file mode 100644 index 000000000..a5b0287c2 --- /dev/null +++ b/config/crd/bases/mongodb.com_mongodb.yaml @@ -0,0 +1,2662 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodb.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDB + listKind: MongoDBList + plural: mongodb + shortNames: + - mdb + singular: mongodb + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment. + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server. + jsonPath: .status.version + name: Version + type: string + - description: The type of MongoDB deployment. One of 'ReplicaSet', 'ShardedCluster' + and 'Standalone'. + jsonPath: .spec.type + name: Type + type: string + - description: The time since the MongoDB resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for the + mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the mongodb + processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + backup: + description: |- + Backup contains configuration options for configuring + backup for this MongoDB resource + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + autoTerminateOnDeletion: + description: |- + AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + when the MongoDB CR is deleted + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + client: + description: KMIP Client configuration + properties: + clientCertificatePrefix: + description: |- + A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + The names are generated using the following pattern: + KMIP Client Certificate (TLS Secret): + --kmip-client + KMIP Client Certificate Password: + --kmip-client-password + The expected key inside is called "password". + type: string + type: object + required: + - client + type: object + type: object + mode: + enum: + - enabled + - disabled + - terminated + type: string + snapshotSchedule: + properties: + clusterCheckpointIntervalMin: + enum: + - 15 + - 30 + - 60 + type: integer + dailySnapshotRetentionDays: + description: Number of days to retain daily snapshots. Setting + 0 will disable this rule. + maximum: 365 + minimum: 0 + type: integer + fullIncrementalDayOfWeek: + description: Day of the week when Ops Manager takes a full + snapshot. This ensures a recent complete backup. Ops Manager + sets the default value to SUNDAY. + enum: + - SUNDAY + - MONDAY + - TUESDAY + - WEDNESDAY + - THURSDAY + - FRIDAY + - SATURDAY + type: string + monthlySnapshotRetentionMonths: + description: Number of months to retain weekly snapshots. + Setting 0 will disable this rule. + maximum: 36 + minimum: 0 + type: integer + pointInTimeWindowHours: + description: Number of hours in the past for which a point-in-time + snapshot can be created. + enum: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 15 + - 30 + - 60 + - 90 + - 120 + - 180 + - 360 + type: integer + referenceHourOfDay: + description: Hour of the day to schedule snapshots using a + 24-hour clock, in UTC. + maximum: 23 + minimum: 0 + type: integer + referenceMinuteOfHour: + description: Minute of the hour to schedule snapshots, in + UTC. + maximum: 59 + minimum: 0 + type: integer + snapshotIntervalHours: + description: Number of hours between snapshots. + enum: + - 6 + - 8 + - 12 + - 24 + type: integer + snapshotRetentionDays: + description: Number of days to keep recent snapshots. + maximum: 365 + minimum: 1 + type: integer + weeklySnapshotRetentionWeeks: + description: Number of weeks to retain weekly snapshots. Setting + 0 will disable this rule + maximum: 365 + minimum: 0 + type: integer + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + format: hostname + type: string + configServerCount: + type: integer + configSrv: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + configSrvPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + duplicateServiceObjects: + description: |- + In few service mesh options for ex: Istio, by default we would need to duplicate the + service objects created per pod in all the clusters to enable DNS resolution. Users can + however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + type: boolean + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing MongoDB + to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added to the + externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + logLevel: + enum: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities and + tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Amount of members for this MongoDB Replica Set + type: integer + mongodsPerShardCount: + type: integer + mongos: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + mongosCount: + type: integer + mongosPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + persistent: + type: boolean + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + prometheus: + description: Prometheus configurations. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth Password. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode that + the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with a + CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + service: + description: |- + DEPRECATED please use `spec.statefulSet.spec.serviceName` to provide a custom service name. + this is an optional service, it will get the name "-service" in case not provided + type: string + shard: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + shardCount: + type: integer + shardOverrides: + description: |- + ShardOverrides allow for overriding the configuration of a specific shard. + It replaces deprecated spec.shard.shardSpecificPodSpec field. When spec.shard.shardSpecificPodSpec is still defined then + spec.shard.shardSpecificPodSpec is applied first to the particular shard and then spec.shardOverrides is applied on top + of that (if defined for the same shard). + items: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation + for the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItemOverride is almost exact copy of ClusterSpecItem object. + The object is used in ClusterSpecList in ShardedClusterComponentOverrideSpec in shard overrides. + The difference lies in some fields being optional, e.g. Members to make it possible to NOT override fields and rely on + what was set in top level shard configuration. + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + type: array + memberConfig: + description: Process configuration override for this shard. + Used in SingleCluster only. The number of items specified + must be >= spec.mongodsPerShardCount or spec.shardOverride.members. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Number of member nodes in this shard. Used only + in SingleCluster. For MultiCluster the number of members is + specified in ShardOverride.ClusterSpecList. + type: integer + podSpec: + description: The following override fields work for SingleCluster + only. For MultiCluster - fields from specific clusters are + used. + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + shardNames: + items: + type: string + minItems: 1 + type: array + statefulSet: + description: Statefulset override for this particular shard. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - shardNames + type: object + type: array + shardPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + shardSpecificPodSpec: + description: |- + ShardSpecificPodSpec allows you to provide a Statefulset override per shard. + DEPRECATED please use spec.shard.shardOverrides instead + items: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array + statefulSet: + description: |- + StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + the global one + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of MongoDB resources + It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - credentials + - type + - version + type: object + x-kubernetes-preserve-unknown-fields: true + status: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + configServerCount: + type: integer + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + members: + type: integer + message: + type: string + mongodsPerShardCount: + type: integer + mongosCount: + type: integer + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + shardCount: + type: integer + sizeStatusInClusters: + description: MongodbShardedSizeStatusInClusters describes the number + and sizes of replica sets members deployed across member clusters + properties: + configServerMongodsInClusters: + additionalProperties: + type: integer + type: object + mongosCountInClusters: + additionalProperties: + type: integer + type: object + shardMongodsInClusters: + additionalProperties: + type: integer + type: object + shardOverridesInClusters: + additionalProperties: + additionalProperties: + type: integer + type: object + type: object + type: object + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/mongodb.com_mongodbmulticluster.yaml b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml new file mode 100644 index 000000000..0e1d65480 --- /dev/null +++ b/config/crd/bases/mongodb.com_mongodbmulticluster.yaml @@ -0,0 +1,1074 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodbmulticluster.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBMultiCluster + listKind: MongoDBMultiClusterList + plural: mongodbmulticluster + shortNames: + - mdbmc + singular: mongodbmulticluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment. + jsonPath: .status.phase + name: Phase + type: string + - description: The time since the MongoDBMultiCluster resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for the + mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the mongodb + processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + backup: + description: |- + Backup contains configuration options for configuring + backup for this MongoDB resource + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + autoTerminateOnDeletion: + description: |- + AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + when the MongoDB CR is deleted + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + client: + description: KMIP Client configuration + properties: + clientCertificatePrefix: + description: |- + A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + The names are generated using the following pattern: + KMIP Client Certificate (TLS Secret): + --kmip-client + KMIP Client Certificate Password: + --kmip-client-password + The expected key inside is called "password". + type: string + type: object + required: + - client + type: object + type: object + mode: + enum: + - enabled + - disabled + - terminated + type: string + snapshotSchedule: + properties: + clusterCheckpointIntervalMin: + enum: + - 15 + - 30 + - 60 + type: integer + dailySnapshotRetentionDays: + description: Number of days to retain daily snapshots. Setting + 0 will disable this rule. + maximum: 365 + minimum: 0 + type: integer + fullIncrementalDayOfWeek: + description: Day of the week when Ops Manager takes a full + snapshot. This ensures a recent complete backup. Ops Manager + sets the default value to SUNDAY. + enum: + - SUNDAY + - MONDAY + - TUESDAY + - WEDNESDAY + - THURSDAY + - FRIDAY + - SATURDAY + type: string + monthlySnapshotRetentionMonths: + description: Number of months to retain weekly snapshots. + Setting 0 will disable this rule. + maximum: 36 + minimum: 0 + type: integer + pointInTimeWindowHours: + description: Number of hours in the past for which a point-in-time + snapshot can be created. + enum: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 15 + - 30 + - 60 + - 90 + - 120 + - 180 + - 360 + type: integer + referenceHourOfDay: + description: Hour of the day to schedule snapshots using a + 24-hour clock, in UTC. + maximum: 23 + minimum: 0 + type: integer + referenceMinuteOfHour: + description: Minute of the hour to schedule snapshots, in + UTC. + maximum: 59 + minimum: 0 + type: integer + snapshotIntervalHours: + description: Number of hours between snapshots. + enum: + - 6 + - 8 + - 12 + - 24 + type: integer + snapshotRetentionDays: + description: Number of days to keep recent snapshots. + maximum: 365 + minimum: 1 + type: integer + weeklySnapshotRetentionWeeks: + description: Number of weeks to retain weekly snapshots. Setting + 0 will disable this rule + maximum: 365 + minimum: 0 + type: integer + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + format: hostname + type: string + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added + to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the name + "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + duplicateServiceObjects: + description: |- + In few service mesh options for ex: Istio, by default we would need to duplicate the + service objects created per pod in all the clusters to enable DNS resolution. Users can + however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + type: boolean + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing MongoDB + to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added to the + externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + logLevel: + enum: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + type: string + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + persistent: + type: boolean + prometheus: + description: Prometheus configurations. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth Password. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode that + the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with a + CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + statefulSet: + description: |- + StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + the global one + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of MongoDB resources + It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - credentials + - type + - version + type: object + x-kubernetes-preserve-unknown-fields: true + status: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + clusterStatusList: + description: ClusterStatusList holds a list of clusterStatuses corresponding + to each cluster + properties: + clusterStatuses: + items: + description: |- + ClusterStatusItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + lastTransition: + type: string + members: + type: integer + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent + resource which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + warnings: + items: + type: string + type: array + required: + - phase + type: object + type: array + type: object + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/mongodb.com_mongodbusers.yaml b/config/crd/bases/mongodb.com_mongodbusers.yaml new file mode 100644 index 000000000..a81f0d449 --- /dev/null +++ b/config/crd/bases/mongodb.com_mongodbusers.yaml @@ -0,0 +1,179 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodbusers.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBUser + listKind: MongoDBUserList + plural: mongodbusers + shortNames: + - mdbu + singular: mongodbuser + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the MongoDB User. + jsonPath: .status.phase + name: Phase + type: string + - description: The time since the MongoDB User resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + connectionStringSecretName: + type: string + db: + type: string + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + passwordSecretKeyRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + roles: + items: + properties: + db: + type: string + name: + type: string + required: + - db + - name + type: object + type: array + username: + type: string + required: + - db + - username + type: object + status: + properties: + db: + type: string + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + project: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + roles: + items: + properties: + db: + type: string + name: + type: string + required: + - db + - name + type: object + type: array + username: + type: string + warnings: + items: + type: string + type: array + required: + - db + - phase + - project + - username + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/mongodb.com_opsmanagers.yaml b/config/crd/bases/mongodb.com_opsmanagers.yaml new file mode 100644 index 000000000..f758cd571 --- /dev/null +++ b/config/crd/bases/mongodb.com_opsmanagers.yaml @@ -0,0 +1,1950 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: opsmanagers.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBOpsManager + listKind: MongoDBOpsManagerList + plural: opsmanagers + shortNames: + - om + singular: opsmanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The number of replicas of MongoDBOpsManager. + jsonPath: .spec.replicas + name: Replicas + type: integer + - description: The version of MongoDBOpsManager. + jsonPath: .spec.version + name: Version + type: string + - description: The current state of the MongoDBOpsManager. + jsonPath: .status.opsManager.phase + name: State (OpsManager) + type: string + - description: The current state of the MongoDBOpsManager Application Database. + jsonPath: .status.applicationDatabase.phase + name: State (AppDB) + type: string + - description: The current state of the MongoDBOpsManager Backup Daemon. + jsonPath: .status.backup.phase + name: State (Backup) + type: string + - description: The time since the MongoDBOpsManager resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Warnings. + jsonPath: .status.warnings + name: Warnings + type: string + name: v1 + schema: + openAPIV3Schema: + description: The MongoDBOpsManager resource allows you to deploy Ops Manager + within your Kubernetes cluster + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + adminCredentials: + description: |- + AdminSecret is the secret for the first admin user to create + has the fields: "Username", "Password", "FirstName", "LastName" + type: string + applicationDatabase: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig are additional configurations that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: specify configuration like startup flags and automation + config settings for the AutomationAgent and MonitoringAgent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + automationConfig: + description: |- + AutomationConfigOverride holds any fields that will be merged on top of the Automation Config + that the operator creates for the AppDB. Currently only the process.disabled and logRotate field is recognized. + properties: + processes: + items: + description: OverrideProcess contains fields that we can + override on the AutomationConfig processes. + properties: + disabled: + type: boolean + logRotate: + description: CrdLogRotate is the crd definition of LogRotate + including fields in strings while the agent supports + them as float64 + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + name: + type: string + required: + - disabled + - name + type: object + type: array + replicaSet: + properties: + settings: + description: |- + MapWrapper is a wrapper for a map to be used by other structs. + The CRD generator does not support map[string]interface{} + on the top level and hence we need to work around this with + a wrapping struct. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + type: string + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added + to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica Set + maximum: 50 + minimum: 3 + type: integer + monitoringAgent: + description: |- + Specify configuration like startup flags just for the MonitoringAgent. + These take precedence over + the flags set in AutomationAgent + properties: + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + required: + - startupOptions + type: object + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + passwordSecretKeyRef: + description: |- + PasswordSecretKeyRef contains a reference to the secret which contains the password + for the mongodb-ops-manager SCRAM-SHA user + properties: + key: + type: string + name: + type: string + required: + - name + type: object + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + prometheus: + description: Enables Prometheus integration on the AppDB. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth + Password. + properties: + key: + description: Key is the key in the secret storing this + password. Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this + password. Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode + that the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of + auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with + a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + service: + description: this is an optional service, it will get the name + "-svc" in case not provided + type: string + topology: + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - version + type: object + backup: + description: Backup + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + blockStores: + items: + description: |- + DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores + Optionally references the user if the Mongodb is configured with authentication + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + required: + - mongodbResourceRef + - name + type: object + type: array + enabled: + description: Enabled indicates if Backups will be enabled for + this Ops Manager. + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + server: + description: KMIP Server configuration + properties: + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used for KMIP authentication + type: string + url: + description: |- + KMIP Server url in the following format: hostname:port + Valid examples are: + 10.10.10.3:5696 + my-kmip-server.mycorp.com:5696 + kmip-svc.svc.cluster.local:5696 + pattern: '[^\:]+:[0-9]{0,5}' + type: string + required: + - ca + - url + type: object + required: + - server + type: object + type: object + externalServiceEnabled: + type: boolean + fileSystemStores: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + headDB: + description: HeadDB specifies configuration options for the HeadDB + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + jvmParameters: + items: + type: string + type: array + logging: + properties: + LogBackAccessRef: + description: LogBackAccessRef points at a ConfigMap/key with + the logback access configuration file to mount on the Pod + properties: + name: + type: string + type: object + LogBackRef: + description: LogBackRef points at a ConfigMap/key with the + logback configuration file to mount on the Pod + properties: + name: + type: string + type: object + type: object + members: + description: Members indicate the number of backup daemon pods + to create. + minimum: 1 + type: integer + opLogStores: + description: OplogStoreConfigs describes the list of oplog store + configs used for backup + items: + description: |- + DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores + Optionally references the user if the Mongodb is configured with authentication + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + required: + - mongodbResourceRef + - name + type: object + type: array + queryableBackupSecretRef: + description: |- + QueryableBackupSecretRef references the secret which contains the pem file which is used + for queryable backup. This will be mounted into the Ops Manager pod. + properties: + name: + type: string + required: + - name + type: object + s3OpLogStores: + description: S3OplogStoreConfigs describes the list of s3 oplog + store configs used for backup. + items: + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + customCertificate: + description: |- + Set this to "true" to use the appDBCa as a CA to access S3. + Deprecated: This has been replaced by CustomCertificateSecretRefs, + In the future all custom certificates, which includes the appDBCa + for s3Config should be configured in CustomCertificateSecretRefs instead. + type: boolean + customCertificateSecretRefs: + description: |- + CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + that apply to the associated S3 bucket. + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + irsaEnabled: + description: |- + This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + type: boolean + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + pathStyleAccessEnabled: + type: boolean + s3BucketEndpoint: + type: string + s3BucketName: + type: string + s3RegionOverride: + type: string + s3SecretRef: + description: |- + S3SecretRef is the secret that contains the AWS credentials used to access S3 + It is optional because the credentials can be provided via AWS IRSA + properties: + name: + type: string + required: + - name + type: object + required: + - name + - pathStyleAccessEnabled + - s3BucketEndpoint + - s3BucketName + type: object + type: array + s3Stores: + items: + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + customCertificate: + description: |- + Set this to "true" to use the appDBCa as a CA to access S3. + Deprecated: This has been replaced by CustomCertificateSecretRefs, + In the future all custom certificates, which includes the appDBCa + for s3Config should be configured in CustomCertificateSecretRefs instead. + type: boolean + customCertificateSecretRefs: + description: |- + CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + that apply to the associated S3 bucket. + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + irsaEnabled: + description: |- + This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + type: boolean + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + pathStyleAccessEnabled: + type: boolean + s3BucketEndpoint: + type: string + s3BucketName: + type: string + s3RegionOverride: + type: string + s3SecretRef: + description: |- + S3SecretRef is the secret that contains the AWS credentials used to access S3 + It is optional because the credentials can be provided via AWS IRSA + properties: + name: + type: string + required: + - name + type: object + required: + - name + - pathStyleAccessEnabled + - s3BucketEndpoint + - s3BucketName + type: object + type: array + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - enabled + type: object + clusterDomain: + format: hostname + type: string + clusterName: + description: |- + Deprecated: This has been replaced by the ClusterDomain which should be + used instead + format: hostname + type: string + clusterSpecList: + items: + description: ClusterSpecOMItem defines members cluster details for + Ops Manager multi-cluster deployment. + properties: + backup: + description: |- + Backup contains settings to override from top-level `spec.backup` for this member cluster. + If the value is not set here, then the value is taken from `spec.backup`. + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + headDB: + description: HeadDB specifies configuration options for + the HeadDB + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + jvmParameters: + items: + type: string + type: array + members: + description: Members indicate the number of backup daemon + pods to create. + minimum: 0 + type: integer + statefulSet: + description: StatefulSetConfiguration specified optional + overrides for backup datemon statefulset. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + clusterDomain: + description: Cluster domain to override the default *.svc.cluster.local + if the default cluster domain has been changed on a cluster + level. + format: hostname + type: string + clusterName: + description: |- + ClusterName is name of the cluster where the Ops Manager Statefulset will be scheduled. + The operator is using ClusterName to find API credentials in `mongodb-enterprise-operator-member-list` config map to use for this member cluster. + If the credentials are not found, then the member cluster is considered unreachable and ignored in the reconcile process. + type: string + configuration: + additionalProperties: + type: string + description: |- + The configuration properties passed to Ops Manager and Backup Daemon in this cluster. + If specified (not empty) then this field overrides `spec.configuration` field entirely. + If not specified, then `spec.configuration` field is used for the Ops Manager and Backup Daemon instances in this cluster. + type: object + externalConnectivity: + description: |- + MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + accessing Ops Manager instances in this member cluster from outside the Kubernetes cluster. + If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be + directly passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this + Service when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to + this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, + this applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + jvmParameters: + description: |- + JVM parameters to pass to Ops Manager and Backup Daemon instances in this member cluster. + If specified (not empty) then this field overrides `spec.jvmParameters` field entirely. + If not specified, then `spec.jvmParameters` field is used for the Ops Manager and Backup Daemon instances in this cluster. + items: + type: string + type: array + members: + description: Number of Ops Manager instances in this member + cluster. + type: integer + statefulSet: + description: |- + Configure custom StatefulSet configuration to override in Ops Manager's statefulset in this member cluster. + If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + configuration: + additionalProperties: + type: string + description: The configuration properties passed to Ops Manager/Backup + Daemon + type: object + externalConnectivity: + description: |- + MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + accessing this Ops Manager resource from outside the Kubernetes cluster. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be directly + passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this Service + when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, this + applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + internalConnectivity: + description: |- + InternalConnectivity if set allows for overriding the settings of the default service + used for internal connectivity to the OpsManager servers. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be directly + passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this Service + when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, this + applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + jvmParameters: + description: Custom JVM parameters passed to the Ops Manager JVM + items: + type: string + type: array + logging: + properties: + LogBackAccessRef: + description: LogBackAccessRef points at a ConfigMap/key with the + logback access configuration file to mount on the Pod + properties: + name: + type: string + type: object + LogBackRef: + description: LogBackRef points at a ConfigMap/key with the logback + configuration file to mount on the Pod + properties: + name: + type: string + type: object + type: object + opsManagerURL: + description: |- + OpsManagerURL specified the URL with which the operator and AppDB monitoring agent should access Ops Manager instance (or instances). + When not set, the operator is using FQDN of Ops Manager's headless service `{name}-svc.{namespace}.svc.cluster.local` to connect to the instance. If that URL cannot be used, then URL in this field should be provided for the operator to connect to Ops Manager instances. + type: string + replicas: + minimum: 1 + type: integer + security: + description: Configure HTTPS. + properties: + certsSecretPrefix: + type: string + tls: + properties: + ca: + type: string + secretRef: + properties: + name: + type: string + required: + - name + type: object + type: object + type: object + statefulSet: + description: Configure custom StatefulSet configuration + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of Ops Manager deployment. + It defaults (and if not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + version: + type: string + required: + - applicationDatabase + - version + type: object + status: + properties: + applicationDatabase: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + clusterStatusList: + items: + properties: + clusterName: + type: string + members: + type: integer + type: object + type: array + configServerCount: + type: integer + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + members: + type: integer + message: + type: string + mongodsPerShardCount: + type: integer + mongosCount: + type: integer + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + shardCount: + type: integer + sizeStatusInClusters: + description: MongodbShardedSizeStatusInClusters describes the + number and sizes of replica sets members deployed across member + clusters + properties: + configServerMongodsInClusters: + additionalProperties: + type: integer + type: object + mongosCountInClusters: + additionalProperties: + type: integer + type: object + shardMongodsInClusters: + additionalProperties: + type: integer + type: object + shardOverridesInClusters: + additionalProperties: + additionalProperties: + type: integer + type: object + type: object + type: object + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + backup: + properties: + clusterStatusList: + items: + properties: + clusterName: + type: string + replicas: + type: integer + type: object + type: array + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + type: object + opsManager: + properties: + clusterStatusList: + items: + properties: + clusterName: + type: string + replicas: + type: integer + type: object + type: array + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + replicas: + type: integer + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + url: + type: string + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 25eecc05f..4d3100512 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,7 +2,10 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +- bases/mongodb.com_mongodb.yaml +- bases/mongodb.com_mongodbusers.yaml +- bases/mongodb.com_opsmanagers.yaml +- bases/mongodb.com_mongodbmulticluster.yaml # +kubebuilder:scaffold:crdkustomizeresource # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index bd972fd91..83ba96021 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,6 +1,6 @@ -namePrefix: "" - resources: - - ../crd - - ../rbac - - ../manager +- ../crd +- ../rbac +- ../manager +- ../scorecard +- ../webhooks diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index cb74a8d0e..4fe5ccaa3 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -3,9 +3,3 @@ resources: generatorOptions: disableNameSuffixHash: true - -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: mongodb-kubernetes-operator - newName: quay.io/mongodb/mongodb-kubernetes-operator:0.5.0 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 9013a8451..276933180 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,74 +1,323 @@ +--- +# Source: enterprise-operator/templates/operator.yaml apiVersion: apps/v1 kind: Deployment metadata: - annotations: - email: support@mongodb.com - labels: - owner: mongodb - name: mongodb-kubernetes-operator + name: mongodb-enterprise-operator + namespace: mongodb spec: replicas: 1 selector: matchLabels: - name: mongodb-kubernetes-operator - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator + app.kubernetes.io/instance: mongodb-enterprise-operator template: metadata: labels: - name: mongodb-kubernetes-operator + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator + app.kubernetes.io/instance: mongodb-enterprise-operator spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: name - operator: In - values: - - mongodb-kubernetes-operator - topologyKey: kubernetes.io/hostname + serviceAccountName: mongodb-enterprise-operator containers: - - command: - - /usr/local/bin/entrypoint - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: mongodb-kubernetes-operator - - name: AGENT_IMAGE - value: quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1 - - name: VERSION_UPGRADE_HOOK_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 - - name: READINESS_PROBE_IMAGE - value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22 - - name: MONGODB_IMAGE - value: mongodb-community-server - - name: MONGODB_REPO_URL - value: quay.io/mongodb - image: quay.io/mongodb/mongodb-kubernetes-operator:0.12.0 - imagePullPolicy: Always - name: mongodb-kubernetes-operator - resources: - limits: - cpu: 1100m - memory: 1Gi - requests: - cpu: 500m - memory: 200Mi - securityContext: - readOnlyRootFilesystem: true - runAsUser: 2000 - allowPrivilegeEscalation: false - securityContext: - seccompProfile: - type: RuntimeDefault - serviceAccountName: mongodb-kubernetes-operator + - name: mongodb-enterprise-operator + image: "quay.io/mongodb/mongodb-enterprise-operator-ubi:1.32.0" + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + command: + - /usr/local/bin/mongodb-enterprise-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: prod + - name: MDB_DEFAULT_ARCHITECTURE + value: non-static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MANAGED_SECURITY_CONTEXT + value: 'true' + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1h" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/mongodb-enterprise-database-ubi + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-database-ubi + - name: INIT_DATABASE_VERSION + value: 1.32.0 + - name: DATABASE_VERSION + value: 1.32.0 + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi + - name: INIT_OPS_MANAGER_VERSION + value: 1.32.0 + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-appdb-ubi + - name: INIT_APPDB_VERSION + value: 1.32.0 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent-ubi" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: ubi8 + - name: PERFORM_FAILOVER + value: 'true' + - name: MDB_WEBHOOK_REGISTER_CONFIGURATION + value: "false" + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "1" + - name: RELATED_IMAGE_MONGODB_ENTERPRISE_DATABASE_IMAGE_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-database-ubi:1.32.0" + - name: RELATED_IMAGE_INIT_DATABASE_IMAGE_REPOSITORY_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-init-database-ubi:1.32.0" + - name: RELATED_IMAGE_INIT_OPS_MANAGER_IMAGE_REPOSITORY_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi:1.32.0" + - name: RELATED_IMAGE_INIT_APPDB_IMAGE_REPOSITORY_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-init-appdb-ubi:1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_2_8729_1 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1_1.32.0" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_25 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.25" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_26 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.26" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_27 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.27" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_7_0_13 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:7.0.13" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_7_0_14 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:7.0.14" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_7_0_15 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:7.0.15" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_8_0_4 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:8.0.4" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_8_0_5 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:8.0.5" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_8_0_6 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:8.0.6" + # since the official server images end with a different suffix we can re-use the same $mongodbImageEnv + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_1_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.1-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_2_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.2-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_3_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.3-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_4_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.4-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_5_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.5-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_6_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.6-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_7_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.7-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_8_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.8-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_9_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.9-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_10_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.10-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_11_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.11-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_12_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.12-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_13_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.13-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_14_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.14-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_15_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.15-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_16_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.16-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_17_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.17-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_18_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.18-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_19_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.19-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_20_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.20-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_21_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.21-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_1_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.1-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_2_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.2-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_3_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.3-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_4_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.4-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_5_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.5-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_6_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.6-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_7_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.7-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_8_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.8-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_9_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.9-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_10_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.10-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_11_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.11-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_12_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.12-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_13_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.13-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_14_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.14-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_15_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.15-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_16_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.16-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_17_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.17-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_18_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.18-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_1_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.1-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_2_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.2-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_3_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.3-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_4_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.4-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_5_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.5-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_8_0_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:8.0.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_8_0_0_ubi9 + value: "quay.io/mongodb/mongodb-enterprise-server:8.0.0-ubi9" + # mongodbLegacyAppDb will be deleted in 1.23 release + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_11_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.11-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_2_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.2-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_24_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.24-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_6_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.6-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_8_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.8-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_0_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.0-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_11_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.11-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_4_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.4-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_21_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.21-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_1_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.1-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_5_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.5-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_6_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.6-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_7_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.7-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_14_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.14-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_18_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.18-ent" diff --git a/config/manifests/bases/mongodb-enterprise.clusterserviceversion.yaml b/config/manifests/bases/mongodb-enterprise.clusterserviceversion.yaml new file mode 100644 index 000000000..4833d8f6d --- /dev/null +++ b/config/manifests/bases/mongodb-enterprise.clusterserviceversion.yaml @@ -0,0 +1,447 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: '[]' + capabilities: Deep Insights + categories: Database + certified: "true" + containerImage: quay.io/mongodb/mongodb-enterprise-operator-ubi:1.32.0 + createdAt: "" + description: The MongoDB Enterprise Kubernetes Operator enables easy deploys of + MongoDB into Kubernetes clusters, using our management, monitoring and backup + platforms, Ops Manager and Cloud Manager. + features.operators.openshift.io/disconnected: "true" + features.operators.openshift.io/fips-compliant: "false" + features.operators.openshift.io/proxy-aware: "false" + features.operators.openshift.io/tls-profiles: "false" + features.operators.openshift.io/token-auth-aws: "false" + features.operators.openshift.io/token-auth-azure: "false" + features.operators.openshift.io/token-auth-gcp: "false" + repository: https://github.com/mongodb/mongodb-enterprise-kubernetes + support: support@mongodb.com + name: mongodb-enterprise.v0.0.0 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - description: MongoDB Deployment + displayName: MongoDB Deployment + kind: MongoDB + name: mongodb.mongodb.com + resources: + - kind: StatefulSet + name: StatefulSet holding the Pod with MongoDB + version: apps/v1 + - kind: Service + name: "" + version: v1 + specDescriptors: + - displayName: MongoDB Deployment Type + path: type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:Standalone + - urn:alm:descriptor:com.tectonic.ui:select:ReplicaSet + - urn:alm:descriptor:com.tectonic.ui:select:ShardedCluster + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - description: Version of MongoDB to use. + displayName: MongoDB Version + path: version + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - description: In a Replica Set deployment type, specifies the amount of members. + displayName: Members of a Replica Set + path: members + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:podCount + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - displayName: Cloud/Ops Manager credentials + path: credentials + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfig + - urn:alm:descriptor:io.kubernetes:Secret + - description: Project configuration for this deployment + displayName: Ops Manager project configuration + path: opsManager + - description: Name of the ConfigMap with the configuration for this project + displayName: Ops Manager Project Configuration + path: opsManager.configMapRef.name + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfig + - urn:alm:descriptor:io.kubernetes:ConfigMap + - description: Enable Persistent Storage with Volume Claims + displayName: Persistent Storage + path: persistent + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - description: Optional. Name of a Kubernetes Cluster Domain. + displayName: Name of Kubernetes Cluster Domain + path: clusterDomain + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Enable TLS + path: security.tls.enabled + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:security + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - displayName: Custom CA Config Map + path: security.tls.ca + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:security + - urn:alm:descriptor:io.kubernetes:ConfigMap + - displayName: Enable authentication + path: security.authentication.enabled + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Authentication + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - displayName: Authentication Mode + path: security.authentication.modes + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Authentication + - urn:alm:descriptor:com.tectonic.ui:select:X509 + - urn:alm:descriptor:com.tectonic.ui:select:SCRAM + - urn:alm:descriptor:com.tectonic.ui:select:LDAP + - displayName: Authentication Mode used for Inter cluster communication + path: security.authentication.internalCluster + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Authentication + - urn:alm:descriptor:com.tectonic.ui:select:X509 + - urn:alm:descriptor:com.tectonic.ui:select:SCRAM + - urn:alm:descriptor:com.tectonic.ui:select:LDAP + - description: Number of Config Servers in Replica + displayName: Number of Config Servers + path: configServerCount + - description: Number of Shards in a Sharded Cluster + displayName: Number of Shards + path: shardCount + - description: Number of MongoDB Servers per Shard + displayName: Number of MongoDB Servers + path: mongodsPerShardCount + - description: Number of Mongo routers, in total, for the whole cluster + displayName: Number of Mongos + path: mongosCount + statusDescriptors: + - description: | + Phase the MongoDB Deployment is currently on. It can be any of Running, Pending, Failed. + displayName: Phase + path: phase + - description: | + Type describes the deployment type this MongoDB resource. Posible values + are Standalone, ReplicaSet or ShardedCluster. + displayName: Type + path: type + - description: | + Timestamp of last transition + displayName: Last Transition + path: lastTransition + - description: | + Current version of MongoDB + displayName: MongoDB Version + path: version + version: v1 + - description: MongoDB Multi Deployment + displayName: MongoDB Multi Deployment + kind: MongoDBMultiCluster + name: mongodbmulticluster.mongodb.com + resources: + - kind: StatefulSet + name: StatefulSet holding the Pod with MongoDB + version: apps/v1 + - kind: Service + name: "" + version: v1 + specDescriptors: + - displayName: MongoDB Deployment Type + path: type + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:select:ReplicaSet + - description: Version of MongoDB to use. + displayName: MongoDB Version + path: version + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - description: In a Replica Set deployment type, specifies the amount of members. + displayName: Members of a Replica Set + path: members + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:podCount + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - displayName: Cloud/Ops Manager credentials + path: credentials + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfig + - urn:alm:descriptor:io.kubernetes:Secret + - description: Project configuration for this deployment + displayName: Ops Manager project configuration + path: opsManager + - description: Name of the ConfigMap with the configuration for this project + displayName: Ops Manager Project Configuration + path: opsManager.configMapRef.name + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfig + - urn:alm:descriptor:io.kubernetes:ConfigMap + - description: Enable Persistent Storage with Volume Claims + displayName: Persistent Storage + path: persistent + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - description: Optional. Specify whether to duplicate service objects among + different Kubernetes clusters. + displayName: Duplicate Service Objects + path: duplicateServiceObjects + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration + - description: Optional. Name of a Kubernetes Cluster Domain. + displayName: Name of Kubernetes Cluster Domain + path: clusterDomain + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Enable TLS + path: security.tls.enabled + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:security + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - displayName: Custom CA Config Map + path: security.tls.ca + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:security + - urn:alm:descriptor:io.kubernetes:ConfigMap + - displayName: Enable authentication + path: security.authentication.enabled + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Authentication + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - displayName: Authentication Mode + path: security.authentication.modes + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Authentication + - urn:alm:descriptor:com.tectonic.ui:select:SCRAM + - description: Spec for each cluster that comprises MongoDB Replicaset + displayName: Cluster SpecList + path: clusterSpecList + statusDescriptors: + - description: | + Phase the MongoDB Deployment is currently on. It can be any of Running, Pending, Failed. + displayName: Phase + path: phase + version: v1 + - description: MongoDB x509 User + displayName: MongoDB User + kind: MongoDBUser + name: mongodbusers.mongodb.com + resources: + - kind: Secret + name: "" + version: v1 + specDescriptors: + - displayName: Name of the database user. + path: username + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Name of the database that stores usernames. + path: db + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - displayName: Secret Name that user stores the user’s password. + path: passwordSecretKeyRef.name + x-descriptors: + - urn:alm:descriptor:io.kubernetes:Secret + - displayName: Name of the MongoDB resource to which this user is associated. + path: mongodbResourceRef.name + x-descriptors: + - urn:alm:descriptor:io.kubernetes:mongodb + - displayName: Database on which the role can act. + path: roles.db + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:arrayFieldGroup:Roles + - displayName: Name of the role to grant the database user. + path: roles.name + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:arrayFieldGroup:Roles + - description: MongoDB resource this user belongs to + displayName: MongoDB resource + path: mongodbResourceRef + - description: Roles this user will have + displayName: MongoDB roles + path: roles + statusDescriptors: + - description: | + The current state of the MongoDB User + displayName: State + path: phase + version: v1 + - description: MongoDB Ops Manager + displayName: MongoDB Ops Manager + kind: MongoDBOpsManager + name: opsmanagers.mongodb.com + resources: + - kind: StatefulSet + name: "" + version: apps/v1 + - kind: Service + name: "" + version: v1 + - kind: ConfigMap + name: "" + version: v1 + specDescriptors: + - displayName: The version of Ops Manager to deploy. + path: version + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfiguration + - displayName: Number of Ops Manager instances. + path: replicas + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfiguration + - displayName: Secret containing admin user credentials. + path: adminCredentials + x-descriptors: + - urn:alm:descriptor:io.kubernetes:Secret + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfiguration + - displayName: Secret to enable TLS for Ops Manager allowing it to serve traffic + over HTTPS. + path: security.tls.secretRef.name + x-descriptors: + - urn:alm:descriptor:io.kubernetes:Secret + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfiguration + - displayName: Number of ReplicaSet nodes for Application Database. + path: applicationDatabase.members + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ApplicationDatabase + - displayName: Secret containing the TLS certificate signed by known or custom + CA. + path: applicationDatabase.security.tls.secretRef.name + x-descriptors: + - urn:alm:descriptor:io.kubernetes:Secret + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ApplicationDatabase + - displayName: ConfigMap with CA for Custom TLS Certificate + path: applicationDatabase.security.tls.ca + x-descriptors: + - urn:alm:descriptor:io.kubernetes:ConfigMap + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ApplicationDatabase + - displayName: Enable Backup Infrastructure + path: backup.enabled + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:BackupInfrastructure + - description: Application Database configuration + displayName: Application Database + path: applicationDatabase + - description: configuration + displayName: configuration + path: configuration + - description: Configures external connectivity + displayName: External Connectivity + path: externalConnectivity + statusDescriptors: + - description: | + The current state of the MongoDBOpsManager. + displayName: Phase + path: opsManager.phase + - description: Type of deployment + displayName: Type + path: type + version: v1 + description: | + The MongoDB Enterprise Kubernetes Operator enables easy deploys of MongoDB + into Kubernetes clusters, using our management, monitoring and backup + platforms, Ops Manager and Cloud Manager. + + ## Before You Start + + To start using the operator you''ll need an account in MongoDB Cloud Manager or + a MongoDB Ops Manager deployment. + + * [Create a Secret with your OpsManager API key](https://docs.mongodb.com/kubernetes-operator/stable/tutorial/create-operator-credentials/#procedure) + + * [Create a ConfigMap with your OpsManager project ID and URL](https://docs.mongodb.com/kubernetes-operator/stable/tutorial/create-project-using-configmap/) + + By installing this integration, you will be able to deploy MongoDB instances + with a single simple command. + + ## Required Parameters + + * `opsManager` or `cloudManager` - Enter the name of the ConfigMap containing project information + * `credentials` - Enter the name of the Secret containing your OpsManager credentials + * `type` - Enter MongoDB Deployment Types ("Standalone", "ReplicaSet", "ShardedCluster" + + ## Supported MongoDB Deployment Types ## + + * Standalone: An instance of mongod that is running as a single server and + not as part of a replica set, this is, it does not do any kind of + replication. + + * Replica Set: A replica set in MongoDB is a group of mongod processes that + maintain the same data set. Replica sets provide redundancy and high + availability, and are the basis for all production deployments. This section + introduces replication in MongoDB as well as the components and architecture + of replica sets. The section also provides tutorials for common tasks + related to replica sets. + + * Sharded Cluster: The set of nodes comprising a sharded MongoDB deployment. + A sharded cluster consists of config servers, shards, and one or more mongos + routing processes. Sharding is a A database architecture that partitions + data by key ranges and distributes the data among two or more database + instances. Sharding enables horizontal scaling. + + ## Requirements for deploying MongoDB OpsManager + + * In order to deploy resources of type MongoDB OpsManager, you will need to + create a secret containing the [credentials](https://docs.mongodb.com/kubernetes-operator/stable/tutorial/plan-om-resource/#om-rsrc-prereqs) + for the Global Owner user + + ## Security ## + + The operator can enable TLS for all traffic between servers and also between + clients and servers. Before enabling `security.tls.enabled` to `true` you + should create your certificates. or you can leave the operator to create all + the certificates for you. The operator ability to create certs is been + deprecated due to Kubernetes API changes. + + For more information, please read the official MongoDB + Kubernetes Operator [docs](https://docs.mongodb.com/kubernetes-operator/stable/). + displayName: MongoDB Enterprise Operator + icon: + - base64data: + iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAJEXpUWHRSYXcgcHJvZmlsZSB0eXBlIGV4aWYAAHjarVhtdiMpDPzPKfYIDUIIHYfP9/YGe/wtQXcnsZ1JMjP2xLQBg1CVSmLc+O/f6f7BiwIFF1ly0pQOvKJGDQUP+divsj79EdfnesVzCN8/9Lt7IKCL0NL+mtM5/+r39wK7KXjidwvldg7UjwN67hDyw0LnRmQWBTz0cyE9F6KwB/y5QNnHOpJmeX+EOnbbr5Pk/efsI7VjHcSfo4/fo8B7nbEPhTDI04HPQHEbQPbnHRUbwCe+YKKnjOe4ejxdlsAhr/x0vLPKPaJyP/lP+h9AobT7HTo+OjPd7ct+z6+d75aL3+1M7d75Qz/3oz4e5/qbs2c359inKzHBpek81HWU9YSJWCTS+lnCW/DHeJb1VryzA3sbIO9Hw44Vz+oDvD999N0XP/1YbfMNJsYwgqANoQEb68skQUOjwxk29vYzCCl1oBaoAV5Cb7ht8WtfXds1n7Fx95gZPBbzK9bs42+8P11oTqO890e+fQW7ggUFzDDk7BOzAIifF494Ofh6P74MVwKCvNycccBy1L1EZX9yy3hEC2jCREa7Y81LPxeAi7A3wxhPQOBIntgnf0gI4j38mIFPwUIZQRMqIPDMocPKEIkSwMnB9sZvxK+5gcPuhmYBCKZEAmiUCrCKEDbwR2IGhwoTR2ZOLJxZuSRKMXFKSZKJXxGSKCxJRLKolEw5Zs4pS84uay4alCCOrElFs6qWgk0LVi74dcGEUmqoVGPlmqrUXLWWBvq02LilJi27pq300KlDJ3rq0nPXXoYfoNKIg0caMvLQUSaoNmnGyTNNmXnqLDdq3m1Yn97fR81fqIWFlE2UGzX8VORawpucsGEGxEL0QFwMARA6GGZH9jEGZ9AZZocGRAUHWMkGTveGGBCMwwee/sbuDbkPuLkY/wi3cCHnDLq/gZwz6D5B7hm3F6h1yzbtILcQsjA0px6E8MOEkUvIxZLat1t3d9QCRxsxap9zbTJnSpC9Ujts4Njb6FI9zspJeXbVkeaYtbVJSEezUW6JaKAvwg/D5hQZLDanrtM00jbEY0rHKkDDT6qjjyI1Tvi0x0mumC00PWvDJgQFlzlr6JBLDpCAfhT8JmmB17ocZZ0GOWg/HHfrHjt+t10LAbGArAzLYWMFIjiYSgUyBMqQThxLoUockGq0iRauh56ughvMVW77wZ9+oOWHXtjDEyFKmyAyYgHI19rzRglrZxYvpcA/8Ec1h7rT63Q63Tw690qqSBQJdCs5llETtVGW9VzNejNAzPo0VWt1MD+hwMgT1lTWuj1MBWGlfqQ8kPXMvgMxs56QdF+17rOBX7WS9IlLzsj0nkswang2SsLdcyIt4xRwm+8UBaGTU0gRkaOh10kbtJLBoye6g78sscDpBA9P6YMn4ngidXfgQR1AIWLLjFyG1Mbw/UzR2d7Z2yfcx6EhKA+P6DfFAW1nywjatUeUGk5/Hc+t+2zgkxYhUnAuglk6BGE0m4lCmm4eaSwCwWjITao1orWjGS3EjpZENeNoxg6Qc0pZEYQv5m4m+E+rg/b47bE2dXwVCQDlNY2me6QRBA1iGCEhRbBjNe8F0L/N03a/bc8FWAUaKJ7FAsVBF7mPWO/Ahnz+XNZCdu86wOgwYwXw4fSOAb+8M1bowkooSoXgmAKCKaaBSwER/RBBCHJR5F0klsyWSyrl2vVkchv+ay0Z5IgTNARSNpvOJbKgdkog+dGr8b23CUVLwm3MXGAv9zf5i0grEqY2dchhniumDwkX78a3afXWuruDC3R9mMCg2ZH4pFQxsNVXIAEKVghKRpe2vqIfodLqTwXAD0EOsNTbjSm4FrCboDvIQtJa77P5ihzfpOrk0jpKqQEZ7DHj30T4X6IfnjjiviTJynfQ74d8NyRZ9rkzoXsbghrGJoIikuGb1hDza7FCQ/LrfeLpbnpOR3Asbg+2S4ERh9mALLv3h+dZXowU1hkdQYwG7ohDpp6qnEf9eXpzI9cWdmgiBua6CmmpVo28HNFiAtLnGDi/IqehYLLd3Urk7acMROiNULaywxE4lTNlYaszIj8MXSMIAxMLMiO81TxpLxc+CIX7plJ8UvScIGDEPQ49k2B8RYKHQut9i9BqjOQWhtomW3G6pguDF2NuDWpCnjZpyP5zL/y6dd8IhbzrPyQdZJhmjcKstRWoSBtK9xFbVKVqmeuN+i+Z/1TdVUuQfAgywAEVaqBb5jGvGCf+AbMfNsTNwZtkGeOslliVhF3371oCOWdAc1jWzoXOnfdCFO6VqDKjipiVCMkYgm2VSwIM1S8Fr33UuDLJhwg2GbEQRgIFRCgbAvlCuOD03tu7Qu8SSNxJSi3FYFjpE76mhtw+vUM+N0WU2lNeBwpqB4ofqpRdBsYiKONYcc3BfWosqbYCLxy8q5HfqNnu2s3qCbWCytHwsH1WvnPmihPU+zgkNxTMioQiqPKROhd1/PDXWS0Fn7nOvWNDLB3FmJYHN24vKtdqBTMuc/gFLogWAJRONyL636yEhYjY7Uv7T7q5vYnIXaXI4a12X+6Ezxni0lHxJpgdU+jNVbkDq+bfqkNeRT8KUJzPWBRn64tFuCcNAotWugWLirEIpXvd1MX+DaXc8K6Q/U9WkwT7ruqDnuh2+ukAQWQJ6SNBGIVWhI7g1qpdEMsDPMINBJBdGLWMKxhmwIhVoOPeYSGyrx28rx0dlxoL9WTGIj1ZjYIyEXV5UsKN/SqRUBi27+vRd9sa5fQjoqPf0ejoDEdZ4UjI0kdWVC3mRZArW4GP0hO6hmi+a2a6auawa2bU2YKyMMAD+2qGKrJ4lNuofE7Zhg1LnMnSI1IGDg0esfENVp1sQ7J0F91M8I1uCJakKNxHE/C0FNw+Ajg3QhWWmrsdcIR5ak2cp9aIA03kpImJTclWlaYGPtVWWk0HfmBnOq84dF1xglVxGWdK2GuVx4o8mvyRO7pD+0Up9evW/TleGy73BV77WqdpX0Is8iEsdgnx+yZeJ0hmIupmwlUcl5BT7SKus9BBm/ft6+xqXfwzibyq3OxgyhFHqt/IHuuMUMrBHLhVjyI/7AoDgDkkjh8GiTETsfU/ZHuEtrDMfYEAAAGFaUNDUElDQyBwcm9maWxlAAB4nH2RPUjDQBzFX1O1UioiVhBxyFB1sSAq4qhVKEKFUCu06mBy6YfQpCFJcXEUXAsOfixWHVycdXVwFQTBDxA3NydFFynxf2mhRYwHx/14d+9x9w4QqkWmWW1jgKbbZjIeE9OZFTHwiiD60IMRdMjMMmYlKQHP8XUPH1/vojzL+9yfo0vNWgzwicQzzDBt4nXiqU3b4LxPHGYFWSU+Jx416YLEj1xX6vzGOe+ywDPDZio5RxwmFvMtrLQwK5ga8SRxRNV0yhfSdVY5b3HWimXWuCd/YSirLy9xneYg4ljAIiSIUFDGBoqwEaVVJ8VCkvZjHv4B1y+RSyHXBhg55lGCBtn1g//B726t3MR4PSkUA9pfHOdjCAjsArWK43wfO07tBPA/A1d601+qAtOfpFeaWuQI6N4GLq6bmrIHXO4A/U+GbMqu5Kcp5HLA+xl9UwbovQWCq/XeGvs4fQBS1FXiBjg4BIbzlL3m8e7O1t7+PdPo7wdVb3KbaWTEXAAADRxpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+Cjx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDQuNC4wLUV4aXYyIj4KIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvbW0vIgogICAgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIKICAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIKICAgIHhtbG5zOkdJTVA9Imh0dHA6Ly93d3cuZ2ltcC5vcmcveG1wLyIKICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIgogICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICB4bXBNTTpEb2N1bWVudElEPSJnaW1wOmRvY2lkOmdpbXA6ZDk1YjhmMjctMWM0NS00YjU1LWEwZTMtNmNmMjM0Yzk1ZWVkIgogICB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOmVhMGY5MTI5LWJlMDItNDVjOS1iNGU4LTU3N2MxZTBiZGJhNyIKICAgeG1wTU06T3JpZ2luYWxEb2N1bWVudElEPSJ4bXAuZGlkOjcyNmY4ZGFlLTM4ZTYtNGQ4Ni1hNTI4LWM0NTc4ZGE4ODA0NSIKICAgZGM6Rm9ybWF0PSJpbWFnZS9wbmciCiAgIEdJTVA6QVBJPSIyLjAiCiAgIEdJTVA6UGxhdGZvcm09Ik1hYyBPUyIKICAgR0lNUDpUaW1lU3RhbXA9IjE2MzQ4MzgwMTYyMTQ2MTMiCiAgIEdJTVA6VmVyc2lvbj0iMi4xMC4yNCIKICAgdGlmZjpPcmllbnRhdGlvbj0iMSIKICAgeG1wOkNyZWF0b3JUb29sPSJHSU1QIDIuMTAiPgogICA8eG1wTU06SGlzdG9yeT4KICAgIDxyZGY6U2VxPgogICAgIDxyZGY6bGkKICAgICAgc3RFdnQ6YWN0aW9uPSJzYXZlZCIKICAgICAgc3RFdnQ6Y2hhbmdlZD0iLyIKICAgICAgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo1YWNhZmVhMC0xZmY5LTRiMmUtYmY0NC02NTM3MzYwMGQzNjEiCiAgICAgIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkdpbXAgMi4xMCAoTWFjIE9TKSIKICAgICAgc3RFdnQ6d2hlbj0iMjAyMS0xMC0yMVQxODo0MDoxNiswMTowMCIvPgogICAgPC9yZGY6U2VxPgogICA8L3htcE1NOkhpc3Rvcnk+CiAgPC9yZGY6RGVzY3JpcHRpb24+CiA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgCjw/eHBhY2tldCBlbmQ9InciPz6528V0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAB3RJTUUH5QoVESgQ+iToFAAAA8xJREFUeNrlW01PU0EUPTPV+oqb4h+wENYKXbmzsjLEKPAHwB1xQ6N7adiboBtrSAT5AaQmBpuYSN25MS17k5Zf0MemFGznungttCkf782bmTels2w6mbnnnnPv3DvzYrBhrMytIT01gz9/f5temkVv/NMUwKsg1MFEGvlizeTy3ALj9zuuGAf4T2QzydEBACwHINXzwwSOE29N7iAWqe7BsoOYsEdITx2ZigcsIupnzqh/8SC0/6Wx+aNy8yTg6X7rWsfEbu96/71JAGQzyY7n/Rg2AcZ3dQdFswA0Exs+je8KYUZ3UDQXA1bmlgFsScwkMFrEx++F4QXgPN/LaZpQR6IxiY2SO6QSGMj3Qd00jpPE5+FkgDz1B3kAMYt8sTQ8AGQzSTTHyqG83z+qcBpplVLQK4Hm2KpC473U2BzLDgcDwgY+QwFRIwP4knLjuwFRIQv0MGB5PgnntKwFAMUs0MMA53Rem/Ge25I4ufvCXgkQVrVXsSSW7JTAq7lpCJQNnK4IEJNhW2jqGdDGsrH6QrB5GyXwWMKXLoi5gdnL8dwuCXjRvy4xs0vjVGDonMa9MNlALQPiJxlJOcvruOlM2yMBzuQ3Q3Al44BFADA8lJ9LrtSKnD2wBwAhe/hhIVIZpWxiQJgG5qHkohYBoPP4q6tks2Qfh1GBzu3xhWQckM0eWgAIfprrBE+SN4LZBACTNIQzF4KO5EAnmxgQwhtckj2WMeBA8gARpqQ9sAcAAfnrbLk4QGBUsQcAHmIzXFLLrbZFDMgXS1KZoN2W1DHVwj6iUH8O4FQKPCcWc3t6AkGCTin0dpUDQPhq6OREgNixD4BmvBBYBlKNTaqpuChVD8B2wQWj98EnOrVA3hf4YHExJLb1l3FUsBeAfLEG0Bef//Y8H28FqSW2VT2p1VgNUi5QLKC4z1qCqoBYt78fkC/WfMWCwMUM21H5oFrzA4n4xrUt724xQy0fxRRVkd/LKQ0lWgHYLrgAvfQXN1vXSYAAmlUeS7VH63yxBMIVUvDdB1jX8S2BmZbYp70scNkRmXtXaQkOXN4b3FJNfbMAAEDzzoLcFRhV4TReaztOGAPAiwdPLgDh8OqUR7M6XoiaB6CbGtts4cLzwbtv1N8Z7hiv+Rsi823xzb0KRB8T7gMA3jxj59dcZoz3snBUY+VpCmD7nautXGcva2Aog8Siqa/Hov1sbuAxJZXgHC/o1Hz0Ehgsmn71/FIxaXz0AAwS8sj0ihYAcBb5CVJ9weFnwLnR1K6PHgC9FyJsFCVwq+9afAQlIITbnxXMjv+6222dh4/VtAAAAABJRU5ErkJggg== + mediatype: image/png + install: + spec: + deployments: + strategy: "" + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - mongodb + - databases + - nosql + links: + - name: Documentation + url: + https://docs.opsmanager.mongodb.com/current/tutorial/install-k8s-operator/index.html + maintainers: + - email: support@mongodb.com + name: MongoDB, Inc + maturity: stable + provider: + name: MongoDB, Inc + replaces: mongodb-enterprise.v1.31.0 + version: 0.0.0 diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml new file mode 100644 index 000000000..ec2c16556 --- /dev/null +++ b/config/manifests/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../default +- ../samples diff --git a/config/rbac/database-roles-patch-namespace.yaml b/config/rbac/database-roles-patch-namespace.yaml new file mode 100644 index 000000000..cffc6d024 --- /dev/null +++ b/config/rbac/database-roles-patch-namespace.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/subjects/0/namespace" diff --git a/config/rbac/database-roles.yaml b/config/rbac/database-roles.yaml new file mode 100644 index 000000000..98bf2b64b --- /dev/null +++ b/config/rbac/database-roles.yaml @@ -0,0 +1,58 @@ +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-database-pods + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-ops-manager + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-appdb +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-appdb + namespace: mongodb diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index f1fe88a33..7fc63841f 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,7 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + resources: -- role.yaml -- role_binding.yaml -- service_account.yaml -- service_account_database.yaml -- role_binding_database.yaml -- role_database.yaml + - database-roles.yaml + - operator-roles.yaml + +# we have to remove service account namespace from RoleBinding as OLM is not overriding it +patchesJson6902: + - target: + version: v1 + group: rbac.authorization.k8s.io + kind: RoleBinding + name: mongodb-enterprise-appdb + path: database-roles-patch-namespace.yaml + diff --git a/config/rbac/operator-roles.yaml b/config/rbac/operator-roles.yaml new file mode 100644 index 000000000..177421e5d --- /dev/null +++ b/config/rbac/operator-roles.yaml @@ -0,0 +1,140 @@ +--- +# Source: enterprise-operator/templates/operator-roles.yaml +--- +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-operator +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6a9c42070..75b26fe48 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,24 +1,34 @@ --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: - name: mongodb-kubernetes-operator + name: manager-role rules: - apiGroups: - - "" + - admissionregistration.k8s.io resources: - - pods - - services - - configmaps - - secrets + - validatingwebhookconfigurations verbs: - create - delete - get - - list - - patch - update +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - create + - get + - list - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: manager-role + namespace: placeholder +rules: - apiGroups: - apps resources: @@ -28,19 +38,74 @@ rules: - delete - get - list - - patch - update - watch - apiGroups: - - mongodbcommunity.mongodb.com + - "" resources: - - mongodbcommunity - - mongodbcommunity/status - - mongodbcommunity/spec - - mongodbcommunity/finalizers + - configmaps + - secrets verbs: + - create + - delete - get - - patch - list - update - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - mongodb.com + resources: + - mongodb + - mongodb/finalizers + - mongodb/status + verbs: + - '*' +- apiGroups: + - mongodb.com + resources: + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodbmulticluster/status + verbs: + - '*' +- apiGroups: + - mongodb.com + resources: + - mongodbusers + - mongodbusers/finalizers + - mongodbusers/status + verbs: + - '*' +- apiGroups: + - mongodb.com + resources: + - opsmanagers + - opsmanagers/finalizers + - opsmanagers/status + verbs: + - '*' diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index b444f2d67..d89f995ef 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,11 +1,12 @@ -kind: RoleBinding +--- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding metadata: - name: mongodb-kubernetes-operator -subjects: -- kind: ServiceAccount - name: mongodb-kubernetes-operator + name: manager-rolebinding roleRef: - kind: Role - name: mongodb-kubernetes-operator apiGroup: rbac.authorization.k8s.io + kind: Role + name: manager-role +subjects: +- kind: ServiceAccount + name: mongodb-enterprise-operator diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 000000000..3f2f94737 --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- mongodb-replicaset.yaml +- mongodb-sharded.yaml +- mongodb-user.yaml +- mongodb-om.yaml +- mongodb-multi.yaml diff --git a/config/samples/mongodb-multi.yaml b/config/samples/mongodb-multi.yaml new file mode 100644 index 000000000..90bcfd38d --- /dev/null +++ b/config/samples/mongodb-multi.yaml @@ -0,0 +1,20 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + persistent: true + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: e2e.cluster1.mongokubernetes.com + members: 2 + - clusterName: e2e.cluster2.mongokubernetes.com + members: 1 + - clusterName: e2e.cluster3.mongokubernetes.com + members: 2 diff --git a/config/samples/mongodb-om.yaml b/config/samples/mongodb-om.yaml new file mode 100644 index 000000000..964a58161 --- /dev/null +++ b/config/samples/mongodb-om.yaml @@ -0,0 +1,15 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + version: 6.0.3 + adminCredentials: ops-manager-admin + configuration: + mms.fromEmailAddr: admin@thecompany.com + externalConnectivity: + type: LoadBalancer + applicationDatabase: + members: 3 + podSpec: + cpu: 1 diff --git a/config/samples/mongodb-replicaset.yaml b/config/samples/mongodb-replicaset.yaml new file mode 100644 index 000000000..19c9839ed --- /dev/null +++ b/config/samples/mongodb-replicaset.yaml @@ -0,0 +1,13 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + credentials: my-credentials + members: 3 + opsManager: + configMapRef: + name: my-project + type: ReplicaSet + version: 4.4.0-ent + persistent: true diff --git a/config/samples/mongodb-sharded.yaml b/config/samples/mongodb-sharded.yaml new file mode 100644 index 000000000..4ae92fc4e --- /dev/null +++ b/config/samples/mongodb-sharded.yaml @@ -0,0 +1,16 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sample-sharded-cluster +spec: + version: 4.4.0-ent + type: ShardedCluster + configServerCount: 3 + credentials: my-credentials + mongodsPerShardCount: 3 + mongosCount: 2 + persistent: true + opsManager: + configMapRef: + name: my-project + shardCount: 1 diff --git a/config/samples/mongodb-user.yaml b/config/samples/mongodb-user.yaml new file mode 100644 index 000000000..0c8eadeeb --- /dev/null +++ b/config/samples/mongodb-user.yaml @@ -0,0 +1,14 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-replica-set-x509-user +spec: + db: $external + mongodbResourceRef: + name: my-replica-set + roles: + - db: admin + name: dbOwner + username: CN=my-replica-set-x509-user,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US +status: + phase: Updated diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml new file mode 100644 index 000000000..c77047841 --- /dev/null +++ b/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml new file mode 100644 index 000000000..d73509ee7 --- /dev/null +++ b/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +# +kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml new file mode 100644 index 000000000..c04db317d --- /dev/null +++ b/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: basic + test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml new file mode 100644 index 000000000..122f70310 --- /dev/null +++ b/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.12.0 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/config/webhooks/kustomization.yaml b/config/webhooks/kustomization.yaml new file mode 100644 index 000000000..132026c41 --- /dev/null +++ b/config/webhooks/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- webhooks.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhooks/kustomizeconfig.yaml b/config/webhooks/kustomizeconfig.yaml new file mode 100644 index 000000000..aa156e370 --- /dev/null +++ b/config/webhooks/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +nameReference: + - kind: Service + version: v1 + fieldSpecs: + # this is for operator-sdk to glue service, webhook definitions and put it in CSV + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name diff --git a/config/webhooks/service.yaml b/config/webhooks/service.yaml new file mode 100644 index 000000000..ff9cbaf91 --- /dev/null +++ b/config/webhooks/service.yaml @@ -0,0 +1,14 @@ +# to output CSV, the operator-sdk is matching together service, webhook and deployment in an opaque and undocumented way. To learn more look into the actual code: +# https://github.com/operator-framework/operator-sdk/blob/4abd483aa4004af83b3c05ddf9d7a7e4d4653cc7/internal/generate/clusterserviceversion/clusterserviceversion_updaters.go#L484 +apiVersion: v1 +kind: Service +metadata: + name: mongodb-enterprise-operator-service + namespace: placeholder +spec: + ports: + - port: 443 + name: "webhook-port" + targetPort: 1993 + selector: + app.kubernetes.io/name: mongodb-enterprise-operator diff --git a/config/webhooks/webhooks.yaml b/config/webhooks/webhooks.yaml new file mode 100644 index 000000000..2ce8b2178 --- /dev/null +++ b/config/webhooks/webhooks.yaml @@ -0,0 +1,75 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: mdbpolicy.mongodb.com + annotations: + service.beta.openshift.io/inject-cabundle: "true" +webhooks: + - name: validate-mongodb.mongodb.com + admissionReviewVersions: + - v1 + clientConfig: + caBundle: Cg== + service: + name: mongodb-enterprise-operator + namespace: placeholder + path: /validate-mongodb-com-v1-mongodb + rules: + - apiGroups: + - mongodb.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - mongodb + failurePolicy: Ignore + sideEffects: None + timeoutSeconds: 5 + + - name: validate-mongodbmulticluster.mongodb.com + admissionReviewVersions: + - v1 + clientConfig: + caBundle: Cg== + service: + name: mongodb-enterprise-operator + namespace: placeholder + path: /validate-mongodb-com-v1-mongodbmulticluster + rules: + - apiGroups: + - mongodb.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - mongodbmulticluster + failurePolicy: Ignore + sideEffects: None + timeoutSeconds: 5 + + - name: validate-opsmanagers.mongodb.com + admissionReviewVersions: + - v1 + clientConfig: + caBundle: Cg== + service: + name: mongodb-enterprise-operator + namespace: placeholder + path: /validate-mongodb-com-v1-mongodbopsmanager + rules: + - apiGroups: + - mongodb.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - opsmanagers + failurePolicy: Ignore + sideEffects: None + timeoutSeconds: 5 diff --git a/controllers/om/agent.go b/controllers/om/agent.go new file mode 100644 index 000000000..858b9fc33 --- /dev/null +++ b/controllers/om/agent.go @@ -0,0 +1,61 @@ +package om + +import ( + "strings" + "time" + + "go.uber.org/zap" +) + +// Checks if the agents have registered. + +type AutomationAgentStatusResponse struct { + OMPaginated + AutomationAgents []AgentStatus `json:"results"` +} + +type AgentStatus struct { + ConfCount int `json:"confCount"` + Hostname string `json:"hostname"` + LastConf string `json:"lastConf"` + StateName string `json:"stateName"` + TypeName string `json:"typeName"` +} + +var _ Paginated = AutomationAgentStatusResponse{} + +// IsRegistered will return true if this given agent has `hostname_prefix` as a +// prefix. This is needed to check if the given agent has registered. +func (agent AgentStatus) IsRegistered(hostnamePrefix string, log *zap.SugaredLogger) bool { + lastPing, err := time.Parse(time.RFC3339, agent.LastConf) + if err != nil { + log.Error("Wrong format for lastConf field: expected UTC format but the value is " + agent.LastConf) + return false + } + if strings.HasPrefix(agent.Hostname, hostnamePrefix) { + // Any pings earlier than 1 minute ago are signs that agents are in trouble, so we cannot consider them as + // registered (maybe we should decrease this to ~5-10 seconds?) + if lastPing.Add(time.Minute).Before(time.Now()) { + log.Debugw("Agent is registered but its last ping was more than 1 minute ago", "ping", + lastPing, "hostname", agent.Hostname) + return false + } + log.Debugw("Agent is already registered", "hostname", agent.Hostname) + return true + } + + return false +} + +// Results are needed to fulfil the Paginated interface +func (aar AutomationAgentStatusResponse) Results() []interface{} { + ans := make([]interface{}, len(aar.AutomationAgents)) + for i, aa := range aar.AutomationAgents { + ans[i] = aa + } + return ans +} + +type Status interface { + IsRegistered(hostnamePrefix string, log *zap.SugaredLogger) bool +} diff --git a/controllers/om/api/admin.go b/controllers/om/api/admin.go new file mode 100644 index 000000000..eec5a4c9d --- /dev/null +++ b/controllers/om/api/admin.go @@ -0,0 +1,427 @@ +package api + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +type Key struct { + Description string `json:"desc"` + ID string `json:"id"` + Links []map[string]string `json:"links"` + PrivateKey string `json:"privateKey"` + PublicKey string `json:"publicKey"` + Roles []map[string]string `json:"roles"` +} + +type GlobalApiKeyRequest struct { + Description string `json:"desc"` + Roles []string `json:"roles"` +} + +type KeyResponse struct { + ApiKeys []Key `json:"results"` +} + +type Whitelist struct { + CidrBlock string `json:"cidrBlock"` + Description string `json:"description"` +} + +type S3OplogStoreAdmin interface { + // ReadS3OplogStoreConfigs returns a list of all Oplog S3Configs + ReadS3OplogStoreConfigs() ([]backup.S3Config, error) + + // UpdateS3OplogConfig updates the given Oplog S3Config + UpdateS3OplogConfig(s3Config backup.S3Config) error + + // CreateS3OplogStoreConfig creates the given Oplog S3Config + CreateS3OplogStoreConfig(s3Config backup.S3Config) error + + // DeleteS3OplogStoreConfig removes an Oplog S3Config by id + DeleteS3OplogStoreConfig(id string) error +} + +type S3StoreBlockStoreAdmin interface { + // CreateS3Config creates the given S3Config + CreateS3Config(s3Config backup.S3Config) error + + // UpdateS3Config updates the given S3Config + UpdateS3Config(s3Config backup.S3Config) error + + // ReadS3Configs returns a list of all S3Configs + ReadS3Configs() ([]backup.S3Config, error) + + // DeleteS3Config removes an s3config by id + DeleteS3Config(id string) error +} + +type BlockStoreAdmin interface { + // ReadBlockStoreConfigs returns all Block stores registered in Ops Manager + ReadBlockStoreConfigs() ([]backup.DataStoreConfig, error) + + // CreateBlockStoreConfig creates a Block store in Ops Manager + CreateBlockStoreConfig(config backup.DataStoreConfig) error + + // UpdateBlockStoreConfig updates the Block store in Ops Manager + UpdateBlockStoreConfig(config backup.DataStoreConfig) error + + // DeleteBlockStoreConfig removes the Block store by its ID + DeleteBlockStoreConfig(id string) error +} + +type OplogStoreAdmin interface { + // ReadOplogStoreConfigs returns all oplog stores registered in Ops Manager + ReadOplogStoreConfigs() ([]backup.DataStoreConfig, error) + + // CreateOplogStoreConfig creates an oplog store in Ops Manager + CreateOplogStoreConfig(config backup.DataStoreConfig) error + + // UpdateOplogStoreConfig updates the oplog store in Ops Manager + UpdateOplogStoreConfig(config backup.DataStoreConfig) error + + // DeleteOplogStoreConfig removes the oplog store by its ID + DeleteOplogStoreConfig(id string) error +} + +// OpsManagerAdmin (imported as 'api.OpsManagerAdmin') is the client to all "administrator" related operations with Ops Manager +// which do not relate to specific groups (that's why it's different from 'om.Connection'). The only state expected +// to be encapsulated is baseUrl, user and key +type OpsManagerAdmin interface { + S3OplogStoreAdmin + S3StoreBlockStoreAdmin + BlockStoreAdmin + OplogStoreAdmin + // ReadDaemonConfig returns the daemon config by hostname and head db path + ReadDaemonConfig(hostName, headDbDir string) (backup.DaemonConfig, error) + + // UpdateDaemonConfig updates the daemon config + UpdateDaemonConfig(backup.DaemonConfig) error + + // CreateDaemonConfig creates the daemon config with specified hostname and head db path + CreateDaemonConfig(hostName, headDbDir string, assignmentLabels []string) error + + // ReadFileSystemStoreConfigs reads the FileSystemSnapshot store by its ID + ReadFileSystemStoreConfigs() ([]backup.DataStoreConfig, error) + + // ReadGlobalAPIKeys reads the global API Keys in Ops Manager + ReadGlobalAPIKeys() ([]Key, error) + + // CreateGlobalAPIKey creates a new Global API Key in Ops Manager + CreateGlobalAPIKey(description string) (Key, error) + + // ReadOpsManagerVersion reads the version returned in the Header + ReadOpsManagerVersion() (versionutil.OpsManagerVersion, error) +} + +// AdminProvider is a function which returns an instance of OpsManagerAdmin interface initialized with connection parameters. +// The parameters can be moved to a separate struct when they grow (e.g. tls is added) +type AdminProvider func(baseUrl, user, publicApiKey string, ca *string) OpsManagerAdmin + +// DefaultOmAdmin is the default (production) implementation of OpsManagerAdmin interface +type DefaultOmAdmin struct { + BaseURL string + User string + PrivateAPIKey string + CA *string +} + +var ( + _ OpsManagerAdmin = &DefaultOmAdmin{} + _ OpsManagerAdmin = &MockedOmAdmin{} +) + +func NewOmAdmin(baseUrl, user, privateKey string, ca *string) OpsManagerAdmin { + return &DefaultOmAdmin{BaseURL: baseUrl, User: user, PrivateAPIKey: privateKey, CA: ca} +} + +func (a *DefaultOmAdmin) ReadDaemonConfig(hostName, headDbDir string) (backup.DaemonConfig, error) { + ans, _, apiErr := a.get("admin/backup/daemon/configs/%s/%s", hostName, url.QueryEscape(headDbDir)) + if apiErr != nil { + return backup.DaemonConfig{}, apiErr + } + daemonConfig := &backup.DaemonConfig{} + if err := json.Unmarshal(ans, daemonConfig); err != nil { + return backup.DaemonConfig{}, apierror.New(err) + } + + return *daemonConfig, nil +} + +func (a *DefaultOmAdmin) UpdateDaemonConfig(config backup.DaemonConfig) error { + _, _, err := a.put("admin/backup/daemon/configs/%s/%s", config, url.QueryEscape(config.Machine.MachineHostName), url.QueryEscape(config.Machine.HeadRootDirectory)) + if err != nil { + return err + } + return nil +} + +func (a *DefaultOmAdmin) CreateDaemonConfig(hostName, headDbDir string, assignmentLabels []string) error { + config := backup.NewDaemonConfig(hostName, headDbDir, assignmentLabels) + // dev note, for creation we don't specify the second path parameter (head db) - it's used only during update + _, _, err := a.put("admin/backup/daemon/configs/%s", config, hostName) + if err != nil { + return err + } + return nil +} + +// ReadOplogStoreConfigs returns all oplog stores registered in Ops Manager +// Some assumption: while the API returns the paginated source we don't handle it to make api simpler (quite improbable +// to have 500+ configs) +func (a *DefaultOmAdmin) ReadOplogStoreConfigs() ([]backup.DataStoreConfig, error) { + res, _, err := a.get("admin/backup/oplog/mongoConfigs/") + if err != nil { + return nil, err + } + + dataStoreConfigResponse := &backup.DataStoreConfigResponse{} + if err = json.Unmarshal(res, dataStoreConfigResponse); err != nil { + return nil, apierror.New(err) + } + + return dataStoreConfigResponse.DataStoreConfigs, nil +} + +// CreateOplogStoreConfig creates an oplog store in Ops Manager +func (a *DefaultOmAdmin) CreateOplogStoreConfig(config backup.DataStoreConfig) error { + _, _, err := a.post("admin/backup/oplog/mongoConfigs/", config) + return err +} + +// UpdateOplogStoreConfig updates an oplog store in Ops Manager +func (a *DefaultOmAdmin) UpdateOplogStoreConfig(config backup.DataStoreConfig) error { + _, _, err := a.put("admin/backup/oplog/mongoConfigs/%s", config, config.Id) + return err +} + +// DeleteOplogStoreConfig removes the oplog store by its ID +func (a *DefaultOmAdmin) DeleteOplogStoreConfig(id string) error { + return a.delete("admin/backup/oplog/mongoConfigs/%s", id) +} + +func (a *DefaultOmAdmin) ReadS3OplogStoreConfigs() ([]backup.S3Config, error) { + res, _, err := a.get("admin/backup/oplog/s3Configs") + if err != nil { + return nil, err + } + + s3Configs := &backup.S3ConfigResponse{} + if err = json.Unmarshal(res, s3Configs); err != nil { + return nil, apierror.New(err) + } + + return s3Configs.S3Configs, nil +} + +func (a *DefaultOmAdmin) UpdateS3OplogConfig(s3Config backup.S3Config) error { + _, _, err := a.put("admin/backup/oplog/s3Configs/%s", s3Config, s3Config.Id) + return err +} + +func (a *DefaultOmAdmin) CreateS3OplogStoreConfig(s3Config backup.S3Config) error { + _, _, err := a.post("admin/backup/oplog/s3Configs", s3Config) + return err +} + +func (a *DefaultOmAdmin) DeleteS3OplogStoreConfig(id string) error { + return a.delete("admin/backup/oplog/s3Configs/%s", id) +} + +// ReadBlockStoreConfigs returns all Block stores registered in Ops Manager +// Some assumption: while the API returns the paginated source we don't handle it to make api simpler (quite improbable +// to have 500+ configs) +func (a *DefaultOmAdmin) ReadBlockStoreConfigs() ([]backup.DataStoreConfig, error) { + res, _, err := a.get("admin/backup/snapshot/mongoConfigs/") + if err != nil { + return nil, err + } + + dataStoreConfigResponse := &backup.DataStoreConfigResponse{} + if err = json.Unmarshal(res, dataStoreConfigResponse); err != nil { + return nil, apierror.New(err) + } + + return dataStoreConfigResponse.DataStoreConfigs, nil +} + +// CreateBlockStoreConfig creates a Block store in Ops Manager +func (a *DefaultOmAdmin) CreateBlockStoreConfig(config backup.DataStoreConfig) error { + _, _, err := a.post("admin/backup/snapshot/mongoConfigs/", config) + return err +} + +// UpdateBlockStoreConfig updates a Block store in Ops Manager +func (a *DefaultOmAdmin) UpdateBlockStoreConfig(config backup.DataStoreConfig) error { + _, _, err := a.put("admin/backup/snapshot/mongoConfigs/%s", config, config.Id) + return err +} + +// DeleteBlockStoreConfig removes the Block store by its ID +func (a *DefaultOmAdmin) DeleteBlockStoreConfig(id string) error { + return a.delete("admin/backup/snapshot/mongoConfigs/%s", id) +} + +// S3 related methods +func (a *DefaultOmAdmin) CreateS3Config(s3Config backup.S3Config) error { + _, _, err := a.post("admin/backup/snapshot/s3Configs", s3Config) + return err +} + +func (a *DefaultOmAdmin) UpdateS3Config(s3Config backup.S3Config) error { + _, _, err := a.put("admin/backup/snapshot/s3Configs/%s", s3Config, s3Config.Id) + return err +} + +func (a *DefaultOmAdmin) ReadS3Configs() ([]backup.S3Config, error) { + res, _, err := a.get("admin/backup/snapshot/s3Configs") + if err != nil { + return nil, apierror.New(err) + } + s3ConfigResponse := &backup.S3ConfigResponse{} + if err = json.Unmarshal(res, s3ConfigResponse); err != nil { + return nil, apierror.New(err) + } + + return s3ConfigResponse.S3Configs, nil +} + +func (a *DefaultOmAdmin) DeleteS3Config(id string) error { + return a.delete("admin/backup/snapshot/s3Configs/%s", id) +} + +func (a *DefaultOmAdmin) ReadFileSystemStoreConfigs() ([]backup.DataStoreConfig, error) { + res, _, err := a.get("admin/backup/snapshot/fileSystemConfigs/") + if err != nil { + return nil, err + } + + dataStoreConfigResponse := &backup.DataStoreConfigResponse{} + if err = json.Unmarshal(res, dataStoreConfigResponse); err != nil { + return nil, apierror.New(err) + } + + return dataStoreConfigResponse.DataStoreConfigs, nil +} + +// ReadGlobalAPIKeys reads the global API Keys in Ops Manager +func (a *DefaultOmAdmin) ReadGlobalAPIKeys() ([]Key, error) { + res, _, err := a.get("admin/apiKeys") + if err != nil { + return nil, err + } + + apiKeyResponse := &KeyResponse{} + if err = json.Unmarshal(res, apiKeyResponse); err != nil { + return nil, apierror.New(err) + } + + return apiKeyResponse.ApiKeys, nil +} + +// addWhitelistEntryIfItDoesntExist adds a whitelist through OM API. If it already exists, in just return +func (a *DefaultOmAdmin) addWhitelistEntryIfItDoesntExist(cidrBlock string, description string) error { + _, _, err := a.post("admin/whitelist", Whitelist{ + CidrBlock: cidrBlock, + Description: description, + }) + if apierror.NewNonNil(err).ErrorCode == apierror.DuplicateWhitelistEntry { + return err + } + return nil +} + +// CreateGlobalAPIKey creates a new Global API Key in Ops Manager. +func (a *DefaultOmAdmin) CreateGlobalAPIKey(description string) (Key, error) { + if err := a.addWhitelistEntryIfItDoesntExist("0.0.0.0/1", description); err != nil { + return Key{}, err + } + if err := a.addWhitelistEntryIfItDoesntExist("128.0.0.0/1", description); err != nil { + return Key{}, err + } + + newKeyBytes, _, err := a.post("admin/apiKeys", GlobalApiKeyRequest{ + Description: description, + Roles: []string{"GLOBAL_OWNER"}, + }) + if err != nil { + return Key{}, err + } + + apiKey := &Key{} + if err := json.Unmarshal(newKeyBytes, apiKey); err != nil { + return Key{}, err + } + return *apiKey, nil +} + +// ReadOpsManagerVersion read the version returned in the Header. +func (a *DefaultOmAdmin) ReadOpsManagerVersion() (versionutil.OpsManagerVersion, error) { + _, header, err := a.get("") + if err != nil { + return versionutil.OpsManagerVersion{}, err + } + return versionutil.OpsManagerVersion{ + VersionString: versionutil.GetVersionFromOpsManagerApiHeader(header.Get("X-MongoDB-Service-Version")), + }, nil +} + +//********************************** Private methods ******************************************************************* + +func (a *DefaultOmAdmin) get(path string, params ...interface{}) ([]byte, http.Header, error) { + return a.httpVerb("GET", path, nil, params...) +} + +func (a *DefaultOmAdmin) put(path string, v interface{}, params ...interface{}) ([]byte, http.Header, error) { + return a.httpVerb("PUT", path, v, params...) +} + +func (a *DefaultOmAdmin) post(path string, v interface{}, params ...interface{}) ([]byte, http.Header, error) { + return a.httpVerb("POST", path, v, params...) +} + +func (a *DefaultOmAdmin) delete(path string, params ...interface{}) error { + _, _, err := a.httpVerb("DELETE", path, nil, params...) + return err +} + +func (a *DefaultOmAdmin) httpVerb(method, path string, v interface{}, params ...interface{}) ([]byte, http.Header, error) { + client, err := CreateOMHttpClient(a.CA, &a.User, &a.PrivateAPIKey) + if err != nil { + return nil, nil, apierror.New(err) + } + + path = fmt.Sprintf("/api/public/v1.0/%s", path) + path = fmt.Sprintf(path, params...) + + return client.Request(method, a.BaseURL, path, v) +} + +// CreateOMHttpClient creates the om http client with auth. The client will add digest if the provided creds exist. +func CreateOMHttpClient(ca *string, user *string, key *string) (*Client, error) { + var opts []func(*Client) error + + if ca != nil { + opts = append(opts, OptionCAValidate(*ca)) + } + if user != nil && key != nil { + opts = append(opts, OptionDigestAuth(*user, *key)) + } + + if env.ReadBoolOrDefault("OM_DEBUG_HTTP", false) { // nolint:forbidigo + opts = append(opts, OptionDebug) + } + + client, err := NewHTTPClient(opts...) + if err != nil { + return nil, err + } + return client, nil +} diff --git a/controllers/om/api/digest.go b/controllers/om/api/digest.go new file mode 100644 index 000000000..13f030ac6 --- /dev/null +++ b/controllers/om/api/digest.go @@ -0,0 +1,70 @@ +package api + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// parseAPIError +func parseAPIError(statusCode int, method, url string, body []byte) *apierror.Error { + // If nobody - returning the error object with only HTTP status + if body == nil { + return &apierror.Error{ + Status: &statusCode, + Detail: fmt.Sprintf("%s %v failed with status %d with no response body", method, url, statusCode), + } + } + // If response body exists - trying to parse it + errorObject := &apierror.Error{} + if err := json.Unmarshal(body, errorObject); err != nil { + // If parsing has failed - returning just the general error with status code + return &apierror.Error{ + Status: &statusCode, + Detail: fmt.Sprintf("%s %v failed with status %d with response body: %s", method, url, statusCode, string(body)), + } + } + + return errorObject +} + +func digestParts(resp *http.Response) map[string]string { + result := map[string]string{} + if len(resp.Header["Www-Authenticate"]) > 0 { + wantedHeaders := []string{"nonce", "realm", "qop"} + responseHeaders := strings.Split(resp.Header["Www-Authenticate"][0], ",") + for _, r := range responseHeaders { + for _, w := range wantedHeaders { + if strings.Contains(r, w) { + result[w] = strings.Split(r, `"`)[1] + break + } + } + } + } + return result +} + +func getCnonce() string { + b := make([]byte, 8) + _, _ = io.ReadFull(rand.Reader, b) + return fmt.Sprintf("%x", b)[:16] +} + +func getDigestAuthorization(digestParts map[string]string, method string, url string, user string, token string) string { + d := digestParts + ha1 := util.MD5Hex(user + ":" + d["realm"] + ":" + token) + ha2 := util.MD5Hex(method + ":" + url) + nonceCount := 1 + cnonce := getCnonce() + response := util.MD5Hex(fmt.Sprintf("%s:%s:%v:%s:%s:%s", ha1, d["nonce"], nonceCount, cnonce, d["qop"], ha2)) + authorization := fmt.Sprintf(`Digest username="%s", realm="%s", nonce="%s", uri="%s", cnonce="%s", nc=%v, qop=%s, response="%s", algorithm="MD5"`, + user, d["realm"], d["nonce"], url, cnonce, nonceCount, d["qop"], response) + return authorization +} diff --git a/controllers/om/api/http.go b/controllers/om/api/http.go new file mode 100644 index 000000000..7269cd9f6 --- /dev/null +++ b/controllers/om/api/http.go @@ -0,0 +1,324 @@ +package api + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "io" + "net/http" + "net/http/httputil" + "os" + "strconv" + "time" + + "github.com/hashicorp/go-retryablehttp" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" + "golang.org/x/xerrors" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +const ( + OMClientSubsystem = "om_client" + ResultKey = "requests_total" +) + +var omClient = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: OMClientSubsystem, + Name: ResultKey, + Help: "Number of HTTP requests, partitioned by status code, method, and path.", +}, []string{"code", "method", "path"}) + +func init() { + // Register custom metrics with the global prometheus registry + if util.OperatorEnvironment(os.Getenv(util.OmOperatorEnv)) != util.OperatorEnvironmentProd { // nolint:forbidigo + zap.S().Debugf("collecting operator specific debug metrics") + registerClientMetrics() + } +} + +// registerClientMetrics sets up the operator om client metrics. +func registerClientMetrics() { + // register the metrics with our registry + metrics.Registry.MustRegister(omClient) +} + +const ( + defaultRetryWaitMin = 1 * time.Second + defaultRetryWaitMax = 10 * time.Second + defaultRetryMax = 3 +) + +type Client struct { + *retryablehttp.Client + + // Digest username and password + username string + password string + + // Enable debugging information on this client + debug bool +} + +// NewHTTPClient is a functional options constructor, based on this blog post: +// https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis +// The default clients specifies some important timeouts (some of them are synced with AA one): +// 10 seconds for connection (TLS/non TLS) +// 10 minutes for requests (time to get the first response headers) +func NewHTTPClient(options ...func(*Client) error) (*Client, error) { + client := &Client{ + Client: newDefaultHTTPClient(), + } + + return applyOptions(client, options...) +} + +func newDefaultHTTPClient() *retryablehttp.Client { + return &retryablehttp.Client{ + HTTPClient: &http.Client{Transport: http.DefaultTransport}, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + RetryMax: defaultRetryMax, + // Will retry on all errors + CheckRetry: retryablehttp.DefaultRetryPolicy, + // Exponential backoff based on the attempt number and limited by the provided minimum and maximum durations. + // We don't need Jitter here as we're the only client to the OM, so there's no risk + // of overwhelming it in a peek. + Backoff: retryablehttp.DefaultBackoff, + } +} + +func applyOptions(client *Client, options ...func(*Client) error) (*Client, error) { + for _, op := range options { + err := op(client) + if err != nil { + return nil, err + } + } + + return client, nil +} + +// NewHTTPOptions returns a list of options that can be used to construct an +// *http.Client using `NewHTTPClient`. +func NewHTTPOptions() []func(*Client) error { + return make([]func(*Client) error, 0) +} + +// OptionsDigestAuth enables Digest authentication. +func OptionDigestAuth(username, password string) func(client *Client) error { + return func(client *Client) error { + client.username = username + client.password = password + + return nil + } +} + +// OptionDebug enables debug on the http client. +func OptionDebug(client *Client) error { + client.debug = true + + return nil +} + +// OptionSkipVerify will set the Insecure Skip which means that TLS certs will not be +// verified for validity. +func OptionSkipVerify(client *Client) error { + TLSClientConfig := &tls.Config{InsecureSkipVerify: true} //nolint //Options for switching this on/off are at the CR level. + + transport := client.HTTPClient.Transport.(*http.Transport).Clone() + transport.TLSClientConfig = TLSClientConfig + client.HTTPClient.Transport = transport + + return nil +} + +// OptionCAValidate will use the CA certificate, passed as a string, to validate the +// certificates presented by Ops Manager. +func OptionCAValidate(ca string) func(client *Client) error { + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM([]byte(ca)) + TLSClientConfig := &tls.Config{ + InsecureSkipVerify: false, + RootCAs: caCertPool, + MinVersion: tls.VersionTLS12, + } + + return func(client *Client) error { + transport := client.HTTPClient.Transport.(*http.Transport).Clone() + transport.TLSClientConfig = TLSClientConfig + client.HTTPClient.Transport = transport + + return nil + } +} + +// Request executes an HTTP request, given a series of parameters, over this *Client object. +// It handles Digest when needed and json marshaling of the `v` struct. +func (client *Client) Request(method, hostname, path string, v interface{}) ([]byte, http.Header, error) { + url := hostname + path + + req, err := createHTTPRequest(method, url, v) + if err != nil { + return nil, nil, apierror.New(err) + } + + if client.username != "" && client.password != "" { + // Only add Digest auth when needed. + err = client.authorizeRequest(method, hostname, path, req) + if err != nil { + return nil, nil, err + } + } + + return client.sendRequest(method, url, path, req) +} + +// RequestWithAgentAuth executes an HTTP request using Basic authorization with the agent API key +// This is used for specific endpoints created for the agent. +// We use it for /group/v2/info and /group/v2/addPreferredHostname to manage preferred hostnames +// We have created a ticket for the EA team to add a public or private endpoint +// whilst maintaining the other 2 https://jira.mongodb.org/browse/CLOUDP-308115 +func (client *Client) RequestWithAgentAuth(method, hostname, path string, agentAuth string, v interface{}) ([]byte, http.Header, error) { + url := hostname + path + + req, err := createHTTPRequest(method, url, v) + if err != nil { + return nil, nil, apierror.New(err) + } + + req.Header.Set("Authorization", agentAuth) + + return client.sendRequest(method, url, path, req) +} + +// authorizeRequest executes one request that's meant to be challenged by the +// server in order to build the next one. The `request` parameter is aggregated +// with the required `Authorization` header. +func (client *Client) authorizeRequest(method, hostname, path string, request *retryablehttp.Request) error { + url := hostname + path + + digestRequest, err := retryablehttp.NewRequest(method, url, nil) + if err != nil { + return err + } + resp, err := client.Do(digestRequest) + if err != nil { + return err + } + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + + _, err = io.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusUnauthorized { + return apierror.New( + xerrors.Errorf( + "Received status code '%v' (%v) but expected the '%d', requested url: %v", + resp.StatusCode, + resp.Status, + http.StatusUnauthorized, + digestRequest.URL, + ), + ) + } + digestParts := digestParts(resp) + digestAuth := getDigestAuthorization(digestParts, method, path, client.username, client.password) + + request.Header.Set("Authorization", digestAuth) + + return nil +} + +// createHTTPRequest +func createHTTPRequest(method string, url string, v interface{}) (*retryablehttp.Request, error) { + buffer, err := serializeToBuffer(v) + if err != nil { + return nil, err + } + + req, err := retryablehttp.NewRequest(method, url, buffer) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", "application/json; charset=UTF-8") + req.Header.Add("Provider", "KUBERNETES") + + return req, nil +} + +func (client *Client) sendRequest(method, url, path string, req *retryablehttp.Request) ([]byte, http.Header, error) { + // we need to limit size of request/response dump, because automation config request can't have over 1MB size + const maxDumpSize = 10000 // 1 MB + client.RequestLogHook = func(logger retryablehttp.Logger, request *http.Request, i int) { + if client.debug { + dumpRequest, _ := httputil.DumpRequest(request, true) + if len(dumpRequest) > maxDumpSize { + dumpRequest = dumpRequest[:maxDumpSize] + } + zap.S().Debugf("Ops Manager request (%d): %s %s\n \n %s", i, method, path, dumpRequest) + } else { + zap.S().Debugf("Ops Manager request: %s %s", method, url) + } + } + + client.ResponseLogHook = func(logger retryablehttp.Logger, response *http.Response) { + if client.debug { + dumpResponse, _ := httputil.DumpResponse(response, true) + if len(dumpResponse) > maxDumpSize { + dumpResponse = dumpResponse[:maxDumpSize] + } + zap.S().Debugf("Ops Manager response: %s %s\n \n %s", method, path, dumpResponse) + } + } + + resp, err := client.Do(req) + if err != nil { + return nil, nil, apierror.New(xerrors.Errorf("error sending %s request to %s: %w", method, url, err)) + } + + omClient.WithLabelValues(strconv.Itoa(resp.StatusCode), method, path).Inc() + + // need to clear hooks, because otherwise they will be persisted for the subsequent calls + // resulting in logging authorizeRequest + client.RequestLogHook = nil + client.ResponseLogHook = nil + + // It is required for the body to be read completely for the connection to be reused. + // https://stackoverflow.com/a/17953506/75928 + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return nil, nil, apierror.New(xerrors.Errorf("Error reading response body from %s to %v status=%v", method, url, resp.StatusCode)) + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + apiError := parseAPIError(resp.StatusCode, method, url, body) + return nil, nil, apiError + } + + return body, resp.Header, nil +} + +// serializeToBuffer takes any object and tries to serialize it to the buffer +func serializeToBuffer(v interface{}) (io.Reader, error) { + var buffer io.Reader + if v != nil { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + buffer = bytes.NewBuffer(b) + } + return buffer, nil +} diff --git a/controllers/om/api/http_test.go b/controllers/om/api/http_test.go new file mode 100644 index 000000000..1d6f868f4 --- /dev/null +++ b/controllers/om/api/http_test.go @@ -0,0 +1,20 @@ +package api + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreateHttpRequest(t *testing.T) { + httpRequest, e := createHTTPRequest("POST", "http://some.com", nil) + u, _ := url.Parse("http://some.com") + + assert.NoError(t, e) + assert.Equal(t, []string{"application/json; charset=UTF-8"}, httpRequest.Header["Content-Type"]) + assert.Equal(t, []string{"KUBERNETES"}, httpRequest.Header["Provider"]) + assert.Equal(t, "POST", httpRequest.Method) + assert.Equal(t, u, httpRequest.URL) + assert.Nil(t, httpRequest.Body) +} diff --git a/controllers/om/api/initializer.go b/controllers/om/api/initializer.go new file mode 100644 index 000000000..2b8a29533 --- /dev/null +++ b/controllers/om/api/initializer.go @@ -0,0 +1,166 @@ +package api + +import ( + "encoding/json" + "fmt" + "io" + "net/url" + + "github.com/blang/semver" + "golang.org/x/xerrors" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +// This is a separate om functionality needed for OM controller +// The 'TryCreateUser' method doesn't use authentication so doesn't use the digest auth. +// That's why it's not added into the 'omclient.go' file + +// Initializer knows how to make calls to Ops Manager to create a first user +type Initializer interface { + // TryCreateUser makes the call to Ops Manager to create the first admin user. Returns the public API key or an + // error if the user already exists for example + TryCreateUser(omUrl string, omVersion string, user User, ca *string) (OpsManagerKeyPair, error) +} + +// DefaultInitializer is the "production" implementation of 'Initializer'. Seems we don't need to keep any state +// as the clients won't call the 'TryCreateUser' more than once after struct instantiation +type DefaultInitializer struct{} + +// User is a json struct corresponding to mms 'ApiAppUserView' object +type User struct { + Username string `json:"username"` + Password string `json:"password"` + FirstName string `json:"firstName"` + LastName string `json:"lastName"` +} + +// UserKeys is a json struct corresponding to mms 'ApiAppUserAndApiKeyView' +// object +type UserKeys struct { + ApiKey string `json:"apiKey"` +} + +// ResultProgrammaticAPIKey struct that deserializes to the result of +// calling `unauth/users` Ops Manager endpoint. +type ResultProgrammaticAPIKey struct { + ProgrammaticAPIKey OpsManagerKeyPair `json:"programmaticApiKey"` +} + +// OpsManagerKeyPair convenient type we use to fetch credentials from different +// versions of Ops Manager API. +type OpsManagerKeyPair struct { + PrivateKey string `json:"privateKey"` + PublicKey string `json:"publicKey"` +} + +// TryCreateUser makes the call to the special OM endpoint '/unauth/users' which +// creates a GLOBAL_ADMIN user and generates public API token. +// +// If the endpoint has been called already and a user already exist, then it +// will return HTTP-409. +// +// If this endpoint is called once again with a different `username` the user +// will be created but with no `GLOBAL_ADMIN` role. +// More here: https://www.mongodb.com/docs/ops-manager/current/reference/api/user-create-first/ +func (o *DefaultInitializer) TryCreateUser(omUrl string, omVersion string, user User, ca *string) (OpsManagerKeyPair, error) { + buffer, err := serializeToBuffer(user) + if err != nil { + return OpsManagerKeyPair{}, err + } + + client, err := CreateOMHttpClient(ca, nil, nil) + if err != nil { + return OpsManagerKeyPair{}, err + } + // dev note: we are doing many similar things that 'http.go' does - though we cannot reuse that now as current + // request is not a digest one + endpoint := buildOMUnauthEndpoint(omUrl) + resp, err := client.Post(endpoint, "application/json; charset=UTF-8", buffer) + if err != nil { + return OpsManagerKeyPair{}, xerrors.Errorf("Error sending POST request to %s: %w", endpoint, err) + } + + var body []byte + if resp.Body != nil { + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + body, err = io.ReadAll(resp.Body) + if err != nil { + return OpsManagerKeyPair{}, xerrors.Errorf("Error reading response body from %v status=%v", omUrl, resp.StatusCode) + } + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + apiError := parseAPIError(resp.StatusCode, "post", omUrl, body) + return OpsManagerKeyPair{}, apiError + } + + return fetchOMCredentialsFromResponse(body, omVersion, user) +} + +// buildOMUnauthEndpoint returns a string pointing at the unauth/users endpoint +// needed to create the first global owner user. +func buildOMUnauthEndpoint(baseUrl string) string { + u, err := url.Parse(baseUrl) + if err != nil { + panic(fmt.Sprintf("Could not parse %s as a url", baseUrl)) + } + + q := u.Query() + q.Add("whitelist", "0.0.0.0/1") + q.Add("whitelist", "128.0.0.0/1") + q.Add("pretty", "true") + + u.Path = "/api/public/v1.0/unauth/users" + u.RawQuery = q.Encode() + + return u.String() +} + +// fetchOMCredentialsFromResponse returns a `OpsManagerKeyPair` which consist of +// a public and private parts. +// +// This function deserializes the result of calling the `unauth/users` endpoint +// and returns a generic credentials object that can work for both 5.0 and +// pre-5.0 OM versions. +// +// One important detail about the returned value is that in the old-style user +// APIs, the entry called `PrivateAPIKey` corresponds to `PrivateAPIKey` in +// programmatic key, and `Username` corresponds to `PublicAPIKey`. +func fetchOMCredentialsFromResponse(body []byte, omVersion string, user User) (OpsManagerKeyPair, error) { + version, err := versionutil.StringToSemverVersion(omVersion) + if err != nil { + return OpsManagerKeyPair{}, err + } + + if semver.MustParseRange(">=5.0.0")(version) { + // Ops Manager 5.0.0+ returns a Programmatic API Key + apiKey := &ResultProgrammaticAPIKey{} + if err := json.Unmarshal(body, apiKey); err != nil { + return OpsManagerKeyPair{}, err + } + + if apiKey.ProgrammaticAPIKey.PrivateKey != "" && apiKey.ProgrammaticAPIKey.PublicKey != "" { + return apiKey.ProgrammaticAPIKey, nil + } + + return OpsManagerKeyPair{}, xerrors.Errorf("Could not fetch credentials from Ops Manager") + } + + // OpsManager up to 4.4.x return a user API key + u := &UserKeys{} + if err := json.Unmarshal(body, u); err != nil { + return OpsManagerKeyPair{}, err + } + + if u.ApiKey == "" { + return OpsManagerKeyPair{}, xerrors.Errorf("Could not find a Global API key from Ops Manager") + } + + return OpsManagerKeyPair{ + PublicKey: user.Username, + PrivateKey: u.ApiKey, + }, nil +} diff --git a/controllers/om/api/initializer_test.go b/controllers/om/api/initializer_test.go new file mode 100644 index 000000000..6a92e6eb7 --- /dev/null +++ b/controllers/om/api/initializer_test.go @@ -0,0 +1,90 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildOMEndpoint(t *testing.T) { + var endpoint string + + endpoint = buildOMUnauthEndpoint("https://om") + assert.Equal(t, "https://om/api/public/v1.0/unauth/users?pretty=true&whitelist=0.0.0.0%2F1&whitelist=128.0.0.0%2F1", endpoint) + + endpoint = buildOMUnauthEndpoint("http://om:8080") + assert.Equal(t, "http://om:8080/api/public/v1.0/unauth/users?pretty=true&whitelist=0.0.0.0%2F1&whitelist=128.0.0.0%2F1", endpoint) +} + +func TestFetchOMCredentialsFromResponseOMPre50(t *testing.T) { + user := User{ + Username: "some-user", + Password: "whatever", + FirstName: "Some First Name", + LastName: "Some Last Name", + } + + bodyStr := `{"apiKey": "hola"}` + body := []byte(bodyStr) + + credentials, err := fetchOMCredentialsFromResponse(body, "4.2.2", user) + assert.NoError(t, err) + + assert.Equal(t, credentials, + OpsManagerKeyPair{ + PrivateKey: "hola", + PublicKey: "some-user", + }) + + bodyStr = `{}` + body = []byte(bodyStr) + + credentials, err = fetchOMCredentialsFromResponse(body, "4.2.2", user) + assert.Equal(t, err.Error(), "Could not find a Global API key from Ops Manager") + + assert.Equal(t, credentials, + OpsManagerKeyPair{ + PrivateKey: "", + PublicKey: "", + }) +} + +func TestFetchOMCredentialsFromResponseOM50(t *testing.T) { + user := User{ + Username: "some-user", + Password: "whatever", + FirstName: "Some First Name", + LastName: "Some Last Name", + } + + bodyStr := `{"programmaticApiKey": {"privateKey": "returned-private-key", "publicKey": "returned-public-key"}}` + body := []byte(bodyStr) + + expected := OpsManagerKeyPair{ + PrivateKey: "returned-private-key", + PublicKey: "returned-public-key", + } + + var credentials OpsManagerKeyPair + var err error + + credentials, err = fetchOMCredentialsFromResponse(body, "5.0.1", user) + assert.NoError(t, err) + assert.Equal(t, credentials, expected) + + credentials, err = fetchOMCredentialsFromResponse(body, "5.0.2", user) + assert.NoError(t, err) + assert.Equal(t, credentials, expected) + + credentials, err = fetchOMCredentialsFromResponse(body, "6.0.2", user) + assert.NoError(t, err) + assert.Equal(t, credentials, expected) + + expected = OpsManagerKeyPair{ + PrivateKey: "", + PublicKey: "", + } + credentials, err = fetchOMCredentialsFromResponse(body, "4.4.99", user) + assert.Equal(t, err.Error(), "Could not find a Global API key from Ops Manager") + assert.Equal(t, credentials, expected) +} diff --git a/controllers/om/api/mockedomadmin.go b/controllers/om/api/mockedomadmin.go new file mode 100644 index 000000000..ba7946d52 --- /dev/null +++ b/controllers/om/api/mockedomadmin.go @@ -0,0 +1,258 @@ +package api + +import ( + "errors" + "fmt" + "sort" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +// ******************************************************************************************************************** +// This is a mock for om admin. It's created as a normal (not a test) go file to allow different packages use it for +// testing. +// Surely don't use it in production :) +// ******************************************************************************************************************** + +// CurrMockedAdmin is a global variable for current OM admin object that was created by MongoDbOpsManager - just for tests +// It's important to clear the state on time - the lifespan of om admin is supposed to be bound to a lifespan of an +// OM controller instance - once a new OM controller is created the mocked admin state must be cleared +var CurrMockedAdmin *MockedOmAdmin + +type MockedOmAdmin struct { + // These variables are not used internally but are used to check the correctness of parameters passed by the controller + BaseURL string + PublicKey string + PrivateKey string + + daemonConfigs []backup.DaemonConfig + s3Configs map[string]backup.S3Config + s3OpLogConfigs map[string]backup.S3Config + oplogConfigs map[string]backup.DataStoreConfig + blockStoreConfigs map[string]backup.DataStoreConfig + fileSystemStoreConfigs map[string]backup.DataStoreConfig + apiKeys []Key + agentVersion string +} + +func (a *MockedOmAdmin) UpdateDaemonConfig(config backup.DaemonConfig) error { + for _, dc := range a.daemonConfigs { + if dc.Machine.MachineHostName == config.Machine.HeadRootDirectory { + dc.AssignmentEnabled = config.AssignmentEnabled + return nil + } + } + return apierror.NewErrorWithCode(apierror.BackupDaemonConfigNotFound) +} + +// NewMockedAdminProvider is the function creating the admin object. The function returns the existing mocked admin instance +// if it exists - this allows to survive through multiple reconciliations and to keep OM state over them +func NewMockedAdminProvider(baseUrl, publicApiKey, privateApiKey string, withSingleton bool) OpsManagerAdmin { + mockedAdmin := &MockedOmAdmin{} + mockedAdmin.BaseURL = baseUrl + mockedAdmin.PublicKey = publicApiKey + mockedAdmin.PrivateKey = privateApiKey + + mockedAdmin.daemonConfigs = make([]backup.DaemonConfig, 0) + mockedAdmin.s3Configs = make(map[string]backup.S3Config) + mockedAdmin.s3OpLogConfigs = make(map[string]backup.S3Config) + mockedAdmin.oplogConfigs = make(map[string]backup.DataStoreConfig) + mockedAdmin.blockStoreConfigs = make(map[string]backup.DataStoreConfig) + mockedAdmin.apiKeys = []Key{{ + PrivateKey: privateApiKey, + PublicKey: publicApiKey, + }} + + if withSingleton { + CurrMockedAdmin = mockedAdmin + } + + return mockedAdmin +} + +func (a *MockedOmAdmin) Reset() { + NewMockedAdminProvider(a.BaseURL, a.PublicKey, a.PrivateKey, true) +} + +func (a *MockedOmAdmin) ReadDaemonConfigs() ([]backup.DaemonConfig, error) { + return a.daemonConfigs, nil +} + +func (a *MockedOmAdmin) ReadDaemonConfig(hostName, headDbDir string) (backup.DaemonConfig, error) { + for _, v := range a.daemonConfigs { + if v.Machine.HeadRootDirectory == headDbDir && v.Machine.MachineHostName == hostName { + return v, nil + } + } + return backup.DaemonConfig{}, apierror.NewErrorWithCode(apierror.BackupDaemonConfigNotFound) +} + +func (a *MockedOmAdmin) CreateDaemonConfig(hostName, headDbDir string, assignmentLabels []string) error { + config := backup.NewDaemonConfig(hostName, headDbDir, assignmentLabels) + + for _, dConf := range a.daemonConfigs { + // Ops Manager should never be performing Update Operations, only Creations + if dConf.Machine.HeadRootDirectory == headDbDir && dConf.Machine.MachineHostName == hostName { + panic(fmt.Sprintf("Config %s, %s already exists", hostName, headDbDir)) + } + } + + a.daemonConfigs = append(a.daemonConfigs, config) + return nil +} + +func (a *MockedOmAdmin) ReadS3Configs() ([]backup.S3Config, error) { + allConfigs := make([]backup.S3Config, 0) + for _, v := range a.s3Configs { + allConfigs = append(allConfigs, v) + } + + sort.SliceStable(allConfigs, func(i, j int) bool { + return allConfigs[i].Id < allConfigs[j].Id + }) + + return allConfigs, nil +} + +func (a *MockedOmAdmin) DeleteS3Config(id string) error { + if _, ok := a.s3Configs[id]; !ok { + return errors.New("failed to remove as the s3 config doesn't exist") + } + delete(a.s3Configs, id) + return nil +} + +func (a *MockedOmAdmin) CreateS3Config(s3Config backup.S3Config) error { + a.s3Configs[s3Config.Id] = s3Config + return nil +} + +func (a *MockedOmAdmin) UpdateS3Config(s3Config backup.S3Config) error { + return a.CreateS3Config(s3Config) +} + +func (a *MockedOmAdmin) ReadOplogStoreConfigs() ([]backup.DataStoreConfig, error) { + allConfigs := make([]backup.DataStoreConfig, 0) + for _, v := range a.oplogConfigs { + allConfigs = append(allConfigs, v) + } + + sort.SliceStable(allConfigs, func(i, j int) bool { + return allConfigs[i].Id < allConfigs[j].Id + }) + + return allConfigs, nil +} + +func (a *MockedOmAdmin) CreateOplogStoreConfig(config backup.DataStoreConfig) error { + // Note, that backup API doesn't throw an error if the config already exists - it just updates it + a.oplogConfigs[config.Id] = config + return nil +} + +func (a *MockedOmAdmin) UpdateOplogStoreConfig(config backup.DataStoreConfig) error { + a.oplogConfigs[config.Id] = config + // OM backup service doesn't throw any errors if the config is not there + return nil +} + +func (a *MockedOmAdmin) DeleteOplogStoreConfig(id string) error { + if _, ok := a.oplogConfigs[id]; !ok { + return errors.New("failed to remove as the oplog doesn't exist") + } + delete(a.oplogConfigs, id) + return nil +} + +func (a *MockedOmAdmin) ReadS3OplogStoreConfigs() ([]backup.S3Config, error) { + allConfigs := make([]backup.S3Config, 0) + for _, v := range a.s3OpLogConfigs { + allConfigs = append(allConfigs, v) + } + + return allConfigs, nil +} + +func (a *MockedOmAdmin) UpdateS3OplogConfig(s3Config backup.S3Config) error { + a.s3OpLogConfigs[s3Config.Id] = s3Config + return nil +} + +func (a *MockedOmAdmin) CreateS3OplogStoreConfig(s3Config backup.S3Config) error { + return a.UpdateS3OplogConfig(s3Config) +} + +func (a *MockedOmAdmin) DeleteS3OplogStoreConfig(id string) error { + if _, ok := a.s3OpLogConfigs[id]; !ok { + return errors.New("failed to remove as the s3 oplog doesn't exist") + } + delete(a.s3OpLogConfigs, id) + return nil +} + +func (a *MockedOmAdmin) ReadBlockStoreConfigs() ([]backup.DataStoreConfig, error) { + allConfigs := make([]backup.DataStoreConfig, 0) + for _, v := range a.blockStoreConfigs { + allConfigs = append(allConfigs, v) + } + + sort.SliceStable(allConfigs, func(i, j int) bool { + return allConfigs[i].Id < allConfigs[j].Id + }) + + return allConfigs, nil +} + +func (a *MockedOmAdmin) CreateBlockStoreConfig(config backup.DataStoreConfig) error { + a.blockStoreConfigs[config.Id] = config + return nil +} + +func (a *MockedOmAdmin) UpdateBlockStoreConfig(config backup.DataStoreConfig) error { + a.blockStoreConfigs[config.Id] = config + // OM backup service doesn't throw any errors if the config is not there + return nil +} + +func (a *MockedOmAdmin) DeleteBlockStoreConfig(id string) error { + if _, ok := a.blockStoreConfigs[id]; !ok { + return errors.New("failed to remove as the block store doesn't exist") + } + delete(a.blockStoreConfigs, id) + return nil +} + +func (a *MockedOmAdmin) ReadFileSystemStoreConfigs() ([]backup.DataStoreConfig, error) { + allConfigs := make([]backup.DataStoreConfig, len(a.blockStoreConfigs)) + for _, v := range a.fileSystemStoreConfigs { + allConfigs = append(allConfigs, v) + } + return allConfigs, nil +} + +func (a *MockedOmAdmin) ReadGlobalAPIKeys() ([]Key, error) { + return a.apiKeys, nil +} + +func (a *MockedOmAdmin) CreateGlobalAPIKey(description string) (Key, error) { + newKey := Key{ + Description: description, + Roles: []map[string]string{{"role_name": "GLOBAL_ONWER"}}, + } + a.apiKeys = append(a.apiKeys, newKey) + return newKey, nil +} + +func (a *MockedOmAdmin) ReadOpsManagerVersion() (versionutil.OpsManagerVersion, error) { + return versionutil.OpsManagerVersion{}, nil +} + +func (a *MockedOmAdmin) UpdateAgentVersion(version string) { + a.agentVersion = version +} + +func (a *MockedOmAdmin) ReadAgentVersion() (string, error) { + return a.agentVersion, nil +} diff --git a/controllers/om/apierror/api_error.go b/controllers/om/apierror/api_error.go new file mode 100644 index 000000000..d637f7b53 --- /dev/null +++ b/controllers/om/apierror/api_error.go @@ -0,0 +1,96 @@ +package apierror + +import ( + "errors" + "fmt" +) + +const ( + // Error codes that Ops Manager may return that we are concerned about + OrganizationNotFound = "ORG_NAME_NOT_FOUND" + ProjectNotFound = "GROUP_NAME_NOT_FOUND" + BackupDaemonConfigNotFound = "DAEMON_MACHINE_CONFIG_NOT_FOUND" + UserAlreadyExists = "USER_ALREADY_EXISTS" + DuplicateWhitelistEntry = "DUPLICATE_GLOBAL_WHITELIST_ENTRY" +) + +// Error is the error extension that contains the details of OM error if OM returned the error. This allows the +// code using Connection methods to do more fine-grained exception handling depending on exact error that happened. +// The class has to encapsulate the usual error (non-OM one) as well as the error may happen at any stage before/after +// OM request (failing to (de)serialize json object for example) so in this case all fields except for 'Detail' will be +// empty +type Error struct { + Status *int `json:"error"` + Reason string `json:"reason"` + Detail string `json:"detail"` + ErrorCode string `json:"errorCode"` +} + +// New returns either the error itself if it's of type 'api.Error' or an 'Error' created from a normal string +func New(err error) error { + if err == nil { + return nil + } + var v *Error + switch { + case errors.As(err, &v): + return v + default: + return &Error{Detail: err.Error()} + } +} + +// NewNonNil returns empty 'Error' if the incoming parameter is nil. This allows to perform the checks for +// error code without risks to get nil pointer +// Unfortunately we have to do this as we cannot return *Error directly in our method signatures +// (https://golang.org/doc/faq#nil_error) +func NewNonNil(err error) *Error { + if err == nil { + return &Error{} + } + return New(err).(*Error) +} + +// NewErrorWithCode returns the Error initialized with the code passed. This is convenient for testing. +func NewErrorWithCode(code string) *Error { + return &Error{ErrorCode: code} +} + +// Error +func (e *Error) Error() string { + if e.Status != nil || e.ErrorCode != "" { + msg := "" + if e.Status != nil { + msg += fmt.Sprintf("Status: %d", *e.Status) + } + if e.Reason != "" { + msg += fmt.Sprintf(" (%s)", e.Reason) + } + if e.ErrorCode != "" { + msg += fmt.Sprintf(", ErrorCode: %s", e.ErrorCode) + } + if e.Detail != "" { + msg += fmt.Sprintf(", Detail: %s", e.Detail) + } + return msg + } + return e.Detail +} + +// ErrorBackupDaemonConfigIsNotFound returns whether the api-error is of not found. Sometimes OM only returns the +// http code. +func (e *Error) ErrorBackupDaemonConfigIsNotFound() bool { + if e == nil { + return false + } + + if e.Status != nil && *e.Status == 404 { + return true + } + + if e.ErrorCode == BackupDaemonConfigNotFound { + return true + } + + return false +} diff --git a/controllers/om/automation_config.go b/controllers/om/automation_config.go new file mode 100644 index 000000000..7dbaa1f67 --- /dev/null +++ b/controllers/om/automation_config.go @@ -0,0 +1,447 @@ +package om + +import ( + "encoding/json" + + "github.com/google/go-cmp/cmp" + "github.com/spf13/cast" + "k8s.io/apimachinery/pkg/api/equality" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/generate" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" +) + +// AutomationConfig maintains the raw map in the Deployment field +// and constructs structs to make use of go's type safety +// Dev notes: actually, this object is just a wrapper for the `Deployment` object which is received from Ops Manager, +// and it's not equal to the AutomationConfig object from mms! It contains some transient struct fields for easier +// configuration which are merged into the `Deployment` object before sending it back to Ops Manager. +// As of right now only support configuring LogRotate for monitoring and backup via dedicated endpoints. +type AutomationConfig struct { + Auth *Auth + AgentSSL *AgentSSL + Deployment Deployment + Ldap *ldap.Ldap +} + +// Apply merges the state of all concrete structs into the Deployment (map[string]interface{}) +func (ac *AutomationConfig) Apply() error { + return applyInto(*ac, &ac.Deployment) +} + +// applyInto is a helper method that does not mutate AutomationConfig "a", but only Deployment "deployment" +func applyInto(a AutomationConfig, into *Deployment) error { + // applies all changes made to the Auth struct and merges with the corresponding map[string]interface{} + // inside the Deployment + if _, ok := a.Deployment["auth"]; ok { + mergedAuth, err := util.MergeWith(a.Auth, a.Deployment["auth"].(map[string]interface{}), &util.AutomationConfigTransformer{}) + if err != nil { + return err + } + (*into)["auth"] = mergedAuth + } + // the same applies for the ssl object and map + if _, ok := a.Deployment["tls"]; ok { + mergedTLS, err := util.MergeWith(a.AgentSSL, a.Deployment["tls"].(map[string]interface{}), &util.AutomationConfigTransformer{}) + if err != nil { + return err + } + (*into)["tls"] = mergedTLS + } + + if _, ok := a.Deployment["ldap"]; ok { + mergedLdap, err := util.MergeWith(a.Ldap, a.Deployment["ldap"].(map[string]interface{}), &util.AutomationConfigTransformer{}) + if err != nil { + return err + } + (*into)["ldap"] = mergedLdap + } + return nil +} + +// EqualsWithoutDeployment returns true if two AutomationConfig objects are meaningful equal by following the following conditions: +// - Not taking AutomationConfig.Deployment into consideration. +// - Serializing ac A and ac B to ensure that we remove util.MergoDelete before comparing those two. +// +// Comparing Deployments will not work correctly in current AutomationConfig implementation. Helper +// structs, such as AutomationConfig.AgentSSL or AutomationConfig.Auth use non-pointer fields (without `omitempty`). +// When merging them into AutomationConfig.deployment, JSON unmarshaller renders them into their representations, +// and they get into the final result. Sadly, some tests (especially TestLDAPIsMerged) relies on this behavior. +func (ac *AutomationConfig) EqualsWithoutDeployment(b AutomationConfig) bool { + deploymentsComparer := cmp.Comparer(func(x, y Deployment) bool { + return true + }) + + acA, err := getSerializedAC(*ac) + if err != nil { + return false + } + + acB, err := getSerializedAC(b) + if err != nil { + return false + } + + return cmp.Equal(acA, acB, deploymentsComparer) +} + +// getSerializedAC calls apply on a deepCopy which decodes the internal struct into the Deployment map. After decoding, +// we encode the map into its internal representation again. Doing that removes util.MergoDelete for proper comparison +func getSerializedAC(original AutomationConfig) (AutomationConfig, error) { + empty := AutomationConfig{} + deepCopy, err := util.MapDeepCopy(original.Deployment) + if err != nil { + return empty, err + } + err = applyInto(original, (*Deployment)(&deepCopy)) + if err != nil { + return empty, err + } + + ac, err := BuildAutomationConfigFromDeployment(deepCopy) + if err != nil { + return empty, err + } + return *ac, nil +} + +// isEqualAfterModification returns true if two Deployment objects are equal ignoring their underlying custom types. +// depFunc might change the Deployment or might only change the types. In both cases it will fail the comparison +// as long as we don't ignore the types. +func isEqualAfterModification(changeDeploymentFunc func(Deployment) error, deployment Deployment) (bool, error) { + original, err := util.MapDeepCopy(deployment) // original over the wire does not contain any types + if err != nil { + return false, err + } + if err := changeDeploymentFunc(deployment); err != nil { // might change types as well + return false, err + } + + deploymentWithoutTypes, err := maputil.StructToMap(deployment) + if err != nil { + return false, err + } + + if equality.Semantic.DeepEqual(deploymentWithoutTypes, original) { + return true, nil + } + return false, nil +} + +// NewAutomationConfig returns an AutomationConfig instance with all reference +// types initialized with non-nil values +func NewAutomationConfig(deployment Deployment) *AutomationConfig { + return &AutomationConfig{AgentSSL: &AgentSSL{}, Auth: NewAuth(), Deployment: deployment} +} + +// NewAuth returns an empty Auth reference with all reference types initialised to non nil values +func NewAuth() *Auth { + return &Auth{ + KeyFile: util.AutomationAgentKeyFilePathInContainer, + KeyFileWindows: util.AutomationAgentWindowsKeyFilePath, + Users: make([]*MongoDBUser, 0), + AutoAuthMechanisms: make([]string, 0), + DeploymentAuthMechanisms: make([]string, 0), + AutoAuthMechanism: "MONGODB-CR", + Disabled: true, + AuthoritativeSet: true, + AutoUser: util.AutomationAgentName, + } +} + +// SetVersion is needed only for the cluster config file when we use a headless agent +func (ac *AutomationConfig) SetVersion(configVersion int64) *AutomationConfig { + ac.Deployment["version"] = configVersion + return ac +} + +// SetOptionsDownloadBase is needed only for the cluster config file when we use a headless agent +func (ac *AutomationConfig) SetOptionsDownloadBase(downloadBase string) *AutomationConfig { + ac.Deployment["options"] = map[string]string{"downloadBase": downloadBase} + + return ac +} + +// SetMongodbVersions is needed only for the cluster config file when we use a headless agent +func (ac *AutomationConfig) SetMongodbVersions(versionConfigs []MongoDbVersionConfig) *AutomationConfig { + ac.Deployment["mongoDbVersions"] = versionConfigs + + return ac +} + +func (ac *AutomationConfig) MongodbVersions() []MongoDbVersionConfig { + return ac.Deployment["mongoDbVersions"].([]MongoDbVersionConfig) +} + +// SetBaseUrlForAgents is needed only for the cluster config file when we use a headless agent +func (ac *AutomationConfig) SetBaseUrlForAgents(baseUrl string) *AutomationConfig { + for _, v := range ac.Deployment.getBackupVersions() { + cast.ToStringMap(v)["baseUrl"] = baseUrl + } + for _, v := range ac.Deployment.getMonitoringVersions() { + cast.ToStringMap(v)["baseUrl"] = baseUrl + } + return ac +} + +func (ac *AutomationConfig) Serialize() ([]byte, error) { + return ac.Deployment.Serialize() +} + +// GetAgentAuthMode returns the agentAuthMode of the given automationConfig. If empty or nil we return the empty string. +func (ac *AutomationConfig) GetAgentAuthMode() string { + if ac == nil || ac.Auth == nil { + return "" + } + return ac.Auth.AutoAuthMechanism +} + +type Auth struct { + // Users is a list which contains the desired users at the project level. + Users []*MongoDBUser `json:"usersWanted,omitempty"` + Disabled bool `json:"disabled"` + // AuthoritativeSet indicates if the MongoDBUsers should be synced with the current list of Users + AuthoritativeSet bool `json:"authoritativeSet"` + // AutoAuthMechanisms is a list of auth mechanisms the Automation Agent is able to use + AutoAuthMechanisms []string `json:"autoAuthMechanisms,omitempty"` + + // AutoAuthMechanism is the currently active agent authentication mechanism. This is a read only + // field + AutoAuthMechanism string `json:"autoAuthMechanism"` + // DeploymentAuthMechanisms is a list of possible auth mechanisms that can be used within deployments + DeploymentAuthMechanisms []string `json:"deploymentAuthMechanisms,omitempty"` + // AutoUser is the MongoDB Automation Agent user, when x509 is enabled, it should be set to the subject of the AA's certificate + AutoUser string `json:"autoUser,omitempty"` + // Key is the contents of the KeyFile, the automation agent will ensure this a KeyFile with these contents exists at the `KeyFile` path + Key string `json:"key,omitempty"` + // KeyFile is the path to a keyfile with read & write permissions. It is a required field if `Disabled=false` + KeyFile string `json:"keyfile,omitempty"` + // KeyFileWindows is required if `Disabled=false` even if the value is not used + KeyFileWindows string `json:"keyfileWindows,omitempty"` + // AutoPwd is a required field when going from `Disabled=false` to `Disabled=true` + AutoPwd string `json:"autoPwd,omitempty"` + // NewAutoPwd is used for rotating the agent password + NewAutoPwd string `json:"newAutoPwd,omitempty"` + // LdapGroupDN is required when enabling LDAP authz and agents authentication on $external + LdapGroupDN string `json:"autoLdapGroupDN,omitempty"` +} + +// IsEnabled is a convenience function to aid readability +func (a *Auth) IsEnabled() bool { + return !a.Disabled +} + +// Enable is a convenience function to aid readability +func (a *Auth) Enable() { + a.Disabled = false +} + +// AddUser updates the Users list with the specified user +func (a *Auth) AddUser(user MongoDBUser) { + a.Users = append(a.Users, &user) +} + +// HasUser returns true if a user exists with the specified username and password +// or false if the user does not exist +func (a *Auth) HasUser(username, db string) bool { + _, user := a.GetUser(username, db) + return user != nil +} + +// GetUser returns the index of the user with the given username and password +// and the user itself. -1 and a nil user are returned if the user does not exist +func (a *Auth) GetUser(username, db string) (int, *MongoDBUser) { + for i, u := range a.Users { + if u != nil && u.Username == username && u.Database == db { + return i, u + } + } + return -1, nil +} + +// UpdateUser accepts a user ad updates the corresponding existing user. +// the user to update is identified by user.Username and user.Database +func (a *Auth) UpdateUser(user MongoDBUser) bool { + i, foundUser := a.GetUser(user.Username, user.Database) + if foundUser == nil { + return false + } + a.Users[i] = &user + return true +} + +// EnsureUser adds the user to the Users list if it does not exist, +// it will update the existing user if it is already present. +func (a *Auth) EnsureUser(user MongoDBUser) { + if a.HasUser(user.Username, user.Database) { + a.UpdateUser(user) + } else { + a.AddUser(user) + } +} + +// EnsureUserRemoved will remove user of given username and password. A boolean +// indicating whether the underlying array was modified will be returned +func (a *Auth) EnsureUserRemoved(username, db string) bool { + if a.HasUser(username, db) { + a.RemoveUser(username, db) + return true + } + return false +} + +// RemoveUser assigns a nil value to the user. This nil value +// will flag this user for deletion when merging, see mergo_utils.go +func (a *Auth) RemoveUser(username, db string) { + i, _ := a.GetUser(username, db) + a.Users[i] = nil +} + +// AgentSSL contains fields related to configuration Automation +// Agent SSL & authentication. +type AgentSSL struct { + CAFilePath string `json:"CAFilePath,omitempty"` + AutoPEMKeyFilePath string `json:"autoPEMKeyFilePath,omitempty"` + ClientCertificateMode string `json:"clientCertificateMode,omitempty"` +} + +type MongoDBUser struct { + Mechanisms []string `json:"mechanisms"` + Roles []*Role `json:"roles"` + Username string `json:"user"` + Database string `json:"db"` + AuthenticationRestrictions []string `json:"authenticationRestrictions"` + + // The clear text password to be assigned to the user + InitPassword string `json:"initPwd,omitempty"` + + // ScramShaCreds are generated by the operator. + ScramSha256Creds *ScramShaCreds `json:"scramSha256Creds"` + ScramSha1Creds *ScramShaCreds `json:"scramSha1Creds"` +} + +type ScramShaCreds struct { + IterationCount int `json:"iterationCount"` + Salt string `json:"salt"` + ServerKey string `json:"serverKey"` + StoredKey string `json:"storedKey"` +} + +func (u *MongoDBUser) AddRole(role *Role) { + u.Roles = append(u.Roles, role) +} + +type Role struct { + Role string `json:"role"` + Database string `json:"db"` +} + +type BuildConfig struct { + Platform string `json:"platform"` + Url string `json:"url"` + GitVersion string `json:"gitVersion"` + Architecture string `json:"architecture"` + Flavor string `json:"flavor"` + MinOsVersion string `json:"minOsVersion"` + MaxOsVersion string `json:"maxOsVersion"` + Modules []string `json:"modules"` + // Note, that we are not including all "windows" parameters like "Win2008plus" as such distros won't be used +} + +type MongoDbVersionConfig struct { + Name string `json:"name"` + Builds []*BuildConfig `json:"builds"` +} + +// EnsureKeyFileContents makes sure a valid keyfile is generated and used for internal cluster authentication +func (ac *AutomationConfig) EnsureKeyFileContents() error { + if ac.Auth.Key == "" || ac.Auth.Key == util.InvalidKeyFileContents { + keyfileContents, err := generate.KeyFileContents() + if err != nil { + return err + } + ac.Auth.Key = keyfileContents + } + return nil +} + +// EnsurePassword makes sure that there is an Automation Agent password +// that the agents will use to communicate with the deployments. The password +// is returned, so it can be provided to the other agents +func (ac *AutomationConfig) EnsurePassword() (string, error) { + if ac.Auth.AutoPwd == "" || ac.Auth.AutoPwd == util.InvalidAutomationAgentPassword { + automationAgentBackupPassword, err := generate.KeyFileContents() + if err != nil { + return "", err + } + ac.Auth.AutoPwd = automationAgentBackupPassword + return automationAgentBackupPassword, nil + } + return ac.Auth.AutoPwd, nil +} + +func (ac *AutomationConfig) CanEnableX509ProjectAuthentication() (bool, string) { + if !ac.Deployment.AllProcessesAreTLSEnabled() { + return false, "not all processes are TLS enabled, unable to enable x509 authentication" + } + return true, "" +} + +func BuildAutomationConfigFromDeployment(deployment Deployment) (*AutomationConfig, error) { + finalAutomationConfig := &AutomationConfig{Deployment: deployment} + finalAutomationConfig.Auth = &Auth{} + + authMap, ok := deployment["auth"] + if ok { + authMarshalled, err := json.Marshal(authMap) + if err != nil { + return nil, err + } + auth := &Auth{} + if err := json.Unmarshal(authMarshalled, auth); err != nil { + return nil, err + } + finalAutomationConfig.Auth = auth + } + + tlsMap, ok := deployment["tls"] + if ok { + sslMarshalled, err := json.Marshal(tlsMap) + if err != nil { + return nil, err + } + ssl := &AgentSSL{} + if err := json.Unmarshal(sslMarshalled, ssl); err != nil { + return nil, err + } + finalAutomationConfig.AgentSSL = ssl + } + + ldapMap, ok := deployment["ldap"] + if ok { + ldapMarshalled, err := json.Marshal(ldapMap) + if err != nil { + return nil, err + } + acLdap := &ldap.Ldap{} + if err := json.Unmarshal(ldapMarshalled, acLdap); err != nil { + return nil, err + } + finalAutomationConfig.Ldap = acLdap + } + + return finalAutomationConfig, nil +} + +// BuildAutomationConfigFromBytes takes in jsonBytes representing the Deployment +// and constructs an instance of AutomationConfig with all the concrete structs +// filled out. +func BuildAutomationConfigFromBytes(jsonBytes []byte) (*AutomationConfig, error) { + deployment, err := BuildDeploymentFromBytes(jsonBytes) + if err != nil { + return nil, err + } + return BuildAutomationConfigFromDeployment(deployment) +} diff --git a/controllers/om/automation_config_test.go b/controllers/om/automation_config_test.go new file mode 100644 index 000000000..7e84d8658 --- /dev/null +++ b/controllers/om/automation_config_test.go @@ -0,0 +1,876 @@ +package om + +import ( + "encoding/json" + "testing" + + "github.com/spf13/cast" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/equality" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +var originalAutomationConfig = *getTestAutomationConfig() + +func getTestAutomationConfig() *AutomationConfig { + a, _ := BuildAutomationConfigFromBytes(loadBytesFromTestData("automation_config.json")) + return a +} + +func TestScramShaCreds_AreRemovedCorrectly(t *testing.T) { + ac := getTestAutomationConfig() + user := ac.Auth.Users[0] + user.ScramSha256Creds = nil + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + deploymentUser := getUser(ac.Deployment, 0) + assert.NotContains(t, deploymentUser, "scramSha256Creds") +} + +func TestUntouchedFieldsAreNotDeleted(t *testing.T) { + a := getTestAutomationConfig() + a.Auth.AutoUser = "some-user" + a.AgentSSL.ClientCertificateMode = util.RequireClientCertificates + + if err := a.Apply(); err != nil { + t.Fatal(err) + } + + auth := cast.ToStringMap(a.Deployment["auth"]) + originalAuth := cast.ToStringMap(originalAutomationConfig.Deployment["auth"]) + + // ensure values aren't overridden + assert.Equal(t, auth["usersDeleted"], originalAuth["usersDeleted"]) + assert.Equal(t, auth["autoAuthMechanisms"], originalAuth["autoAuthMechanisms"]) + assert.Equal(t, auth["autoAuthRestrictions"], originalAuth["autoAuthRestrictions"]) + assert.Equal(t, auth["disabled"], originalAuth["disabled"]) + assert.Equal(t, auth["usersWanted"], originalAuth["usersWanted"]) + + // ensure values we specified are overridden + assert.Equal(t, auth["autoUser"], "some-user") + tls := cast.ToStringMap(a.Deployment["tls"]) + assert.Equal(t, tls["clientCertificateMode"], util.RequireClientCertificates) + + // ensures fields in nested fields we don't know about are retained + scramSha256Creds := cast.ToStringMap(getUser(a.Deployment, 0)["scramSha256Creds"]) + assert.Equal(t, float64(15000), scramSha256Creds["iterationCount"]) + assert.Equal(t, "I570PanWIx1eNUTo7j4ROl2/zIqMsVd6CcIE+A==", scramSha256Creds["salt"]) + assert.Equal(t, "M4/jskiMM0DpvG/qgMELWlfReqV2ZmwdU8+vJZ/4prc=", scramSha256Creds["serverKey"]) + assert.Equal(t, "m1dXf5hHJk7EOAAyJBxfsZvFx1HwtTdda6pFPm0BlOE=", scramSha256Creds["storedKey"]) + + // ensure values we know nothing about aren't touched + options := cast.ToStringMap(a.Deployment["options"]) + assert.Equal(t, options["downloadBase"], "/var/lib/mongodb-mms-automation") + assert.Equal(t, options["downloadBaseWindows"], "%SystemDrive%\\MMSAutomation\\versions") +} + +func TestUserIsAddedToTheEnd(t *testing.T) { + a := getTestAutomationConfig() + + a.Auth.AddUser(MongoDBUser{ + Database: "my-db", + Username: "my-user", + Roles: []*Role{{Role: "my-role", Database: "role-db"}}, + }) + + if err := a.Apply(); err != nil { + t.Fatal(err) + } + + assert.Len(t, getUsers(a.Deployment), 4) + + lastUser := getUser(a.Deployment, 3) + + assert.Equal(t, "my-db", lastUser["db"]) + assert.Equal(t, "my-user", lastUser["user"]) + roles := cast.ToSlice(lastUser["roles"]) + role := cast.ToStringMap(roles[0]) + assert.Equal(t, "my-role", role["role"]) + assert.Equal(t, "role-db", role["db"]) +} + +func TestUserIsUpdated_AndOtherUsersDontGetAffected(t *testing.T) { + ac := getTestAutomationConfig() + + originalUser := getUser(ac.Deployment, 0) + + assert.Equal(t, "testDb0", originalUser["db"]) + assert.Equal(t, "testUser0", originalUser["user"]) + + // change the fields on the struct + user := ac.Auth.Users[0] + + // struct fields should be read correctly + assert.Equal(t, "testDb0", user.Database) + assert.Equal(t, "testUser0", user.Username) + + user.Database = "new-db" + user.Username = "new-user" + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + userFromDep := getUser(ac.Deployment, 0) + assert.Equal(t, "new-db", userFromDep["db"]) + assert.Equal(t, "new-user", userFromDep["user"]) + + allUsers := getUsers(ac.Deployment) + for _, user := range allUsers[1:] { + userMap := cast.ToStringMap(user) + assert.NotEqual(t, "new-db", userMap["db"]) + assert.NotEqual(t, "new-user", userMap["user"]) + } +} + +func TestCanPrependUser(t *testing.T) { + ac := getTestAutomationConfig() + + newUser := &MongoDBUser{ + Database: "myDatabase", + Username: "myUsername", + AuthenticationRestrictions: []string{}, + Roles: []*Role{ + { + Role: "myRole", + Database: "myDb", + }, + }, + } + + assert.Len(t, getUsers(ac.Deployment), 3) + assert.Len(t, ac.Auth.Users, 3) + + ac.Auth.Users = append([]*MongoDBUser{newUser}, ac.Auth.Users...) + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + dep := ac.Deployment + + firstUser := getUser(dep, 0) + firstUsersRole := getRole(dep, 0, 0) + secondUser := getUser(dep, 1) + secondUsersRoles := getRoles(dep, 1) + + // the user added to the start of the list should be the first element + assert.Equal(t, "myUsername", firstUser["user"]) + assert.Equal(t, "myDatabase", firstUser["db"]) + + // it should have the single role provided + assert.Equal(t, "myRole", firstUsersRole["role"]) + assert.Equal(t, "myDb", firstUsersRole["db"]) + + // the already existing user should be the second element + assert.Equal(t, "testUser0", secondUser["user"]) + assert.Equal(t, "testDb0", secondUser["db"]) + + assert.Len(t, secondUsersRoles, 3, "second user should not have an additional role") + + // already existing user should not have been granted the additional role + for _, omRoleInterface := range secondUsersRoles { + omRoleMap := cast.ToStringMap(omRoleInterface) + assert.False(t, omRoleMap["db"] == "myDb") + assert.False(t, omRoleMap["role"] == "myRole") + } + + allUsers := getUsers(ac.Deployment) + for _, user := range allUsers[1:] { + userMap := cast.ToStringMap(user) + assert.NotEqual(t, "new-db", userMap["db"]) + assert.NotEqual(t, "new-user", userMap["user"]) + } +} + +func TestUserIsDeleted(t *testing.T) { + a := getTestAutomationConfig() + + a.Auth.Users[1] = nil + a.Auth.Users[2] = nil + + if err := a.Apply(); err != nil { + t.Fatal(err) + } + + assert.Len(t, getUsers(a.Deployment), 1) +} + +func TestUnknownFields_AreNotMergedWithOtherElements(t *testing.T) { + ac := getTestAutomationConfig() + + userToDelete := getUser(ac.Deployment, 1) + assert.Contains(t, userToDelete, "unknownFieldOne") + assert.Contains(t, userToDelete, "unknownFieldTwo") + + ac.Auth.Users[1] = nil + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + users := getUsers(ac.Deployment) + + // user got removed + assert.Len(t, users, 2) + + // other users didn't accidentally get unknown fields merged into them + for _, user := range users { + userMap := cast.ToStringMap(user) + assert.NotContains(t, userMap, "unknownFieldOne") + assert.NotContains(t, userMap, "unknownFieldTwo") + } +} + +func TestSettingFieldInListToNil_RemovesElement(t *testing.T) { + ac := getTestAutomationConfig() + + ac.Auth.Users[1] = nil + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + assert.Len(t, getUsers(ac.Deployment), 2) +} + +func TestRoleIsAddedToTheEnd(t *testing.T) { + ac := getTestAutomationConfig() + + roles := getRoles(ac.Deployment, 0) + assert.Len(t, roles, 3) + + ac.Auth.Users[0].AddRole(&Role{ + Database: "admin", + Role: "some-new-role", + }) + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + roles = getRoles(ac.Deployment, 0) + assert.Len(t, roles, 4) + + lastRole := cast.ToStringMap(roles[len(roles)-1]) + + assert.Equal(t, "admin", lastRole["db"]) + assert.Equal(t, "some-new-role", lastRole["role"]) +} + +func TestRoleIsUpdated(t *testing.T) { + ac := getTestAutomationConfig() + + originalRole := getRole(ac.Deployment, 0, 0) + + assert.Equal(t, "admin", originalRole["db"]) + assert.Equal(t, "backup", originalRole["role"]) + + role := ac.Auth.Users[0].Roles[0] + role.Database = "updated-db" + role.Role = "updated-role" + + if err := ac.Apply(); err != nil { + assert.Fail(t, "Error applying changes") + } + + actualRole := getRole(ac.Deployment, 0, 0) + assert.Equal(t, "updated-role", actualRole["role"]) + assert.Equal(t, "updated-db", actualRole["db"]) +} + +func TestMiddleRoleIsCorrectlyDeleted(t *testing.T) { + a := getTestAutomationConfig() + + a.Auth.Users[0].Roles = remove(a.Auth.Users[0].Roles, 1) + + if err := a.Apply(); err != nil { + t.Fatal(err) + } + + roles := getRoles(a.Deployment, 0) + assert.Len(t, roles, 2) + + firstRole := cast.ToStringMap(roles[0]) + secondRole := cast.ToStringMap(roles[1]) + + // first role from automation_config.json + assert.Equal(t, "admin", firstRole["db"]) + assert.Equal(t, "backup", firstRole["role"]) + + // third role from automation_config.json + assert.Equal(t, "admin", secondRole["db"]) + assert.Equal(t, "automation", secondRole["role"]) +} + +func TestAllRolesAreDeleted(t *testing.T) { + a := getTestAutomationConfig() + a.Auth.Users[0].Roles = []*Role{} + if err := a.Apply(); err != nil { + t.Fatal(err) + } + + roles := getRoles(a.Deployment, 0) + assert.Len(t, roles, 0) +} + +func TestRoleIsDeletedAndAppended(t *testing.T) { + a := getTestAutomationConfig() + + a.Auth.Users[0].Roles = remove(a.Auth.Users[0].Roles, 2) + + newRole := &Role{ + Database: "updated-db", + Role: "updated-role", + } + a.Auth.Users[0].Roles = append(a.Auth.Users[0].Roles, newRole) + + if err := a.Apply(); err != nil { + t.Fatal(err) + } + + actualRole := getRole(a.Deployment, 0, 2) + assert.Equal(t, "updated-role", actualRole["role"]) + assert.Equal(t, "updated-db", actualRole["db"]) +} + +func TestNoAdditionalFieldsAreAddedToAgentSSL(t *testing.T) { + ac := getTestAutomationConfig() + ac.AgentSSL = &AgentSSL{ + CAFilePath: util.MergoDelete, + ClientCertificateMode: util.OptionalClientCertficates, + AutoPEMKeyFilePath: util.MergoDelete, + } + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + tls := cast.ToStringMap(ac.Deployment["tls"]) + assert.Contains(t, tls, "clientCertificateMode") + + assert.NotContains(t, tls, "autoPEMKeyFilePath") + assert.NotContains(t, tls, "CAFilePath") +} + +func TestCanResetAgentSSL(t *testing.T) { + ac := getTestAutomationConfig() + ac.AgentSSL = &AgentSSL{ + ClientCertificateMode: util.OptionalClientCertficates, + CAFilePath: util.CAFilePathInContainer, + AutoPEMKeyFilePath: util.AutomationAgentPemFilePath, + } + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + tls := cast.ToStringMap(ac.Deployment["tls"]) + assert.Equal(t, tls["clientCertificateMode"], util.OptionalClientCertficates) + assert.Equal(t, tls["autoPEMKeyFilePath"], util.AutomationAgentPemFilePath) + assert.Equal(t, tls["CAFilePath"], util.CAFilePathInContainer) + + ac.AgentSSL = &AgentSSL{ + CAFilePath: util.MergoDelete, + AutoPEMKeyFilePath: util.MergoDelete, + ClientCertificateMode: util.OptionalClientCertficates, + } + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + tls = cast.ToStringMap(ac.Deployment["tls"]) + assert.Equal(t, tls["clientCertificateMode"], util.OptionalClientCertficates) + assert.NotContains(t, tls, "autoPEMKeyFilePath") + assert.NotContains(t, tls, "CAFilePath") +} + +func TestVersionsAndBuildsRetained(t *testing.T) { + ac := getTestAutomationConfig() + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + dep := ac.Deployment + versions := getMongoDbVersions(dep) + assert.Len(t, versions, 2) + + // ensure no elements lost from array + builds1 := getVersionBuilds(dep, 0) + builds2 := getVersionBuilds(dep, 1) + + // ensure the correct number of elements in each nested array + assert.Len(t, builds1, 6) + assert.Len(t, builds2, 11) + + /* + { + "architecture": "amd64", + "bits": 64, + "flavor": "suse", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "12", + "minOsVersion": "11", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-suse11-3.2.0.tgz" + } + */ + // ensure correct values for nested fields + build1 := getVersionBuild(dep, 1, 4) + assert.Equal(t, "amd64", build1["architecture"]) + assert.Equal(t, float64(64), build1["bits"]) + assert.Equal(t, "suse", build1["flavor"]) + assert.Equal(t, "45d947729a0315accb6d4f15a6b06be6d9c19fe7", build1["gitVersion"]) + assert.Equal(t, "12", build1["maxOsVersion"]) + assert.Equal(t, "11", build1["minOsVersion"]) + assert.Equal(t, "linux", build1["platform"]) + assert.Equal(t, "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-suse11-3.2.0.tgz", build1["url"]) + + // nested list maintains untouched + modulesList := build1["modules"].([]interface{}) + assert.Equal(t, "enterprise", modulesList[0]) +} + +func TestMergoDeleteWorksInNestedMapsWithFieldsNotReturnedByAutomationConfig(t *testing.T) { + ac := getTestAutomationConfig() + + ac.Auth.Users[0].Database = util.MergoDelete + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + user := getUser(ac.Deployment, 0) + + assert.NotContains(t, user, "db") // specify value to delete, it does not remain in final map + assert.Contains(t, user, "user") // value untouched remains +} + +func TestDeletionOfMiddleElements(t *testing.T) { + ac := getTestAutomationConfig() + + ac.Auth.AddUser(MongoDBUser{ + Database: "my-db", + Username: "my-user", + Roles: []*Role{{Role: "my-role", Database: "role-db"}}, + }) + + ac.Auth.AddUser(MongoDBUser{ + Database: "my-db-1", + Username: "my-user-1", + Roles: []*Role{{Role: "my-role", Database: "role-db"}}, + }) + + ac.Auth.AddUser(MongoDBUser{ + Database: "my-db-2", + Username: "my-user-2", + Roles: []*Role{{Role: "my-role", Database: "role-db"}}, + }) + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + assert.Len(t, getUsers(ac.Deployment), 6) + + // remove the 3rd element of the list + ac.Auth.Users[2] = nil + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + assert.Len(t, getUsers(ac.Deployment), 5) + + users := getUsers(ac.Deployment) + lastUser := cast.ToStringMap(users[len(users)-1]) + assert.Equal(t, lastUser["user"], "my-user-2") + assert.Equal(t, lastUser["db"], "my-db-2") + + // my-user-1 was correctly removed from between the other two elements + secondLastUser := cast.ToStringMap(users[len(users)-2]) + assert.Equal(t, "my-user-1", secondLastUser["user"]) + assert.Equal(t, "my-db-1", secondLastUser["db"]) +} + +func TestDeleteLastElement(t *testing.T) { + ac := getTestAutomationConfig() + ac.Auth.Users[len(ac.Auth.Users)-1] = nil + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + users := getUsers(ac.Deployment) + assert.Len(t, users, 2) + + lastUser := cast.ToStringMap(users[1]) + assert.Equal(t, "testDb1", lastUser["db"]) + assert.Equal(t, "testUser1", lastUser["user"]) +} + +func TestCanDeleteUsers_AndAddNewOnes_InSingleOperation(t *testing.T) { + ac := getTestAutomationConfig() + + for i := range ac.Auth.Users { + ac.Auth.Users[i] = nil + } + + ac.Auth.AddUser(MongoDBUser{ + Database: "my-added-db", + Username: "my-added-user", + Roles: []*Role{}, + }) + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + users := getUsers(ac.Deployment) + assert.Len(t, users, 1) + + addedUser := cast.ToStringMap(users[0]) + assert.Equal(t, "my-added-db", addedUser["db"]) + assert.Equal(t, "my-added-user", addedUser["user"]) +} + +func TestOneUserDeleted_OneUserUpdated(t *testing.T) { + ac := getTestAutomationConfig() + ac.Auth.Users[1] = nil + ac.Auth.Users[2].Database = "updated-database" + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + users := getUsers(ac.Deployment) + + assert.Len(t, users, 2) + updatedUser := cast.ToStringMap(users[1]) + assert.Equal(t, "updated-database", updatedUser["db"]) +} + +func TestAssigningListsReassignsInDeployment(t *testing.T) { + ac := getTestAutomationConfig() + + ac.Auth.AutoAuthMechanisms = append(ac.Auth.AutoAuthMechanisms, "one", "two", "three") + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + auth := cast.ToStringMap(ac.Deployment["auth"]) + authMechanisms := cast.ToSlice(auth["autoAuthMechanisms"]) + assert.Len(t, authMechanisms, 3) + + ac.Auth.AutoAuthMechanisms = []string{"two"} + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + auth = cast.ToStringMap(ac.Deployment["auth"]) + authMechanisms = cast.ToSlice(auth["autoAuthMechanisms"]) + assert.Len(t, authMechanisms, 1) + assert.Contains(t, authMechanisms, "two") + + ac.Auth.AutoAuthMechanisms = []string{} + + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + + auth = cast.ToStringMap(ac.Deployment["auth"]) + authMechanisms = cast.ToSlice(auth["autoAuthMechanisms"]) + assert.Len(t, authMechanisms, 0) +} + +func TestAutomationConfigEquality(t *testing.T) { + deployment1 := NewDeployment() + deployment1.setReplicaSets([]ReplicaSet{NewReplicaSet("1", "5.0.0")}) + + deployment2 := NewDeployment() + deployment2.setReplicaSets([]ReplicaSet{NewReplicaSet("2", "5.0.0")}) + + authConfig := Auth{ + Users: []*MongoDBUser{ + { + Roles: []*Role{ + { + Role: "root", + Database: "db1", + }, + }, + }, + }, + Disabled: false, + } + authConfig2 := authConfig + + agentSSLConfig := AgentSSL{ + CAFilePath: "/tmp/mypath", + } + agentSSLConfig2 := agentSSLConfig + + ldapConfig := ldap.Ldap{ + Servers: "server1", + } + ldapConfig2 := ldapConfig + + tests := map[string]struct { + a *AutomationConfig + b *AutomationConfig + expectedEquality bool + }{ + "Two empty configs are equal": { + a: &AutomationConfig{}, + b: &AutomationConfig{}, + expectedEquality: true, + }, + "Two different configs are not equal": { + a: getTestAutomationConfig(), + b: &AutomationConfig{}, + expectedEquality: false, + }, + "Two different configs are equal apart from the deployment": { + a: &AutomationConfig{ + Deployment: deployment1, + }, + b: &AutomationConfig{ + Deployment: deployment2, + }, + expectedEquality: true, + }, + "Two the same configs created using the same structs are the same": { + a: &AutomationConfig{ + Auth: &authConfig, + AgentSSL: &agentSSLConfig, + Deployment: deployment1, + Ldap: &ldapConfig, + }, + b: &AutomationConfig{ + Auth: &authConfig, + AgentSSL: &agentSSLConfig, + Deployment: deployment1, + Ldap: &ldapConfig, + }, + expectedEquality: true, + }, + "Two the same configs created using deep copy (and structs with different addresses) are the same": { + a: &AutomationConfig{ + Auth: &authConfig, + AgentSSL: &agentSSLConfig, + Ldap: &ldapConfig, + }, + b: &AutomationConfig{ + Auth: &authConfig2, + AgentSSL: &agentSSLConfig2, + Ldap: &ldapConfig2, + }, + expectedEquality: true, + }, + "Same configs, except for MergoDelete, which is ignored": { + a: &AutomationConfig{ + Auth: &Auth{ + NewAutoPwd: util.MergoDelete, + LdapGroupDN: "abc", + }, + Ldap: &ldapConfig, + }, + b: &AutomationConfig{ + Auth: &Auth{ + LdapGroupDN: "abc", + }, + AgentSSL: &AgentSSL{ + AutoPEMKeyFilePath: util.MergoDelete, + }, + Ldap: &ldapConfig2, + }, + expectedEquality: true, + }, + } + for testName, testParameters := range tests { + t.Run(testName, func(t *testing.T) { + result := testParameters.a.EqualsWithoutDeployment(*testParameters.b) + assert.Equalf(t, testParameters.expectedEquality, result, "Expected %v, got %v", testParameters.expectedEquality, result) + }) + } +} + +func getUsers(deployment map[string]interface{}) []interface{} { + auth := deployment["auth"].(map[string]interface{}) + if users, ok := auth["usersWanted"]; ok { + return users.([]interface{}) + } + return make([]interface{}, 0) +} + +func getUser(deployment map[string]interface{}, i int) map[string]interface{} { + users := getUsers(deployment) + return users[i].(map[string]interface{}) +} + +func getRoles(deployment map[string]interface{}, userIdx int) []interface{} { + user := getUser(deployment, userIdx) + return user["roles"].([]interface{}) +} + +func getRole(deployment map[string]interface{}, userIdx, roleIdx int) map[string]interface{} { + roles := getRoles(deployment, userIdx) + return roles[roleIdx].(map[string]interface{}) +} + +func remove(slice []*Role, i int) []*Role { + copy(slice[i:], slice[i+1:]) + return slice[:len(slice)-1] +} + +func getMongoDbVersions(deployment map[string]interface{}) []interface{} { + return deployment["mongoDbVersions"].([]interface{}) +} + +func getVersionBuilds(deployment map[string]interface{}, versionIndex int) []interface{} { + versions := deployment["mongoDbVersions"].([]interface{}) + return versions[versionIndex].(map[string]interface{})["builds"].([]interface{}) +} + +func getVersionBuild(deployment map[string]interface{}, versionIndex, buildIndex int) map[string]interface{} { + return getVersionBuilds(deployment, versionIndex)[buildIndex].(map[string]interface{}) +} + +func TestLDAPIsMerged(t *testing.T) { + ac := getTestAutomationConfig() + ac.Ldap = &ldap.Ldap{ + AuthzQueryTemplate: "AuthzQueryTemplate", + BindMethod: "", + BindQueryUser: "BindQueryUser", + BindSaslMechanisms: "BindSaslMechanisms", + Servers: "", + TransportSecurity: "TransportSecurity", + UserToDnMapping: "UserToDnMapping", + ValidateLDAPServerConfig: false, + BindQueryPassword: "", + TimeoutMS: 1000, + UserCacheInvalidationInterval: 60, + CaFileContents: "", + } + if err := ac.Apply(); err != nil { + t.Fatal(err) + } + ldapMap := cast.ToStringMap(ac.Deployment["ldap"]) + assert.Equal(t, "AuthzQueryTemplate", ldapMap["authzQueryTemplate"]) + assert.Equal(t, "BindQueryUser", ldapMap["bindQueryUser"]) + assert.Equal(t, "BindSaslMechanisms", ldapMap["bindSaslMechanisms"]) + assert.Equal(t, "TransportSecurity", ldapMap["transportSecurity"]) + // ldap.Ldap is being merged by marshalling it to a map first, so ints end up as float64 + assert.Equal(t, float64(1000), ldapMap["timeoutMS"]) + assert.Equal(t, float64(60), ldapMap["userCacheInvalidationInterval"]) + // ensure zero value fields are added + assert.Contains(t, ldapMap, "bindMethod") + assert.Contains(t, ldapMap, "servers") + assert.Contains(t, ldapMap, "validateLDAPServerConfig") + assert.Contains(t, ldapMap, "bindQueryPassword") + assert.Contains(t, ldapMap, "CAFileContents") +} + +func TestApplyInto(t *testing.T) { + config := AutomationConfig{ + Auth: NewAuth(), + AgentSSL: &AgentSSL{ + CAFilePath: util.MergoDelete, + ClientCertificateMode: "test", + }, + Deployment: Deployment{"tls": map[string]interface{}{"test": ""}}, + Ldap: nil, + } + deepCopy := Deployment{"tls": map[string]interface{}{}} + err := applyInto(config, &deepCopy) + assert.NoError(t, err) + + // initial config.Deployment did not change + assert.NotEqual(t, config.Deployment, deepCopy) + + // new deployment is the merge result of the previous config.Deployment + config + assert.Equal(t, Deployment{"tls": map[string]interface{}{"clientCertificateMode": "test", "test": ""}}, deepCopy) +} + +func changeTypes(deployment Deployment) error { + rs := deployment.getReplicaSets() + deployment.setReplicaSets(rs) + return nil +} + +func TestIsEqual(t *testing.T) { + type args struct { + depFunc func(Deployment) error + deployment Deployment + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "depFunc does not do anything", + args: args{ + depFunc: func(deployment Deployment) error { + return nil + }, + deployment: getDeploymentWithRSOverTheWire(t), + }, + want: true, + }, + { + name: "depFunc does changes types, but content does not change", + args: args{ + depFunc: changeTypes, + deployment: getDeploymentWithRSOverTheWire(t), + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := isEqualAfterModification(tt.args.depFunc, tt.args.deployment) + assert.NoError(t, err) + assert.Equalf(t, tt.want, got, "isEqualAfterModification(%v, %v)", tt.args.depFunc, tt.args.deployment) + }) + } +} + +// TestIsEqualNotWorkingWithTypeChanges is a test that shows that deep equality does not work if our depFunc changes +// the underlying types as we mostly do. +func TestIsEqualNotWorkingWithTypeChanges(t *testing.T) { + t.Run("is not working", func(t *testing.T) { + overTheWire := getDeploymentWithRSOverTheWire(t) + + original, err := util.MapDeepCopy(overTheWire) + assert.NoError(t, err) + + _ = changeTypes(overTheWire) + + equal := equality.Semantic.DeepEqual(original, overTheWire) + assert.False(t, equal) + }) +} + +func getDeploymentWithRSOverTheWire(t *testing.T) Deployment { + overTheWire := getTestAutomationConfig().Deployment + overTheWire.addReplicaSet(NewReplicaSet("rs-1", "3.2.0")) + overTheWire.addReplicaSet(NewReplicaSet("rs-2", "3.2.0")) + marshal, err := json.Marshal(overTheWire) // as we get it over the wire, we need to reflect that + assert.NoError(t, err) + err = json.Unmarshal(marshal, &overTheWire) + assert.NoError(t, err) + return overTheWire +} diff --git a/controllers/om/automation_status.go b/controllers/om/automation_status.go new file mode 100644 index 000000000..4353ea660 --- /dev/null +++ b/controllers/om/automation_status.go @@ -0,0 +1,133 @@ +package om + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "sort" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +const automationAgentKubeUpgradePlan = "ChangeVersionKube" + +// AutomationStatus represents the status of automation agents registered with Ops Manager +type AutomationStatus struct { + GoalVersion int `json:"goalVersion"` + Processes []ProcessStatus `json:"processes"` +} + +// ProcessStatus status of the process and what's the last version achieved +type ProcessStatus struct { + Hostname string `json:"hostname"` + Name string `json:"name"` + LastGoalVersionAchieved int `json:"lastGoalVersionAchieved"` + Plan []string `json:"plan"` +} + +func buildAutomationStatusFromBytes(b []byte) (*AutomationStatus, error) { + as := &AutomationStatus{} + if err := json.Unmarshal(b, &as); err != nil { + return nil, err + } + + return as, nil +} + +// WaitForReadyState waits until the agents for relevant processes reach their state +func WaitForReadyState(oc Connection, processNames []string, supressErrors bool, log *zap.SugaredLogger) error { + if len(processNames) == 0 { + log.Infow("Not waiting for MongoDB agents to reach READY state (no expected processes to wait for)") + return nil + } + + log.Infow("Waiting for MongoDB agents to reach READY state...", "processes", processNames) + reachStateFunc := func() (string, bool) { + as, lastErr := oc.ReadAutomationStatus() + if lastErr != nil { + return fmt.Sprintf("Error reading Automation Agents status: %s", lastErr), false + } + + if allReachedGoalState, msg := checkAutomationStatusIsGoal(as, processNames, log); allReachedGoalState { + return msg, true + } else { + return fmt.Sprintf("MongoDB agents haven't reached READY state; %s", msg), false + } + } + ok, msg := util.DoAndRetry(reachStateFunc, log, 30, 3) + if !ok { + if supressErrors { + log.Warnf("automation agents haven't reached READY state but the error is supressed") + return nil + } + return apierror.New(xerrors.Errorf("automation agents haven't reached READY state during defined interval: %s", msg)) + } + log.Info("MongoDB agents have reached READY state") + return nil +} + +// CheckAutomationStatusIsGoal returns true if all the relevant processes are in Goal +// state. +// Note, that the function is quite tolerant to any situations except for non-matching goal state, for example +// if one of the requested processes doesn't exist in the list of OM status processes - this is considered as ok +// (maybe we are doing the scale down for the RS and some members were removed from OM manually - this is ok as the Operator +// will fix this later) +func checkAutomationStatusIsGoal(as *AutomationStatus, relevantProcesses []string, log *zap.SugaredLogger) (bool, string) { + if areAnyAgentsInKubeUpgradeMode(as, relevantProcesses, log) { + return true, "" + } + + goalsNotAchievedMap := map[string]int{} + goalsAchievedMap := map[string]int{} + for _, p := range as.Processes { + if !stringutil.Contains(relevantProcesses, p.Name) { + continue + } + if p.LastGoalVersionAchieved == as.GoalVersion { + goalsAchievedMap[p.Name] = p.LastGoalVersionAchieved + } else { + goalsNotAchievedMap[p.Name] = p.LastGoalVersionAchieved + } + } + + var goalsNotAchievedMsgList []string + for processName, goalAchieved := range goalsNotAchievedMap { + goalsNotAchievedMsgList = append(goalsNotAchievedMsgList, fmt.Sprintf("%s@%d", processName, goalAchieved)) + } + goalsAchievedMsgList := slices.Collect(maps.Keys(goalsAchievedMap)) + sort.Strings(goalsAchievedMsgList) + + if len(goalsNotAchievedMap) > 0 { + return false, fmt.Sprintf("%d processes waiting to reach automation config goal state (version=%d): %s, %d processes reached goal state: %s", + len(goalsNotAchievedMap), as.GoalVersion, goalsNotAchievedMsgList, len(goalsAchievedMsgList), goalsAchievedMsgList) + } else if len(goalsAchievedMap) == 0 { + return true, "there were no processes in automation config matched with the processes to wait for" + } else { + return true, fmt.Sprintf("processes that reached goal state: %s", goalsAchievedMsgList) + } +} + +func areAnyAgentsInKubeUpgradeMode(as *AutomationStatus, relevantProcesses []string, log *zap.SugaredLogger) bool { + for _, p := range as.Processes { + if !stringutil.Contains(relevantProcesses, p.Name) { + continue + } + for _, plan := range p.Plan { + // This means the following: + // - the cluster is in static architecture + // - the agents are in a dedicated upgrade process, waiting for their binaries to be replaced by kubernetes + // - this can only happen if the statefulset is ready, therefore we are returning ready here + if plan == automationAgentKubeUpgradePlan { + log.Debug("cluster is in changeVersionKube mode, returning the agent is ready.") + return true + } + } + } + return false +} diff --git a/controllers/om/automation_status_test.go b/controllers/om/automation_status_test.go new file mode 100644 index 000000000..7ac7f3dcf --- /dev/null +++ b/controllers/om/automation_status_test.go @@ -0,0 +1,121 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func TestCheckAutomationStatusIsGoal(t *testing.T) { + type args struct { + as *AutomationStatus + relevantProcesses []string + } + tests := []struct { + name string + args args + expectedResult bool + expectedMsg string + }{ + { + name: "all in goal state", + args: args{ + as: &AutomationStatus{ + Processes: []ProcessStatus{ + { + Name: "a", + Plan: []string{"FCV"}, + LastGoalVersionAchieved: 1, + }, + { + Name: "b", + Plan: []string{"FCV"}, + LastGoalVersionAchieved: 1, + }, + }, + GoalVersion: 1, + }, + relevantProcesses: []string{"a", "b"}, + }, + expectedResult: true, + // We can not check for the full message as the ordering of the processes won't be deterministic (stored in a map) + expectedMsg: "processes that reached goal state: [a b]", + }, { + name: "one not in goal state", + args: args{ + as: &AutomationStatus{ + Processes: []ProcessStatus{ + { + Name: "a", + Plan: []string{"FCV"}, + LastGoalVersionAchieved: 0, + }, + { + Name: "b", + Plan: []string{"FCV"}, + LastGoalVersionAchieved: 1, + }, + }, + GoalVersion: 1, + }, + relevantProcesses: []string{"a", "b"}, + }, + expectedResult: false, + expectedMsg: "1 processes waiting to reach automation config goal state (version=1): [a@0], 1 processes reached goal state: [b]", + }, { + name: "one not in goal state but at least one is in kube upgrade", + args: args{ + as: &AutomationStatus{ + Processes: []ProcessStatus{ + { + Name: "a", + Plan: []string{"FCV", "something-else"}, + LastGoalVersionAchieved: 0, + }, + { + Name: "b", + Plan: []string{"FCV", automationAgentKubeUpgradePlan}, + LastGoalVersionAchieved: 1, + }, + }, + GoalVersion: 1, + }, + relevantProcesses: []string{"a", "b"}, + }, + expectedResult: true, + // we don't return any msg for agentKubeUpgradePlan + expectedMsg: "", + }, { + name: "none of the processes matched with AC", + args: args{ + as: &AutomationStatus{ + Processes: []ProcessStatus{ + { + Name: "a", + Plan: []string{"X", "Y"}, + LastGoalVersionAchieved: 1, + }, + { + Name: "b", + Plan: []string{"Y", "Z"}, + LastGoalVersionAchieved: 1, + }, + }, + GoalVersion: 1, + }, + relevantProcesses: []string{"c", "d"}, + }, + // we return true when there weren't any processes to wait for in AC + expectedResult: true, + expectedMsg: "there were no processes in automation config matched with the processes to wait for", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + goal, msg := checkAutomationStatusIsGoal(tt.args.as, tt.args.relevantProcesses, zap.S()) + assert.Equalf(t, tt.expectedResult, goal, "checkAutomationStatusIsGoal(%v, %v)", tt.args.as, tt.args.relevantProcesses) + assert.Contains(t, msg, tt.expectedMsg) + }) + } +} diff --git a/controllers/om/backup/backup.go b/controllers/om/backup/backup.go new file mode 100644 index 000000000..824e4c31d --- /dev/null +++ b/controllers/om/backup/backup.go @@ -0,0 +1,163 @@ +package backup + +import ( + "errors" + "fmt" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +type MongoDbResourceType string + +const ( + ReplicaSetType MongoDbResourceType = "ReplicaSet" + ShardedClusterType MongoDbResourceType = "ShardedCluster" +) + +/* +for sharded cluster: + + { + "clusterName": "shannon", + "groupId": "5ba0c398a957713d7f8653bd", + "id": "5ba3d344a957713d7f8f43fd", + "lastHeartbeat": "2018-09-20T17:12:28Z", + "links": [ ... ], + "shardName": "shannon-0", + "typeName": "SHARDED_REPLICA_SET" + } + +for sharded cluster member: + + { + "clusterName": "shannon", + "groupId": "5ba0c398a957713d7f8653bd", + "id": "5ba4ec37a957713d7f9bcba0", + "lastHeartbeat": "2018-09-24T12:41:05Z", + "links": [ ... ], + "replicaSetName": "shannon-0", + "shardName": "shannon-0", + "typeName": "REPLICA_SET" + } + +for replica set: + + { + "clusterName": "liffey", + "groupId": "5ba0c398a957713d7f8653bd", + "id": "5ba8db64a957713d7fa5018b", + "lastHeartbeat": "2018-09-24T12:41:08Z", + "links": [ ... ], + "replicaSetName": "liffey", + "typeName": "REPLICA_SET" + } +*/ +type HostCluster struct { + ReplicaSetName string `json:"replicaSetName"` + ClusterName string `json:"clusterName"` + ShardName string `json:"shardName"` + TypeName string `json:"typeName"` +} + +type HostClusterReader interface { + ReadHostCluster(clusterID string) (*HostCluster, error) +} + +// StopBackupIfEnabled tries to find backup configuration for specified resource (can be Replica Set or Sharded Cluster - +// Ops Manager doesn't backup Standalones) and disable it. +func StopBackupIfEnabled(readUpdater ConfigHostReadUpdater, hostClusterReader HostClusterReader, name string, resourceType MongoDbResourceType, log *zap.SugaredLogger) error { + response, err := readUpdater.ReadBackupConfigs() + if err != nil { + // If the operator can't read BackupConfigs, it might indicate that the Pods were removed before establishing + // or activating monitoring for the deployment. But if this is a deletion process of the MDB resource, it needs + // to be removed anyway, so we are logging the Error and continuing. + // TODO: Discussion. To avoid removing dependant objects in a DELETE operation, a finalizer should be implemented + // This finalizer would be required to add a "delay" to the deletion of the StatefulSet waiting for monitoring + // to be activated at the project. + var v *apierror.Error + if errors.As(err, &v) { + if v.ErrorCode == "CANNOT_GET_BACKUP_CONFIG_INVALID_STATE" { + log.Warnf("Could not read backup configs for this deployment. Will continue with the removal of the objects. %s", err) + return nil + } + } + return err + } + + for _, config := range response.Configs { + l := log.With("cluster id", config.ClusterId) + + l.Debugw("Found backup/host config", "status", config.Status) + // Any status except for inactive will result in API rejecting the deletion of resource - we need to disable backup + if config.Status != Inactive { + cluster, err := hostClusterReader.ReadHostCluster(config.ClusterId) + if err != nil { + l.Errorf("Failed to read information about HostCluster: %s", err) + } else { + l.Debugw("Read cluster information", "details", cluster) + } + // We need to compare the returned backup type with the given type because for sharded_clusters we + // have 4 configs. + // Three replica_sets and one sharded_replica_set. + // We only want to disable the backup for the sharded_replica_set + if cluster.ClusterName == name && + (resourceType == ReplicaSetType && cluster.TypeName == "REPLICA_SET" || + resourceType == ShardedClusterType && cluster.TypeName == "SHARDED_REPLICA_SET") { + err = disableBackup(readUpdater, config, l) + if err != nil { + return err + } + l.Infow("Disabled backup for host cluster in Ops Manager", "host cluster name", cluster.ClusterName) + } + } + } + return nil +} + +func disableBackup(readUpdater ConfigHostReadUpdater, backupConfig *Config, log *zap.SugaredLogger) error { + if backupConfig.Status == Started { + err := readUpdater.UpdateBackupStatus(backupConfig.ClusterId, Stopped) + if err != nil { + log.Errorf("Failed to stop backup for host cluster: %s", err) + } else { + if ok, msg := waitUntilBackupReachesStatus(readUpdater, backupConfig, Stopped, log); ok { + log.Debugw("Stopped backup for host cluster") + } else { + log.Warnf("Failed to stop backup for host cluster in Ops Manager (timeout exhausted): %s", msg) + } + } + } + // We try to terminate in any case (it will fail if the backup config is not stopped) + err := readUpdater.UpdateBackupStatus(backupConfig.ClusterId, Terminating) + if err != nil { + return err + } + if ok, msg := waitUntilBackupReachesStatus(readUpdater, backupConfig, Inactive, log); !ok { + return xerrors.Errorf("Failed to disable backup for host cluster in Ops Manager (timeout exhausted): %s", msg) + } + return nil +} + +func waitUntilBackupReachesStatus(configReader ConfigReader, backupConfig *Config, status Status, log *zap.SugaredLogger) (bool, string) { + waitSeconds := env.ReadIntOrPanic(util.BackupDisableWaitSecondsEnv) // nolint:forbidigo + retries := env.ReadIntOrPanic(util.BackupDisableWaitRetriesEnv) // nolint:forbidigo + + backupStatusFunc := func() (string, bool) { + config, err := configReader.ReadBackupConfig(backupConfig.ClusterId) + if err != nil { + return fmt.Sprintf("Unable to read from OM API: %s", err), false + } + + if config.Status == status { + return "", true + } + return fmt.Sprintf("Current status: %s", config.Status), false + } + + return util.DoAndRetry(backupStatusFunc, log, retries, waitSeconds) +} diff --git a/controllers/om/backup/backup_config.go b/controllers/om/backup/backup_config.go new file mode 100644 index 000000000..c474eff1a --- /dev/null +++ b/controllers/om/backup/backup_config.go @@ -0,0 +1,65 @@ +package backup + +type Status string + +const ( + Inactive Status = "INACTIVE" + Started Status = "STARTED" + Stopped Status = "STOPPED" + Terminating Status = "TERMINATING" + wiredTigerStorageEngine string = "WIRED_TIGER" +) + +type ConfigReader interface { + // ReadBackupConfigs returns all host clusters registered in OM. If there's no backup enabled the status is supposed + // to be Inactive + ReadBackupConfigs() (*ConfigsResponse, error) + + // ReadBackupConfig reads an individual backup config by cluster id + ReadBackupConfig(clusterID string) (*Config, error) + + ReadSnapshotSchedule(clusterID string) (*SnapshotSchedule, error) +} + +// ConfigUpdater is something can update an existing Backup Config +type ConfigUpdater interface { + UpdateBackupConfig(config *Config) (*Config, error) + UpdateBackupStatus(clusterID string, status Status) error + UpdateSnapshotSchedule(clusterID string, schedule *SnapshotSchedule) error +} + +type ConfigHostReadUpdater interface { + ConfigReader + ConfigUpdater + HostClusterReader +} + +/* + { + "authMechanismName": "NONE", + "clusterId": "5ba4ec37a957713d7f9bcb9a", + "encryptionEnabled": false, + "excludedNamespaces": [], + "groupId": "5ba0c398a957713d7f8653bd", + "links": [ + ... + ], + "sslEnabled": false, + "statusName": "INACTIVE" + } +*/ +type ConfigsResponse struct { + Configs []*Config `json:"results"` +} + +type Config struct { + ClusterId string `json:"clusterId"` + EncryptionEnabled bool `json:"encryptionEnabled"` + ExcludedNamespaces []string `json:"excludedNamespaces"` + IncludedNamespaces []string `json:"includedNamespaces"` + Provisioned bool `json:"provisioned"` + Status Status `json:"statusName"` + StorageEngineName string `json:"storageEngineName"` + SyncSource string `json:"syncSource"` + ProjectId string `json:"groupId"` +} diff --git a/controllers/om/backup/daemonconfig.go b/controllers/om/backup/daemonconfig.go new file mode 100644 index 000000000..c2d77e183 --- /dev/null +++ b/controllers/om/backup/daemonconfig.go @@ -0,0 +1,35 @@ +package backup + +type DaemonConfig struct { + Machine MachineConfig `json:"machine"` + AssignmentEnabled bool `json:"assignmentEnabled"` + BackupJobsEnabled bool `json:"backupJobsEnabled"` + ResourceUsageEnabled bool `json:"resourceUsageEnabled"` + GarbageCollectionEnabled bool `json:"garbageCollectionEnabled"` + RestoreQueryableJobsEnabled bool `json:"restoreQueryableJobsEnabled"` + Configured bool `json:"configured"` + Labels []string `json:"labels"` +} + +type MachineConfig struct { + HeadRootDirectory string `json:"headRootDirectory"` + MachineHostName string `json:"machine"` +} + +// NewDaemonConfig creates the 'DaemonConfig' fully initialized +func NewDaemonConfig(hostName, headDbDir string, assignmentLabels []string) DaemonConfig { + return DaemonConfig{ + Machine: MachineConfig{ + HeadRootDirectory: headDbDir, + MachineHostName: hostName, + }, + AssignmentEnabled: true, + Labels: assignmentLabels, + BackupJobsEnabled: true, + ResourceUsageEnabled: true, + GarbageCollectionEnabled: true, + RestoreQueryableJobsEnabled: true, + // TODO is this ok to have daemon configured with may be lacking oplog stores for example? + Configured: true, + } +} diff --git a/controllers/om/backup/datastoreconfig.go b/controllers/om/backup/datastoreconfig.go new file mode 100644 index 000000000..2cf3a3a48 --- /dev/null +++ b/controllers/om/backup/datastoreconfig.go @@ -0,0 +1,66 @@ +package backup + +import ( + "fmt" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type DataStoreConfigResponse struct { + DataStoreConfigs []DataStoreConfig `json:"results"` +} + +// DataStoreConfig corresponds to 'ApiBackupDataStoreConfigView' in mms. It's shared by all configs which relate to +// MongoDB (oplogStore, blockStore) +type DataStoreConfig struct { + // These are the fields managed by the Operator + Id string `json:"id"` + Uri string `json:"uri"` + UseSSL bool `json:"ssl"` + + // These are all the rest fields + LoadFactor int `json:"loadFactor,omitempty"` + WriteConcern string `json:"writeConcern,omitempty"` + UsedSize int64 `json:"usedSize,omitempty"` + AssignmentEnabled bool `json:"assignmentEnabled,omitempty"` + MaxCapacityGB int64 `json:"maxCapacityGB,omitempty"` + Labels []string `json:"labels"` + EncryptedCredentials bool `json:"encryptedCredentials,omitempty"` +} + +// NewDataStoreConfig returns the new 'DataStoreConfig' object initializing the default values +func NewDataStoreConfig(id, uri string, tls bool, assignmentLabels []string) DataStoreConfig { + ret := DataStoreConfig{ + Id: id, + Uri: uri, + UseSSL: tls, + + // Default values + AssignmentEnabled: true, + } + + // The assignment labels has been set in the CR - so the CR becomes a source of truth for them + if assignmentLabels != nil { + ret.Labels = assignmentLabels + } + + return ret +} + +func (s DataStoreConfig) Identifier() interface{} { + return s.Id +} + +// MergeIntoOpsManagerConfig performs the merge operation of the Operator config view ('s') into the OM owned one +// ('opsManagerConfig') +func (s DataStoreConfig) MergeIntoOpsManagerConfig(opsManagerConfig DataStoreConfig) DataStoreConfig { + opsManagerConfig.Id = s.Id + opsManagerConfig.Uri = s.Uri + opsManagerConfig.UseSSL = s.UseSSL + opsManagerConfig.Labels = s.Labels + return opsManagerConfig +} + +func (s DataStoreConfig) String() string { + return fmt.Sprintf("id: %s, uri: %s, ssl: %v", s.Id, util.RedactMongoURI(s.Uri), s.UseSSL) +} diff --git a/controllers/om/backup/group_config.go b/controllers/om/backup/group_config.go new file mode 100644 index 000000000..eec847b05 --- /dev/null +++ b/controllers/om/backup/group_config.go @@ -0,0 +1,48 @@ +package backup + +// GroupConfigReader reads the Group Backup Config +type GroupConfigReader interface { + // ReadGroupBackupConfig reads project level backup configuration + // See: https://www.mongodb.com/docs/ops-manager/v6.0/reference/api/admin/backup/groups/get-one-backup-group-configuration-by-id/ + ReadGroupBackupConfig() (GroupBackupConfig, error) +} + +// GroupConfigUpdater updates the existing Group Backup Config +type GroupConfigUpdater interface { + // UpdateGroupBackupConfig updates project level backup configuration + // See: https://www.mongodb.com/docs/ops-manager/v6.0/reference/api/admin/backup/groups/update-one-backup-group-configuration/ + UpdateGroupBackupConfig(config GroupBackupConfig) ([]byte, error) +} + +// DaemonFilter corresponds to the "daemonFilter" from the "Project Backup Jobs Configuration" from Ops Manager +// See: https://www.mongodb.com/docs/ops-manager/v6.0/reference/api/admin/backup/groups/update-one-backup-group-configuration/ +type DaemonFilter struct { + HeadRootDirectory *string `json:"headRootDirectory,omitempty"` + Machine *string `json:"machine,omitempty"` +} + +// OplogStoreFilter corresponds to the "oplogStoreFilter" from the "Project Backup Jobs Configuration" from Ops Manager +// See: https://www.mongodb.com/docs/ops-manager/v6.0/reference/api/admin/backup/groups/update-one-backup-group-configuration/ +type OplogStoreFilter struct { + Id *string `json:"id,omitempty"` + Type *string `json:"type,omitempty"` +} + +// SnapshotStoreFilter corresponds to the "snapshotStoreFilter" from the "Project Backup Jobs Configuration" from Ops Manager +// See: https://www.mongodb.com/docs/ops-manager/v6.0/reference/api/admin/backup/groups/update-one-backup-group-configuration/ +type SnapshotStoreFilter struct { + OplogStoreFilter `json:",inline"` +} + +// GroupBackupConfig corresponds to the "Project Backup Jobs Configuration" from Ops Manager +// See: https://www.mongodb.com/docs/ops-manager/v6.0/reference/api/admin/backup/groups/update-one-backup-group-configuration/ +type GroupBackupConfig struct { + DaemonFilter []DaemonFilter `json:"daemonFilter,omitempty"` + Id *string `json:"id,omitempty"` + KmipClientCertPassword *string `json:"kmipClientCertPassword,omitempty"` + KmipClientCertPath *string `json:"kmipClientCertPath,omitempty"` + LabelFilter []string `json:"labelFilter"` + OplogStoreFilter []OplogStoreFilter `json:"oplogStoreFilter,omitempty"` + SnapshotStoreFilter []SnapshotStoreFilter `json:"snapshotStoreFilter,omitempty"` + SyncStoreFilter []string `json:"syncStoreFilter,omitempty"` +} diff --git a/controllers/om/backup/mongodbresource_backup.go b/controllers/om/backup/mongodbresource_backup.go new file mode 100644 index 000000000..00346ec9b --- /dev/null +++ b/controllers/om/backup/mongodbresource_backup.go @@ -0,0 +1,311 @@ +package backup + +import ( + "context" + "reflect" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type ConfigReaderUpdater interface { + GetBackupSpec() *mdbv1.Backup + GetResourceType() mdbv1.ResourceType + GetResourceName() string + v1.CustomResourceReadWriter +} + +// EnsureBackupConfigurationInOpsManager updates the backup configuration based on the MongoDB resource +// specification. +func EnsureBackupConfigurationInOpsManager(ctx context.Context, mdb ConfigReaderUpdater, secretsReader secrets.SecretClient, projectId string, configReadUpdater ConfigHostReadUpdater, groupConfigReader GroupConfigReader, groupConfigUpdater GroupConfigUpdater, log *zap.SugaredLogger) (workflow.Status, []status.Option) { + if mdb.GetBackupSpec() == nil { + return workflow.OK(), nil + } + + desiredConfig := getMongoDBBackupConfig(mdb.GetBackupSpec(), projectId) + + configs, err := configReadUpdater.ReadBackupConfigs() + if err != nil { + return workflow.Failed(err), nil + } + + projectConfigs := configs.Configs + + if len(projectConfigs) == 0 { + return workflow.Pending("Waiting for backup configuration to be created in Ops Manager").WithRetry(10), nil + } + + err = ensureGroupConfig(ctx, mdb, secretsReader, groupConfigReader, groupConfigUpdater) + if err != nil { + return workflow.Failed(err), nil + } + + return ensureBackupConfigStatuses(mdb, projectConfigs, desiredConfig, log, configReadUpdater) +} + +func ensureGroupConfig(ctx context.Context, mdb ConfigReaderUpdater, secretsReader secrets.SecretClient, reader GroupConfigReader, updater GroupConfigUpdater) error { + if mdb.GetBackupSpec() == nil || (mdb.GetBackupSpec().AssignmentLabels == nil && mdb.GetBackupSpec().Encryption == nil) { + return nil + } + + assignmentLabels := mdb.GetBackupSpec().AssignmentLabels + kmip := mdb.GetBackupSpec().GetKmip() + + config, err := reader.ReadGroupBackupConfig() + if err != nil { + return err + } + + requiresUpdate := false + + if kmip != nil { + desiredPath := util.KMIPClientSecretsHome + "/" + kmip.Client.ClientCertificateSecretName(mdb.GetName()) + kmip.Client.ClientCertificateSecretKeyName() + if config.KmipClientCertPath == nil || desiredPath != *config.KmipClientCertPath { + config.KmipClientCertPath = &desiredPath + requiresUpdate = true + } + + // The password is optional, so we propagate the error only if something abnormal happens + kmipPasswordSecret, err := secretsReader.GetSecret(ctx, types.NamespacedName{ + Name: kmip.Client.ClientCertificatePasswordSecretName(mdb.GetName()), + Namespace: mdb.GetNamespace(), + }) + if err == nil { + desiredPassword := string(kmipPasswordSecret.Data[kmip.Client.ClientCertificatePasswordKeyName()]) + if config.KmipClientCertPassword == nil || desiredPassword != *config.KmipClientCertPassword { + config.KmipClientCertPassword = &desiredPassword + requiresUpdate = true + } + } else if !apiErrors.IsNotFound(err) { + return err + } + } + + if assignmentLabels != nil { + if config.LabelFilter == nil || !reflect.DeepEqual(config.LabelFilter, assignmentLabels) { + config.LabelFilter = mdb.GetBackupSpec().AssignmentLabels + requiresUpdate = true + } + } + + if requiresUpdate { + _, err = updater.UpdateGroupBackupConfig(config) + } + return err +} + +// ensureBackupConfigStatuses makes sure that every config in the project has reached the desired state. +func ensureBackupConfigStatuses(mdb ConfigReaderUpdater, projectConfigs []*Config, desiredConfig *Config, log *zap.SugaredLogger, configReadUpdater ConfigHostReadUpdater) (workflow.Status, []status.Option) { + result := workflow.OK() + + for _, config := range projectConfigs { + desiredConfig.ClusterId = config.ClusterId + + cluster, err := configReadUpdater.ReadHostCluster(config.ClusterId) + if err != nil { + return workflow.Failed(err), nil + } + + // There is one HostConfig per component of the deployment being backed up. + // E.g. a sharded cluster with 2 shards is composed of 4 backup configurations. + // 1x CONFIG_SERVER_REPLICA_SET (config server) + // 2x REPLICA_SET (each shard) + // 1x SHARDED_REPLICA_SET (the source of truth for sharded cluster configuration) + // Only the SHARDED_REPLICA_SET can be configured, we need to ensure that based on the cluster wide + // we care about we are only updating the config if the type and name are correct. + resourceType := MongoDbResourceType(mdb.GetResourceType()) + nameIsEqual := cluster.ClusterName == mdb.GetResourceName() + isReplicaSet := resourceType == ReplicaSetType && cluster.TypeName == "REPLICA_SET" + isShardedCluster := resourceType == ShardedClusterType && cluster.TypeName == "SHARDED_REPLICA_SET" + shouldUpdateBackupConfiguration := nameIsEqual && (isReplicaSet || isShardedCluster) + // If we are configuring a sharded cluster, we must only update the config of the whole cluster, not each individual shard. + // Status: 409 (Conflict), ErrorCode: CANNOT_MODIFY_SHARD_BACKUP_CONFIG, Detail: Cannot modify backup configuration for individual shard; use cluster ID 611a63f668d22f4e2e62c2e3 for entire cluster. + if !shouldUpdateBackupConfiguration { + continue + } + + // config.Status = current backup status in OM + // desiredConfig.Status = spec.backup.mode from CR, mapped as: + // status in CR | status in OM + // ---------------------------- + // enabled Started, + // disabled Stopped, + // terminated Terminating, + // desiredStatus here is desiredConfig.Status modified to potentially handle intermediate steps according to what the user specified in spec.backup.mode + desiredStatus := getDesiredStatus(desiredConfig, config) + + intermediateStepRequired := desiredStatus != desiredConfig.Status + if intermediateStepRequired { + result.Requeue() + } + + desiredConfig.Status = desiredStatus + + // If backup was never enabled and the deployment has `spec.backup.mode=disabled` specified + // we don't send this state to OM, or we will get + // CANNOT_STOP_BACKUP_INVALID_STATE, Detail: Cannot stop backup unless the cluster is in the STARTED state. + if desiredConfig.Status == Stopped && config.Status == Inactive { + continue + } + + // When mdb is newly created or backup is being enabled from terminated state, it is not possible to send snapshot schedule to OM, + // so the first run will be skipped. Update will be executed again at the end of this method when the backup reaches valid status. + // When the backup is already configured (not in INACTIVE state) and it is not changing its status, then this execution will update snapshot schedule. + // When both status and snapshot schedule is changing (e.g. backup starting from stopped), then this method will update snapshot schedule twice, which is harmless. + if err := updateSnapshotSchedule(mdb.GetBackupSpec().SnapshotSchedule, configReadUpdater, config, log); err != nil { + return workflow.Failed(err), nil + } + + if desiredConfig.Status == config.Status { + log.Debug("Config is already in the desired state, not updating configuration") + + // we are already in the desired state, nothing to change + // if we attempt to send the desired state again we get + // CANNOT_START_BACKUP_INVALID_STATE: Cannot start backup unless the cluster is in the INACTIVE or STOPPED state. + continue + } + + updatedConfig, err := configReadUpdater.UpdateBackupConfig(desiredConfig) + if err != nil { + return workflow.Failed(err), nil + } + + log.Debugw("Project Backup Configuration", "desiredConfig", desiredConfig, "updatedConfig", updatedConfig) + + if ok, msg := waitUntilBackupReachesStatus(configReadUpdater, updatedConfig, desiredConfig.Status, log); !ok { + log.Debugf("wait error message: %s", msg) + statusOpts, err := getCurrentBackupStatusOption(configReadUpdater, config.ClusterId) + if err != nil { + return workflow.Failed(err), nil + } + return workflow.Pending("Backup configuration %s has not yet reached the desired status", updatedConfig.ClusterId).WithRetry(1), statusOpts + } + + log.Debugf("Backup has reached the desired state of %s", desiredConfig.Status) + + // second run for cases when backup was inactive (see comment above) + if err := updateSnapshotSchedule(mdb.GetBackupSpec().SnapshotSchedule, configReadUpdater, desiredConfig, log); err != nil { + return workflow.Failed(err), nil + } + + backupOpts, err := getCurrentBackupStatusOption(configReadUpdater, desiredConfig.ClusterId) + if err != nil { + return workflow.Failed(err), nil + } + return result, backupOpts + } + + return result, nil +} + +func updateSnapshotSchedule(specSnapshotSchedule *mdbv1.SnapshotSchedule, configReadUpdater ConfigHostReadUpdater, config *Config, log *zap.SugaredLogger) error { + if specSnapshotSchedule == nil { + return nil + } + + // in Inactive state we cannot update snapshot schedule in OM + if config.Status == Inactive { + log.Debugf("Skipping updating backup snapshot schedule due to Inactive status") + return nil + } + + omSnapshotSchedule, err := configReadUpdater.ReadSnapshotSchedule(config.ClusterId) + if err != nil { + return xerrors.Errorf("failed to read snapshot schedule: %w", err) + } + + snapshotSchedule := mergeExistingScheduleWithSpec(*omSnapshotSchedule, *specSnapshotSchedule) + + // we need to use DeepEqual in order to compare pointers' underlying values + if !reflect.DeepEqual(snapshotSchedule, *omSnapshotSchedule) { + if err := configReadUpdater.UpdateSnapshotSchedule(snapshotSchedule.ClusterID, &snapshotSchedule); err != nil { + return xerrors.Errorf("failed to update backup snapshot schedule for cluster %s: %w", config.ClusterId, err) + } + log.Debugf("Updated backup snapshot schedule with: %s", snapshotSchedule) + } else { + log.Infof("Backup snapshot schedule is up to date") + } + + return nil +} + +// getCurrentBackupStatusOption fetches the latest information from the backup config +// with the given cluster id and returns the relevant status Options. +func getCurrentBackupStatusOption(configReader ConfigReader, clusterId string) ([]status.Option, error) { + config, err := configReader.ReadBackupConfig(clusterId) + if err != nil { + return nil, err + } + return []status.Option{ + status.NewBackupStatusOption( + string(config.Status), + ), + }, nil +} + +// getMongoDBBackupConfig builds the backup configuration from the given MongoDB resource +func getMongoDBBackupConfig(backupSpec *mdbv1.Backup, projectId string) *Config { + mappings := getStatusMappings() + return &Config{ + // the encryptionEnabled field is also only used in old backup, 4.2 backup will copy all files whether they are encrypted + // the encryption happens at the mongod level and should be managed by the customer + EncryptionEnabled: false, + + // 4.2 backup does not yet support filtering namespaces, both excluded and included namespaces will be ignored + ExcludedNamespaces: []string{}, + // validation requires exactly one of these being set + // INVALID_FILTERLIST, Detail: Backup configuration cannot specify both included namespaces and excluded namespaces. + IncludedNamespaces: nil, + + // we map our more declarative API to the values required by backup + Status: mappings[string(backupSpec.Mode)], + + // with 4.2 backup we only need to support wired tiger + StorageEngineName: wiredTigerStorageEngine, + // syncSource is only required on pre-4.2 backup, the value is still validated however, so we can just send primary + SyncSource: "PRIMARY", + ProjectId: projectId, + } +} + +// getStatusMappings returns a map which maps the fields exposed on the CRD +// to the fields expected by the Backup API +func getStatusMappings() map[string]Status { + return map[string]Status{ + "enabled": Started, + "disabled": Stopped, + "terminated": Terminating, + } +} + +// getDesiredStatus takes the desired config and the current config and returns the Status +// that the operator should try to configure for this reconciliation +func getDesiredStatus(desiredConfig, currentConfig *Config) Status { + if currentConfig == nil { + return desiredConfig.Status + } + // valid transitions can be found here https://github.com/10gen/mms/blob/7487cf31e775a38703ca6ef247b31b4d10c78c41/server/src/main/com/xgen/svc/mms/api/res/ApiBackupConfigsResource.java#L186 + // transitioning from Started to Terminating is not a valid transition + // we need to first go to Stopped. + if desiredConfig.Status == Terminating && currentConfig.Status == Started { + return Stopped + } + + // transitioning from Stopped to Terminating is not possible, it is only possible through + // Stopped -> Started -> Terminating + if desiredConfig.Status == Stopped && currentConfig.Status == Terminating { + return Started + } + + return desiredConfig.Status +} diff --git a/controllers/om/backup/mongodbresource_backup_test.go b/controllers/om/backup/mongodbresource_backup_test.go new file mode 100644 index 000000000..acd81a762 --- /dev/null +++ b/controllers/om/backup/mongodbresource_backup_test.go @@ -0,0 +1,47 @@ +package backup + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetDesiredStatus(t *testing.T) { + t.Run("Test transition from enabled to disabled", func(t *testing.T) { + desired := Config{ + Status: Stopped, + } + current := Config{ + Status: Started, + } + assert.Equal(t, Stopped, getDesiredStatus(&desired, ¤t)) + }) + t.Run("Test transition from disabled to enabled", func(t *testing.T) { + desired := Config{ + Status: Started, + } + current := Config{ + Status: Stopped, + } + assert.Equal(t, Started, getDesiredStatus(&desired, ¤t)) + }) + t.Run("Test transition from enabled to terminated", func(t *testing.T) { + desired := Config{ + Status: Terminating, + } + current := Config{ + Status: Started, + } + assert.Equal(t, Stopped, getDesiredStatus(&desired, ¤t)) + }) + + t.Run("Test transition from terminated to disabled", func(t *testing.T) { + desired := Config{ + Status: Stopped, + } + current := Config{ + Status: Terminating, + } + assert.Equal(t, Started, getDesiredStatus(&desired, ¤t)) + }) +} diff --git a/controllers/om/backup/s3config.go b/controllers/om/backup/s3config.go new file mode 100644 index 000000000..a29beaa10 --- /dev/null +++ b/controllers/om/backup/s3config.go @@ -0,0 +1,174 @@ +package backup + +import ( + "fmt" + + "go.uber.org/zap" + + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +type authmode string + +const ( + KEYS authmode = "KEYS" + IAM authmode = "IAM_ROLE" +) + +type S3ConfigResponse struct { + S3Configs []S3Config `json:"results"` +} + +// https://docs.opsmanager.mongodb.com/current/reference/api/admin/backup/snapshot/s3Configs/create-one-s3-blockstore-configuration/#request-body-parameters +type S3Config struct { + S3Bucket + S3Credentials + + // Flag indicating the style of this endpoint. + // true: Path-style URL endpoint eg. s3.amazonaws.com/ + // false: Virtual-host-style URL endpoint e.g. .s3.amazonaws.com + PathStyleAccessEnabled bool `json:"pathStyleAccessEnabled"` + + // Flag indicating whether you can assign backup jobs to this data store. + AssignmentEnabled bool `json:"assignmentEnabled"` + + // Flag indicating whether you accepted the terms of service for using S3-compatible stores with Ops Manager. + // If this is false, the request results in an error and Ops Manager doesn't create the S3-compatible store. + AcceptedTos bool `json:"acceptedTos"` + + // Unique name that labels this S3 Snapshot Store. + Id string `json:"id"` + + // Positive integer indicating the maximum number of connections to this S3 blockstore. + MaxConnections int `json:"s3MaxConnections"` + + // Comma-separated list of hosts in the format that can access this S3 blockstore. + Uri string `json:"uri"` + + // Fields the operator will not configure. All of these can be changed via the UI and the operator + // will not reset their values on reconciliation + EncryptedCredentials bool `json:"encryptedCredentials"` + Labels []string `json:"labels"` + LoadFactor int `json:"loadFactor,omitempty"` + + AuthMethod string `json:"s3AuthMethod"` + + WriteConcern string `json:"writeConcern,omitempty"` + + // Region where the S3 bucket resides. + // This is currently set to the empty string to avoid HELP-22791. + S3RegionOverride *string `json:"s3RegionOverride,omitempty"` + + // Flag indicating whether this S3 blockstore enables server-side encryption. + SseEnabled bool `json:"sseEnabled"` + + // Required for OM 4.4 + DisableProxyS3 *bool `json:"disableProxyS3,omitempty"` + + // Flag indicating whether this S3 blockstore only accepts connections encrypted using TLS. + Ssl bool `json:"ssl"` + + // CustomCertificates is a list of valid Certificate Authority certificates that apply to the associated S3 bucket. + CustomCertificates []S3CustomCertificate `json:"customCertificates,omitempty"` +} + +// S3CustomCertificate stores the filename or contents of a custom certificate PEM file. +type S3CustomCertificate struct { + // Filename identifies the Certificate Authority PEM file. + Filename string `json:"filename"` + // CertString contains the contents of the Certificate Authority PEM file that comprise your Certificate Authority chain. + CertString string `json:"certString"` +} + +type S3Bucket struct { + // Name of the S3 bucket that hosts the S3 blockstore. + Name string `json:"s3BucketName"` + // URL used to access this AWS S3 or S3-compatible bucket. + Endpoint string `json:"s3BucketEndpoint"` +} + +type S3Credentials struct { + // AWS Access Key ID that can access the S3 bucket specified in s3BucketName. + AccessKey string `json:"awsAccessKey"` + + // AWS Secret Access Key that can access the S3 bucket specified in s3BucketName. + SecretKey string `json:"awsSecretKey"` +} + +func NewS3Config(opsManager *omv1.MongoDBOpsManager, s3Config omv1.S3Config, uri string, s3CustomCertificates []S3CustomCertificate, bucket S3Bucket, s3Creds *S3Credentials) S3Config { + authMode := IAM + cred := S3Credentials{} + + if s3Creds != nil { + authMode = KEYS + cred = *s3Creds + } + + config := S3Config{ + S3Bucket: bucket, + S3Credentials: cred, + AcceptedTos: true, + AssignmentEnabled: true, // defaults to true. This will not be overridden on merge, so it can be manually disabled in UI. + SseEnabled: false, + DisableProxyS3: nil, + Id: s3Config.Name, + Uri: uri, + MaxConnections: util.DefaultS3MaxConnections, // can be configured in UI + Labels: s3Config.AssignmentLabels, + EncryptedCredentials: false, + PathStyleAccessEnabled: true, + AuthMethod: string(authMode), + S3RegionOverride: &s3Config.S3RegionOverride, + } + + if _, err := versionutil.StringToSemverVersion(opsManager.Spec.Version); err == nil { + config.DisableProxyS3 = util.BooleanRef(false) + + for _, certificate := range s3CustomCertificates { + + if s3Config.CustomCertificate { + zap.S().Warn("CustomCertificate is deprecated. Please switch to customCertificates to add your appDB-CA") + } + + // Historically, if s3Config.CustomCertificate was set to true, then we would use the appDBCa for s3Config. + if !s3Config.CustomCertificate && certificate.Filename == omv1.GetAppDBCaPemPath() { + continue + } + + // Attributes that are only available in 5.0+ version of Ops Manager. + // Both filename and path need to be provided. + if certificate.CertString != "" && certificate.Filename != "" { + // CustomCertificateSecretRefs needs to be a pointer for it to not be + // passed as part of the API request. + config.CustomCertificates = append(config.CustomCertificates, certificate) + } + } + } + + return config +} + +func (s S3Config) Identifier() interface{} { + return s.Id +} + +// MergeIntoOpsManagerConfig performs the merge operation of the Operator config view ('s') into the OM owned one +// ('opsManagerS3Config') +func (s S3Config) MergeIntoOpsManagerConfig(opsManagerS3Config S3Config) S3Config { + opsManagerS3Config.Id = s.Id + opsManagerS3Config.S3Credentials = s.S3Credentials + opsManagerS3Config.S3Bucket = s.S3Bucket + opsManagerS3Config.PathStyleAccessEnabled = s.PathStyleAccessEnabled + opsManagerS3Config.Uri = s.Uri + opsManagerS3Config.S3RegionOverride = s.S3RegionOverride + opsManagerS3Config.Labels = s.Labels + opsManagerS3Config.CustomCertificates = s.CustomCertificates + return opsManagerS3Config +} + +func (s S3Config) String() string { + return fmt.Sprintf("id %s, uri: %s, enabled: %t, awsAccessKey: %s, awsSecretKey: %s, bucketEndpoint: %s, bucketName: %s, pathStyleAccessEnabled: %t", + s.Id, util.RedactMongoURI(s.Uri), s.AssignmentEnabled, util.Redact(s.AccessKey), util.Redact(s.SecretKey), s.Endpoint, s.Name, s.PathStyleAccessEnabled) +} diff --git a/controllers/om/backup/snapshot_schedule.go b/controllers/om/backup/snapshot_schedule.go new file mode 100644 index 000000000..df770ea14 --- /dev/null +++ b/controllers/om/backup/snapshot_schedule.go @@ -0,0 +1,79 @@ +package backup + +import ( + "fmt" + "strings" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +// SnapshotSchedule object represents request and response body object of get and update snapshot schedule request +// https://www.mongodb.com/docs/ops-manager/master/reference/api/backup/update-one-snapshot-schedule-by-cluster-id/#request-body-parameters +type SnapshotSchedule struct { + GroupID string `json:"groupId,omitempty"` + ClusterID string `json:"clusterId,omitempty"` + ClusterCheckpointIntervalMin *int `json:"clusterCheckpointIntervalMin,omitempty"` + DailySnapshotRetentionDays *int `json:"dailySnapshotRetentionDays,omitempty"` + FullIncrementalDayOfWeek *string `json:"fullIncrementalDayOfWeek,omitempty"` + MonthlySnapshotRetentionMonths *int `json:"monthlySnapshotRetentionMonths,omitempty"` + PointInTimeWindowHours *int `json:"pointInTimeWindowHours,omitempty"` + ReferenceHourOfDay *int `json:"referenceHourOfDay,omitempty"` + ReferenceMinuteOfHour *int `json:"referenceMinuteOfHour,omitempty"` + SnapshotIntervalHours *int `json:"snapshotIntervalHours,omitempty"` + SnapshotRetentionDays *int `json:"snapshotRetentionDays,omitempty"` + WeeklySnapshotRetentionWeeks *int `json:"weeklySnapshotRetentionWeeks,omitempty"` + // ReferenceTimeZoneOffset is not handled deliberately, because OM converts ReferenceHourOfDay to UTC and saves always +0000 offset, + // and this would cause constant updates, due to always different timezone offset. + // ReferenceTimeZoneOffset *string `json:"referenceTimeZoneOffset,omitempty"` +} + +func ptrToStr[T any](val *T) string { + if val != nil { + return fmt.Sprintf("%v", *val) + } + return "nil" +} + +func (s SnapshotSchedule) String() string { + str := strings.Builder{} + _, _ = str.WriteString("GroupID: " + s.GroupID) + _, _ = str.WriteString(", ClusterID: " + s.ClusterID) + _, _ = str.WriteString(", ClusterCheckpointIntervalMin: " + ptrToStr(s.ClusterCheckpointIntervalMin)) + _, _ = str.WriteString(", DailySnapshotRetentionDays: " + ptrToStr(s.DailySnapshotRetentionDays)) + _, _ = str.WriteString(", FullIncrementalDayOfWeek: " + ptrToStr(s.FullIncrementalDayOfWeek)) + _, _ = str.WriteString(", MonthlySnapshotRetentionMonths: " + ptrToStr(s.MonthlySnapshotRetentionMonths)) + _, _ = str.WriteString(", PointInTimeWindowHours: " + ptrToStr(s.PointInTimeWindowHours)) + _, _ = str.WriteString(", ReferenceHourOfDay: " + ptrToStr(s.ReferenceHourOfDay)) + _, _ = str.WriteString(", ReferenceMinuteOfHour: " + ptrToStr(s.ReferenceMinuteOfHour)) + _, _ = str.WriteString(", SnapshotIntervalHours: " + ptrToStr(s.SnapshotIntervalHours)) + _, _ = str.WriteString(", SnapshotRetentionDays: " + ptrToStr(s.SnapshotRetentionDays)) + _, _ = str.WriteString(", WeeklySnapshotRetentionWeeks: " + ptrToStr(s.WeeklySnapshotRetentionWeeks)) + + return str.String() +} + +func mergeExistingScheduleWithSpec(existingSnapshotSchedule SnapshotSchedule, specSnapshotSchedule mdb.SnapshotSchedule) SnapshotSchedule { + snapshotSchedule := SnapshotSchedule{} + snapshotSchedule.ClusterID = existingSnapshotSchedule.ClusterID + snapshotSchedule.GroupID = existingSnapshotSchedule.GroupID + snapshotSchedule.ClusterCheckpointIntervalMin = pickFirstIfNotNil(specSnapshotSchedule.ClusterCheckpointIntervalMin, existingSnapshotSchedule.ClusterCheckpointIntervalMin) + snapshotSchedule.DailySnapshotRetentionDays = pickFirstIfNotNil(specSnapshotSchedule.DailySnapshotRetentionDays, existingSnapshotSchedule.DailySnapshotRetentionDays) + snapshotSchedule.FullIncrementalDayOfWeek = pickFirstIfNotNil(specSnapshotSchedule.FullIncrementalDayOfWeek, existingSnapshotSchedule.FullIncrementalDayOfWeek) + snapshotSchedule.MonthlySnapshotRetentionMonths = pickFirstIfNotNil(specSnapshotSchedule.MonthlySnapshotRetentionMonths, existingSnapshotSchedule.MonthlySnapshotRetentionMonths) + snapshotSchedule.PointInTimeWindowHours = pickFirstIfNotNil(specSnapshotSchedule.PointInTimeWindowHours, existingSnapshotSchedule.PointInTimeWindowHours) + snapshotSchedule.ReferenceHourOfDay = pickFirstIfNotNil(specSnapshotSchedule.ReferenceHourOfDay, existingSnapshotSchedule.ReferenceHourOfDay) + snapshotSchedule.ReferenceMinuteOfHour = pickFirstIfNotNil(specSnapshotSchedule.ReferenceMinuteOfHour, existingSnapshotSchedule.ReferenceMinuteOfHour) + snapshotSchedule.SnapshotIntervalHours = pickFirstIfNotNil(specSnapshotSchedule.SnapshotIntervalHours, existingSnapshotSchedule.SnapshotIntervalHours) + snapshotSchedule.SnapshotRetentionDays = pickFirstIfNotNil(specSnapshotSchedule.SnapshotRetentionDays, existingSnapshotSchedule.SnapshotRetentionDays) + snapshotSchedule.WeeklySnapshotRetentionWeeks = pickFirstIfNotNil(specSnapshotSchedule.WeeklySnapshotRetentionWeeks, existingSnapshotSchedule.WeeklySnapshotRetentionWeeks) + + return snapshotSchedule +} + +func pickFirstIfNotNil[T any](first *T, second *T) *T { + if first != nil { + return first + } else { + return second + } +} diff --git a/controllers/om/backup/snapshot_schedule_test.go b/controllers/om/backup/snapshot_schedule_test.go new file mode 100644 index 000000000..8cc359c1b --- /dev/null +++ b/controllers/om/backup/snapshot_schedule_test.go @@ -0,0 +1,65 @@ +package backup + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +func TestMergeExistingScheduleWithSpec(t *testing.T) { + existingSchedule := SnapshotSchedule{ + GroupID: "a", + ClusterID: "b", + DailySnapshotRetentionDays: ptr.To(2), + FullIncrementalDayOfWeek: ptr.To("c"), + MonthlySnapshotRetentionMonths: ptr.To(3), + PointInTimeWindowHours: ptr.To(4), + ReferenceHourOfDay: ptr.To(5), + ReferenceMinuteOfHour: ptr.To(6), + SnapshotIntervalHours: ptr.To(8), + SnapshotRetentionDays: ptr.To(9), + WeeklySnapshotRetentionWeeks: ptr.To(10), + ClusterCheckpointIntervalMin: ptr.To(11), + } + + specSchedule := mdb.SnapshotSchedule{ + SnapshotIntervalHours: ptr.To(11), + SnapshotRetentionDays: ptr.To(12), + DailySnapshotRetentionDays: ptr.To(13), + WeeklySnapshotRetentionWeeks: ptr.To(14), + MonthlySnapshotRetentionMonths: ptr.To(15), + PointInTimeWindowHours: ptr.To(16), + ReferenceHourOfDay: ptr.To(17), + ReferenceMinuteOfHour: ptr.To(18), + FullIncrementalDayOfWeek: ptr.To("cc"), + ClusterCheckpointIntervalMin: ptr.To(11), + } + + merged := mergeExistingScheduleWithSpec(existingSchedule, specSchedule) + assert.Equal(t, specSchedule.SnapshotIntervalHours, merged.SnapshotIntervalHours) + assert.Equal(t, specSchedule.SnapshotRetentionDays, merged.SnapshotRetentionDays) + assert.Equal(t, specSchedule.DailySnapshotRetentionDays, merged.DailySnapshotRetentionDays) + assert.Equal(t, specSchedule.WeeklySnapshotRetentionWeeks, merged.WeeklySnapshotRetentionWeeks) + assert.Equal(t, specSchedule.MonthlySnapshotRetentionMonths, merged.MonthlySnapshotRetentionMonths) + assert.Equal(t, specSchedule.PointInTimeWindowHours, merged.PointInTimeWindowHours) + assert.Equal(t, specSchedule.ReferenceHourOfDay, merged.ReferenceHourOfDay) + assert.Equal(t, specSchedule.ReferenceMinuteOfHour, merged.ReferenceMinuteOfHour) + assert.Equal(t, specSchedule.FullIncrementalDayOfWeek, merged.FullIncrementalDayOfWeek) + assert.Equal(t, specSchedule.ClusterCheckpointIntervalMin, merged.ClusterCheckpointIntervalMin) + + emptySpecSchedule := mdb.SnapshotSchedule{} + merged = mergeExistingScheduleWithSpec(existingSchedule, emptySpecSchedule) + assert.Equal(t, existingSchedule.SnapshotIntervalHours, merged.SnapshotIntervalHours) + assert.Equal(t, existingSchedule.SnapshotRetentionDays, merged.SnapshotRetentionDays) + assert.Equal(t, existingSchedule.DailySnapshotRetentionDays, merged.DailySnapshotRetentionDays) + assert.Equal(t, existingSchedule.WeeklySnapshotRetentionWeeks, merged.WeeklySnapshotRetentionWeeks) + assert.Equal(t, existingSchedule.MonthlySnapshotRetentionMonths, merged.MonthlySnapshotRetentionMonths) + assert.Equal(t, existingSchedule.PointInTimeWindowHours, merged.PointInTimeWindowHours) + assert.Equal(t, existingSchedule.ReferenceHourOfDay, merged.ReferenceHourOfDay) + assert.Equal(t, existingSchedule.ReferenceMinuteOfHour, merged.ReferenceMinuteOfHour) + assert.Equal(t, existingSchedule.FullIncrementalDayOfWeek, merged.FullIncrementalDayOfWeek) + assert.Equal(t, existingSchedule.ClusterCheckpointIntervalMin, merged.ClusterCheckpointIntervalMin) +} diff --git a/controllers/om/backup_agent_config.go b/controllers/om/backup_agent_config.go new file mode 100644 index 000000000..7a5fea3f4 --- /dev/null +++ b/controllers/om/backup_agent_config.go @@ -0,0 +1,100 @@ +package om + +import ( + "encoding/json" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type BackupAgentTemplate struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + SSLPemKeyFile string `json:"sslPEMKeyFile,omitempty"` + LdapGroupDN string `json:"ldapGroupDN,omitempty"` + LogRotate mdbv1.LogRotateForBackupAndMonitoring `json:"logRotate,omitempty"` +} + +type BackupAgentConfig struct { + BackupAgentTemplate *BackupAgentTemplate + BackingMap map[string]interface{} +} + +func (bac *BackupAgentConfig) Apply() error { + merged, err := util.MergeWith(bac.BackupAgentTemplate, bac.BackingMap, &util.AutomationConfigTransformer{}) + if err != nil { + return err + } + bac.BackingMap = merged + return nil +} + +func (bac *BackupAgentConfig) SetAgentUserName(backupAgentSubject string) { + bac.BackupAgentTemplate.Username = backupAgentSubject +} + +func (bac *BackupAgentConfig) SetLogRotate(logRotate mdbv1.LogRotateForBackupAndMonitoring) { + bac.BackupAgentTemplate.LogRotate = logRotate +} + +func (bac *BackupAgentConfig) UnsetAgentUsername() { + bac.BackupAgentTemplate.Username = util.MergoDelete +} + +func (bac *BackupAgentConfig) SetAgentPassword(pwd string) { + bac.BackupAgentTemplate.Password = pwd +} + +func (bac *BackupAgentConfig) UnsetAgentPassword() { + bac.BackupAgentTemplate.Password = util.MergoDelete +} + +func (bac *BackupAgentConfig) EnableX509Authentication(backupAgentSubject string) { + bac.BackupAgentTemplate.SSLPemKeyFile = util.AutomationAgentPemFilePath + bac.SetAgentUserName(backupAgentSubject) +} + +func (bac *BackupAgentConfig) DisableX509Authentication() { + bac.BackupAgentTemplate.SSLPemKeyFile = util.MergoDelete + bac.UnsetAgentUsername() +} + +func (bac *BackupAgentConfig) EnableLdapAuthentication(backupAgentSubject string, backupAgentPwd string) { + bac.SetAgentUserName(backupAgentSubject) + bac.SetAgentPassword(backupAgentPwd) +} + +func (bac *BackupAgentConfig) DisableLdapAuthentication() { + bac.UnsetAgentUsername() + bac.UnsetAgentPassword() + bac.UnsetLdapGroupDN() +} + +func (bac *BackupAgentConfig) SetLdapGroupDN(ldapGroupDn string) { + bac.BackupAgentTemplate.LdapGroupDN = ldapGroupDn +} + +func (bac *BackupAgentConfig) UnsetLdapGroupDN() { + bac.BackupAgentTemplate.LdapGroupDN = util.MergoDelete +} + +func BuildBackupAgentConfigFromBytes(jsonBytes []byte) (*BackupAgentConfig, error) { + fullMap := make(map[string]interface{}) + if err := json.Unmarshal(jsonBytes, &fullMap); err != nil { + return nil, err + } + + config := &BackupAgentConfig{BackingMap: fullMap} + template := &BackupAgentTemplate{} + if username, ok := fullMap["username"].(string); ok { + template.Username = username + } + + if sslPemKeyfile, ok := fullMap["sslPEMKeyFile"].(string); ok { + template.SSLPemKeyFile = sslPemKeyfile + } + + config.BackupAgentTemplate = template + + return config, nil +} diff --git a/controllers/om/backup_agent_test.go b/controllers/om/backup_agent_test.go new file mode 100644 index 000000000..f8c9429b5 --- /dev/null +++ b/controllers/om/backup_agent_test.go @@ -0,0 +1,79 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +var testBackupAgentConfig = *getTestBackupConfig() + +func getLinuxUrls(config BackupAgentConfig) map[string]interface{} { + return config.BackingMap["urls"].(map[string]interface{})["linux"].(map[string]interface{}) +} + +func getTestBackupConfig() *BackupAgentConfig { + bac, _ := BuildBackupAgentConfigFromBytes(loadBytesFromTestData("backup_config.json")) + return bac +} + +func TestFieldsAreUpdatedBackupConfig(t *testing.T) { + config := getTestBackupConfig() + config.BackupAgentTemplate.Username = "my-backup-user-name" + config.BackupAgentTemplate.SSLPemKeyFile = "my-backup-pem-file" + + _ = config.Apply() + + assert.Equal(t, config.BackingMap["username"], "my-backup-user-name") + assert.Equal(t, config.BackingMap["sslPEMKeyFile"], "my-backup-pem-file") +} + +func TestBackupFieldsAreNotLost(t *testing.T) { + config := getTestBackupConfig() + config.EnableX509Authentication("namespace") + + assert.Contains(t, config.BackingMap, "logPath") + assert.Contains(t, config.BackingMap, "logRotate") + assert.Contains(t, config.BackingMap, "urls") + + _ = config.Apply() + + assert.Equal(t, config.BackingMap["logPath"], testBackupAgentConfig.BackingMap["logPath"]) + assert.Equal(t, config.BackingMap["logRotate"], testBackupAgentConfig.BackingMap["logRotate"]) + assert.Equal(t, config.BackingMap["urls"], testBackupAgentConfig.BackingMap["urls"]) +} + +func TestNestedFieldsAreNotLost(t *testing.T) { + config := getTestBackupConfig() + + config.EnableX509Authentication("namespace") + + _ = config.Apply() + + urls := config.BackingMap["urls"].(map[string]interface{}) + + assert.Contains(t, urls, "linux") + assert.Contains(t, urls, "osx") + assert.Contains(t, urls, "windows") + + linuxUrls := urls["linux"].(map[string]interface{}) + + testUrls := getLinuxUrls(testBackupAgentConfig) + + assert.Equal(t, linuxUrls["default"], testUrls["default"]) + assert.Equal(t, linuxUrls["ppc64le_rhel7"], testUrls["ppc64le_rhel7"]) + assert.Equal(t, linuxUrls["ppc64le_ubuntu1604"], testUrls["ppc64le_ubuntu1604"]) +} + +func TestFieldsCanBeDeleted(t *testing.T) { + config := getTestBackupConfig() + + config.BackupAgentTemplate.SSLPemKeyFile = util.MergoDelete + + _ = config.Apply() + + assert.Equal(t, config.BackingMap["username"], testBackupAgentConfig.BackingMap["username"]) + assert.NotContains(t, config.BackingMap, "sslPEMKeyFile") +} diff --git a/controllers/om/backup_test.go b/controllers/om/backup_test.go new file mode 100644 index 000000000..08f62df6b --- /dev/null +++ b/controllers/om/backup_test.go @@ -0,0 +1,26 @@ +package om + +import ( + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// TestBackupWaitsForTermination tests that 'StopBackupIfEnabled' procedure waits for backup statuses on each stage +// (STARTED -> STOPPED, STOPPED -> INACTIVE) +func TestBackupWaitsForTermination(t *testing.T) { + t.Setenv(util.BackupDisableWaitSecondsEnv, "1") + t.Setenv(util.BackupDisableWaitRetriesEnv, "3") + + connection := NewMockedOmConnection(NewDeployment()) + connection.EnableBackup("test", backup.ReplicaSetType, uuid.New().String()) + err := backup.StopBackupIfEnabled(connection, connection, "test", backup.ReplicaSetType, zap.S()) + assert.NoError(t, err) + + connection.CheckResourcesAndBackupDeleted(t, "test") +} diff --git a/controllers/om/deployment.go b/controllers/om/deployment.go new file mode 100644 index 000000000..aabaff472 --- /dev/null +++ b/controllers/om/deployment.go @@ -0,0 +1,1219 @@ +package om + +import ( + "encoding/gob" + "encoding/json" + "fmt" + "math" + "regexp" + + "github.com/blang/semver" + "github.com/spf13/cast" + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +type DeploymentType int + +const ( + // Note that the default version constants shouldn't need to be changed often + // as the AutomationAgent upgrades both other agents automatically + + MonitoringAgentDefaultVersion = "11.12.0.7388-1" + + BackupAgentDefaultVersion = "11.12.0.7388-1" + + // Default listen address for Prometheus + ListenAddress = "0.0.0.0" +) + +func init() { + // gob is used to implement a deep copy internally. If a data type is part of + // a deep copy performed using util.MapDeepCopy—which includes anything used + // as part of a "process" object embedded within a deployment—then it must be + // registered below as otherwise the operator will successfully compile and + // run but be completely broken. + // TODO should we move this to main.go? + gob.Register(map[string]interface{}{}) + gob.Register([]interface{}{}) + gob.Register(map[string]int{}) + gob.Register(map[string]string{}) + gob.Register([]interface{}{}) + gob.Register([]Process{}) + gob.Register([]ReplicaSet{}) + gob.Register([]ReplicaSetMember{}) + gob.Register([]ShardedCluster{}) + gob.Register([]MongoDbVersionConfig{}) + gob.Register([]Shard{}) + gob.Register(ProcessTypeMongos) + gob.Register(mdbv1.MongoDBHorizonConfig{}) + + gob.Register(tls.Require) + gob.Register(tls.Prefer) + gob.Register(tls.Allow) + gob.Register(tls.Disabled) + gob.Register([]mdbv1.MongoDbRole{}) + gob.Register([]automationconfig.MemberOptions{}) +} + +// Deployment is a map representing the automation agent's cluster configuration. +// For more information see the following documentation: +// https://docs.opsmanager.mongodb.com/current/reference/cluster-configuration/ +// https://github.com/10gen/mms-automation/blob/master/go_planner/config_specs/clusterConfig_spec.md +// +// Dev note: it's important to keep to the following principle during development: we don't use structs for json +// (de)serialization as we don't want to own the schema and synchronize it with the api one constantly. +// Also, we don't want to override any configuration provided by OM by accident. +// The Operator only sets the configuration it "owns" but +// keeps the other one that was set by the user in Ops Manager if any +type Deployment map[string]interface{} + +func BuildDeploymentFromBytes(jsonBytes []byte) (Deployment, error) { + deployment := Deployment{} + err := json.Unmarshal(jsonBytes, &deployment) + return deployment, err +} + +func NewDeployment() Deployment { + ans := Deployment{} + ans.setProcesses(make([]Process, 0)) + ans.setReplicaSets(make([]ReplicaSet, 0)) + ans.setShardedClusters(make([]ShardedCluster, 0)) + ans.setMonitoringVersions(make([]interface{}, 0)) + ans.setBackupVersions(make([]interface{}, 0)) + + // these keys are required to exist for mergo to merge + // correctly + ans["auth"] = make(map[string]interface{}) + ans["tls"] = map[string]interface{}{ + "clientCertificateMode": util.OptionalClientCertficates, + "CAFilePath": util.CAFilePathInContainer, + } + return ans +} + +// TLSConfigurationWillBeDisabled checks that if applying this security configuration the Deployment +// TLS configuration will go from Enabled -> Disabled. +func (d Deployment) TLSConfigurationWillBeDisabled(security *mdbv1.Security) bool { + tlsIsCurrentlyEnabled := false + + // To detect that TLS is enabled, it is sufficient to check for the + // d["tls"]["CAFilePath"] attribute to have a value different from nil. + if tlsMap, ok := d["tls"]; ok { + if caFilePath, ok := tlsMap.(map[string]interface{})["CAFilePath"]; ok { + if caFilePath != nil { + tlsIsCurrentlyEnabled = true + } + } + } + + return tlsIsCurrentlyEnabled && !security.IsTLSEnabled() +} + +// ConfigureTLS configures the deployment's TLS settings from the security +// specification provided by the user in the mongodb resource. +func (d Deployment) ConfigureTLS(security *mdbv1.Security, caFilePath string) { + if !security.IsTLSEnabled() { + d["tls"] = map[string]any{"clientCertificateMode": string(automationconfig.ClientCertificateModeOptional)} + return + } + + tlsConfig := util.ReadOrCreateMap(d, "tls") + // ClientCertificateMode detects if Ops Manager requires client certification - may be there will be no harm + // setting this to "REQUIRED" always (need to check). Otherwise, this should be configurable + // see OM configurations that affects this setting from AA side: + // https://docs.opsmanager.mongodb.com/current/reference/configuration/#mms.https.ClientCertificateMode + // sslConfig["ClientCertificateMode"] = "OPTIONAL" + // sslConfig["AutoPEMKeyFilePath"] = util.PEMKeyFilePathInContainer + + tlsConfig["CAFilePath"] = caFilePath +} + +// MergeStandalone merges "operator" standalone ('standaloneMongo') to "OM" deployment ('d'). If we found the process +// with the same name - update some fields there. Otherwise, add the new one +func (d Deployment) MergeStandalone(standaloneMongo Process, specArgs26, prevArgs26 map[string]interface{}, l *zap.SugaredLogger) { + if l == nil { + l = zap.S() + } + log := l.With("standalone", standaloneMongo) + + // merging process in case exists, otherwise adding it + for _, pr := range d.getProcesses() { + if pr.Name() == standaloneMongo.Name() { + pr.mergeFrom(standaloneMongo, specArgs26, prevArgs26) + log.Debug("Merged process into existing one") + return + } + } + d.addProcess(standaloneMongo) + log.Debug("Added process as current OM deployment didn't have it") +} + +// MergeReplicaSet merges the "operator" replica set and its members to the "OM" deployment ("d"). If "alien" RS members are +// removed after merge - corresponding processes are removed as well. +func (d Deployment) MergeReplicaSet(operatorRs ReplicaSetWithProcesses, specArgs26, prevArgs26 map[string]interface{}, l *zap.SugaredLogger) { + log := l.With("replicaSet", operatorRs.Rs.Name()) + + r := d.getReplicaSetByName(operatorRs.Rs.Name()) + // If the new replica set is bigger than old one - we need to copy first member to positions of new members so that + // they were merged with operator replica sets on next step + // (in case OM made any changes to existing processes - these changes must be propagated to new members). + if r != nil && len(operatorRs.Rs.Members()) > len(r.Members()) { + if err := d.copyFirstProcessToNewPositions(operatorRs.Processes, len(r.Members()), l); err != nil { + // I guess this error is not so serious to fail the whole process - RS will be scaled up anyway + log.Error("Failed to copy first process (so new replica set processes may miss Ops Manager changes done to "+ + "existing replica set processes): %s", err) + } + } + + // Merging all RS processes + for _, p := range operatorRs.Processes { + d.MergeStandalone(p, specArgs26, prevArgs26, log) + } + + if r == nil { + // Adding a new Replicaset + d.addReplicaSet(operatorRs.Rs) + log.Debugw("Added replica set as current OM deployment didn't have it") + } else { + + processesToRemove := r.mergeFrom(operatorRs.Rs) + log.Debugw("Merged replica set into existing one") + + if len(processesToRemove) > 0 { + d.removeProcesses(processesToRemove, log) + log.Debugw("Removed processes as they were removed from replica set", "processesToRemove", processesToRemove) + } + } + + // In both cases (the new replicaset was added to OM deployment, or it was merged with OM one) we need to make sure + // there are no more than 7 voting members + d.limitVotingMembers(operatorRs.Rs.Name()) +} + +// ConfigurePrometheus adds Prometheus configuration to `Deployment` resource. +// +// If basic auth is enabled, then `hash` and `salt` need to be calculated by caller and passed in. +func (d Deployment) ConfigurePrometheus(prom *mdbcv1.Prometheus, hash string, salt string, certName string) automationconfig.Prometheus { + if prom == nil { + // No prometheus configuration this time + return automationconfig.Prometheus{} + } + + promConfig := automationconfig.NewDefaultPrometheus(prom.Username) + + if prom.TLSSecretRef.Name != "" { + promConfig.TLSPemPath = util.SecretVolumeMountPathPrometheus + "/" + certName + promConfig.Scheme = "https" + } else { + promConfig.TLSPemPath = "" + promConfig.Scheme = "http" + } + + promConfig.PasswordHash = hash + promConfig.PasswordSalt = salt + + if prom.Port > 0 { + promConfig.ListenAddress = fmt.Sprintf("%s:%d", ListenAddress, prom.Port) + } + + if prom.MetricsPath != "" { + promConfig.MetricsPath = prom.MetricsPath + } + + d["prometheus"] = promConfig + + return promConfig +} + +// DeploymentShardedClusterMergeOptions contains all the required values to update the ShardedCluster +// in the automation config. These values should be provided my the MongoDB resource. +type DeploymentShardedClusterMergeOptions struct { + Name string + MongosProcesses []Process + ConfigServerRs ReplicaSetWithProcesses + Shards []ReplicaSetWithProcesses + Finalizing bool + ConfigServerAdditionalOptionsDesired map[string]interface{} + MongosAdditionalOptionsDesired map[string]interface{} + ShardAdditionalOptionsDesired map[string]interface{} + ConfigServerAdditionalOptionsPrev map[string]interface{} + MongosAdditionalOptionsPrev map[string]interface{} + ShardAdditionalOptionsPrev map[string]interface{} +} + +// MergeShardedCluster merges "operator" sharded cluster into "OM" deployment ("d"). Mongos, config servers and all shards +// are all merged one by one. +// 'shardsToRemove' is an array containing names of shards which should be removed. +func (d Deployment) MergeShardedCluster(opts DeploymentShardedClusterMergeOptions) (bool, error) { + log := zap.S().With("sharded cluster", opts.Name) + + err := d.mergeMongosProcesses(opts, log) + if err != nil { + return false, err + } + + d.mergeConfigReplicaSet(opts, log) + + shardsScheduledForRemoval := d.mergeShards(opts, log) + + return shardsScheduledForRemoval, nil +} + +// AddMonitoringAndBackup adds monitoring and backup agents to each process +// The automation agent will update the agents versions to the latest version automatically +// Note, that these two are deliberately combined as all clients (standalone, rs etc.) need both backup and monitoring +// together +func (d Deployment) AddMonitoringAndBackup(log *zap.SugaredLogger, tls bool, caFilepath string) { + if len(d.getProcesses()) == 0 { + return + } + d.AddMonitoring(log, tls, caFilepath) + d.addBackup(log) +} + +// DEPRECATED: this shouldn't be used as it may panic because of different underlying type; use getReplicaSets instead +func (d Deployment) ReplicaSets() []ReplicaSet { + return d["replicaSets"].([]ReplicaSet) +} + +func (d Deployment) GetReplicaSetByName(name string) ReplicaSet { + for _, rs := range d.getReplicaSets() { + if rs.Name() == name { + return rs + } + } + return nil +} + +// AddMonitoring adds monitoring agents for all processes in the deployment +func (d Deployment) AddMonitoring(log *zap.SugaredLogger, tls bool, caFilePath string) { + if len(d.getProcesses()) == 0 { + return + } + monitoringVersions := d.getMonitoringVersions() + for _, p := range d.getProcesses() { + found := false + var monitoringVersion map[string]interface{} + for _, m := range monitoringVersions { + monitoringVersion = m.(map[string]interface{}) + if monitoringVersion["hostname"] == p.HostName() { + found = true + break + } + } + + if !found { + monitoringVersion = map[string]interface{}{ + "hostname": p.HostName(), + "name": MonitoringAgentDefaultVersion, + } + log.Debugw("Added monitoring agent configuration", "host", p.HostName(), "tls", tls) + monitoringVersions = append(monitoringVersions, monitoringVersion) + } + + monitoringVersion["hostname"] = p.HostName() + + if tls { + additionalParams := map[string]string{ + "useSslForAllConnections": "true", + "sslTrustedServerCertificates": caFilePath, + } + + pemKeyFile := p.EnsureTLSConfig()["PEMKeyFile"] + if pemKeyFile != nil { + additionalParams["sslClientCertificate"] = pemKeyFile.(string) + } + + monitoringVersion["additionalParams"] = additionalParams + } + + } + d.setMonitoringVersions(monitoringVersions) +} + +// RemoveMonitoringAndBackup removes both monitoring and backup agent configurations. This must be called when the +// Mongodb resource is being removed, otherwise UI will show non-existing agents in the "servers" tab +func (d Deployment) RemoveMonitoringAndBackup(names []string, log *zap.SugaredLogger) { + d.removeMonitoring(names) + d.removeBackup(names, log) +} + +func (d Deployment) DisableProcesses(processNames []string) { + for _, p := range processNames { + d.getProcessByName(p).SetDisabled(true) + } +} + +func (d Deployment) MarkRsMembersUnvoted(rsName string, rsMembers []string) error { + rs := d.getReplicaSetByName(rsName) + if rs == nil { + return xerrors.New("Failed to find Replica Set " + rsName) + } + + failedMembers := "" + for _, m := range rsMembers { + rsMember := rs.findMemberByName(m) + if rsMember == nil { + failedMembers += m + } else { + rsMember.setVotes(0).setPriority(0) + } + } + if failedMembers != "" { + return xerrors.Errorf("failed to find the following members of Replica Set %s: %v", rsName, failedMembers) + } + return nil +} + +// RemoveProcessByName removes the process from deployment +// Note, that the backup and monitoring configs are also cleaned up +func (d Deployment) RemoveProcessByName(name string, log *zap.SugaredLogger) error { + s := d.getProcessByName(name) + if s == nil { + return xerrors.Errorf("Standalone %s does not exist", name) + } + + d.removeProcesses([]string{s.Name()}, log) + + return nil +} + +// RemoveReplicaSetByName removes replica set and all relevant processes from deployment +// Note, that the backup and monitoring configs are also cleaned up +func (d Deployment) RemoveReplicaSetByName(name string, log *zap.SugaredLogger) error { + rs := d.getReplicaSetByName(name) + if rs == nil { + return xerrors.New("ReplicaSet does not exist") + } + + currentRs := d.getReplicaSets() + toKeep := make([]ReplicaSet, len(currentRs)-1) + i := 0 + for _, el := range currentRs { + if el.Name() != name { + toKeep[i] = el + i++ + } + } + + d.setReplicaSets(toKeep) + + members := rs.Members() + processNames := make([]string, len(members)) + for i, el := range members { + processNames[i] = el.Name() + } + d.removeProcesses(processNames, log) + + return nil +} + +// RemoveShardedClusterByName removes the sharded cluster element, all relevant replica sets and all processes. +// Note, that the backup and monitoring configs are also cleaned up +func (d Deployment) RemoveShardedClusterByName(clusterName string, log *zap.SugaredLogger) error { + sc := d.getShardedClusterByName(clusterName) + if sc == nil { + return xerrors.New("sharded Cluster does not exist") + } + + // 1. Remove the sharded cluster + toKeep := make([]ShardedCluster, 0) + for _, el := range d.getShardedClusters() { + if el.Name() != clusterName { + toKeep = append(toKeep, el) + } + } + + d.setShardedClusters(toKeep) + + // 2. Remove all replicasets and their processes for shards + shards := sc.shards() + shardNames := make([]string, len(shards)) + for _, el := range shards { + shardNames = append(shardNames, el.id()) + } + d.removeReplicaSets(shardNames, log) + + // 3. Remove config server replicaset + _ = d.RemoveReplicaSetByName(sc.ConfigServerRsName(), log) + + // 4. Remove mongos processes for cluster + d.removeProcesses(d.getMongosProcessesNames(clusterName), log) + + return nil +} + +// GetProcessNames returns an array of all the process names relevant to the given deployment +// these processes are the only ones checked for goal state when updating the +// deployment +func (d Deployment) GetProcessNames(kind interface{}, name string) []string { + switch kind.(type) { + case ShardedCluster: + return d.getShardedClusterProcessNames(name) + case ReplicaSet: + return d.getReplicaSetProcessNames(name) + case Standalone: + return []string{name} + default: + panic(xerrors.Errorf("unexpected kind: %v", kind)) + } +} + +// ConfigureInternalClusterAuthentication configures all processes in processNames to have the corresponding +// clusterAuthenticationMode enabled +func (d Deployment) ConfigureInternalClusterAuthentication(processNames []string, clusterAuthenticationMode string, internalClusterPath string) { + for _, p := range processNames { + if process := d.getProcessByName(p); process != nil { + process.ConfigureClusterAuthMode(clusterAuthenticationMode, internalClusterPath) + } + } +} + +// GetInternalClusterFilePath returns the first InternalClusterFilepath for the given list of processes. +func (d Deployment) GetInternalClusterFilePath(processNames []string) string { + for _, p := range processNames { + if process := d.getProcessByName(p); process != nil { + if !process.IsTLSEnabled() { + return "" + } + tlsConf := process.EnsureTLSConfig() + if v, ok := tlsConf["clusterFile"]; ok { + return v.(string) + } + } + } + return "" +} + +// SetInternalClusterFilePathOnlyIfItThePathHasChanged sets the internal cluster path for the given process names only if it has changed and has been set before. +func (d Deployment) SetInternalClusterFilePathOnlyIfItThePathHasChanged(names []string, filePath string, clusterAuth string, isRecovering bool) { + if currPath := d.GetInternalClusterFilePath(names); isRecovering || currPath != filePath && currPath != "" { + d.ConfigureInternalClusterAuthentication(names, clusterAuth, filePath) + } +} + +// MinimumMajorVersion returns the lowest major version in the entire deployment. +// this includes feature compatibility version. This can be used to determine +// which version of SCRAM-SHA the deployment can enable. +func (d Deployment) MinimumMajorVersion() uint64 { + if len(d.getProcesses()) == 0 { + return 0 + } + minimumMajorVersion := semver.Version{Major: math.MaxUint64} + for _, p := range d.getProcesses() { + if p.FeatureCompatibilityVersion() != "" { + fcv := fmt.Sprintf("%s.0", util.StripEnt(p.FeatureCompatibilityVersion())) + semverFcv, _ := semver.Make(fcv) + if semverFcv.LE(minimumMajorVersion) { + minimumMajorVersion = semverFcv + } + } else { + semverVersion, _ := semver.Make(util.StripEnt(p.Version())) + if semverVersion.LE(minimumMajorVersion) { + minimumMajorVersion = semverVersion + } + } + } + + return minimumMajorVersion.Major +} + +// allProcessesAreTLSEnabled ensures that every process in the given deployment is TLS enabled +// it is not possible to enable x509 authentication at the project level if a single process +// does not have TLS enabled. +func (d Deployment) AllProcessesAreTLSEnabled() bool { + for _, p := range d.getProcesses() { + if !p.IsTLSEnabled() { + return false + } + } + return true +} + +func (d Deployment) GetAllHostnames() []string { + hostnames := make([]string, d.NumberOfProcesses()) + for idx, p := range d.getProcesses() { + hostnames[idx] = p.HostName() + } + + return hostnames +} + +func (d Deployment) NumberOfProcesses() int { + return len(d.getProcesses()) +} + +// anyProcessHasInternalClusterAuthentication determines if at least one process +// has internal cluster authentication enabled. If this is true, it is impossible to disable +// x509 authentication +func (d Deployment) AnyProcessHasInternalClusterAuthentication() bool { + return d.processesHaveInternalClusterAuthentication(d.getProcesses()) +} + +func (d Deployment) ExistingProcessesHaveInternalClusterAuthentication(processes []Process) bool { + deploymentProcesses := make([]Process, 0) + for _, p := range processes { + deploymentProcess := d.getProcessByName(p.Name()) + if deploymentProcess != nil { + deploymentProcesses = append(deploymentProcesses, *deploymentProcess) + } + } + return d.processesHaveInternalClusterAuthentication(deploymentProcesses) +} + +func (d Deployment) Serialize() ([]byte, error) { + return json.Marshal(d) +} + +// ToCanonicalForm performs serialization/deserialization to get a map without struct members +// This may be useful if the Operator version of Deployment (which may contain structs) needs to be compared with +// a deployment deserialized from json +func (d Deployment) ToCanonicalForm() Deployment { + bytes, err := d.Serialize() + if err != nil { + // dev error + panic(err) + } + var canonical Deployment + canonical, err = BuildDeploymentFromBytes(bytes) + if err != nil { + panic(err) + } + return canonical +} + +func (d Deployment) Version() int64 { + if _, ok := d["version"]; !ok { + return -1 + } + return cast.ToInt64(d["version"]) +} + +// ProcessBelongsToResource determines if `processName` belongs to `resourceName`. +func (d Deployment) ProcessBelongsToResource(processName, resourceName string) bool { + if stringutil.Contains(d.GetProcessNames(ShardedCluster{}, resourceName), processName) { + return true + } + if stringutil.Contains(d.GetProcessNames(ReplicaSet{}, resourceName), processName) { + return true + } + if stringutil.Contains(d.GetProcessNames(Standalone{}, resourceName), processName) { + return true + } + + return false +} + +// GetNumberOfExcessProcesses calculates how many processes do not belong to +// this resource. +func (d Deployment) GetNumberOfExcessProcesses(resourceName string) int { + processNames := d.GetAllProcessNames() + excessProcesses := len(processNames) + for _, p := range processNames { + if d.ProcessBelongsToResource(p, resourceName) { + excessProcesses -= 1 + } + } + // Edge case: for sharded cluster it's ok to have junk replica sets during scale down - we consider them as + // belonging to sharded cluster + if d.getShardedClusterByName(resourceName) != nil { + for _, r := range d.findReplicaSetsRemovedFromShardedCluster(resourceName) { + excessProcesses -= len(d.GetProcessNames(ReplicaSet{}, r)) + } + } + + return excessProcesses +} + +func (d Deployment) SetRoles(roles []mdbv1.MongoDbRole) { + d["roles"] = roles +} + +func (d Deployment) GetRoles() []mdbv1.MongoDbRole { + val, ok := d["roles"].([]mdbv1.MongoDbRole) + if !ok { + return []mdbv1.MongoDbRole{} + } + return val +} + +// GetAgentVersion returns the current version of all Agents in the deployment. It's empty until the +// 'automationConfig/updateAgentVersions' endpoint is called the first time +func (d Deployment) GetAgentVersion() string { + agentVersionMap := util.ReadOrCreateMap(d, "agentVersion") + return maputil.ReadMapValueAsString(agentVersionMap, "name") +} + +func (d Deployment) Debug(l *zap.SugaredLogger) { + dep := Deployment{} + for key, value := range d { + if key != "mongoDbVersions" { + dep[key] = value + } + } + b, err := json.MarshalIndent(dep, "", " ") + if err != nil { + fmt.Println("error:", err) + } + l.Debugf(">> Deployment: \n %s \n", string(b)) +} + +// ProcessesCopy returns the COPY of processes in the deployment. +func (d Deployment) ProcessesCopy() []Process { + return d.deepCopy().getProcesses() +} + +// ReplicaSetsCopy returns the COPY of replicasets in the deployment. +func (d Deployment) ReplicaSetsCopy() []ReplicaSet { + return d.deepCopy().getReplicaSets() +} + +// ShardedClustersCopy returns the COPY of sharded clusters in the deployment. +func (d Deployment) ShardedClustersCopy() []ShardedCluster { + return d.deepCopy().getShardedClusters() +} + +// MonitoringVersionsCopy returns the COPY of monitoring versions in the deployment. +func (d Deployment) MonitoringVersionsCopy() []interface{} { + return d.deepCopy().getMonitoringVersions() +} + +// BackupVersionsCopy returns the COPY of backup versions in the deployment. +func (d Deployment) BackupVersionsCopy() []interface{} { + return d.deepCopy().getBackupVersions() +} + +// ***************************************** Private methods *********************************************************** + +func (d Deployment) getReplicaSetProcessNames(name string) []string { + processNames := make([]string, 0) + if rs := d.getReplicaSetByName(name); rs != nil { + for _, member := range rs.Members() { + processNames = append(processNames, member.Name()) + } + } + return processNames +} + +// GetShardedClusterShardProcessNames returns the process names for sharded cluster named "name" of index "shardNum". +func (d Deployment) GetShardedClusterShardProcessNames(name string, shardNum int) []string { + if sc := d.getShardedClusterByName(name); sc != nil { + if shardNum < 0 || shardNum >= len(sc.shards()) { + return nil + } + return d.getReplicaSetProcessNames(sc.shards()[shardNum].rs()) + } + return nil +} + +// getShardedClusterShardsProcessNames returns the process names fo sharded cluster named "name". +func (d Deployment) getShardedClusterShardsProcessNames(name string) []string { + processNames := make([]string, 0) + if sc := d.getShardedClusterByName(name); sc != nil { + for i := range sc.shards() { + processNames = append(processNames, d.GetShardedClusterShardProcessNames(name, i)...) + } + } + return processNames +} + +// GetShardedClusterConfigProcessNames returns the process names of config servers of the sharded cluster named "name" +func (d Deployment) GetShardedClusterConfigProcessNames(name string) []string { + if sc := d.getShardedClusterByName(name); sc != nil { + return d.getReplicaSetProcessNames(sc.ConfigServerRsName()) + } + return nil +} + +// GetShardedClusterMongosProcessNames returns the process names of mongos of the sharded cluster named "name" +func (d Deployment) GetShardedClusterMongosProcessNames(name string) []string { + if sc := d.getShardedClusterByName(name); sc != nil { + return d.getMongosProcessesNames(name) + } + return nil +} + +func (d Deployment) getShardedClusterProcessNames(name string) []string { + processNames := d.getShardedClusterShardsProcessNames(name) + processNames = append(processNames, d.GetShardedClusterConfigProcessNames(name)...) + processNames = append(processNames, d.GetShardedClusterMongosProcessNames(name)...) + return processNames +} + +func (d Deployment) mergeMongosProcesses(opts DeploymentShardedClusterMergeOptions, log *zap.SugaredLogger) error { + // First removing old mongos processes + for _, p := range d.getMongosProcessesNames(opts.Name) { + found := false + for _, v := range opts.MongosProcesses { + if p == v.Name() { + found = true + break + } + } + if !found { + d.removeProcesses([]string{p}, log) + log.Debugw("Removed redundant mongos process", "name", p) + } + } + // Making sure changes to existing mongos processes are propagated to new ones + if cntMongosProcesses := len(d.getMongosProcessesNames(opts.Name)); cntMongosProcesses > 0 && cntMongosProcesses < len(opts.MongosProcesses) { + if err := d.copyFirstProcessToNewPositions(opts.MongosProcesses, cntMongosProcesses, log); err != nil { + // I guess this error is not so serious to fail the whole process - mongoses will be scaled up anyway + log.Error("Failed to copy first mongos process (so new mongos processes may miss Ops Manager changes done to "+ + "existing mongos processes): %w", err) + } + } + + // Then merging mongos processes with existing ones + for _, p := range opts.MongosProcesses { + if p.ProcessType() != ProcessTypeMongos { + return xerrors.New(`all mongos processes must have processType="mongos"`) + } + p.setCluster(opts.Name) + d.MergeStandalone(p, opts.MongosAdditionalOptionsDesired, opts.MongosAdditionalOptionsPrev, log) + } + return nil +} + +func (d Deployment) getMongosProcessesNames(clusterName string) []string { + processNames := make([]string, 0) + for _, p := range d.getProcesses() { + if p.ProcessType() == ProcessTypeMongos && p.cluster() == clusterName { + processNames = append(processNames, p.Name()) + } + } + return processNames +} + +func (d Deployment) mergeConfigReplicaSet(opts DeploymentShardedClusterMergeOptions, l *zap.SugaredLogger) { + for _, p := range opts.ConfigServerRs.Processes { + p.setClusterRoleConfigSrv() + } + + d.MergeReplicaSet(opts.ConfigServerRs, opts.ConfigServerAdditionalOptionsDesired, opts.ConfigServerAdditionalOptionsPrev, l) +} + +// mergeShards does merge of replicasets for shards (which in turn merge each process) and merge or add the sharded cluster +// element as well +func (d Deployment) mergeShards(opts DeploymentShardedClusterMergeOptions, log *zap.SugaredLogger) bool { + // First merging the individual replica sets for each shard + for _, v := range opts.Shards { + d.MergeReplicaSet(v, opts.ShardAdditionalOptionsDesired, opts.ShardAdditionalOptionsPrev, log) + } + cluster := NewShardedCluster(opts.Name, opts.ConfigServerRs.Rs.Name(), opts.Shards) + + // Merging "sharding" json value + for _, s := range d.getShardedClusters() { + if s.Name() == opts.Name { + s.mergeFrom(cluster) + log.Debug("Merged sharded cluster into existing one") + + return d.handleShardsRemoval(opts.Finalizing, s, log) + } + } + // Adding the new sharded cluster + d.addShardedCluster(cluster) + log.Debug("Added sharded cluster as current OM deployment didn't have it") + return false +} + +// handleShardsRemoval is a complicated method handling different scenarios. +// - 'draining' array is empty and no extra shards were found in OM which should be removed - return +// - if 'finalizing' == false - this means that this is the 1st phase of the process - when the shards are due to be removed +// or have already been removed and their replica sets are added/already sit in the 'draining' array. Note, that this +// method can be called many times while in the 1st phase and 'draining' array is not empty - this means that the agent +// is performing the shards rebalancing +// - if 'finalizing' == true - this means that this is the 2nd phase of the process - when the shards were removed +// from the sharded cluster and their data was rebalanced to the rest of the shards. Now we can remove the replica sets +// and their processes and clean the 'draining' array. +func (d Deployment) handleShardsRemoval(finalizing bool, s ShardedCluster, log *zap.SugaredLogger) bool { + junkReplicaSets := d.findReplicaSetsRemovedFromShardedCluster(s.Name()) + + if len(junkReplicaSets) == 0 { + return false + } + + if !finalizing { + if len(junkReplicaSets) > 0 { + s.addToDraining(junkReplicaSets) + } + log.Infof("The following shards are scheduled for removal: %s", s.draining()) + return true + } else if len(junkReplicaSets) > 0 { + // Cleaning replica sets which used to be shards in past iterations. + s.removeDraining() + d.removeReplicaSets(junkReplicaSets, log) + log.Debugw("Removed replica sets as they were removed from sharded cluster", "replica sets", junkReplicaSets) + } + return false +} + +// GetAllProcessNames returns a list of names of processes in this deployment. This is, the names of all processes +// in the `processes` attribute of the deployment object. +func (d Deployment) GetAllProcessNames() (names []string) { + for _, p := range d.getProcesses() { + names = append(names, p.Name()) + } + return +} + +func (d Deployment) getProcesses() []Process { + if _, ok := d["processes"]; !ok { + return []Process{} + } + switch v := d["processes"].(type) { + case []Process: + return v + case []interface{}: + // seems we cannot directly cast the array of interfaces to array of Processes - have to manually copy references + ans := make([]Process, len(v)) + for i, val := range v { + ans[i] = NewProcessFromInterface(val) + } + return ans + default: + panic(fmt.Sprintf("Unexpected type of processes variable: %T", v)) + } +} + +func (d Deployment) getProcessesHostNames(names []string) []string { + ans := make([]string, len(names)) + + for i, n := range names { + if p := d.getProcessByName(n); p != nil { + ans[i] = p.HostName() + } + } + return ans +} + +func (d Deployment) setProcesses(processes []Process) { + d["processes"] = processes +} + +func (d Deployment) addProcess(p Process) { + d.setProcesses(append(d.getProcesses(), p)) +} + +func (d Deployment) removeProcesses(processNames []string, log *zap.SugaredLogger) { + // (CLOUDP-37709) implementation ideas: we remove agents for the processes if they are removed. Note, that + // processes removal happens also during merge operations - so hypothetically if OM added some processes that were + // removed by the Operator on merge - the agents will be removed from config as well. Seems this is quite safe and + // in the Operator-managed environment we'll never get the situation when some agents reside on the hosts which are + // not some processes. + d.RemoveMonitoringAndBackup(processNames, log) + + processes := make([]Process, 0) + + for _, p := range d.getProcesses() { + found := false + for _, p2 := range processNames { + if p.Name() == p2 { + found = true + break + } + } + if !found { + processes = append(processes, p) + } + } + + d.setProcesses(processes) +} + +func (d Deployment) removeReplicaSets(replicaSets []string, log *zap.SugaredLogger) { + for _, v := range replicaSets { + _ = d.RemoveReplicaSetByName(v, log) + } +} + +func (d Deployment) getProcessByName(name string) *Process { + for _, p := range d.getProcesses() { + if p.Name() == name { + return &p + } + } + + return nil +} + +func (d Deployment) getReplicaSetByName(name string) *ReplicaSet { + for _, r := range d.getReplicaSets() { + if r.Name() == name { + return &r + } + } + + return nil +} + +func (d Deployment) getShardedClusterByName(name string) *ShardedCluster { + for _, s := range d.getShardedClusters() { + if s.Name() == name { + return &s + } + } + + return nil +} + +func (d Deployment) getReplicaSets() []ReplicaSet { + switch v := d["replicaSets"].(type) { + case []ReplicaSet: + return v + case []interface{}: + ans := make([]ReplicaSet, len(v)) + for i, val := range v { + ans[i] = NewReplicaSetFromInterface(val) + } + return ans + default: + panic(fmt.Sprintf("Unexpected type of replicasets variable: %T", v)) + } +} + +func (d Deployment) setReplicaSets(replicaSets []ReplicaSet) { + d["replicaSets"] = replicaSets +} + +func (d Deployment) addReplicaSet(rs ReplicaSet) { + d.setReplicaSets(append(d.getReplicaSets(), rs)) +} + +func (d Deployment) getShardedClusters() []ShardedCluster { + switch v := d["sharding"].(type) { + case []ShardedCluster: + return v + case []interface{}: + ans := make([]ShardedCluster, len(v)) + for i, val := range v { + ans[i] = NewShardedClusterFromInterface(val) + } + return ans + default: + panic(fmt.Sprintf("Unexpected type of sharding variable: %T", v)) + } +} + +func (d Deployment) setShardedClusters(shardedClusters []ShardedCluster) { + d["sharding"] = shardedClusters +} + +func (d Deployment) addShardedCluster(shardedCluster ShardedCluster) { + d.setShardedClusters(append(d.getShardedClusters(), shardedCluster)) +} + +func (d Deployment) getMonitoringVersions() []interface{} { + return d["monitoringVersions"].([]interface{}) +} + +func (d Deployment) getBackupVersions() []interface{} { + return d["backupVersions"].([]interface{}) +} + +func (d Deployment) setMonitoringVersions(monitoring []interface{}) { + d["monitoringVersions"] = monitoring +} + +func (d Deployment) setBackupVersions(backup []interface{}) { + d["backupVersions"] = backup +} + +func (d Deployment) getTLS() map[string]interface{} { + return util.ReadOrCreateMap(d, "tls") +} + +// findReplicaSetsRemovedFromShardedCluster finds all replica sets which look like shards that have been removed from +// the sharded cluster. +// To make this method work correctly the shards MUST have the same prefix as a shard (which is true for the +// Operator-created resource) +func (d Deployment) findReplicaSetsRemovedFromShardedCluster(clusterName string) []string { + shardedCluster := d.getShardedClusterByName(clusterName) + clusterReplicaSets := shardedCluster.getAllReplicaSets() + var ans []string + + for _, v := range d.getReplicaSets() { + if !stringutil.Contains(clusterReplicaSets, v.Name()) && isShardOfShardedCluster(clusterName, v.Name()) { + ans = append(ans, v.Name()) + } + } + return ans +} + +func isShardOfShardedCluster(clusterName, rsName string) bool { + return regexp.MustCompile(`^` + clusterName + `-[0-9]+$`).MatchString(rsName) +} + +// removeMonitoring removes the monitoring agent configuration that match any of processes hosts 'processNames' parameter +// Note, that by contract there will be only one monitoring agent, but the method tries to be maximum safe and clean +// all matches (maybe someone "hacked" the automation config manually and added the monitoring agents there) +// Note 2: it's ok if nothing was removed as the processes in the array may be from replica set from sharded cluster +// which doesn't have a monitoring agents (one monitoring agent per cluster) +func (d Deployment) removeMonitoring(processNames []string) { + monitoringVersions := d.getMonitoringVersions() + updatedMonitoringVersions := make([]interface{}, 0) + hostNames := d.getProcessesHostNames(processNames) + for _, m := range monitoringVersions { + monitoring := m.(map[string]interface{}) + hostname := monitoring["hostname"].(string) + if !stringutil.Contains(hostNames, hostname) { + updatedMonitoringVersions = append(updatedMonitoringVersions, m) + } else { + hostNames = stringutil.Remove(hostNames, hostname) + } + } + + d.setMonitoringVersions(updatedMonitoringVersions) +} + +// addBackup adds backup agent configuration for each of the processes of deployment +func (d Deployment) addBackup(log *zap.SugaredLogger) { + backupVersions := d.getBackupVersions() + for _, p := range d.getProcesses() { + found := false + var backupVersion map[string]interface{} + for _, b := range backupVersions { + backupVersion = b.(map[string]interface{}) + if backupVersion["hostname"] == p.HostName() { + found = true + break + } + } + + if !found { + backupVersion = map[string]interface{}{"hostname": p.HostName(), "name": BackupAgentDefaultVersion} + backupVersions = append(backupVersions, backupVersion) + log.Debugw("Added backup agent configuration", "host", p.HostName()) + } + } + d.setBackupVersions(backupVersions) +} + +// removeBackup removes the backup versions from Deployment that are in 'hosts' array parameter +func (d Deployment) removeBackup(processNames []string, log *zap.SugaredLogger) { + backupVersions := d.getBackupVersions() + updatedBackupVersions := make([]interface{}, 0) + initialLength := len(processNames) + hostNames := d.getProcessesHostNames(processNames) + for _, b := range backupVersions { + backup := b.(map[string]interface{}) + hostname := backup["hostname"].(string) + if !stringutil.Contains(hostNames, hostname) { + updatedBackupVersions = append(updatedBackupVersions, b) + } else { + hostNames = stringutil.Remove(hostNames, hostname) + } + } + + if len(hostNames) != 0 { + // Note, that we don't error/warn here as there can be plenty of reasons why the config is not here (e.g. some + // process added to OM deployment manually that doesn't have corresponding backup config). Warn prints the + // stacktrace which looks quite scary + log.Infof("The following hosts were not removed from backup config as they were not found: %s", hostNames) + } else { + log.Debugf("Removed backup agent configuration for %d host(s)", initialLength) + } + d.setBackupVersions(updatedBackupVersions) +} + +// copyFirstProcessToNewPositions is used when scaling up replica set / set of mongos processes. Its main goal is to clone +// the sample existing deployment process as many times as the number of new processes to be added. The copies get +// the names of the "new" processes so that the following "mergeStandalone" operation could merge "Operator owned" information +// back into copies. This allows to keep all changes made by OM to existing processes and overwrite only the fields that +// Operator is responsible for. +// So if current RS deployment that came from OM has processes A, B, C and operator wants to scale up on 2 more members +// (meaning it wants to add X, Y processes) - then in the end of this function deployment will contain processes A, B, C, +// and X, Y where X, Y will be complete copies of A instead of names and aliases. +// "processes" is the array of "Operator view" processes (so for the example above they will be "A, B, C, X, Y") +// "idxOfFirstNewMember" is the index of the first NEW member. So for the example above it will be 3 +func (d Deployment) copyFirstProcessToNewPositions(processes []Process, idxOfFirstNewMember int, log *zap.SugaredLogger) error { + newProcesses := processes[idxOfFirstNewMember:] + + var sampleProcess Process + + // The sample process must be the one that exist in OM deployment - so if for some reason OM added some + // processes to Deployment (and they won't get into merged deployment) - we must find the first one matching + // As an example let's consider the RS that contained processes A, B, C and then OM UI removed the processes A and B + // So the "processes" array (which is Kubernetes view of RS) will still contain A, B, C, … so in the end the sample + // process will be C (as this is the only process that intersects in the "OM view" and "Kubernetes view" and it will + // get into final deployment + for _, v := range processes { + if d.getProcessByName(v.Name()) != nil { + sampleProcess = *d.getProcessByName(v.Name()) + break + } + } + // If sample process has not been found - that means that all processes in OM deployment are some fake - we'll remove + // them anyway and there is no need in merging + // Example: OM UI removed A, B, C and added P, T, R, but Kubernetes will still try to create the RS of A, B, C - and + // will remove faked processes in the end. So no OM "sample" would exist in this case as all processes will be brand + // new + if sampleProcess == nil { + return nil + } + + for _, p := range newProcesses { + sampleProcessCopy, err := sampleProcess.DeepCopy() + if err != nil { + return xerrors.Errorf("failed to make a copy of Process %s: %w", sampleProcess.Name(), err) + } + sampleProcessCopy.setName(p.Name()) + + // add here other attributes that mustn't be copied (maybe some others should be added here) + delete(sampleProcessCopy, "alias") + + // This is just fool protection - if for some reason the process already exists in deployment (it mustn't actually) + // then we don't add the copy of sample one + if d.getProcessByName(p.Name()) == nil { + d.addProcess(sampleProcessCopy) + log.Debugw("Added the copy of the process to the end of deployment processes", "process name", + sampleProcess.Name(), "new process name", sampleProcessCopy.Name()) + } + } + return nil +} + +func (d Deployment) processesHaveInternalClusterAuthentication(processes []Process) bool { + for _, p := range processes { + if p.HasInternalClusterAuthentication() { + return true + } + } + return false +} + +// limitVotingMembers ensures the number of voting members in the replica set is not more than 7 members +func (d Deployment) limitVotingMembers(rsName string) { + r := d.getReplicaSetByName(rsName) + + numberOfVotingMembers := 0 + for _, v := range r.Members() { + if v.Votes() > 0 { + numberOfVotingMembers++ + } + if numberOfVotingMembers > 7 { + v.setVotes(0).setPriority(0) + } + } +} + +func (d Deployment) deepCopy() Deployment { + var depCopy Deployment + + depCopy, err := util.MapDeepCopy(d) + if err != nil { + panic(err) + } + return depCopy +} diff --git a/controllers/om/deployment/om_deployment.go b/controllers/om/deployment/om_deployment.go new file mode 100644 index 000000000..845ec3d59 --- /dev/null +++ b/controllers/om/deployment/om_deployment.go @@ -0,0 +1,10 @@ +package deployment + +import ( + "fmt" +) + +// Link returns the deployment link given the baseUrl and groupId. +func Link(url, groupId string) string { + return fmt.Sprintf("%s/v2/%s", url, groupId) +} diff --git a/controllers/om/deployment/om_deployment_test.go b/controllers/om/deployment/om_deployment_test.go new file mode 100644 index 000000000..78619e408 --- /dev/null +++ b/controllers/om/deployment/om_deployment_test.go @@ -0,0 +1,40 @@ +package deployment + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/replicaset" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) + mock.InitDefaultEnvVariables() +} + +// TestPrepareScaleDown_OpsManagerRemovedMember tests the situation when during scale down some replica set member doesn't +// exist (this can happen when for example the member was removed from Ops Manager manually). The exception is handled +// and only the existing member is marked as unvoted +func TestPrepareScaleDown_OpsManagerRemovedMember(t *testing.T) { + // This is deployment with 2 members (emulating that OpsManager removed the 3rd one) + rs := mdbv1.NewReplicaSetBuilder().SetName("bam").SetMembers(2).Build() + oldDeployment := CreateFromReplicaSet("fake-mongoDBImage", false, rs) + mockedOmConnection := om.NewMockedOmConnection(oldDeployment) + + // We try to prepare two members for scale down, but one of them will fail (bam-2) + rsWithThreeMembers := map[string][]string{"bam": {"bam-1", "bam-2"}} + assert.NoError(t, replicaset.PrepareScaleDownFromMap(mockedOmConnection, rsWithThreeMembers, rsWithThreeMembers["bam"], zap.S())) + + expectedDeployment := CreateFromReplicaSet("fake-mongoDBImage", false, rs) + + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted("bam", []string{"bam-1"})) + + mockedOmConnection.CheckNumberOfUpdateRequests(t, 1) + mockedOmConnection.CheckDeployment(t, expectedDeployment) +} diff --git a/controllers/om/deployment/testing_utils.go b/controllers/om/deployment/testing_utils.go new file mode 100644 index 000000000..c77600257 --- /dev/null +++ b/controllers/om/deployment/testing_utils.go @@ -0,0 +1,43 @@ +package deployment + +import ( + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/replicaset" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +// CreateFromReplicaSet builds the replica set for the automation config +// based on the given MongoDB replica set. +// NOTE: This method is only used for testing. +// But we can't move in a *_test file since it is called from tests in +// different packages. And test files are only compiled +// when testing that specific package +// https://github.com/golang/go/issues/10184#issuecomment-84465873 +func CreateFromReplicaSet(mongoDBImage string, forceEnterprise bool, rs *mdb.MongoDB) om.Deployment { + sts := construct.DatabaseStatefulSet(*rs, construct.ReplicaSetOptions( + func(options *construct.DatabaseStatefulSetOptions) { + options.PodVars = &env.PodEnvVars{ProjectID: "abcd"} + }, + ), zap.S()) + d := om.NewDeployment() + + lastConfig, err := rs.GetLastAdditionalMongodConfigByType(mdb.ReplicaSetConfig) + if err != nil { + panic(err) + } + + d.MergeReplicaSet( + replicaset.BuildFromStatefulSet(mongoDBImage, forceEnterprise, sts, rs.GetSpec(), rs.Status.FeatureCompatibilityVersion), + rs.Spec.AdditionalMongodConfig.ToMap(), + lastConfig.ToMap(), + zap.S(), + ) + d.AddMonitoringAndBackup(zap.S(), rs.Spec.GetSecurity().IsTLSEnabled(), util.CAFilePathInContainer) + d.ConfigureTLS(rs.Spec.GetSecurity(), util.CAFilePathInContainer) + return d +} diff --git a/controllers/om/deployment_test.go b/controllers/om/deployment_test.go new file mode 100644 index 000000000..569eda9e6 --- /dev/null +++ b/controllers/om/deployment_test.go @@ -0,0 +1,831 @@ +package om + +import ( + "fmt" + "os" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +// First time merge adds the new standalone +// second invocation doesn't add new node as the existing standalone is found (by name) and the data is merged +func TestMergeStandalone(t *testing.T) { + d := NewDeployment() + mergeStandalone(d, createStandalone()) + + assert.Len(t, d.getProcesses(), 1) + + d["version"] = 5 + d.getProcesses()[0]["alias"] = "alias" + d.getProcesses()[0]["hostname"] = "foo" + d.getProcesses()[0]["authSchemaVersion"] = 10 + d.getProcesses()[0]["featureCompatibilityVersion"] = "bla" + + mergeStandalone(d, createStandalone()) + + assert.Len(t, d.getProcesses(), 1) + + expected := createStandalone() + + // fields which are not owned by OM-Kube should be left unchanged + assert.Equal(t, d["version"], 5) + expected["alias"] = "alias" + + assert.Equal(t, &expected, d.getProcessByName(expected.Name())) +} + +// First merge results in just adding the ReplicaSet +// Second merge performs real merge operation +func TestMergeReplicaSet(t *testing.T) { + d := NewDeployment() + mergeReplicaSet(d, "fooRs", createReplicaSetProcesses("fooRs")) + expectedRs := buildRsByProcesses("fooRs", createReplicaSetProcesses("fooRs")) + + assert.Len(t, d.getProcesses(), 3) + assert.Len(t, d.getReplicaSets(), 1) + assert.Len(t, d.getReplicaSets()[0].Members(), 3) + assert.Equal(t, d.getReplicaSets()[0], expectedRs.Rs) + + // Now the deployment "gets updated" from external - new node is added and one is removed - this should be fixed + // by merge + newProcess := NewMongodProcess("foo", "bar", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, &mdbv1.NewStandaloneBuilder().Build().Spec, "", nil, "") + + d.getProcesses()[0]["processType"] = ProcessTypeMongos // this will be overriden + d.getProcesses()[1].EnsureNetConfig()["MaxIncomingConnections"] = 20 // this will be left as-is + d.getReplicaSets()[0]["protocolVersion"] = 10 // this field will be overriden by Operator + d.getReplicaSets()[0].setMembers(d.getReplicaSets()[0].Members()[0:2]) // "removing" the last node in replicaset + d.getReplicaSets()[0].addMember(newProcess, "", automationconfig.MemberOptions{}) // "adding" some new node + d.getReplicaSets()[0].Members()[0]["arbiterOnly"] = true // changing data for first node + + mergeReplicaSet(d, "fooRs", createReplicaSetProcesses("fooRs")) + + assert.Len(t, d.getProcesses(), 3) + assert.Len(t, d.getReplicaSets(), 1) + + expectedRs = buildRsByProcesses("fooRs", createReplicaSetProcesses("fooRs")) + expectedRs.Rs.Members()[0]["arbiterOnly"] = true + expectedRs.Processes[1].EnsureNetConfig()["MaxIncomingConnections"] = 20 + + checkReplicaSet(t, d, expectedRs) +} + +// Checking that on scale down the old processes are removed +func TestMergeReplica_ScaleDown(t *testing.T) { + d := NewDeployment() + + mergeReplicaSet(d, "someRs", createReplicaSetProcesses("someRs")) + assert.Len(t, d.getProcesses(), 3) + assert.Len(t, d.getReplicaSets()[0].Members(), 3) + + // "scale down" + scaledDownRsProcesses := createReplicaSetProcesses("someRs")[0:2] + mergeReplicaSet(d, "someRs", scaledDownRsProcesses) + + assert.Len(t, d.getProcesses(), 2) + assert.Len(t, d.getReplicaSets()[0].Members(), 2) + + // checking that the last member was removed + rsProcesses := buildRsByProcesses("someRs", createReplicaSetProcesses("someRs")).Processes + assert.Contains(t, d.getProcesses(), rsProcesses[0]) + assert.Contains(t, d.getProcesses(), rsProcesses[1]) + assert.NotContains(t, d.getProcesses(), rsProcesses[2]) +} + +// TestMergeReplicaSet_MergeFirstProcess checks that if the replica set is scaled up - then all OM changes to existing +// processes (more precisely - to the first member) are copied to new members +func TestMergeReplicaSet_MergeFirstProcess(t *testing.T) { + d := NewDeployment() + + mergeReplicaSet(d, "fooRs", createReplicaSetProcesses("fooRs")) + mergeReplicaSet(d, "anotherRs", createReplicaSetProcesses("anotherRs")) + + // Now the first process (and usually all others in practice) are changed by OM + d.getProcesses()[0].EnsureNetConfig()["MaxIncomingConnections"] = 20 + d.getProcesses()[0]["backupRestoreUrl"] = "http://localhost:7890" + d.getProcesses()[0]["logRotate"] = map[string]int{"sizeThresholdMB": 3000, "timeThresholdHrs": 12} + d.getProcesses()[0]["kerberos"] = map[string]string{"keytab": "123456"} + + // Now we merged the scaled up RS + mergeReplicaSet(d, "fooRs", createReplicaSetProcessesCount(5, "fooRs")) + + assert.Len(t, d.getProcesses(), 8) + assert.Len(t, d.getReplicaSets(), 2) + + expectedRs := buildRsByProcesses("fooRs", createReplicaSetProcessesCount(5, "fooRs")) + + // Verifying that the first process was merged with new ones + for _, i := range []int{0, 3, 4} { + expectedRs.Processes[i].EnsureNetConfig()["MaxIncomingConnections"] = 20 + expectedRs.Processes[i]["backupRestoreUrl"] = "http://localhost:7890" + expectedRs.Processes[i]["logRotate"] = map[string]int{"sizeThresholdMB": 3000, "timeThresholdHrs": 12} + expectedRs.Processes[i]["kerberos"] = map[string]string{"keytab": "123456"} + } + + // The other replica set must be the same + checkReplicaSet(t, d, buildRsByProcesses("anotherRs", createReplicaSetProcesses("anotherRs"))) + checkReplicaSet(t, d, expectedRs) +} + +func TestConfigureSSL_Deployment(t *testing.T) { + d := Deployment{} + d.ConfigureTLS(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: true}}, util.CAFilePathInContainer) + expectedSSLConfig := map[string]interface{}{ + "CAFilePath": "/mongodb-automation/ca.pem", + } + assert.Equal(t, expectedSSLConfig, d["tls"].(map[string]interface{})) + + d.ConfigureTLS(&mdbv1.Security{}, util.CAFilePathInContainer) + assert.Equal(t, d["tls"], map[string]any{"clientCertificateMode": string(automationconfig.ClientCertificateModeOptional)}) +} + +func TestTLSConfigurationWillBeDisabled(t *testing.T) { + d := Deployment{} + d.ConfigureTLS(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: false}}, util.CAFilePathInContainer) + + assert.False(t, d.TLSConfigurationWillBeDisabled(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: false}})) + assert.False(t, d.TLSConfigurationWillBeDisabled(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: true}})) + + d = Deployment{} + d.ConfigureTLS(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: true}}, util.CAFilePathInContainer) + + assert.False(t, d.TLSConfigurationWillBeDisabled(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: true}})) + assert.True(t, d.TLSConfigurationWillBeDisabled(&mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{Enabled: false}})) +} + +// TestMergeDeployment_BigReplicaset ensures that adding a big replica set (> 7 members) works correctly and no more than +// 7 voting members are added +func TestMergeDeployment_BigReplicaset(t *testing.T) { + omDeployment := NewDeployment() + rs := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(8, "my-rs")) + checkNumberOfVotingMembers(t, rs, 8, 8) + + omDeployment.MergeReplicaSet(rs, nil, nil, zap.S()) + checkNumberOfVotingMembers(t, rs, 7, 8) + + // Now OM user "has changed" votes for some of the members - this must stay the same after merge + omDeployment.getReplicaSets()[0].Members()[2].setVotes(0).setPriority(0) + omDeployment.getReplicaSets()[0].Members()[4].setVotes(0).setPriority(0) + + omDeployment.MergeReplicaSet(rs, nil, nil, zap.S()) + checkNumberOfVotingMembers(t, rs, 5, 8) + + // Now operator scales up by one - the "OM votes" should not suffer, but total number of votes will increase by one + rsToMerge := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(9, "my-rs")) + rsToMerge.Rs.Members()[2].setVotes(0).setPriority(0) + rsToMerge.Rs.Members()[4].setVotes(0).setPriority(0) + rsToMerge.Rs.Members()[7].setVotes(0).setPriority(0) + omDeployment.MergeReplicaSet(rsToMerge, nil, nil, zap.S()) + checkNumberOfVotingMembers(t, rs, 6, 9) + + // Now operator scales up by two - the "OM votes" should not suffer, but total number of votes will increase by one + // only as 7 is the upper limit + rsToMerge = buildRsByProcesses("my-rs", createReplicaSetProcessesCount(11, "my-rs")) + rsToMerge.Rs.Members()[2].setVotes(0).setPriority(0) + rsToMerge.Rs.Members()[4].setVotes(0).setPriority(0) + + omDeployment.MergeReplicaSet(rsToMerge, nil, nil, zap.S()) + checkNumberOfVotingMembers(t, rs, 7, 11) + assert.Equal(t, 0, omDeployment.getReplicaSets()[0].Members()[2].Votes()) + assert.Equal(t, 0, omDeployment.getReplicaSets()[0].Members()[4].Votes()) + assert.Equal(t, float32(0), omDeployment.getReplicaSets()[0].Members()[2].Priority()) + assert.Equal(t, float32(0), omDeployment.getReplicaSets()[0].Members()[4].Priority()) +} + +func TestGetAllProcessNames_MergedReplicaSetsAndShardedClusters(t *testing.T) { + d := NewDeployment() + rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs")) + + d.MergeReplicaSet(rs0, nil, nil, zap.S()) + assert.Equal(t, []string{"my-rs-0", "my-rs-1", "my-rs-2"}, d.GetAllProcessNames()) + + rs1 := buildRsByProcesses("another-rs", createReplicaSetProcessesCount(5, "another-rs")) + d.MergeReplicaSet(rs1, nil, nil, zap.S()) + + assert.Equal( + t, + []string{ + "my-rs-0", "my-rs-1", "my-rs-2", + "another-rs-0", "another-rs-1", "another-rs-2", "another-rs-3", "another-rs-4", + }, + d.GetAllProcessNames()) + + configRs := createConfigSrvRs("configSrv", false) + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + assert.Equal( + t, + []string{ + "my-rs-0", "my-rs-1", "my-rs-2", + "another-rs-0", "another-rs-1", "another-rs-2", "another-rs-3", "another-rs-4", + "pretty0", "pretty1", "pretty2", + "configSrv-0", "configSrv-1", "configSrv-2", + "myShard-0-0", "myShard-0-1", "myShard-0-2", + "myShard-1-0", "myShard-1-1", "myShard-1-2", + "myShard-2-0", "myShard-2-1", "myShard-2-2", + }, + d.GetAllProcessNames()) +} + +func TestGetAllProcessNames_MergedShardedClusters(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + assert.Equal( + t, + []string{ + "pretty0", "pretty1", "pretty2", + "configSrv-0", "configSrv-1", "configSrv-2", + "myShard-0-0", "myShard-0-1", "myShard-0-2", + "myShard-1-0", "myShard-1-1", "myShard-1-2", + "myShard-2-0", "myShard-2-1", "myShard-2-2", + }, + d.GetAllProcessNames(), + ) + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "anotherCluster", + MongosProcesses: createMongosProcesses(3, "anotherMongos", ""), + ConfigServerRs: configRs, + Shards: createShards("anotherClusterSh"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + assert.Equal( + t, + []string{ + "pretty0", "pretty1", "pretty2", + "configSrv-0", "configSrv-1", "configSrv-2", + "myShard-0-0", "myShard-0-1", "myShard-0-2", + "myShard-1-0", "myShard-1-1", "myShard-1-2", + "myShard-2-0", "myShard-2-1", "myShard-2-2", + "anotherMongos0", "anotherMongos1", "anotherMongos2", + "anotherClusterSh-0-0", "anotherClusterSh-0-1", "anotherClusterSh-0-2", + "anotherClusterSh-1-0", "anotherClusterSh-1-1", "anotherClusterSh-1-2", + "anotherClusterSh-2-0", "anotherClusterSh-2-1", "anotherClusterSh-2-2", + }, + d.GetAllProcessNames(), + ) +} + +func TestDeploymentCountIsCorrect(t *testing.T) { + d := NewDeployment() + + rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs")) + d.MergeReplicaSet(rs0, nil, nil, zap.S()) + + excessProcesses := d.GetNumberOfExcessProcesses("my-rs") + // There's only one resource in this deployment + assert.Equal(t, 0, excessProcesses) + + rs1 := buildRsByProcesses("my-rs-second", createReplicaSetProcessesCount(3, "my-rs-second")) + d.MergeReplicaSet(rs1, nil, nil, zap.S()) + excessProcesses = d.GetNumberOfExcessProcesses("my-rs") + + // another replica set was added to the deployment. 3 processes do not belong to this one + assert.Equal(t, 3, excessProcesses) + + configRs := createConfigSrvRs("config", false) + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "sc001", + MongosProcesses: createMongosProcesses(3, "mongos", ""), + ConfigServerRs: configRs, + Shards: createShards("sc001"), + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + excessProcesses = d.GetNumberOfExcessProcesses("my-rs") + + // a Sharded Cluster was added, plenty of processes do not belong to "my-rs" anymore + assert.Equal(t, 18, excessProcesses) + + // This unknown process does not belong in here + excessProcesses = d.GetNumberOfExcessProcesses("some-unknown-name") + + // a Sharded Cluster was added, plenty of processes do not belong to "my-rs" anymore + assert.Equal(t, 21, excessProcesses) + + excessProcesses = d.GetNumberOfExcessProcesses("sc001") + // There are 6 processes that do not belong to the sc001 sharded cluster + assert.Equal(t, 6, excessProcesses) +} + +func TestGetNumberOfExcessProcesses_ShardedClusterScaleDown(t *testing.T) { + d := NewDeployment() + configRs := createConfigSrvRs("config", false) + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "sc001", + MongosProcesses: createMongosProcesses(3, "mongos", ""), + ConfigServerRs: configRs, + Shards: createShards("sc001"), + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + assert.Len(t, d.getShardedClusterByName("sc001").shards(), 3) + assert.Len(t, d.getReplicaSets(), 4) + assert.Equal(t, 0, d.GetNumberOfExcessProcesses("sc001")) + + // Now we are "scaling down" the sharded cluster - so junk replica sets will appear - this is still ok + twoShards := createShards("sc001")[0:2] + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "sc001", + MongosProcesses: createMongosProcesses(3, "mongos", ""), + ConfigServerRs: configRs, + Shards: twoShards, + Finalizing: false, + } + + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + assert.Len(t, d.getShardedClusterByName("sc001").shards(), 2) + assert.Len(t, d.getReplicaSets(), 4) + + assert.Equal(t, 0, d.GetNumberOfExcessProcesses("sc001")) +} + +func TestIsShardOf(t *testing.T) { + clusterName := "my-shard" + assert.True(t, isShardOfShardedCluster(clusterName, "my-shard-0")) + assert.True(t, isShardOfShardedCluster(clusterName, "my-shard-3")) + assert.True(t, isShardOfShardedCluster(clusterName, "my-shard-9")) + assert.True(t, isShardOfShardedCluster(clusterName, "my-shard-10")) + assert.True(t, isShardOfShardedCluster(clusterName, "my-shard-23")) + assert.True(t, isShardOfShardedCluster(clusterName, "my-shard-452")) + + assert.False(t, isShardOfShardedCluster(clusterName, "my-shard")) + assert.False(t, isShardOfShardedCluster(clusterName, "my-my-shard")) + assert.False(t, isShardOfShardedCluster(clusterName, "my-shard-s")) + assert.False(t, isShardOfShardedCluster(clusterName, "my-shard-1-0")) + assert.False(t, isShardOfShardedCluster(clusterName, "mmy-shard-1")) + assert.False(t, isShardOfShardedCluster(clusterName, "my-shard-1s")) +} + +func TestProcessBelongsToReplicaSet(t *testing.T) { + d := NewDeployment() + rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs")) + d.MergeReplicaSet(rs0, nil, nil, zap.S()) + + assert.True(t, d.ProcessBelongsToResource("my-rs-0", "my-rs")) + assert.True(t, d.ProcessBelongsToResource("my-rs-1", "my-rs")) + assert.True(t, d.ProcessBelongsToResource("my-rs-2", "my-rs")) + + // Process does not belong if resource does not exist + assert.False(t, d.ProcessBelongsToResource("unknown-0", "unknown")) +} + +func TestProcessBelongsToShardedCluster(t *testing.T) { + d := NewDeployment() + configRs := createConfigSrvRs("config", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "sh001", + MongosProcesses: createMongosProcesses(3, "mongos", ""), + ConfigServerRs: configRs, + Shards: createShards("shards"), + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // Config Servers + assert.True(t, d.ProcessBelongsToResource("config-0", "sh001")) + assert.True(t, d.ProcessBelongsToResource("config-1", "sh001")) + assert.True(t, d.ProcessBelongsToResource("config-2", "sh001")) + + // Does not belong! + assert.False(t, d.ProcessBelongsToResource("config-3", "sh001")) + + // Mongos + assert.True(t, d.ProcessBelongsToResource("mongos0", "sh001")) + assert.True(t, d.ProcessBelongsToResource("mongos1", "sh001")) + assert.True(t, d.ProcessBelongsToResource("mongos2", "sh001")) + // Does not belong! + assert.False(t, d.ProcessBelongsToResource("mongos3", "sh001")) + + // Shard members + assert.True(t, d.ProcessBelongsToResource("shards-0-0", "sh001")) + assert.True(t, d.ProcessBelongsToResource("shards-0-1", "sh001")) + assert.True(t, d.ProcessBelongsToResource("shards-0-2", "sh001")) + + // Does not belong! + assert.False(t, d.ProcessBelongsToResource("shards-0-3", "sh001")) + // Shard members + assert.True(t, d.ProcessBelongsToResource("shards-1-0", "sh001")) + assert.True(t, d.ProcessBelongsToResource("shards-1-1", "sh001")) + assert.True(t, d.ProcessBelongsToResource("shards-1-2", "sh001")) + + // Does not belong! + assert.False(t, d.ProcessBelongsToResource("shards-1-3", "sh001")) + // Shard members + assert.True(t, d.ProcessBelongsToResource("shards-2-0", "sh001")) + assert.True(t, d.ProcessBelongsToResource("shards-2-1", "sh001")) + assert.True(t, d.ProcessBelongsToResource("shards-2-2", "sh001")) + + // Does not belong! + assert.False(t, d.ProcessBelongsToResource("shards-2-3", "sh001")) +} + +func TestDeploymentMinimumMajorVersion(t *testing.T) { + d0 := NewDeployment() + rs0Processes := createReplicaSetProcessesCount(3, "my-rs") + rs0 := buildRsByProcesses("my-rs", rs0Processes) + d0.MergeReplicaSet(rs0, nil, nil, zap.S()) + + assert.Equal(t, uint64(3), d0.MinimumMajorVersion()) + + d1 := NewDeployment() + rs1Processes := createReplicaSetProcessesCount(3, "my-rs") + rs1Processes[0]["featureCompatibilityVersion"] = "2.4" + rs1 := buildRsByProcesses("my-rs", rs1Processes) + d1.MergeReplicaSet(rs1, nil, nil, zap.S()) + + assert.Equal(t, uint64(2), d1.MinimumMajorVersion()) + + d2 := NewDeployment() + rs2Processes := createReplicaSetProcessesCountEnt(3, "my-rs") + rs2 := buildRsByProcesses("my-rs", rs2Processes) + d2.MergeReplicaSet(rs2, nil, nil, zap.S()) + + assert.Equal(t, uint64(3), d2.MinimumMajorVersion()) +} + +// TestConfiguringTlsProcessFromOpsManager ensures that if OM sends 'tls' fields for processes and deployments - +// they are moved to 'ssl' +func TestConfiguringTlsProcessFromOpsManager(t *testing.T) { + data, err := os.ReadFile("testdata/deployment_tls.json") + assert.NoError(t, err) + deployment, err := BuildDeploymentFromBytes(data) + assert.NoError(t, err) + + assert.Contains(t, deployment, "tls") + + for _, p := range deployment.getProcesses() { + assert.Contains(t, p.EnsureNetConfig(), "tls") + } +} + +func TestAddMonitoring(t *testing.T) { + d := NewDeployment() + + rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs")) + d.MergeReplicaSet(rs0, nil, nil, zap.S()) + d.AddMonitoring(zap.S(), false, util.CAFilePathInContainer) + + expectedMonitoringVersions := []interface{}{ + map[string]interface{}{"hostname": "my-rs-0.some.host", "name": MonitoringAgentDefaultVersion}, + map[string]interface{}{"hostname": "my-rs-1.some.host", "name": MonitoringAgentDefaultVersion}, + map[string]interface{}{"hostname": "my-rs-2.some.host", "name": MonitoringAgentDefaultVersion}, + } + assert.Equal(t, expectedMonitoringVersions, d.getMonitoringVersions()) + + // adding again - nothing changes + d.AddMonitoring(zap.S(), false, util.CAFilePathInContainer) + assert.Equal(t, expectedMonitoringVersions, d.getMonitoringVersions()) +} + +func TestAddMonitoringTls(t *testing.T) { + d := NewDeployment() + + rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs")) + d.MergeReplicaSet(rs0, nil, nil, zap.S()) + d.AddMonitoring(zap.S(), true, util.CAFilePathInContainer) + + expectedAdditionalParams := map[string]string{ + "useSslForAllConnections": "true", + "sslTrustedServerCertificates": util.CAFilePathInContainer, + } + + expectedMonitoringVersions := []interface{}{ + map[string]interface{}{"hostname": "my-rs-0.some.host", "name": MonitoringAgentDefaultVersion, "additionalParams": expectedAdditionalParams}, + map[string]interface{}{"hostname": "my-rs-1.some.host", "name": MonitoringAgentDefaultVersion, "additionalParams": expectedAdditionalParams}, + map[string]interface{}{"hostname": "my-rs-2.some.host", "name": MonitoringAgentDefaultVersion, "additionalParams": expectedAdditionalParams}, + } + assert.Equal(t, expectedMonitoringVersions, d.getMonitoringVersions()) + + // adding again - nothing changes + d.AddMonitoring(zap.S(), false, util.CAFilePathInContainer) + assert.Equal(t, expectedMonitoringVersions, d.getMonitoringVersions()) +} + +func TestAddBackup(t *testing.T) { + d := NewDeployment() + + rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs")) + d.MergeReplicaSet(rs0, nil, nil, zap.S()) + d.addBackup(zap.S()) + + expectedBackupVersions := []interface{}{ + map[string]interface{}{"hostname": "my-rs-0.some.host", "name": BackupAgentDefaultVersion}, + map[string]interface{}{"hostname": "my-rs-1.some.host", "name": BackupAgentDefaultVersion}, + map[string]interface{}{"hostname": "my-rs-2.some.host", "name": BackupAgentDefaultVersion}, + } + assert.Equal(t, expectedBackupVersions, d.getBackupVersions()) + + // adding again - nothing changes + d.addBackup(zap.S()) + assert.Equal(t, expectedBackupVersions, d.getBackupVersions()) +} + +// ************************ Methods for checking deployment units + +func checkShardedCluster(t *testing.T, d Deployment, expectedCluster ShardedCluster, replicaSetWithProcesses []ReplicaSetWithProcesses) { + checkShardedClusterCheckExtraReplicaSets(t, d, expectedCluster, replicaSetWithProcesses, true) +} + +func checkShardedClusterCheckExtraReplicaSets(t *testing.T, d Deployment, expectedCluster ShardedCluster, + expectedReplicaSets []ReplicaSetWithProcesses, checkExtraReplicaSets bool, +) { + cluster := d.getShardedClusterByName(expectedCluster.Name()) + + require.NotNil(t, cluster) + + assert.Equal(t, expectedCluster, *cluster) + + checkReplicaSets(t, d, expectedReplicaSets, checkExtraReplicaSets) + + if checkExtraReplicaSets { + // checking that no previous replica sets are left. For this we take the name of first shard and remove the last digit + firstShardName := expectedReplicaSets[0].Rs.Name() + i := 0 + for _, r := range d.getReplicaSets() { + if strings.HasPrefix(r.Name(), firstShardName[0:len(firstShardName)-1]) { + i++ + } + } + assert.Equal(t, len(expectedReplicaSets), i) + } +} + +func checkReplicaSets(t *testing.T, d Deployment, replicaSetWithProcesses []ReplicaSetWithProcesses, checkExtraProcesses bool) { + for _, r := range replicaSetWithProcesses { + if checkExtraProcesses { + checkReplicaSet(t, d, r) + } else { + checkReplicaSetCheckExtraProcesses(t, d, r, false) + } + } +} + +func checkReplicaSetCheckExtraProcesses(t *testing.T, d Deployment, expectedRs ReplicaSetWithProcesses, checkExtraProcesses bool) { + rs := d.getReplicaSetByName(expectedRs.Rs.Name()) + + require.NotNil(t, rs) + + assert.Equal(t, expectedRs.Rs, *rs) + rsPrefix := expectedRs.Rs.Name() + + found := 0 + totalMongods := 0 + for _, p := range d.getProcesses() { + for i, e := range expectedRs.Processes { + if p.ProcessType() == ProcessTypeMongod && p.Name() == e.Name() { + assert.Equal(t, e, p, "Process %d (%s) doesn't match! \nExpected: %v, \nReal: %v", i, p.Name(), e.json(), p.json()) + found++ + } + } + if p.ProcessType() == ProcessTypeMongod && strings.HasPrefix(p.Name(), rsPrefix) { + totalMongods++ + } + } + assert.Equalf(t, len(expectedRs.Processes), found, "Not all %s replicaSet processes are found!", expectedRs.Rs.Name()) + if checkExtraProcesses { + assert.Equalf(t, len(expectedRs.Processes), totalMongods, "Some excessive mongod processes are found for %s replicaSet!", expectedRs.Rs.Name()) + } +} + +func checkReplicaSet(t *testing.T, d Deployment, expectedRs ReplicaSetWithProcesses) { + checkReplicaSetCheckExtraProcesses(t, d, expectedRs, true) +} + +func checkProcess(t *testing.T, d Deployment, expectedProcess Process) { + assert.NotNil(t, d.getProcessByName(expectedProcess.Name())) + + for _, p := range d.getProcesses() { + if p.Name() == expectedProcess.Name() { + assert.Equal(t, expectedProcess, p) + break + } + } +} + +func checkMongoSProcesses(t *testing.T, allProcesses []Process, expectedMongosProcesses []Process) { + found := 0 + totalMongoses := 0 + + mongosPrefix := expectedMongosProcesses[0].Name()[0 : len(expectedMongosProcesses[0].Name())-1] + + for _, p := range allProcesses { + for _, e := range expectedMongosProcesses { + if p.ProcessType() == ProcessTypeMongos && p.Name() == e.Name() { + assert.Equal(t, e, p, "Actual: %v\n, Expected: %v", p.json(), e.json()) + found++ + } + } + if p.ProcessType() == ProcessTypeMongos && strings.HasPrefix(p.Name(), mongosPrefix) { + totalMongoses++ + } + } + assert.Equal(t, len(expectedMongosProcesses), found, "Not all mongos processes are found!") + assert.Equal(t, len(expectedMongosProcesses), totalMongoses, "Some excessive mongos processes are found!") +} + +func checkShardedClusterRemoved(t *testing.T, d Deployment, sc ShardedCluster, configRs ReplicaSetWithProcesses, shards []ReplicaSetWithProcesses) { + assert.Nil(t, d.getShardedClusterByName(sc.Name())) + + checkReplicaSetRemoved(t, d, configRs) + + for _, s := range shards { + checkReplicaSetRemoved(t, d, s) + } + + assert.Len(t, d.getMongosProcessesNames(sc.Name()), 0) +} + +func checkReplicaSetRemoved(t *testing.T, d Deployment, rs ReplicaSetWithProcesses) { + assert.Nil(t, d.getReplicaSetByName(rs.Rs.Name())) + + for _, p := range rs.Processes { + checkProcessRemoved(t, d, p.Name()) + } +} + +func checkProcessRemoved(t *testing.T, d Deployment, p string) { + assert.Nil(t, d.getProcessByName(p)) +} + +func checkNumberOfVotingMembers(t *testing.T, rs ReplicaSetWithProcesses, expectedNumberOfVotingMembers, totalNumberOfMembers int) { + count := 0 + for _, m := range rs.Rs.Members() { + if m.Votes() > 0 && m.Priority() > 0 { + count++ + } + } + assert.Equal(t, expectedNumberOfVotingMembers, count) + assert.Len(t, rs.Rs.Members(), totalNumberOfMembers) +} + +func createShards(name string) []ReplicaSetWithProcesses { + return createSpecificNumberOfShards(3, name) +} + +// ******************************** Methods for creating deployment units + +func createSpecificNumberOfShards(count int, name string) []ReplicaSetWithProcesses { + return createSpecificNumberOfShardsAndMongods(count, 3, name) +} + +func createShardsSpecificNumberOfMongods(count int, name string) []ReplicaSetWithProcesses { + return createSpecificNumberOfShardsAndMongods(3, count, name) +} + +func createSpecificNumberOfShardsAndMongods(countShards, countMongods int, name string) []ReplicaSetWithProcesses { + shards := make([]ReplicaSetWithProcesses, countShards) + for i := 0; i < countShards; i++ { + rsName := fmt.Sprintf("%s-%d", name, i) + options := make([]automationconfig.MemberOptions, countMongods) + shards[i] = NewReplicaSetWithProcesses( + NewReplicaSet(rsName, "3.6.3"), + createReplicaSetProcessesCount(countMongods, rsName), + options, + ) + } + return shards +} + +func buildRsByProcesses(rsName string, processes []Process) ReplicaSetWithProcesses { + options := make([]automationconfig.MemberOptions, len(processes)) + return NewReplicaSetWithProcesses( + NewReplicaSet(rsName, "3.6.3"), + processes, + options, + ) +} + +func createStandalone() Process { + return NewMongodProcess("merchantsStandalone", "mongo1.some.host", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, defaultMongoDBVersioned("3.6.3"), "", nil, "") +} + +func createMongosProcesses(num int, name, clusterName string) []Process { + mongosProcesses := make([]Process, num) + + for i := 0; i < num; i++ { + idx := strconv.Itoa(i) + mongosProcesses[i] = NewMongosProcess(name+idx, "mongoS"+idx+".some.host", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, defaultMongoDBVersioned("3.6.3"), "", nil, "") + if clusterName != "" { + mongosProcesses[i].setCluster(clusterName) + } + } + return mongosProcesses +} + +func createReplicaSetProcesses(rsName string) []Process { + return createReplicaSetProcessesCount(3, rsName) +} + +func createReplicaSetProcessesCount(count int, rsName string) []Process { + rsMembers := make([]Process, count) + + for i := 0; i < count; i++ { + rsMembers[i] = NewMongodProcess(fmt.Sprintf("%s-%d", rsName, i), fmt.Sprintf("%s-%d.some.host", rsName, i), "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, defaultMongoDBVersioned("3.6.3"), "", nil, "") + // Note that we don't specify the replicaset config for process + } + return rsMembers +} + +func createReplicaSetProcessesCountEnt(count int, rsName string) []Process { + rsMembers := make([]Process, count) + + for i := 0; i < count; i++ { + rsMembers[i] = NewMongodProcess(fmt.Sprintf("%s-%d", rsName, i), fmt.Sprintf("%s-%d.some.host", rsName, i), "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, defaultMongoDBVersioned("3.6.3-ent"), "", nil, "") + // Note that we don't specify the replicaset config for process + } + return rsMembers +} + +func createConfigSrvRs(name string, check bool) ReplicaSetWithProcesses { + options := make([]automationconfig.MemberOptions, 3) + replicaSetWithProcesses := NewReplicaSetWithProcesses( + NewReplicaSet(name, "3.6.3"), + createReplicaSetProcesses(name), + options, + ) + + if check { + for _, p := range replicaSetWithProcesses.Processes { + p.setClusterRoleConfigSrv() + } + } + return replicaSetWithProcesses +} + +func createConfigSrvRsCount(count int, name string, check bool) ReplicaSetWithProcesses { + options := make([]automationconfig.MemberOptions, count) + replicaSetWithProcesses := NewReplicaSetWithProcesses( + NewReplicaSet(name, "3.6.3"), + createReplicaSetProcessesCount(count, name), + options, + ) + + if check { + for _, p := range replicaSetWithProcesses.Processes { + p.setClusterRoleConfigSrv() + } + } + return replicaSetWithProcesses +} + +func mergeReplicaSet(d Deployment, rsName string, rsProcesses []Process) ReplicaSetWithProcesses { + rs := buildRsByProcesses(rsName, rsProcesses) + d.MergeReplicaSet(rs, nil, nil, zap.S()) + return rs +} + +func mergeStandalone(d Deployment, s Process) Process { + d.MergeStandalone(s, nil, nil, zap.S()) + return s +} + +func defaultMongoDBVersioned(version string) mdbv1.DbSpec { + spec := mdbv1.NewReplicaSetBuilder().SetVersion(version).Build().Spec + return &spec +} diff --git a/controllers/om/depshardedcluster_test.go b/controllers/om/depshardedcluster_test.go new file mode 100644 index 000000000..d0a317981 --- /dev/null +++ b/controllers/om/depshardedcluster_test.go @@ -0,0 +1,531 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// TestMergeShardedClusterNoExisting that just merges the Sharded cluster into an empty deployment +func TestMergeShardedCluster_New(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + shards := createShards("myShard") + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + assert.NoError(t, err) + + require.Len(t, d.getProcesses(), 15) + require.Len(t, d.getReplicaSets(), 4) + for i := 0; i < 4; i++ { + require.Len(t, d.getReplicaSets()[i].Members(), 3) + } + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(3, "pretty", "cluster")) + checkReplicaSet(t, d, createConfigSrvRs("configSrv", true)) + checkShardedCluster(t, d, NewShardedCluster("cluster", configRs.Rs.Name(), shards), createShards("myShard")) +} + +func TestMergeShardedCluster_ProcessesModified(t *testing.T) { + d := NewDeployment() + + shards := createShards("cluster") + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: createConfigSrvRs("configSrv", false), + Shards: shards, + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // OM "made" some changes (should not be overriden) + (*d.getProcessByName("pretty0"))["logRotate"] = map[string]int{"sizeThresholdMB": 1000, "timeThresholdHrs": 24} + + // These OM changes must be overriden + (*d.getProcessByName("configSrv-1")).Args()["sharding"] = map[string]interface{}{"clusterRole": "shardsrv", "archiveMovedChunks": true} + (*d.getProcessByName("cluster-1-1"))["hostname"] = "rubbish" + (*d.getProcessByName("pretty2")).SetLogPath("/doesnt/exist") + + // Final check - we create the expected configuration, add there correct OM changes and check for equality with merge + // result + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: createConfigSrvRs("configSrv", false), + Shards: createShards("cluster"), + Finalizing: false, + } + + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + expectedMongosProcesses := createMongosProcesses(3, "pretty", "cluster") + expectedMongosProcesses[0]["logRotate"] = map[string]int{"sizeThresholdMB": 1000, "timeThresholdHrs": 24} + expectedConfigrs := createConfigSrvRs("configSrv", true) + expectedConfigrs.Processes[1].Args()["sharding"] = map[string]interface{}{"clusterRole": "configsvr", "archiveMovedChunks": true} + + require.Len(t, d.getProcesses(), 15) + checkMongoSProcesses(t, d.getProcesses(), expectedMongosProcesses) + checkReplicaSet(t, d, expectedConfigrs) + checkShardedCluster(t, d, NewShardedCluster("cluster", expectedConfigrs.Rs.Name(), shards), createShards("cluster")) +} + +func TestMergeShardedCluster_ReplicaSetsModified(t *testing.T) { + d := NewDeployment() + + shards := createShards("cluster") + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: createConfigSrvRs("configSrv", false), + Shards: shards, + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // OM "made" some changes (should not be overriden) + (*d.getReplicaSetByName("cluster-0"))["writeConcernMajorityJournalDefault"] = true + + // These OM changes must be overriden + (*d.getReplicaSetByName("cluster-0"))["protocolVersion"] = util.Int32Ref(2) + (*d.getReplicaSetByName("configSrv")).addMember( + NewMongodProcess("foo", "bar", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, mdbv1.NewStandaloneBuilder().Build().GetSpec(), "", nil, ""), "", automationconfig.MemberOptions{}, + ) + (*d.getReplicaSetByName("cluster-2")).setMembers(d.getReplicaSetByName("cluster-2").Members()[0:2]) + + // Final check - we create the expected configuration, add there correct OM changes and check for equality with merge + // result + configRs := createConfigSrvRs("configSrv", false) + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("cluster"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + expectedShards := createShards("cluster") + expectedShards[0].Rs["writeConcernMajorityJournalDefault"] = true + + require.Len(t, d.getProcesses(), 15) + require.Len(t, d.getReplicaSets(), 4) + for i := 0; i < 4; i++ { + require.Len(t, d.getReplicaSets()[i].Members(), 3) + } + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(3, "pretty", "cluster")) + checkReplicaSet(t, d, createConfigSrvRs("configSrv", true)) + checkShardedCluster(t, d, NewShardedCluster("cluster", configRs.Rs.Name(), shards), expectedShards) +} + +func TestMergeShardedCluster_ShardedClusterModified(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + shards := createShards("myShard") + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // OM "made" some changes (should not be overriden) + (*d.getShardedClusterByName("cluster"))["managedSharding"] = true + (*d.getShardedClusterByName("cluster"))["collections"] = []map[string]interface{}{{"_id": "some", "unique": true}} + + // These OM changes must be overriden + (*d.getShardedClusterByName("cluster")).setConfigServerRsName("fake") + (*d.getShardedClusterByName("cluster")).setShards(d.getShardedClusterByName("cluster").shards()[0:2]) + (*d.getShardedClusterByName("cluster")).setShards(append(d.getShardedClusterByName("cluster").shards(), newShard("fakeShard"))) + + mergeReplicaSet(d, "fakeShard", createReplicaSetProcesses("fakeShard")) + + require.Len(t, d.getReplicaSets(), 5) + + // Final check - we create the expected configuration, add there correct OM changes and check for equality with merge + // result + configRs = createConfigSrvRs("configSrv", false) + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + expectedCluster := NewShardedCluster("cluster", configRs.Rs.Name(), shards) + expectedCluster["managedSharding"] = true + expectedCluster["collections"] = []map[string]interface{}{{"_id": "some", "unique": true}} + + // Note, that fake replicaset and it's processes haven't disappeared as we passed 'false' to 'MergeShardedCluster' + // which results in "draining" for redundant shards but not physical removal of replica sets + require.Len(t, d.getProcesses(), 18) + require.Len(t, d.getReplicaSets(), 5) + for i := 0; i < 4; i++ { + require.Len(t, d.getReplicaSets()[i].Members(), 3) + } + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(3, "pretty", "cluster")) + checkReplicaSet(t, d, createConfigSrvRs("configSrv", true)) + checkShardedCluster(t, d, expectedCluster, createShards("myShard")) +} + +// TestMergeShardedCluster_ShardAdded checks the scenario of incrementing the number of shards +func TestMergeShardedCluster_ShardsAdded(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + shards := createShards("cluster") + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + shards = createSpecificNumberOfShards(5, "cluster") + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkShardedCluster(t, d, NewShardedCluster("cluster", configRs.Rs.Name(), shards), shards) +} + +// TestMergeShardedCluster_ShardRemoved checks the scenario of decrementing the number of shards +// It creates a sharded cluster with 5 shards and scales down to 3 +func TestMergeShardedCluster_ShardsRemoved(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + shards := createSpecificNumberOfShards(5, "cluster") + + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(5, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // On first merge the redundant replica sets and processes are not removed, but 'draining' array is populated + shards = createSpecificNumberOfShards(3, "cluster") + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + expectedCluster := NewShardedCluster("cluster", configRs.Rs.Name(), shards) + expectedCluster.setDraining([]string{"cluster-3", "cluster-4"}) + checkShardedClusterCheckExtraReplicaSets(t, d, expectedCluster, shards, false) + + // On second merge the redundant rses and processes will be removed ('draining' array is gone as well) + shards = createSpecificNumberOfShards(3, "cluster") + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: true, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkShardedClusterCheckExtraReplicaSets(t, d, NewShardedCluster("cluster", configRs.Rs.Name(), shards), shards, true) +} + +// TestMergeShardedCluster_MongosCountChanged checks the scenario of incrementing and decrementing the number of mongos +func TestMergeShardedCluster_MongosCountChanged(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(3, "pretty", "cluster")) + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(4, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(4, "pretty", "cluster")) + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(2, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(2, "pretty", "cluster")) +} + +// TestMergeShardedCluster_MongosCountChanged checks the scenario of incrementing and decrementing the number of replicas +// in config server +func TestMergeShardedCluster_ConfigSrvCountChanged(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkReplicaSet(t, d, createConfigSrvRs("configSrv", true)) + + configRs = createConfigSrvRsCount(6, "configSrv", false) + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(4, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkReplicaSet(t, d, createConfigSrvRsCount(6, "configSrv", true)) + + configRs = createConfigSrvRsCount(2, "configSrv", false) + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(4, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + checkReplicaSet(t, d, createConfigSrvRsCount(2, "configSrv", true)) +} + +func TestMergeShardedCluster_ScaleUpShardMergeFirstProcess(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + // creating other cluster to make sure no side effects occur + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "anotherCluster", + MongosProcesses: createMongosProcesses(3, "anotherMongos", ""), + ConfigServerRs: configRs, + Shards: createShards("anotherClusterSh"), + Finalizing: false, + } + + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // Emulating changes to current shards by OM + for _, s := range d.getShardedClusters()[0].shards() { + shardRs := d.getReplicaSetByName(s.rs()) + for _, m := range shardRs.Members() { + process := d.getProcessByName(m.Name()) + process.Args()["security"] = map[string]interface{}{"clusterAuthMode": "sendX509"} + } + } + + // Now we "scale up" mongods from 3 to 4 + shards := createShardsSpecificNumberOfMongods(4, "myShard") + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + expectedShards := createShardsSpecificNumberOfMongods(4, "myShard") + + for _, s := range expectedShards { + for _, p := range s.Processes { + p.Args()["security"] = map[string]interface{}{"clusterAuthMode": "sendX509"} + } + } + + expectedCluster := NewShardedCluster("cluster", configRs.Rs.Name(), expectedShards) + checkShardedCluster(t, d, expectedCluster, expectedShards) + + expectedAnotherCluster := NewShardedCluster("anotherCluster", configRs.Rs.Name(), createShards("anotherClusterSh")) + checkShardedCluster(t, d, expectedAnotherCluster, createShards("anotherClusterSh")) +} + +func TestMergeShardedCluster_ScaleUpMongosMergeFirstProcess(t *testing.T) { + d := NewDeployment() + + configRs := createConfigSrvRs("configSrv", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + mergeOpts.Name = "other" + mergeOpts.MongosProcesses = createMongosProcesses(3, "otherMongos", "") + mergeOpts.Shards = createShards("otherSh") + + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + // Emulating changes to current mongoses by OM + for _, m := range d.getMongosProcessesNames("cluster") { + process := d.getProcessByName(m) + process.Args()["security"] = map[string]interface{}{"clusterAuthMode": "sendX509"} + } + + // Now we "scale up" mongoses from 3 to 5 + mongoses := createMongosProcesses(5, "pretty", "") + + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: mongoses, + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + expectedMongoses := createMongosProcesses(5, "pretty", "cluster") + for _, p := range expectedMongoses { + p.Args()["security"] = map[string]interface{}{"clusterAuthMode": "sendX509"} + } + + checkMongoSProcesses(t, d.getProcesses(), expectedMongoses) + + // "other" mongoses stayed untouched + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(3, "otherMongos", "other")) + + totalMongos := 0 + for _, p := range d.getProcesses() { + if p.ProcessType() == ProcessTypeMongos { + totalMongos++ + } + } + assert.Equal(t, 8, totalMongos) +} + +// TestRemoveShardedClusterByName checks that sharded cluster and all linked artifacts are removed - but existing objects +// should stay untouched +func TestRemoveShardedClusterByName(t *testing.T) { + d := NewDeployment() + configRs := createConfigSrvRs("configSrv", false) + mergeOpts := DeploymentShardedClusterMergeOptions{ + Name: "cluster", + MongosProcesses: createMongosProcesses(3, "pretty", ""), + ConfigServerRs: configRs, + Shards: createShards("myShard"), + Finalizing: false, + } + _, err := d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + configRs2 := createConfigSrvRs("otherConfigSrv", false) + mergeOpts = DeploymentShardedClusterMergeOptions{ + Name: "otherCluster", + MongosProcesses: createMongosProcesses(3, "ugly", ""), + ConfigServerRs: configRs2, + Shards: createShards("otherShard"), + Finalizing: false, + } + _, err = d.MergeShardedCluster(mergeOpts) + assert.NoError(t, err) + + mergeStandalone(d, createStandalone()) + + rs := mergeReplicaSet(d, "fooRs", createReplicaSetProcesses("fooRs")) + + err = d.RemoveShardedClusterByName("otherCluster", zap.S()) + assert.NoError(t, err) + + // First check that all other entities stay untouched + checkProcess(t, d, createStandalone()) + checkReplicaSet(t, d, rs) + checkMongoSProcesses(t, d.getProcesses(), createMongosProcesses(3, "pretty", "cluster")) + checkReplicaSet(t, d, createConfigSrvRs("configSrv", true)) + shards := createShards("myShard") + checkShardedCluster(t, d, NewShardedCluster("cluster", configRs.Rs.Name(), shards), shards) + + // Then check that the sharded cluster and all replica sets were removed + shards2 := createShards("otherShard") + checkShardedClusterRemoved(t, d, NewShardedCluster("otherCluster", configRs2.Rs.Name(), shards2), createConfigSrvRs("otherConfigSrv", false), shards2) +} diff --git a/controllers/om/fullreplicaset.go b/controllers/om/fullreplicaset.go new file mode 100644 index 000000000..28eeb2914 --- /dev/null +++ b/controllers/om/fullreplicaset.go @@ -0,0 +1,93 @@ +package om + +import ( + "strconv" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +// ReplicaSetWithProcesses is a wrapper for replica set and processes that match to it +// The contract for class is that both processes and replica set are guaranteed to match to each other +// Note, that the type modifies the entities directly and doesn't create copies! (seems not a big deal for clients) +type ReplicaSetWithProcesses struct { + Rs ReplicaSet + Processes []Process +} + +// NewReplicaSetWithProcesses is the only correct function for creation ReplicaSetWithProcesses +func NewReplicaSetWithProcesses( + rs ReplicaSet, + processes []Process, + memberOptions []automationconfig.MemberOptions, +) ReplicaSetWithProcesses { + rs.clearMembers() + + for idx, p := range processes { + p.setReplicaSetName(rs.Name()) + var options automationconfig.MemberOptions + if len(memberOptions) > idx { + options = memberOptions[idx] + } + rs.addMember(p, "", options) + } + return ReplicaSetWithProcesses{rs, processes} +} + +// determineNextProcessIdStartingPoint returns the number which should be used as a starting +// point for generating new _ids. +func determineNextProcessIdStartingPoint(desiredProcesses []Process, existingProcessIds map[string]int) int { + // determine the next id, it has to be higher than any previous value + newId := 0 + for _, id := range existingProcessIds { + if id >= newId { + newId = id + 1 + } + } + return newId +} + +// NewMultiClusterReplicaSetWithProcesses Creates processes for a multi cluster deployment. +// This function ensures that new processes which are added never have an overlapping _id with any existing process. +// existing _ids are re-used, and when new processes are added, a new higher number is used. +func NewMultiClusterReplicaSetWithProcesses(rs ReplicaSet, processes []Process, memberOptions []automationconfig.MemberOptions, existingProcessIds map[string]int, connectivity *mdbv1.MongoDBConnectivity) ReplicaSetWithProcesses { + newId := determineNextProcessIdStartingPoint(processes, existingProcessIds) + rs.clearMembers() + for idx, p := range processes { + p.setReplicaSetName(rs.Name()) + var options automationconfig.MemberOptions + if len(memberOptions) > idx { + options = memberOptions[idx] + } + // ensure the process id is not changed if it already exists + if existingId, ok := existingProcessIds[p.Name()]; ok { + rs.addMember(p, strconv.Itoa(existingId), options) + } else { + // otherwise add a new id which is always incrementing + rs.addMember(p, strconv.Itoa(newId), options) + newId++ + } + } + fullRs := ReplicaSetWithProcesses{Rs: rs, Processes: processes} + if connectivity != nil { + fullRs.SetHorizons(connectivity.ReplicaSetHorizons) + } + return fullRs +} + +func (r ReplicaSetWithProcesses) GetProcessNames() []string { + processNames := make([]string, len(r.Processes)) + for i, p := range r.Processes { + processNames[i] = p.Name() + } + return processNames +} + +func (r ReplicaSetWithProcesses) SetHorizons(replicaSetHorizons []mdbv1.MongoDBHorizonConfig) { + if len(replicaSetHorizons) >= len(r.Rs.Members()) { + for i, m := range r.Rs.Members() { + m.setHorizonConfig(replicaSetHorizons[i]) + } + } +} diff --git a/controllers/om/fullreplicaset_test.go b/controllers/om/fullreplicaset_test.go new file mode 100644 index 000000000..f9dfeeaa1 --- /dev/null +++ b/controllers/om/fullreplicaset_test.go @@ -0,0 +1,227 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" + + ac "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" +) + +func TestDetermineNextProcessIdStartingPoint(t *testing.T) { + t.Run("New id should be higher than any other id", func(t *testing.T) { + desiredProcesses := []Process{ + { + "name": "p-0", + }, + { + "name": "p-1", + }, + { + "name": "p-2", + }, + { + "name": "p-3", + }, + } + + existingIds := map[string]int{ + "p-0": 0, + "p-1": 1, + "p-2": 2, + "p-3": 3, + } + + assert.Equal(t, 4, determineNextProcessIdStartingPoint(desiredProcesses, existingIds)) + }) + + t.Run("New id should be higher than other ids even if there are gaps in between", func(t *testing.T) { + desiredProcesses := []Process{ + { + "name": "p-0", + }, + { + "name": "p-1", + }, + { + "name": "p-2", + }, + } + + existingIds := map[string]int{ + "p-0": 0, + "p-1": 5, + "p-2": 3, + } + + assert.Equal(t, 6, determineNextProcessIdStartingPoint(desiredProcesses, existingIds)) + }) +} + +func TestNewMultiClusterReplicaSetWithProcesses(t *testing.T) { + tests := []struct { + name string + processes []Process + memberOptions []ac.MemberOptions + expected ReplicaSetWithProcesses + }{ + { + name: "Same number of processes and member options", + processes: []Process{ + { + "name": "p-0", + }, + { + "name": "p-1", + }, + }, + memberOptions: []ac.MemberOptions{ + { + Votes: ptr.To(1), + Priority: ptr.To("1.3"), + }, + { + Votes: ptr.To(0), + Priority: ptr.To("0.7"), + }, + }, + expected: ReplicaSetWithProcesses{ + Rs: ReplicaSet{ + "_id": "mdb-multi", "members": []ReplicaSetMember{ + {"_id": "0", "host": "p-0", "priority": float32(1.3), "tags": map[string]string{}, "votes": 1}, + {"_id": "1", "host": "p-1", "priority": float32(0.7), "tags": map[string]string{}, "votes": 0}, + }, + "protocolVersion": "1", + }, + Processes: []Process{ + {"name": "p-0", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + {"name": "p-1", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + }, + }, + }, + { + name: "More member options than processes", + processes: []Process{ + { + "name": "p-0", + }, + { + "name": "p-1", + }, + }, + memberOptions: []ac.MemberOptions{ + { + Votes: ptr.To(1), + Priority: ptr.To("1.3"), + }, + { + Votes: ptr.To(0), + Priority: ptr.To("0.7"), + }, + { + Votes: ptr.To(1), + Tags: map[string]string{ + "env": "dev", + }, + }, + }, + expected: ReplicaSetWithProcesses{ + Rs: ReplicaSet{ + "_id": "mdb-multi", "members": []ReplicaSetMember{ + {"_id": "0", "host": "p-0", "priority": float32(1.3), "tags": map[string]string{}, "votes": 1}, + {"_id": "1", "host": "p-1", "priority": float32(0.7), "tags": map[string]string{}, "votes": 0}, + }, + "protocolVersion": "1", + }, + Processes: []Process{ + {"name": "p-0", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + {"name": "p-1", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + }, + }, + }, + { + name: "Less member options than processes", + processes: []Process{ + { + "name": "p-0", + }, + { + "name": "p-1", + }, + }, + memberOptions: []ac.MemberOptions{ + { + Votes: ptr.To(1), + Priority: ptr.To("1.3"), + }, + }, + expected: ReplicaSetWithProcesses{ + Rs: ReplicaSet{ + "_id": "mdb-multi", "members": []ReplicaSetMember{ + {"_id": "0", "host": "p-0", "priority": float32(1.3), "tags": map[string]string{}, "votes": 1}, + // Defaulting priority 1.0 and votes to 1 when no member options are present + {"_id": "1", "host": "p-1", "priority": float32(1.0), "tags": map[string]string{}, "votes": 1}, + }, + "protocolVersion": "1", + }, + Processes: []Process{ + {"name": "p-0", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + {"name": "p-1", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + }, + }, + }, + { + name: "No member options", + processes: []Process{ + { + "name": "p-0", + }, + { + "name": "p-1", + }, + }, + memberOptions: []ac.MemberOptions{}, + expected: ReplicaSetWithProcesses{ + Rs: ReplicaSet{ + "_id": "mdb-multi", "members": []ReplicaSetMember{ + // Defaulting priority 1.0 and votes to 1 when no member options are present + {"_id": "0", "host": "p-0", "priority": float32(1.0), "tags": map[string]string{}, "votes": 1}, + // Defaulting priority 1.0 and votes to 1 when no member options are present + {"_id": "1", "host": "p-1", "priority": float32(1.0), "tags": map[string]string{}, "votes": 1}, + }, + "protocolVersion": "1", + }, + Processes: []Process{ + {"name": "p-0", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + {"name": "p-1", "args2_6": map[string]interface{}{"replication": map[string]interface{}{"replSetName": "mdb-multi"}}}, + }, + }, + }, + { + name: "No processes", + processes: []Process{}, + memberOptions: []ac.MemberOptions{ + { + Votes: ptr.To(1), + Priority: ptr.To("1.3"), + }, + { + Votes: ptr.To(0), + Priority: ptr.To("0.7"), + }, + }, + expected: ReplicaSetWithProcesses{ + Rs: ReplicaSet{"_id": "mdb-multi", "members": []ReplicaSetMember{}, "protocolVersion": "1"}, + Processes: []Process{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := NewMultiClusterReplicaSetWithProcesses(NewReplicaSet("mdb-multi", "5.0.5"), tt.processes, tt.memberOptions, map[string]int{}, nil) + assert.Equal(t, tt.expected, actual) + }) + } +} diff --git a/controllers/om/group.go b/controllers/om/group.go new file mode 100644 index 000000000..de3946d64 --- /dev/null +++ b/controllers/om/group.go @@ -0,0 +1,23 @@ +package om + +type ProjectsResponse struct { + OMPaginated + Groups []*Project `json:"results"` +} + +func (o ProjectsResponse) Results() []interface{} { + // Lack of covariance in Go... :( + ans := make([]interface{}, len(o.Groups)) + for i, org := range o.Groups { + ans[i] = org + } + return ans +} + +type Project struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + OrgID string `json:"orgId,omitempty"` + Tags []string `json:"tags,omitempty"` + AgentAPIKey string `json:"agentApiKey,omitempty"` +} diff --git a/controllers/om/host/hosts.go b/controllers/om/host/hosts.go new file mode 100644 index 000000000..72fd9fa55 --- /dev/null +++ b/controllers/om/host/hosts.go @@ -0,0 +1,40 @@ +package host + +type Host struct { + Password string `json:"password"` + Username string `json:"username"` + Hostname string `json:"hostname"` + Port int `json:"port"` + AuthMechanismName string `json:"authMechanismName"` + Id string `json:"id"` +} + +type Result struct { + Results []Host `json:"results"` +} + +type Getter interface { + GetHosts() (*Result, error) +} + +type Adder interface { + AddHost(host Host) error +} + +type Updater interface { + UpdateHost(host Host) error +} + +type Remover interface { + RemoveHost(hostID string) error +} + +type GetRemover interface { + Getter + Remover +} + +type GetAdder interface { + Getter + Adder +} diff --git a/controllers/om/host/monitoring.go b/controllers/om/host/monitoring.go new file mode 100644 index 000000000..6da0c1f90 --- /dev/null +++ b/controllers/om/host/monitoring.go @@ -0,0 +1,69 @@ +package host + +import ( + "errors" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// StopMonitoring will stop OM monitoring of hosts, which will then +// make OM stop displaying old hosts from Processes view. +// Note, that the method tries to delete as many hosts as possible and doesn't give up on errors, returns +// the last error instead +func StopMonitoring(getRemover GetRemover, hostnames []string, log *zap.SugaredLogger) error { + if len(hostnames) == 0 { + return nil + } + + hosts, err := getRemover.GetHosts() + if err != nil { + return err + } + errorHappened := false + for _, hostname := range hostnames { + found := false + for _, h := range hosts.Results { + if h.Hostname == hostname { + found = true + err = getRemover.RemoveHost(h.Id) + if err != nil { + log.Warnf("Failed to remove host %s from monitoring in Ops Manager: %s", h.Hostname, err) + errorHappened = true + } else { + log.Debugf("Removed the host %s from monitoring in Ops Manager", h.Hostname) + } + break + } + } + if !found { + log.Warnf("Unable to remove monitoring on host %s as it was not found", hostname) + } + } + + if errorHappened { + return errors.New("failed to remove some hosts from monitoring in Ops manager") + } + return nil +} + +// stopMonitoringHosts removes monitoring for this list of hosts from Ops Manager. +func stopMonitoringHosts(getRemover GetRemover, hosts []string, log *zap.SugaredLogger) error { + if len(hosts) == 0 { + return nil + } + + if err := StopMonitoring(getRemover, hosts, log); err != nil { + return xerrors.Errorf("Failed to stop monitoring on hosts %s: %w", hosts, err) + } + + return nil +} + +// CalculateDiffAndStopMonitoringHosts checks hosts that are present in hostsBefore but not hostsAfter, and removes +// monitoring from them. +func CalculateDiffAndStopMonitoring(getRemover GetRemover, hostsBefore, hostsAfter []string, log *zap.SugaredLogger) error { + return stopMonitoringHosts(getRemover, util.FindLeftDifference(hostsBefore, hostsAfter), log) +} diff --git a/controllers/om/mockedomclient.go b/controllers/om/mockedomclient.go new file mode 100644 index 000000000..bc8c5346c --- /dev/null +++ b/controllers/om/mockedomclient.go @@ -0,0 +1,947 @@ +package om + +import ( + "fmt" + "math/rand" + "reflect" + "runtime" + "strconv" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "golang.org/x/xerrors" + + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/pkg/handler" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +// ******************************************************************************************************************** +// Dev notes: +// * this is a mocked implementation of 'Connection' interface which mocks all communication with Ops Manager. It doesn't +// do anything sophisticated - just saves the state that OM is supposed to have after these invocations - for example +// the deployment pushed to it +// * The usual place to start from is the 'NewEmptyMockedOmConnection' method that pre-creates the group - convenient +// in most cases. Should work fine for the "full" reconciliation testing +// * The class tracks the functions called - some methods ('CheckOrderOfOperations', 'CheckOperationsDidntHappen' - more +// can be added) may help to check the communication happened +// * Any overriding of default behavior can be done via functions (e.g. 'CreateGroupFunc', 'UpdateGroupFunc') +// * To emulate the work of real OM it's possible to emulate the agents delay in "reaching" goal state. This can be +// configured using 'AgentsDelayCount' property +// * As Deployment has package access to most of its data to preserve encapsulation (processes, ssl etc.) this class can +// be used as an access point to those fields for testing (see 'getProcesses' as an example) +// ******************************************************************************************************************** + +const ( + TestGroupID = "abcd1234" + TestGroupName = "my-project" + TestOrgID = "xyz9876" + TestAgentKey = "qwerty9876" + TestURL = "http://mycompany.example.com:8080" + TestUser = "test@mycompany.example.com" + TestApiKey = "36lj245asg06s0h70245dstgft" //nolint +) + +type MockedOmConnection struct { + context *OMContext + + deployment Deployment + automationConfig *AutomationConfig + backupAgentConfig *BackupAgentConfig + monitoringAgentConfig *MonitoringAgentConfig + controlledFeature *controlledfeature.ControlledFeature + // hosts are used for both automation agents and monitoring endpoints. + // They are necessary for emulating "agents" are ready behavior as operator checks for hosts for agents to exist + hostResults *host.Result + agentHostnameMap map[string]struct{} + + ReadAutomationStatusFunc func() (*AutomationStatus, error) + ReadAutomationAgentsFunc func(int) (Paginated, error) + + numRequestsSent int + AgentAPIKey string + OrganizationsWithGroups map[*Organization][]*Project + CreateGroupFunc func(group *Project) (*Project, error) + UpdateGroupFunc func(group *Project) (*Project, error) + BackupConfigs map[string]*backup.Config + BackupHostClusters map[string]*backup.HostCluster + UpdateBackupStatusFunc func(clusterId string, status backup.Status) error + AgentAuthMechanism string + SnapshotSchedules map[string]*backup.SnapshotSchedule + Hostnames []string + PreferredHostnames []PreferredHostname + + agentVersion string + agentMinimumVersion string + + // UpdateMonitoringAgentConfigFunc is delegated to if not nil when UpdateMonitoringAgentConfig is called + UpdateMonitoringAgentConfigFunc func(mac *MonitoringAgentConfig, log *zap.SugaredLogger) ([]byte, error) + // AgentsDelayCount is the number of loops to wait until the agents reach the goal + AgentsDelayCount int + // mocked client keeps track of all implemented functions called - uses reflection Func for this to enable type-safety + // and make function names rename easier + history []*runtime.Func +} + +func (oc *MockedOmConnection) GetDeployment() Deployment { + return oc.deployment +} + +func (oc *MockedOmConnection) ReadGroupBackupConfig() (backup.GroupBackupConfig, error) { + return backup.GroupBackupConfig{}, xerrors.Errorf("not implemented") +} + +func (oc *MockedOmConnection) UpdateGroupBackupConfig(config backup.GroupBackupConfig) ([]byte, error) { + return nil, xerrors.Errorf("not implemented") +} + +func (oc *MockedOmConnection) UpdateBackupAgentConfig(mat *BackupAgentConfig, log *zap.SugaredLogger) ([]byte, error) { + return nil, xerrors.Errorf("not implemented") +} + +func (oc *MockedOmConnection) BaseURL() string { + return oc.context.BaseURL +} + +func (oc *MockedOmConnection) GroupID() string { + return oc.context.GroupID +} + +func (oc *MockedOmConnection) GroupName() string { + return oc.context.GroupName +} + +func (oc *MockedOmConnection) OrgID() string { + return oc.context.OrgID +} + +func (oc *MockedOmConnection) PublicKey() string { + return oc.context.PublicKey +} + +func (oc *MockedOmConnection) PrivateKey() string { + return oc.context.PrivateKey +} + +func (oc *MockedOmConnection) ConfigureProject(project *Project) { + oc.context.GroupID = project.ID + oc.context.OrgID = project.OrgID +} + +var _ Connection = &MockedOmConnection{} + +// NewEmptyMockedOmConnection is the standard function for creating mocked connections that is usually used for testing +// "full cycle" mocked controller. It has group created already, but doesn't have the deployment. Also it "survives" +// recreations (as this is what we do in 'ReconcileCommonController.prepareConnection') +func NewEmptyMockedOmConnection(ctx *OMContext) Connection { + connection := NewMockedOmConnection(nil) + connection.OrganizationsWithGroups = make(map[*Organization][]*Project) + connection.OrganizationsWithGroups = map[*Organization][]*Project{ + {ID: TestOrgID, Name: TestGroupName}: {{ + Name: TestGroupName, + ID: TestGroupID, + Tags: []string{util.OmGroupExternallyManagedTag}, + AgentAPIKey: TestAgentKey, + OrgID: TestOrgID, + }}, + } + connection.context = ctx + + return connection +} + +// NewMockedConnection is the simplified connection wrapping some deployment that already exists. Should be used for +// partial functionality (not the "full cycle" controller), for example read-update operation for the deployment +func NewMockedOmConnection(d Deployment) *MockedOmConnection { + connection := MockedOmConnection{deployment: d} + connection.hostResults = buildHostsFromDeployment(d) + connection.BackupConfigs = make(map[string]*backup.Config) + connection.BackupHostClusters = make(map[string]*backup.HostCluster) + connection.SnapshotSchedules = make(map[string]*backup.SnapshotSchedule) + // By default, we don't wait for agents to reach goal + connection.AgentsDelayCount = 0 + // We use a simplified version of context as this is the only thing needed to get lock for the update + connection.context = &OMContext{GroupName: TestGroupName, OrgID: TestOrgID, GroupID: TestGroupID} + connection.AgentAPIKey = TestAgentKey + connection.history = make([]*runtime.Func, 0) + return &connection +} + +func NewEmptyMockedOmConnectionWithAgentVersion(agentVersion string, agentMinimumVersion string) Connection { + connection := NewMockedOmConnection(nil) + connection.agentVersion = agentVersion + connection.agentMinimumVersion = agentMinimumVersion + return connection +} + +// CachedOMConnectionFactory is a wrapper over om.ConnectionFactory that is caching a single instance of om.Connection when it's requested from connectionFactoryFunc. +// It's used to replace globally shared mock.CurrMockedConnection. +// WARNING: while this class alone is thread safe, it's not suitable for concurrent tests because it returns one cached instance of connection and our MockedOMConnection is not thread safe. +// In order to handle concurrent tests it is required to introduce map of connections and refactor all GetConnection usages by adding parameter with e.g. resource/OM project name. +// WARNING #2: This class won't create different connections when different OMContexts are passed into connectionFactoryFunc. It's caching a single instance of OMConnection. +type CachedOMConnectionFactory struct { + connectionFactoryFunc ConnectionFactory + connMap map[string]Connection + resourceToProjectMapping map[string]string + lock sync.Mutex + postCreateHook func(Connection) +} + +func NewCachedOMConnectionFactory(connectionFactoryFunc ConnectionFactory) *CachedOMConnectionFactory { + return &CachedOMConnectionFactory{connectionFactoryFunc: connectionFactoryFunc} +} + +func NewCachedOMConnectionFactoryWithInitializedConnection(conn Connection) *CachedOMConnectionFactory { + return &CachedOMConnectionFactory{connMap: map[string]Connection{conn.GroupID(): conn}} +} + +func NewDefaultCachedOMConnectionFactory() *CachedOMConnectionFactory { + return NewCachedOMConnectionFactory(NewEmptyMockedOmConnection) +} + +// WithResourceToProjectMapping used to provide a mapping between MongoDB/MongoDBMultiCluster resource name and OM project +// Used for tests with multiple OM projects to retrieve appropriate OM connection +func (c *CachedOMConnectionFactory) WithResourceToProjectMapping(resourceToProjectMapping map[string]string) *CachedOMConnectionFactory { + c.resourceToProjectMapping = resourceToProjectMapping + return c +} + +// GetConnectionFunc can be used as om.ConnectionFactory function to return cached instance of OMConnection based on ctx.GroupName +func (c *CachedOMConnectionFactory) GetConnectionFunc(ctx *OMContext) Connection { + c.lock.Lock() + defer c.lock.Unlock() + + if c.connMap == nil { + c.connMap = make(map[string]Connection) + } + + connection, ok := c.connMap[ctx.GroupName] + if !ok { + connection = c.connectionFactoryFunc(ctx) + c.connMap[ctx.GroupName] = connection + + if c.postCreateHook != nil { + c.postCreateHook(connection) + } + + } + + return connection +} + +func (c *CachedOMConnectionFactory) GetConnectionForResource(v *appsv1.StatefulSet) Connection { + // No resourceToProjectMapping provided, assuming it's a single project test + if len(c.resourceToProjectMapping) == 0 { + return c.GetConnection() + } + + ownerResourceName := getOwnerResourceName(v) + + projectName, ok := c.resourceToProjectMapping[ownerResourceName] + if !ok { + panic(fmt.Sprintf("resourceToProjectMapping does not contain project for resource name %s", ownerResourceName)) + } + + return c.GetConnectionFunc(&OMContext{GroupName: projectName}) +} + +// getOwnerResourceName tries to retrieve owner resource that can be used for OM project mapping +func getOwnerResourceName(v *appsv1.StatefulSet) string { + ownerReferences := v.GetOwnerReferences() + if len(ownerReferences) == 1 { + return ownerReferences[0].Name + } + + if mdbMultiResourceName, ok := v.GetAnnotations()[handler.MongoDBMultiResourceAnnotation]; ok { + return mdbMultiResourceName + } + + panic("could not retrieve owner resource name") +} + +func (c *CachedOMConnectionFactory) GetConnection() Connection { + c.lock.Lock() + defer c.lock.Unlock() + + if len(c.connMap) > 1 { + panic("multiple connections available but the resourceToProjectMapping was not provided") + } + + // Get first connection or nil if connMap empty + var conns []Connection + for _, conn := range c.connMap { + conns = append(conns, conn) + } + + if len(conns) == 0 { + return nil + } + + return conns[0] +} + +// SetPostCreateHook is a workaround to alter mocked connection state after it was created by the reconciler. +// It's used e.g. to set initial deployment processes. +// The proper way would be to define om.Connection interceptor but it's impractical due to a large number of methods there. +func (c *CachedOMConnectionFactory) SetPostCreateHook(postCreateHook func(Connection)) { + c.postCreateHook = postCreateHook +} + +func (oc *MockedOmConnection) UpdateDeployment(d Deployment) ([]byte, error) { + oc.addToHistory(reflect.ValueOf(oc.UpdateDeployment)) + oc.numRequestsSent++ + oc.deployment = d + return nil, nil +} + +func (oc *MockedOmConnection) ReadDeployment() (Deployment, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadDeployment)) + if oc.deployment == nil { + return NewDeployment(), nil + } + return oc.deployment, nil +} + +func (oc *MockedOmConnection) ReadUpdateDeployment(depFunc func(Deployment) error, log *zap.SugaredLogger) error { + oc.addToHistory(reflect.ValueOf(oc.ReadUpdateDeployment)) + if oc.deployment == nil { + oc.deployment = NewDeployment() + } + err := depFunc(oc.deployment) + oc.numRequestsSent++ + return err +} + +func (oc *MockedOmConnection) ReadUpdateMonitoringAgentConfig(matFunc func(*MonitoringAgentConfig) error, log *zap.SugaredLogger) error { + oc.addToHistory(reflect.ValueOf(oc.ReadUpdateMonitoringAgentConfig)) + if oc.monitoringAgentConfig == nil { + oc.monitoringAgentConfig = &MonitoringAgentConfig{MonitoringAgentTemplate: &MonitoringAgentTemplate{}} + } + + err := matFunc(oc.monitoringAgentConfig) + if err != nil { + return err + } + _, err = oc.UpdateMonitoringAgentConfig(oc.monitoringAgentConfig, log) + return err +} + +func (oc *MockedOmConnection) UpdateAutomationConfig(ac *AutomationConfig, log *zap.SugaredLogger) error { + oc.addToHistory(reflect.ValueOf(oc.UpdateAutomationConfig)) + oc.deployment = ac.Deployment + oc.automationConfig = ac + return nil +} + +func (oc *MockedOmConnection) ReadAutomationConfig() (*AutomationConfig, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadAutomationConfig)) + if oc.automationConfig == nil { + if oc.deployment == nil { + oc.deployment = NewDeployment() + } + oc.automationConfig = NewAutomationConfig(oc.deployment) + } + return oc.automationConfig, nil +} + +func (oc *MockedOmConnection) ReadUpdateAutomationConfig(modifyACFunc func(ac *AutomationConfig) error, log *zap.SugaredLogger) error { + oc.addToHistory(reflect.ValueOf(oc.ReadUpdateAutomationConfig)) + if oc.automationConfig == nil { + if oc.deployment == nil { + oc.deployment = NewDeployment() + } + oc.automationConfig = NewAutomationConfig(oc.deployment) + } + + // when we update the mocked automation config, update the corresponding deployment + err := modifyACFunc(oc.automationConfig) + + // mock the change of auto auth mechanism that is done based on the provided autoAuthMechanisms + updateAutoAuthMechanism(oc.automationConfig) + + _ = oc.automationConfig.Apply() + oc.deployment = oc.automationConfig.Deployment + return err +} + +func (oc *MockedOmConnection) AddHost(host host.Host) error { + oc.hostResults.Results = append(oc.hostResults.Results, host) + return nil +} + +func (oc *MockedOmConnection) UpdateHost(host host.Host) error { + // assume the host in question exists + for idx := range oc.hostResults.Results { + if oc.hostResults.Results[idx].Hostname == host.Hostname { + oc.hostResults.Results[idx] = host + } + } + return nil +} + +func (oc *MockedOmConnection) MarkProjectAsBackingDatabase(_ BackingDatabaseType) error { + return nil +} + +func (oc *MockedOmConnection) UpgradeAgentsToLatest() (string, error) { + oc.addToHistory(reflect.ValueOf(oc.UpgradeAgentsToLatest)) + return "new-version", nil +} + +func (oc *MockedOmConnection) ReadBackupAgentConfig() (*BackupAgentConfig, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadBackupAgentConfig)) + if oc.backupAgentConfig == nil { + oc.backupAgentConfig = &BackupAgentConfig{BackupAgentTemplate: &BackupAgentTemplate{}} + } + return oc.backupAgentConfig, nil +} + +func (oc *MockedOmConnection) UpdateBackupAgentConfigFromConfigWrapper(bac *BackupAgentConfig, log *zap.SugaredLogger) ([]byte, error) { + oc.addToHistory(reflect.ValueOf(oc.UpdateBackupAgentConfigFromConfigWrapper)) + oc.backupAgentConfig = bac + return nil, nil +} + +func (oc *MockedOmConnection) ReadUpdateBackupAgentConfig(bacFunc func(*BackupAgentConfig) error, log *zap.SugaredLogger) error { + oc.addToHistory(reflect.ValueOf(oc.ReadUpdateBackupAgentConfig)) + if oc.backupAgentConfig == nil { + oc.backupAgentConfig = &BackupAgentConfig{BackupAgentTemplate: &BackupAgentTemplate{}} + } + return bacFunc(oc.backupAgentConfig) +} + +func (oc *MockedOmConnection) ReadUpdateAgentsLogRotation(logRotateSetting mdbv1.AgentConfig, log *zap.SugaredLogger) error { + oc.addToHistory(reflect.ValueOf(oc.ReadUpdateAgentsLogRotation)) + return nil +} + +func (oc *MockedOmConnection) ReadMonitoringAgentConfig() (*MonitoringAgentConfig, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadMonitoringAgentConfig)) + if oc.monitoringAgentConfig == nil { + oc.monitoringAgentConfig = &MonitoringAgentConfig{MonitoringAgentTemplate: &MonitoringAgentTemplate{}} + } + return oc.monitoringAgentConfig, nil +} + +func (oc *MockedOmConnection) UpdateMonitoringAgentConfig(mac *MonitoringAgentConfig, log *zap.SugaredLogger) ([]byte, error) { + oc.addToHistory(reflect.ValueOf(oc.UpdateMonitoringAgentConfig)) + if oc.UpdateMonitoringAgentConfigFunc != nil { + return oc.UpdateMonitoringAgentConfigFunc(mac, log) + } + oc.monitoringAgentConfig = mac + return nil, nil +} + +func (oc *MockedOmConnection) GenerateAgentKey() (string, error) { + oc.addToHistory(reflect.ValueOf(oc.GenerateAgentKey)) + + return oc.AgentAPIKey, nil +} + +func (oc *MockedOmConnection) ReadAutomationStatus() (*AutomationStatus, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadAutomationStatus)) + if oc.ReadAutomationStatusFunc != nil { + return oc.ReadAutomationStatusFunc() + } + + if oc.AgentsDelayCount <= 0 { + // Emulating "agents reached goal state": returning the proper status for all the + // processes in the deployment + return oc.buildAutomationStatusFromDeployment(oc.deployment, true), nil + } + oc.AgentsDelayCount-- + + return oc.buildAutomationStatusFromDeployment(oc.deployment, false), nil +} + +func (oc *MockedOmConnection) ReadAutomationAgents(pageNum int) (Paginated, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadAutomationAgents)) + if oc.ReadAutomationAgentsFunc != nil { + return oc.ReadAutomationAgentsFunc(pageNum) + } + + results := make([]AgentStatus, 0) + for _, r := range oc.hostResults.Results { + results = append(results, + AgentStatus{Hostname: r.Hostname, LastConf: time.Now().Add(time.Second * -1).Format(time.RFC3339)}) + } + + return AutomationAgentStatusResponse{AutomationAgents: results}, nil +} + +func (oc *MockedOmConnection) GetHosts() (*host.Result, error) { + oc.addToHistory(reflect.ValueOf(oc.GetHosts)) + return oc.hostResults, nil +} + +func (oc *MockedOmConnection) RemoveHost(hostID string) error { + oc.addToHistory(reflect.ValueOf(oc.RemoveHost)) + toKeep := make([]host.Host, 0) + for _, v := range oc.hostResults.Results { + if v.Id != hostID { + toKeep = append(toKeep, v) + } + } + oc.hostResults = &host.Result{Results: toKeep} + oc.agentHostnameMap = util.TransformToMap(oc.hostResults.Results, func(obj host.Host, idx int) (string, struct{}) { + return obj.Hostname, struct{}{} + }) + return nil +} + +func (oc *MockedOmConnection) ReadOrganizationsByName(name string) ([]*Organization, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadOrganizationsByName)) + allOrgs := make([]*Organization, 0) + for k := range oc.OrganizationsWithGroups { + if k.Name == name { + allOrgs = append(allOrgs, k) + } + } + if len(allOrgs) == 0 { + return nil, apierror.NewErrorWithCode(apierror.OrganizationNotFound) + } + return allOrgs, nil +} + +func (oc *MockedOmConnection) ReadOrganizations(page int) (Paginated, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadOrganizations)) + // We don't set Next field - so there should be no pagination + allOrgs := make([]*Organization, 0) + for k := range oc.OrganizationsWithGroups { + allOrgs = append(allOrgs, k) + } + response := OrganizationsResponse{Organizations: allOrgs, OMPaginated: OMPaginated{TotalCount: len(oc.OrganizationsWithGroups)}} + return &response, nil +} + +func (oc *MockedOmConnection) ReadOrganization(orgID string) (*Organization, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadOrganization)) + return oc.findOrganization(orgID) +} + +func (oc *MockedOmConnection) ReadProjectsInOrganizationByName(orgID string, name string) ([]*Project, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadProjectsInOrganizationByName)) + org, err := oc.findOrganization(orgID) + if err != nil { + return nil, err + } + projects := make([]*Project, 0) + for _, p := range oc.OrganizationsWithGroups[org] { + if p.Name == name { + projects = append(projects, p) + } + } + return projects, nil +} + +func (oc *MockedOmConnection) ReadProjectsInOrganization(orgID string, page int) (Paginated, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadProjectsInOrganization)) + org, err := oc.findOrganization(orgID) + if err != nil { + return nil, err + } + response := &ProjectsResponse{Groups: oc.OrganizationsWithGroups[org], OMPaginated: OMPaginated{TotalCount: len(oc.OrganizationsWithGroups[org])}} + return response, nil +} + +func (oc *MockedOmConnection) CreateProject(project *Project) (*Project, error) { + oc.addToHistory(reflect.ValueOf(oc.CreateProject)) + if oc.CreateGroupFunc != nil { + return oc.CreateGroupFunc(project) + } + project.ID = TestGroupID + + // We emulate the behavior of Ops Manager: we create the organization with random id and the name matching the project + //nolint + organization := &Organization{ID: strconv.Itoa(rand.Int()), Name: project.Name} + if _, exists := oc.OrganizationsWithGroups[organization]; !exists { + oc.OrganizationsWithGroups[organization] = make([]*Project, 0) + } + project.OrgID = organization.ID + oc.OrganizationsWithGroups[organization] = append(oc.OrganizationsWithGroups[organization], project) + return project, nil +} + +func (oc *MockedOmConnection) UpdateProject(project *Project) (*Project, error) { + oc.addToHistory(reflect.ValueOf(oc.UpdateProject)) + if oc.UpdateGroupFunc != nil { + return oc.UpdateGroupFunc(project) + } + org, err := oc.findOrganization(project.OrgID) + if err != nil { + return nil, err + } + for _, g := range oc.OrganizationsWithGroups[org] { + if g.Name == project.Name { + *g = *project + return project, nil + } + } + return nil, xerrors.Errorf("failed to find project") +} + +func (oc *MockedOmConnection) UpdateBackupConfig(config *backup.Config) (*backup.Config, error) { + oc.addToHistory(reflect.ValueOf(oc.UpdateBackupConfig)) + oc.BackupConfigs[config.ClusterId] = config + return config, nil +} + +func (oc *MockedOmConnection) ReadBackupConfigs() (*backup.ConfigsResponse, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadBackupConfigs)) + + values := make([]*backup.Config, 0, len(oc.BackupConfigs)) + for _, v := range oc.BackupConfigs { + values = append(values, v) + } + return &backup.ConfigsResponse{Configs: values}, nil +} + +func (oc *MockedOmConnection) ReadBackupConfig(clusterId string) (*backup.Config, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadBackupConfig)) + + if config, ok := oc.BackupConfigs[clusterId]; ok { + return config, nil + } + return nil, apierror.New(errors.New("Failed to find backup config")) +} + +func (oc *MockedOmConnection) ReadHostCluster(clusterId string) (*backup.HostCluster, error) { + oc.addToHistory(reflect.ValueOf(oc.ReadHostCluster)) + + if hostCluster, ok := oc.BackupHostClusters[clusterId]; ok { + return hostCluster, nil + } + return nil, apierror.New(errors.New("Failed to find host cluster")) +} + +func (oc *MockedOmConnection) UpdateBackupStatus(clusterId string, newStatus backup.Status) error { + oc.addToHistory(reflect.ValueOf(oc.UpdateBackupStatus)) + + if oc.UpdateBackupStatusFunc != nil { + return oc.UpdateBackupStatusFunc(clusterId, newStatus) + } + + oc.doUpdateBackupStatus(clusterId, newStatus) + return nil +} + +func (oc *MockedOmConnection) UpdateControlledFeature(cf *controlledfeature.ControlledFeature) error { + oc.controlledFeature = cf + return nil +} + +func (oc *MockedOmConnection) GetControlledFeature() (*controlledfeature.ControlledFeature, error) { + if oc.controlledFeature == nil { + oc.controlledFeature = &controlledfeature.ControlledFeature{} + } + return oc.controlledFeature, nil +} + +func (oc *MockedOmConnection) GetAgentAuthMode() (string, error) { + return oc.AgentAuthMechanism, nil +} + +func (oc *MockedOmConnection) ReadSnapshotSchedule(clusterID string) (*backup.SnapshotSchedule, error) { + if snapshotSchedule, ok := oc.SnapshotSchedules[clusterID]; ok { + return snapshotSchedule, nil + } + return nil, apierror.New(errors.New("Failed to find snapshot schedule")) +} + +func (oc *MockedOmConnection) UpdateSnapshotSchedule(clusterID string, snapshotSchedule *backup.SnapshotSchedule) error { + oc.addToHistory(reflect.ValueOf(oc.UpdateSnapshotSchedule)) + oc.SnapshotSchedules[clusterID] = snapshotSchedule + return nil +} + +// SetAgentVersion updates the versions returned by ReadAgentVersion method +func (oc *MockedOmConnection) SetAgentVersion(agentVersion string, agentMinimumVersion string) { + oc.agentVersion = agentVersion + oc.agentMinimumVersion = agentMinimumVersion +} + +// ReadAgentVersion reads the versions from OM API +func (oc *MockedOmConnection) ReadAgentVersion() (AgentsVersionsResponse, error) { + return AgentsVersionsResponse{oc.agentVersion, oc.agentMinimumVersion}, nil +} + +// ************* These are native methods of Mocked client (not implementation of OmConnection) + +func (oc *MockedOmConnection) CheckMonitoredHostsRemoved(t *testing.T, removedHosts []string) { + for _, v := range oc.hostResults.Results { + for _, e := range removedHosts { + assert.NotEqual(t, e, v.Hostname, "Host %s is expected to be removed from monitored", e) + } + } +} + +func (oc *MockedOmConnection) doUpdateBackupStatus(clusterID string, newStatus backup.Status) { + if value, ok := oc.BackupConfigs[clusterID]; ok { + if newStatus == backup.Terminating { + value.Status = backup.Inactive + } else { + value.Status = newStatus + } + } +} + +func (oc *MockedOmConnection) GetProcesses() []Process { + return oc.deployment.getProcesses() +} + +func (oc *MockedOmConnection) GetTLS() map[string]interface{} { + return oc.deployment.getTLS() +} + +func (oc *MockedOmConnection) CheckNumberOfUpdateRequests(t *testing.T, expected int) { + assert.Equal(t, expected, oc.numRequestsSent) +} + +func (oc *MockedOmConnection) CheckDeployment(t *testing.T, expected Deployment, ignoreFields ...string) bool { + succeeded := true + for key := range expected { + if !stringutil.Contains(ignoreFields, key) { + if !assert.Equal(t, expected[key], oc.deployment[key]) { + succeeded = false + } + } + } + + return succeeded +} + +func (oc *MockedOmConnection) CheckResourcesDeleted(t *testing.T) { + oc.CheckResourcesAndBackupDeleted(t, "") +} + +// CheckResourcesDeleted verifies the results of "delete" operations in OM: the deployment and monitoring must be empty, +// backup - inactive (note, that in real life backup config will disappear together with monitoring hosts, but we +// ignore this for the sake of testing) +func (oc *MockedOmConnection) CheckResourcesAndBackupDeleted(t *testing.T, resourceName string) { + // This can be improved for some more complicated scenarios when we have different resources in parallel - so far + // just checking if deployment + assert.Empty(t, oc.deployment.getProcesses()) + assert.Empty(t, oc.deployment.getReplicaSets()) + assert.Empty(t, oc.deployment.getShardedClusters()) + assert.Empty(t, oc.deployment.getMonitoringVersions()) + assert.Empty(t, oc.deployment.getBackupVersions()) + assert.Empty(t, oc.hostResults.Results) + + if resourceName != "" { + assert.NotEmpty(t, oc.BackupHostClusters) + + found := false + for k, v := range oc.BackupHostClusters { + if v.ClusterName == resourceName { + assert.Equal(t, backup.Inactive, oc.BackupConfigs[k].Status) + found = true + } + } + assert.True(t, found) + + oc.CheckOrderOfOperations(t, reflect.ValueOf(oc.ReadBackupConfigs), reflect.ValueOf(oc.ReadHostCluster), + reflect.ValueOf(oc.UpdateBackupStatus)) + } +} + +func (oc *MockedOmConnection) CleanHistory() { + oc.history = make([]*runtime.Func, 0) + oc.numRequestsSent = 0 +} + +// CheckOrderOfOperations verifies the mocked client operations were called in specified order +func (oc *MockedOmConnection) CheckOrderOfOperations(t *testing.T, value ...reflect.Value) { + j := 0 + matched := "" + for _, h := range oc.history { + valueName := runtime.FuncForPC(value[j].Pointer()).Name() + zap.S().Infof("Comparing history func %s with %s (value[%d])", h.Name(), valueName, j) + if h.Name() == valueName { + matched += h.Name() + " " + j++ + } + if j == len(value) { + break + } + } + assert.Equal(t, len(value), j, "Only %d of %d expected operations happened in expected order (%s)", j, len(value), matched) +} + +func (oc *MockedOmConnection) CheckOperationsDidntHappen(t *testing.T, value ...reflect.Value) { + for _, h := range oc.history { + for _, o := range value { + assert.NotEqual(t, h.Name(), runtime.FuncForPC(o.Pointer()).Name(), "Operation %v is not expected to happen", h.Name()) + } + } +} + +// this is internal method only for testing, used by kubernetes mocked client +func (oc *MockedOmConnection) AddHosts(hostnames []string) { + if oc.agentHostnameMap == nil { + oc.agentHostnameMap = map[string]struct{}{} + } + + for _, p := range hostnames { + if _, ok := oc.agentHostnameMap[p]; !ok { + oc.hostResults.Results = append(oc.hostResults.Results, host.Host{Id: strconv.Itoa(len(oc.agentHostnameMap)), Hostname: p}) + oc.agentHostnameMap[p] = struct{}{} + } + } +} + +// this is internal method only for testing, used by kubernetes mocked client +func (oc *MockedOmConnection) ClearHosts() { + oc.agentHostnameMap = map[string]struct{}{} + oc.hostResults = &host.Result{} +} + +func (oc *MockedOmConnection) EnableBackup(resourceName string, resourceType backup.MongoDbResourceType, uuidStr string) { + if resourceType == backup.ReplicaSetType { + config := backup.Config{ClusterId: uuidStr, Status: backup.Started} + cluster := backup.HostCluster{TypeName: "REPLICA_SET", ClusterName: resourceName, ReplicaSetName: resourceName} + oc.BackupConfigs[uuidStr] = &config + oc.BackupHostClusters[uuidStr] = &cluster + } else { + config := backup.Config{ClusterId: uuidStr, Status: backup.Started} + cluster := backup.HostCluster{TypeName: "SHARDED_REPLICA_SET", ClusterName: resourceName, ShardName: resourceName} + oc.BackupConfigs[uuidStr] = &config + oc.BackupHostClusters[uuidStr] = &cluster + + // adding some host clusters for one shard and one config server - we don't care about relevance as they are + // expected top be ignored by Operator + + configUUID := uuid.New().String() + config1 := backup.Config{ClusterId: configUUID, Status: backup.Inactive} + cluster1 := backup.HostCluster{TypeName: "REPLICA_SET", ClusterName: resourceName, ShardName: resourceName + "-0"} + oc.BackupConfigs[configUUID] = &config1 + oc.BackupHostClusters[configUUID] = &cluster1 + + config2UUID := uuid.New().String() + config2 := backup.Config{ClusterId: config2UUID, Status: backup.Inactive} + cluster2 := backup.HostCluster{TypeName: "REPLICA_SET", ClusterName: resourceName, ShardName: resourceName + "-config-rs-0"} + oc.BackupConfigs[config2UUID] = &config2 + oc.BackupHostClusters[config2UUID] = &cluster2 + } +} + +func (oc *MockedOmConnection) addToHistory(value reflect.Value) { + oc.history = append(oc.history, runtime.FuncForPC(value.Pointer())) +} + +func buildHostsFromDeployment(d Deployment) *host.Result { + hosts := make([]host.Host, 0) + if d != nil { + for i, p := range d.getProcesses() { + hosts = append(hosts, host.Host{Id: strconv.Itoa(i), Hostname: p.HostName()}) + } + } + return &host.Result{Results: hosts} +} + +func (oc *MockedOmConnection) buildAutomationStatusFromDeployment(d Deployment, reached bool) *AutomationStatus { + // edge case: if there are no processes - we think that + processStatuses := make([]ProcessStatus, 0) + if d != nil { + for _, p := range d.getProcesses() { + if reached { + processStatuses = append(processStatuses, ProcessStatus{Name: p.Name(), LastGoalVersionAchieved: 1}) + } else { + processStatuses = append(processStatuses, ProcessStatus{Name: p.Name(), LastGoalVersionAchieved: 0}) + } + } + } + return &AutomationStatus{GoalVersion: 1, Processes: processStatuses} +} + +func (oc *MockedOmConnection) CheckGroupInOrganization(t *testing.T, orgName, groupName string) { + for k, v := range oc.OrganizationsWithGroups { + if k.Name == orgName { + for _, g := range v { + if g.Name == groupName { + return + } + } + } + } + assert.Fail(t, fmt.Sprintf("Project %s not found in organization %s", groupName, orgName)) +} + +func (oc *MockedOmConnection) FindGroup(groupName string) *Project { + for _, v := range oc.OrganizationsWithGroups { + for _, g := range v { + if g.Name == groupName { + return g + } + } + } + return nil +} + +func (oc *MockedOmConnection) findOrganization(orgId string) (*Organization, error) { + for k := range oc.OrganizationsWithGroups { + if k.ID == orgId { + return k, nil + } + } + return nil, apierror.New(xerrors.Errorf("Organization with id %s not found", orgId)) +} + +func (oc *MockedOmConnection) OpsManagerVersion() versionutil.OpsManagerVersion { + if oc.context.Version.VersionString != "" { + return oc.context.Version + } + return versionutil.OpsManagerVersion{VersionString: "7.0.0"} +} + +func (oc *MockedOmConnection) GetPreferredHostnames(agentApiKey string) ([]PreferredHostname, error) { + if agentApiKey != oc.AgentAPIKey { + return nil, apierror.New(xerrors.Errorf("Unauthorized")) + } + + return oc.PreferredHostnames, nil +} + +func (oc *MockedOmConnection) AddPreferredHostname(agentApiKey string, value string, isRegexp bool) error { + if agentApiKey != oc.AgentAPIKey { + return apierror.New(xerrors.Errorf("Unauthorized")) + } + + for _, hostname := range oc.PreferredHostnames { + if hostname.Regexp == isRegexp && hostname.Value == value { + return nil + } + } + oc.PreferredHostnames = append(oc.PreferredHostnames, PreferredHostname{ + Regexp: isRegexp, + EndsWith: !isRegexp, + Value: value, + }) + + return nil +} + +// updateAutoAuthMechanism simulates the changes made by Ops Manager and the agents in deciding which +// mechanism will be specified as the "autoAuthMechanisms" +func updateAutoAuthMechanism(ac *AutomationConfig) { + mechanisms := ac.Auth.AutoAuthMechanisms + if stringutil.Contains(mechanisms, "SCRAM-SHA-256") { + ac.Auth.AutoAuthMechanism = "SCRAM-SHA-256" + } else if stringutil.Contains(mechanisms, "MONGODB-CR") { + ac.Auth.AutoAuthMechanism = "MONGODB-CR" + } else if stringutil.Contains(mechanisms, "MONGODB-X509") { + ac.Auth.AutoAuthMechanism = "MONGODB-X509" + } +} diff --git a/controllers/om/monitoring_agent_config.go b/controllers/om/monitoring_agent_config.go new file mode 100644 index 000000000..142a68f14 --- /dev/null +++ b/controllers/om/monitoring_agent_config.go @@ -0,0 +1,99 @@ +package om + +import ( + "encoding/json" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type MonitoringAgentConfig struct { + MonitoringAgentTemplate *MonitoringAgentTemplate + BackingMap map[string]interface{} +} + +type MonitoringAgentTemplate struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + SSLPemKeyFile string `json:"sslPEMKeyFile,omitempty"` + LdapGroupDN string `json:"ldapGroupDN,omitempty"` + LogRotate mdbv1.LogRotateForBackupAndMonitoring `json:"logRotate,omitempty"` +} + +func (m *MonitoringAgentConfig) Apply() error { + merged, err := util.MergeWith(m.MonitoringAgentTemplate, m.BackingMap, &util.AutomationConfigTransformer{}) + if err != nil { + return err + } + m.BackingMap = merged + return nil +} + +func (m *MonitoringAgentConfig) SetAgentUserName(MonitoringAgentSubject string) { + m.MonitoringAgentTemplate.Username = MonitoringAgentSubject +} + +func (m *MonitoringAgentConfig) UnsetAgentUsername() { + m.MonitoringAgentTemplate.Username = util.MergoDelete +} + +func (m *MonitoringAgentConfig) SetAgentPassword(pwd string) { + m.MonitoringAgentTemplate.Password = pwd +} + +func (m *MonitoringAgentConfig) UnsetAgentPassword() { + m.MonitoringAgentTemplate.Password = util.MergoDelete +} + +func (m *MonitoringAgentConfig) EnableX509Authentication(MonitoringAgentSubject string) { + m.MonitoringAgentTemplate.SSLPemKeyFile = util.AutomationAgentPemFilePath + m.SetAgentUserName(MonitoringAgentSubject) +} + +func (m *MonitoringAgentConfig) DisableX509Authentication() { + m.MonitoringAgentTemplate.SSLPemKeyFile = util.MergoDelete + m.UnsetAgentUsername() +} + +func (m *MonitoringAgentConfig) EnableLdapAuthentication(monitoringAgentSubject string, monitoringAgentPwd string) { + m.SetAgentUserName(monitoringAgentSubject) + m.SetAgentPassword(monitoringAgentPwd) +} + +func (m *MonitoringAgentConfig) DisableLdapAuthentication() { + m.UnsetAgentUsername() + m.UnsetAgentPassword() + m.UnsetLdapGroupDN() +} + +func (m *MonitoringAgentConfig) SetLdapGroupDN(ldapGroupDn string) { + m.MonitoringAgentTemplate.LdapGroupDN = ldapGroupDn +} + +func (m *MonitoringAgentConfig) UnsetLdapGroupDN() { + m.MonitoringAgentTemplate.LdapGroupDN = util.MergoDelete +} + +func (m *MonitoringAgentConfig) SetLogRotate(logRotateConfig mdbv1.LogRotateForBackupAndMonitoring) { + m.MonitoringAgentTemplate.LogRotate = logRotateConfig +} + +func BuildMonitoringAgentConfigFromBytes(jsonBytes []byte) (*MonitoringAgentConfig, error) { + fullMap := make(map[string]interface{}) + if err := json.Unmarshal(jsonBytes, &fullMap); err != nil { + return nil, err + } + + config := &MonitoringAgentConfig{BackingMap: fullMap} + template := &MonitoringAgentTemplate{} + if username, ok := fullMap["username"].(string); ok { + template.Username = username + } + + if sslPemKeyfile, ok := fullMap["sslPEMKeyFile"].(string); ok { + template.SSLPemKeyFile = sslPemKeyfile + } + + config.MonitoringAgentTemplate = template + return config, nil +} diff --git a/controllers/om/monitoring_agent_test.go b/controllers/om/monitoring_agent_test.go new file mode 100644 index 000000000..9c24e8d67 --- /dev/null +++ b/controllers/om/monitoring_agent_test.go @@ -0,0 +1,54 @@ +package om + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +var testMonitoringConfig = *getTestMonitoringConfig() + +func getTestMonitoringConfig() *MonitoringAgentConfig { + a, _ := BuildMonitoringAgentConfigFromBytes(loadBytesFromTestData("monitoring_config.json")) + return a +} + +func TestMonitoringAgentConfigApply(t *testing.T) { + config := getTestMonitoringConfig() + config.MonitoringAgentTemplate.Username = "my-user-name" + config.MonitoringAgentTemplate.SSLPemKeyFile = util.MergoDelete + + err := config.Apply() + assert.NoError(t, err) + + modified := config.BackingMap + assert.Equal(t, "my-user-name", modified["username"], "modified values should be reflected in the map") + assert.NotContains(t, modified, "sslPEMKeyFile", "final map should not have keys with empty values") +} + +func TestFieldsAreAddedToMonitoringConfig(t *testing.T) { + config := getTestMonitoringConfig() + config.MonitoringAgentTemplate.SSLPemKeyFile = "my-pem-file" + config.MonitoringAgentTemplate.Username = "my-user-name" + + err := config.Apply() + assert.NoError(t, err) + + modified := config.BackingMap + assert.Equal(t, modified["sslPEMKeyFile"], "my-pem-file") + assert.Equal(t, modified["username"], "my-user-name") +} + +func TestFieldsAreNotRemovedWhenUpdatingMonitoringConfig(t *testing.T) { + config := getTestMonitoringConfig() + config.MonitoringAgentTemplate.SSLPemKeyFile = "my-pem-file" + config.MonitoringAgentTemplate.Username = "my-user-name" + + err := config.Apply() + assert.NoError(t, err) + + assert.Equal(t, config.BackingMap["logPath"], testMonitoringConfig.BackingMap["logPath"]) + assert.Equal(t, config.BackingMap["logPathWindows"], testMonitoringConfig.BackingMap["logPathWindows"]) +} diff --git a/controllers/om/omclient.go b/controllers/om/omclient.go new file mode 100644 index 000000000..5824ccc57 --- /dev/null +++ b/controllers/om/omclient.go @@ -0,0 +1,1072 @@ +package om + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "reflect" + "strconv" + "strings" + "sync" + + "github.com/blang/semver" + "github.com/r3labs/diff/v3" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/utils/ptr" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om/api" + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +// TODO move it to 'api' package + +// Connection is a client interacting with OpsManager API. Note, that all methods returning 'error' return the +// '*Error' in fact, but it's error-prone to declare method as returning specific implementation of error +// (see https://golang.org/doc/faq#nil_error) +type Connection interface { + UpdateDeployment(deployment Deployment) ([]byte, error) + ReadDeployment() (Deployment, error) + + // ReadUpdateDeployment reads Deployment from Ops Manager, applies the update function to it and pushes it back + ReadUpdateDeployment(depFunc func(Deployment) error, log *zap.SugaredLogger) error + ReadUpdateAgentsLogRotation(logRotateSetting mdbv1.AgentConfig, log *zap.SugaredLogger) error + ReadAutomationStatus() (*AutomationStatus, error) + ReadAutomationAgents(page int) (Paginated, error) + MarkProjectAsBackingDatabase(databaseType BackingDatabaseType) error + + ReadOrganizationsByName(name string) ([]*Organization, error) + // ReadOrganizations returns all organizations at specified page + ReadOrganizations(page int) (Paginated, error) + ReadOrganization(orgID string) (*Organization, error) + + ReadProjectsInOrganizationByName(orgID string, name string) ([]*Project, error) + // ReadProjectsInOrganization returns all projects in the organization at the specified page + ReadProjectsInOrganization(orgID string, page int) (Paginated, error) + CreateProject(project *Project) (*Project, error) + UpdateProject(project *Project) (*Project, error) + + ReadAgentVersion() (AgentsVersionsResponse, error) + + GetPreferredHostnames(agentApiKey string) ([]PreferredHostname, error) + AddPreferredHostname(agentApiKey string, value string, isRegexp bool) error + + backup.GroupConfigReader + backup.GroupConfigUpdater + + backup.HostClusterReader + + backup.ConfigReader + backup.ConfigUpdater + + OpsManagerVersion() versionutil.OpsManagerVersion + + AgentKeyGenerator + + AutomationConfigConnection + MonitoringConfigConnection + BackupConfigConnection + HasAgentAuthMode + + host.Adder + host.GetRemover + host.Updater + + controlledfeature.Getter + controlledfeature.Updater + + BaseURL() string + GroupID() string + GroupName() string + OrgID() string + PublicKey() string + PrivateKey() string + + // ConfigureProject configures the OMContext to have the correct project and org ids + ConfigureProject(project *Project) +} + +type MonitoringConfigConnection interface { + ReadMonitoringAgentConfig() (*MonitoringAgentConfig, error) + UpdateMonitoringAgentConfig(mat *MonitoringAgentConfig, log *zap.SugaredLogger) ([]byte, error) + ReadUpdateMonitoringAgentConfig(matFunc func(*MonitoringAgentConfig) error, log *zap.SugaredLogger) error +} + +type BackupConfigConnection interface { + ReadBackupAgentConfig() (*BackupAgentConfig, error) + UpdateBackupAgentConfig(mat *BackupAgentConfig, log *zap.SugaredLogger) ([]byte, error) + ReadUpdateBackupAgentConfig(matFunc func(*BackupAgentConfig) error, log *zap.SugaredLogger) error +} + +type HasAgentAuthMode interface { + GetAgentAuthMode() (string, error) +} + +type AgentKeyGenerator interface { + GenerateAgentKey() (string, error) +} + +// AutomationConfigConnection is an interface that only deals with reading/updating of the AutomationConfig +type AutomationConfigConnection interface { + // UpdateAutomationConfig updates the Automation Config in Ops Manager + // Note, that this method calls *the same* api endpoint as the `OmConnection.UpdateDeployment` - just uses a + // Deployment wrapper (AutomationConfig) as a parameter + UpdateAutomationConfig(ac *AutomationConfig, log *zap.SugaredLogger) error + ReadAutomationConfig() (*AutomationConfig, error) + // ReadAutomationConfig reads the Automation Config from Ops Manager + // Note, that this method calls *the same* api endpoint as the `OmConnection.ReadDeployment` - just wraps the answer + // to the different object + ReadUpdateAutomationConfig(acFunc func(ac *AutomationConfig) error, log *zap.SugaredLogger) error + + // Calls the API to update all the MongoDB Agents in the project to the latest. Returns the new version + UpgradeAgentsToLatest() (string, error) +} + +// omMutexes is the synchronous map of mutexes that provide strict serializability for operations "read-modify-write" +// for Ops Manager. Keys are (group_name + org_id) and values are mutexes. +var omMutexes = sync.Map{} + +// GetMutex creates or reuses the relevant mutex for the group + org +func GetMutex(projectName, orgId string) *sync.Mutex { + lockName := projectName + orgId + mutex, _ := omMutexes.LoadOrStore(lockName, &sync.Mutex{}) + return mutex.(*sync.Mutex) +} + +type BackingDatabaseType string + +const ( + AppDBDatabaseType BackingDatabaseType = "APP_DB" +) + +// ConnectionFactory type defines a function to create a connection to Ops Manager API. +// That's the way we implement some kind of Template Factory pattern to create connections using some incoming parameters +// (groupId, api key etc. - all encapsulated into 'context'). The reconciler later uses this factory to build real +// connections and this allows us to mock out Ops Manager communication during unit testing +type ConnectionFactory func(context *OMContext) Connection + +// OMContext is the convenient way of grouping all OM related information together +type OMContext struct { + BaseURL string + GroupID string + GroupName string + OrgID string + PrivateKey string + PublicKey string + Version versionutil.OpsManagerVersion + + // Will check that the SSL certificate provided by the Ops Manager Server is valid + // I've decided to use a "AllowInvalid" instead of "RequireValid" as the Zero value + // of bool is false. + AllowInvalidSSLCertificate bool + + // CACertificate is the actual certificate as a string, as every "Project" could have + // its own certificate. + CACertificate string +} + +type HTTPOmConnection struct { + context *OMContext + once sync.Once + client *api.Client +} + +func (oc *HTTPOmConnection) ReadUpdateAgentsLogRotation(logRotateSetting mdbv1.AgentConfig, log *zap.SugaredLogger) error { + // We don't have to wait for each step for the agent to reach goal state as setting logrotation does not require order + if logRotateSetting.Mongod.LogRotate == nil && logRotateSetting.MonitoringAgent.LogRotate == nil && + logRotateSetting.BackupAgent.LogRotate == nil && logRotateSetting.Mongod.AuditLogRotate == nil { + return nil + } + + automationConfig, err := oc.ReadAutomationConfig() + if err != nil { + return err + } + + if len(automationConfig.Deployment.getProcesses()) > 0 && logRotateSetting.Mongod.LogRotate != nil { + omVersion, err := oc.OpsManagerVersion().Semver() + if err != nil { + log.Debugw("Failed to fetch OpsManager version: %s", err) + return nil + } + + // We only support process configuration for OM larger than 7.0.4 or 6.0.24 + if !oc.OpsManagerVersion().IsCloudManager() && !omVersion.GTE(semver.MustParse("7.0.4")) && !omVersion.GTE(semver.MustParse("6.0.24")) { + return xerrors.Errorf("configuring log rotation for mongod processes is supported only with Cloud Manager or Ops Manager with versions >= 7.0.4 or >= 6.0.24") + } + + // We only retrieve the first process, since logRotation is configured the same for all processes + process := automationConfig.Deployment.getProcesses()[0] + if err = updateProcessLogRotateIfChanged(logRotateSetting.Mongod.LogRotate, process.GetLogRotate(), oc.UpdateProcessLogRotation); err != nil { + return err + } + if err = updateProcessLogRotateIfChanged(logRotateSetting.Mongod.AuditLogRotate, process.GetAuditLogRotate(), oc.UpdateAuditLogRotation); err != nil { + return err + } + } + + if len(automationConfig.Deployment.getBackupVersions()) > 0 && logRotateSetting.BackupAgent.LogRotate != nil { + err = oc.ReadUpdateBackupAgentConfig(func(config *BackupAgentConfig) error { + config.SetLogRotate(*logRotateSetting.BackupAgent.LogRotate) + return nil + }, log) + } + + if len(automationConfig.Deployment.getMonitoringVersions()) > 0 && logRotateSetting.MonitoringAgent.LogRotate != nil { + err = oc.ReadUpdateMonitoringAgentConfig(func(config *MonitoringAgentConfig) error { + config.SetLogRotate(*logRotateSetting.MonitoringAgent.LogRotate) + return nil + }, log) + } + + return err +} + +func updateProcessLogRotateIfChanged(logRotateSettingFromCRD *automationconfig.CrdLogRotate, logRotationSettingFromWire map[string]interface{}, updateLogRotationSetting func(logRotateSetting automationconfig.AcLogRotate) ([]byte, error)) error { + logRotationToSetInAC := automationconfig.ConvertCrdLogRotateToAC(logRotateSettingFromCRD) + if logRotationToSetInAC == nil { + return nil + } + toMap, err := maputil.StructToMap(logRotationToSetInAC) + if err != nil { + return err + } + + // We only support setting the same log rotation for all agents for the same type the same rotation config + if equality.Semantic.DeepEqual(logRotationSettingFromWire, toMap) { + return nil + } + + _, err = updateLogRotationSetting(*logRotationToSetInAC) + return err +} + +func (oc *HTTPOmConnection) UpdateProcessLogRotation(logRotateSetting automationconfig.AcLogRotate) ([]byte, error) { + return oc.put(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/systemLogRotateConfig", oc.GroupID()), logRotateSetting) +} + +func (oc *HTTPOmConnection) UpdateAuditLogRotation(logRotateSetting automationconfig.AcLogRotate) ([]byte, error) { + return oc.put(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/auditLogRotateConfig", oc.GroupID()), logRotateSetting) +} + +func (oc *HTTPOmConnection) GetAgentAuthMode() (string, error) { + ac, err := oc.ReadAutomationConfig() + if err != nil { + return "", err + } + if ac.Auth == nil { + return "", nil + } + return ac.Auth.AutoAuthMechanism, nil +} + +var _ Connection = &HTTPOmConnection{} + +// NewOpsManagerConnection stores OpsManger api endpoint and authentication credentials. +// It makes it easy to call the API without having to explicitly provide connection details. +func NewOpsManagerConnection(context *OMContext) Connection { + return &HTTPOmConnection{ + context: context, + } +} + +func (oc *HTTPOmConnection) ReadGroupBackupConfig() (backup.GroupBackupConfig, error) { + ans, apiErr := oc.get(fmt.Sprintf("/api/public/v1.0/admin/backup/groups/%s", oc.GroupID())) + + if apiErr != nil { + // This API provides very inconsistent way for obtaining values and authorization. In certain Ops Manager versions + // when there's no Group Backup Config we get 404 (which is inconsistent with the UI, as the endpoints for UI + // always return values). In Ops Manager 6, if this object doesn't yet exist, we get 401. So the only reasonable + // thing to do here is to check whether the error comes from the Ops Manager (if we can parse it), and if we do, + // we just ignore it. + var err *apierror.Error + ok := errors.As(apiErr, &err) + if ok { + return backup.GroupBackupConfig{ + Id: ptr.To(oc.GroupID()), + }, nil + } + return backup.GroupBackupConfig{}, apiErr + } + groupBackupConfig := &backup.GroupBackupConfig{} + if err := json.Unmarshal(ans, groupBackupConfig); err != nil { + return backup.GroupBackupConfig{}, apierror.New(err) + } + + return *groupBackupConfig, nil +} + +func (oc *HTTPOmConnection) UpdateGroupBackupConfig(config backup.GroupBackupConfig) ([]byte, error) { + return oc.put(fmt.Sprintf("/api/public/v1.0/admin/backup/groups/%s", *config.Id), config) +} + +func (oc *HTTPOmConnection) ConfigureProject(project *Project) { + oc.context.GroupID = project.ID + oc.context.OrgID = project.OrgID +} + +// BaseURL returns BaseURL of HTTPOmConnection +func (oc *HTTPOmConnection) BaseURL() string { + return oc.context.BaseURL +} + +// GroupID returns GroupID of HTTPOmConnection +func (oc *HTTPOmConnection) GroupID() string { + return oc.context.GroupID +} + +// GroupName returns GroupName of HTTPOmConnection +func (oc *HTTPOmConnection) GroupName() string { + return oc.context.GroupName +} + +// OrgID returns OrgID of HTTPOmConnection +func (oc *HTTPOmConnection) OrgID() string { + return oc.context.OrgID +} + +// PublicKey returns PublicKey of HTTPOmConnection +func (oc *HTTPOmConnection) PublicKey() string { + return oc.context.PublicKey +} + +// PrivateKey returns PrivateKey of HTTPOmConnection +func (oc *HTTPOmConnection) PrivateKey() string { + return oc.context.PrivateKey +} + +// OpsManagerVersion returns the current Ops Manager version +func (oc *HTTPOmConnection) OpsManagerVersion() versionutil.OpsManagerVersion { + return oc.context.Version +} + +// UpdateDeployment updates a given deployment to the new deployment object passed as parameter. +func (oc *HTTPOmConnection) UpdateDeployment(deployment Deployment) ([]byte, error) { + return oc.put(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig", oc.GroupID()), deployment) +} + +// ReadDeployment returns a Deployment object for this group +func (oc *HTTPOmConnection) ReadDeployment() (Deployment, error) { + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig", oc.GroupID())) + if err != nil { + return nil, err + } + d, e := BuildDeploymentFromBytes(ans) + return d, apierror.New(e) +} + +func (oc *HTTPOmConnection) ReadAutomationConfig() (*AutomationConfig, error) { + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig", oc.GroupID())) + if err != nil { + return nil, err + } + + ac, err := BuildAutomationConfigFromBytes(ans) + + return ac, apierror.New(err) +} + +// ReadUpdateDeployment performs the "read-modify-update" operation on OpsManager Deployment. +// Note, that the mutex locks infinitely (there is no built-in support for timeouts for locks in Go) which seems to be +// ok as OM endpoints are not supposed to hang for long +func (oc *HTTPOmConnection) ReadUpdateDeployment(changeDeploymentFunc func(Deployment) error, log *zap.SugaredLogger) error { + mutex := GetMutex(oc.GroupName(), oc.OrgID()) + mutex.Lock() + defer mutex.Unlock() + deployment, err := oc.ReadDeployment() + if err != nil { + return err + } + + isEqual, err := isEqualAfterModification(changeDeploymentFunc, deployment) + if err != nil { + return err + } + if isEqual { + log.Debug("AutomationConfig has not changed, not pushing changes to Ops Manager") + return nil + } + + _, err = oc.UpdateDeployment(deployment) + if util.ShouldLogAutomationConfigDiff() { + originalDeployment, err := oc.ReadDeployment() + if err != nil { + return apierror.New(err) + } + + changelog, err := diff.Diff(originalDeployment, deployment, diff.AllowTypeMismatch(true)) + if err != nil { + return apierror.New(err) + } + + log.Debug("Deployment diff (%d changes): %+v", len(changelog), changelog) + } + if err != nil { + return apierror.New(err) + } + return nil +} + +func (oc *HTTPOmConnection) UpdateAutomationConfig(ac *AutomationConfig, log *zap.SugaredLogger) error { + err := ac.Apply() + if err != nil { + return err + } + + _, err = oc.UpdateDeployment(ac.Deployment) + if err != nil { + return err + } + return nil +} + +func (oc *HTTPOmConnection) ReadUpdateAutomationConfig(modifyACFunc func(ac *AutomationConfig) error, log *zap.SugaredLogger) error { + mutex := GetMutex(oc.GroupName(), oc.OrgID()) + mutex.Lock() + defer mutex.Unlock() + + ac, err := oc.ReadAutomationConfig() + if err != nil { + log.Errorf("error reading automation config. %s", err) + return err + } + + original, err := BuildAutomationConfigFromDeployment(ac.Deployment) + if err != nil { + return err + } + + if err := modifyACFunc(ac); err != nil { + return apierror.New(err) + } + + if !reflect.DeepEqual(original.Deployment, ac.Deployment) { + panic("It seems you modified the deployment directly. This is not allowed. Please use helper objects instead.") + } + + areEqual := original.EqualsWithoutDeployment(*ac) + if areEqual { + log.Debug("AutomationConfig has not changed, not pushing changes to Ops Manager") + return nil + } + + // we are using UpdateAutomationConfig since we need to apply our changes. + err = oc.UpdateAutomationConfig(ac, log) + if util.ShouldLogAutomationConfigDiff() { + changelog, err := diff.Diff(original.Deployment, ac.Deployment, diff.AllowTypeMismatch(true)) + if err != nil { + return apierror.New(err) + } + + log.Debug("Deployment diff (%d changes): %+v", len(changelog), changelog) + } + if err != nil { + log.Errorf("error updating automation config. %s", err) + return apierror.New(err) + } + return nil +} + +func (oc *HTTPOmConnection) UpgradeAgentsToLatest() (string, error) { + ans, err := oc.post(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/updateAgentVersions", oc.GroupID()), nil) + if err != nil { + return "", err + } + type updateAgentsVersionsResponse struct { + AutomationAgentVersion string `json:"automationAgentVersion"` + } + var response updateAgentsVersionsResponse + if err = json.Unmarshal(ans, &response); err != nil { + return "", apierror.New(err) + } + return response.AutomationAgentVersion, nil +} + +func (oc *HTTPOmConnection) GenerateAgentKey() (string, error) { + data := map[string]string{"desc": "Agent key for Kubernetes"} + ans, err := oc.post(fmt.Sprintf("/api/public/v1.0/groups/%s/agentapikeys", oc.GroupID()), data) + if err != nil { + return "", err + } + + var keyInfo map[string]interface{} + if err := json.Unmarshal(ans, &keyInfo); err != nil { + return "", apierror.New(err) + } + return keyInfo["key"].(string), nil +} + +// ReadAutomationAgents returns the state of the automation agents registered in Ops Manager +func (oc *HTTPOmConnection) ReadAutomationAgents(pageNum int) (Paginated, error) { + // TODO: Add proper testing to this pagination. In order to test it I just used `itemsPerPage=1`, which will make + // the endpoint to be called 3 times in a 3 member replica set. The default itemsPerPage is 100 + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/agents/AUTOMATION?pageNum=%d", oc.GroupID(), pageNum)) + if err != nil { + return nil, err + } + var resp AutomationAgentStatusResponse + if err := json.Unmarshal(ans, &resp); err != nil { + return nil, err + } + return resp, apierror.New(err) +} + +// ReadAutomationStatus returns the state of the automation status, this includes if the agents +// have reached goal state. +func (oc *HTTPOmConnection) ReadAutomationStatus() (*AutomationStatus, error) { + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/automationStatus", oc.GroupID())) + if err != nil { + return nil, err + } + status, e := buildAutomationStatusFromBytes(ans) + return status, apierror.New(e) +} + +// AddHost adds the given host to the project +func (oc *HTTPOmConnection) AddHost(host host.Host) error { + _, err := oc.post(fmt.Sprintf("/api/public/v1.0/groups/%s/hosts", oc.GroupID()), host) + return err +} + +// UpdateHost adds the given host. +func (oc *HTTPOmConnection) UpdateHost(host host.Host) error { + _, err := oc.patch(fmt.Sprintf("/api/public/v1.0/groups/%s/hosts/%s", oc.GroupID(), host.Id), host) + return err +} + +// GetHosts return the hosts in this group +func (oc *HTTPOmConnection) GetHosts() (*host.Result, error) { + mPath := fmt.Sprintf("/api/public/v1.0/groups/%s/hosts/", oc.GroupID()) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + hosts := &host.Result{} + if err := json.Unmarshal(res, hosts); err != nil { + return nil, apierror.New(err) + } + + return hosts, nil +} + +// RemoveHost will remove host, identified by hostID from group +func (oc *HTTPOmConnection) RemoveHost(hostID string) error { + mPath := fmt.Sprintf("/api/public/v1.0/groups/%s/hosts/%s", oc.GroupID(), hostID) + return oc.delete(mPath) +} + +// ReadOrganizationsByName finds the organizations by name. It uses the same endpoint as the 'ReadOrganizations' but +// 'name' and 'page' parameters are not supposed to be used together so having a separate endpoint allows +func (oc *HTTPOmConnection) ReadOrganizationsByName(name string) ([]*Organization, error) { + mPath := fmt.Sprintf("/api/public/v1.0/orgs?name=%s", url.QueryEscape(name)) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + orgsResponse := &OrganizationsResponse{} + if err = json.Unmarshal(res, orgsResponse); err != nil { + return nil, apierror.New(err) + } + + return orgsResponse.Organizations, nil +} + +// ReadOrganizations returns all organizations at the specified page. +func (oc *HTTPOmConnection) ReadOrganizations(page int) (Paginated, error) { + mPath := fmt.Sprintf("/api/public/v1.0/orgs?itemsPerPage=500&pageNum=%d", page) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + orgsResponse := &OrganizationsResponse{} + if err := json.Unmarshal(res, orgsResponse); err != nil { + return nil, apierror.New(err) + } + + return orgsResponse, nil +} + +func (oc *HTTPOmConnection) ReadOrganization(orgID string) (*Organization, error) { + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/orgs/%s", orgID)) + if err != nil { + return nil, err + } + organization := &Organization{} + if err := json.Unmarshal(ans, organization); err != nil { + return nil, apierror.New(err) + } + + return organization, nil +} + +func (oc *HTTPOmConnection) MarkProjectAsBackingDatabase(backingType BackingDatabaseType) error { + _, err := oc.post(fmt.Sprintf("/api/private/v1.0/groups/%s/markAsBackingDatabase", oc.GroupID()), string(backingType)) + if err != nil { + var apiErr *apierror.Error + if errors.As(err, &apiErr) { + if apiErr.Status != nil && *apiErr.Status == 400 && strings.Contains(apiErr.Detail, "INVALID_DOCUMENT") { + return nil + } + } + return err + } + return nil +} + +func (oc *HTTPOmConnection) ReadProjectsInOrganizationByName(orgID string, name string) ([]*Project, error) { + mPath := fmt.Sprintf("/api/public/v1.0/orgs/%s/groups?name=%s", orgID, url.QueryEscape(name)) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + projectsResponse := &ProjectsResponse{} + if err := json.Unmarshal(res, projectsResponse); err != nil { + return nil, apierror.New(err) + } + + return projectsResponse.Groups, nil +} + +// ReadProjectsInOrganization returns all projects inside organization +func (oc *HTTPOmConnection) ReadProjectsInOrganization(orgID string, page int) (Paginated, error) { + mPath := fmt.Sprintf("/api/public/v1.0/orgs/%s/groups?itemsPerPage=500&pageNum=%d", orgID, page) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + projectsResponse := &ProjectsResponse{} + if err := json.Unmarshal(res, projectsResponse); err != nil { + return nil, apierror.New(err) + } + + return projectsResponse, nil +} + +func (oc *HTTPOmConnection) CreateProject(project *Project) (*Project, error) { + res, err := oc.post("/api/public/v1.0/groups", project) + if err != nil { + return nil, err + } + + g := &Project{} + if err := json.Unmarshal(res, g); err != nil { + return nil, apierror.New(err) + } + + return g, nil +} + +func (oc *HTTPOmConnection) UpdateProject(project *Project) (*Project, error) { + path := fmt.Sprintf("/api/public/v1.0/groups/%s", project.ID) + res, err := oc.patch(path, project) + if err != nil { + return nil, err + } + + g := &Project{} + if err := json.Unmarshal(res, g); err != nil { + return nil, apierror.New(err) + } + + return project, nil +} + +func (oc *HTTPOmConnection) ReadBackupConfigs() (*backup.ConfigsResponse, error) { + mPath := fmt.Sprintf("/api/public/v1.0/groups/%s/backupConfigs", oc.GroupID()) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + response := &backup.ConfigsResponse{} + if err := json.Unmarshal(res, response); err != nil { + return nil, apierror.New(err) + } + + return response, nil +} + +func (oc *HTTPOmConnection) ReadBackupConfig(clusterID string) (*backup.Config, error) { + mPath := fmt.Sprintf("/api/public/v1.0/groups/%s/backupConfigs/%s", oc.GroupID(), clusterID) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + response := &backup.Config{} + if err := json.Unmarshal(res, response); err != nil { + return nil, apierror.New(err) + } + + return response, nil +} + +func (oc *HTTPOmConnection) UpdateBackupConfig(config *backup.Config) (*backup.Config, error) { + path := fmt.Sprintf("/api/public/v1.0/groups/%s/backupConfigs/%s", oc.GroupID(), config.ClusterId) + res, err := oc.patch(path, config) + if err != nil { + return nil, err + } + + response := &backup.Config{} + if err := json.Unmarshal(res, response); err != nil { + return nil, apierror.New(err) + } + return response, nil +} + +func (oc *HTTPOmConnection) ReadHostCluster(clusterID string) (*backup.HostCluster, error) { + mPath := fmt.Sprintf("/api/public/v1.0/groups/%s/clusters/%s", oc.GroupID(), clusterID) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + cluster := &backup.HostCluster{} + if err := json.Unmarshal(res, cluster); err != nil { + return nil, apierror.New(err) + } + + return cluster, nil +} + +func (oc *HTTPOmConnection) UpdateBackupStatus(clusterID string, status backup.Status) error { + path := fmt.Sprintf("/api/public/v1.0/groups/%s/backupConfigs/%s", oc.GroupID(), clusterID) + + _, err := oc.patch(path, map[string]interface{}{"statusName": status}) + if err != nil { + return apierror.New(err) + } + + return nil +} + +func (oc *HTTPOmConnection) ReadMonitoringAgentConfig() (*MonitoringAgentConfig, error) { + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/monitoringAgentConfig", oc.GroupID())) + if err != nil { + return nil, err + } + + mat, err := BuildMonitoringAgentConfigFromBytes(ans) + if err != nil { + return nil, err + } + return mat, nil +} + +func (oc *HTTPOmConnection) UpdateMonitoringAgentConfig(mat *MonitoringAgentConfig, log *zap.SugaredLogger) ([]byte, error) { + original, _ := util.MapDeepCopy(mat.BackingMap) + + err := mat.Apply() + if err != nil { + return nil, err + } + + if reflect.DeepEqual(original, mat.BackingMap) { + log.Debug("Monitoring Configuration has not changed, not pushing changes to Ops Manager") + } else { + return oc.put(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/monitoringAgentConfig", oc.GroupID()), mat.BackingMap) + } + return nil, nil +} + +func (oc *HTTPOmConnection) ReadUpdateMonitoringAgentConfig(modifyMonitoringAgentFunction func(*MonitoringAgentConfig) error, log *zap.SugaredLogger) error { + if log == nil { + log = zap.S() + } + mutex := GetMutex(oc.GroupName(), oc.OrgID()) + mutex.Lock() + defer mutex.Unlock() + + mat, err := oc.ReadMonitoringAgentConfig() + if err != nil { + return err + } + + if err := modifyMonitoringAgentFunction(mat); err != nil { + return err + } + + if _, err := oc.UpdateMonitoringAgentConfig(mat, log); err != nil { + return err + } + + return nil +} + +func (oc *HTTPOmConnection) ReadBackupAgentConfig() (*BackupAgentConfig, error) { + ans, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/backupAgentConfig", oc.GroupID())) + if err != nil { + return nil, err + } + + backupAgentConfig, err := BuildBackupAgentConfigFromBytes(ans) + if err != nil { + return nil, err + } + + return backupAgentConfig, nil +} + +func (oc *HTTPOmConnection) UpdateBackupAgentConfig(backup *BackupAgentConfig, log *zap.SugaredLogger) ([]byte, error) { + original, _ := util.MapDeepCopy(backup.BackingMap) + + err := backup.Apply() + if err != nil { + return nil, err + } + + if reflect.DeepEqual(original, backup.BackingMap) { + log.Debug("Backup Configuration has not changed, not pushing changes to Ops Manager") + } else { + return oc.put(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig/backupAgentConfig", oc.GroupID()), backup.BackingMap) + } + + return nil, nil +} + +func (oc *HTTPOmConnection) ReadUpdateBackupAgentConfig(backupFunc func(*BackupAgentConfig) error, log *zap.SugaredLogger) error { + if log == nil { + log = zap.S() + } + mutex := GetMutex(oc.GroupName(), oc.OrgID()) + mutex.Lock() + defer mutex.Unlock() + + backupAgentConfig, err := oc.ReadBackupAgentConfig() + if err != nil { + return err + } + + if err := backupFunc(backupAgentConfig); err != nil { + return err + } + + if _, err := oc.UpdateBackupAgentConfig(backupAgentConfig, log); err != nil { + return err + } + + return nil +} + +func (oc *HTTPOmConnection) UpdateControlledFeature(cf *controlledfeature.ControlledFeature) error { + _, err := oc.put(fmt.Sprintf("/api/public/v1.0/groups/%s/controlledFeature", oc.GroupID()), cf) + return err +} + +func (oc *HTTPOmConnection) GetControlledFeature() (*controlledfeature.ControlledFeature, error) { + res, err := oc.get(fmt.Sprintf("/api/public/v1.0/groups/%s/controlledFeature", oc.GroupID())) + if err != nil { + return nil, err + } + cf := &controlledfeature.ControlledFeature{} + if err := json.Unmarshal(res, cf); err != nil { + return nil, apierror.New(err) + } + return cf, nil +} + +func (oc *HTTPOmConnection) ReadSnapshotSchedule(clusterID string) (*backup.SnapshotSchedule, error) { + mPath := fmt.Sprintf("/api/public/v1.0/groups/%s/backupConfigs/%s/snapshotSchedule", oc.GroupID(), clusterID) + res, err := oc.get(mPath) + if err != nil { + return nil, err + } + + response := &backup.SnapshotSchedule{} + if err := json.Unmarshal(res, response); err != nil { + return nil, apierror.New(err) + } + + // OM returns 0 if not set instead of null or omitted field + if response.ClusterCheckpointIntervalMin != nil && *response.ClusterCheckpointIntervalMin == 0 { + response.ClusterCheckpointIntervalMin = nil + } + + return response, nil +} + +func (oc *HTTPOmConnection) UpdateSnapshotSchedule(clusterID string, snapshotSchedule *backup.SnapshotSchedule) error { + path := fmt.Sprintf("/api/public/v1.0/groups/%s/backupConfigs/%s/snapshotSchedule", oc.GroupID(), clusterID) + res, err := oc.patch(path, snapshotSchedule) + if err != nil { + return err + } + + if err := json.Unmarshal(res, &backup.SnapshotSchedule{}); err != nil { + return apierror.New(err) + } + return nil +} + +type AgentsVersionsResponse struct { + AutomationVersion string `json:"automationVersion"` + AutomationMinimumVersion string `json:"automationMinimumVersion"` +} + +// ReadAgentVersion reads the versions from OM API +func (oc *HTTPOmConnection) ReadAgentVersion() (AgentsVersionsResponse, error) { + body, err := oc.get("/api/public/v1.0/softwareComponents/versions/") + if err != nil { + return AgentsVersionsResponse{}, err + } + agentsVersions := &AgentsVersionsResponse{} + if err := json.Unmarshal(body, agentsVersions); err != nil { + return AgentsVersionsResponse{}, err + } + return *agentsVersions, nil +} + +type PreferredHostname struct { + Regexp bool `json:"regexp"` + EndsWith bool `json:"endsWith"` + Id string `json:"id"` + Value string `json:"value"` +} + +type GroupInfoResponse struct { + PreferredHostnames []PreferredHostname `json:"preferredHostnames"` +} + +// GetPreferredHostnames will call the info endpoint with the agent API key. +// We extract only the preferred hostnames from the response. +func (oc *HTTPOmConnection) GetPreferredHostnames(agentApiKey string) ([]PreferredHostname, error) { + infoPath := fmt.Sprintf("/group/v2/info/%s", oc.GroupID()) + body, err := oc.getWithAgentAuth(infoPath, agentApiKey) + if err != nil { + return nil, err + } + + groupInfo := &GroupInfoResponse{} + if err := json.Unmarshal(body, groupInfo); err != nil { + return nil, err + } + + return groupInfo.PreferredHostnames, nil +} + +// AddPreferredHostname will add a new preferred hostname. +// This does not check for duplicates. That needs to be checked in the consumer of this method. +// Here we also use the agent API key, so we need to configure basic auth. +// isRegex is true if the preferred hostnames is a regex, and false if it is "endsWith". +// We pass only "isRegex" to eliminate edge cases where both are set to the same value. +func (oc *HTTPOmConnection) AddPreferredHostname(agentApiKey string, value string, isRegexp bool) error { + path := fmt.Sprintf("/group/v2/addPreferredHostname/%s?value=%s&isRegexp=%s&isEndsWith=%s", + oc.GroupID(), value, strconv.FormatBool(isRegexp), strconv.FormatBool(!isRegexp)) + _, err := oc.getWithAgentAuth(path, agentApiKey) + if err != nil { + return err + } + return nil +} + +//********************************** Private methods ******************************************************************* + +func (oc *HTTPOmConnection) get(path string) ([]byte, error) { + return oc.httpVerb("GET", path, nil) +} + +func (oc *HTTPOmConnection) post(path string, v interface{}) ([]byte, error) { + return oc.httpVerb("POST", path, v) +} + +func (oc *HTTPOmConnection) put(path string, v interface{}) ([]byte, error) { + return oc.httpVerb("PUT", path, v) +} + +func (oc *HTTPOmConnection) patch(path string, v interface{}) ([]byte, error) { + return oc.httpVerb("PATCH", path, v) +} + +func (oc *HTTPOmConnection) delete(path string) error { + _, err := oc.httpVerb("DELETE", path, nil) + return err +} + +func (oc *HTTPOmConnection) httpVerb(method, path string, v interface{}) ([]byte, error) { + client, err := oc.getHTTPClient() + if err != nil { + return nil, err + } + + response, header, err := client.Request(method, oc.BaseURL(), path, v) + oc.setVersionFromHeader(header) + + return response, err +} + +func (oc *HTTPOmConnection) getWithAgentAuth(path string, agentApiKey string) ([]byte, error) { + client, err := oc.getHTTPClient() + if err != nil { + return nil, err + } + + response, header, err := client.RequestWithAgentAuth("GET", oc.BaseURL(), path, oc.getAgentAuthorization(agentApiKey), nil) + oc.setVersionFromHeader(header) + + return response, err +} + +// getHTTPClient gets a new or an already existing client. +func (oc *HTTPOmConnection) getHTTPClient() (*api.Client, error) { + var err error + + oc.once.Do(func() { + opts := api.NewHTTPOptions() + + if oc.context.CACertificate != "" { + zap.S().Debug("Using CA Certificate") + opts = append(opts, api.OptionCAValidate(oc.context.CACertificate)) + } + + if oc.context.AllowInvalidSSLCertificate { + zap.S().Debug("Allowing insecure certs") + opts = append(opts, api.OptionSkipVerify) + } + + opts = append(opts, api.OptionDigestAuth(oc.PublicKey(), oc.PrivateKey())) + + if env.ReadBoolOrDefault("OM_DEBUG_HTTP", false) { // nolint:forbidigo + zap.S().Debug("Enabling OM_DEBUG_HTTP mode") + opts = append(opts, api.OptionDebug) + } + + oc.client, err = api.NewHTTPClient(opts...) + }) + + return oc.client, err +} + +// getAgentAuthorization generates the basic authorization header +func (oc *HTTPOmConnection) getAgentAuthorization(agentApiKey string) string { + credentials := oc.GroupID() + ":" + agentApiKey + encodedCredentials := base64.StdEncoding.EncodeToString([]byte(credentials)) + return "Basic " + encodedCredentials +} + +func (oc *HTTPOmConnection) setVersionFromHeader(header http.Header) { + if header != nil { + oc.context.Version = versionutil.OpsManagerVersion{ + VersionString: versionutil.GetVersionFromOpsManagerApiHeader(header.Get("X-MongoDB-Service-Version")), + } + } +} diff --git a/controllers/om/omclient_test.go b/controllers/om/omclient_test.go new file mode 100644 index 000000000..09fe0e605 --- /dev/null +++ b/controllers/om/omclient_test.go @@ -0,0 +1,229 @@ +package om + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +func TestReadProjectsInOrganizationByName(t *testing.T) { + projects := []*Project{{ID: "111", Name: "The Project"}} + srv := serverMock(projectsInOrganizationByName("testOrgId", projects)) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL}) + + data, err := connection.ReadProjectsInOrganizationByName("testOrgId", "The Project") + assert.NoError(t, err) + assert.Equal(t, projects, data) +} + +func TestReadOrganizationsByName(t *testing.T) { + organizations := []*Organization{{ID: "111", Name: "The Organization"}} + srv := serverMock(organizationsByName(organizations)) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL}) + + data, err := connection.ReadOrganizationsByName("The Organization") + assert.NoError(t, err) + assert.Equal(t, organizations, data) +} + +func TestGettingAutomationConfig(t *testing.T) { + testAutomationConfig := getTestAutomationConfig() + handleFunc, _ := automationConfig("1", automationConfigResponse{config: testAutomationConfig}) + srv := serverMock(handleFunc) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL, GroupID: "1"}) + data, err := connection.ReadAutomationConfig() + + assert.NoError(t, err) + assert.Equal(t, testAutomationConfig.Deployment, data.Deployment) +} + +func TestNotSendingRequestOnNonModifiedAutomationConfig(t *testing.T) { + logger := zap.NewNop().Sugar() + testAutomationConfig := getTestAutomationConfig() + handleFunc, counters := automationConfig("1", automationConfigResponse{config: testAutomationConfig}) + srv := serverMock(handleFunc) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL, GroupID: "1"}) + err := connection.ReadUpdateAutomationConfig(func(ac *AutomationConfig) error { + return nil + }, logger) + + assert.NoError(t, err) + assert.Equal(t, 1, counters.getHitCount) + assert.Equal(t, 0, counters.putHitCount) +} + +// TestNotSendingRequestOnNonModifiedAutomationConfigWithMergoDelete verifies that util.MergoDelete will be ignored during equality comparisons +func TestNotSendingRequestOnNonModifiedAutomationConfigWithMergoDelete(t *testing.T) { + logger := zap.NewNop().Sugar() + testAutomationConfig := getTestAutomationConfig() + handleFunc, counters := automationConfig("1", automationConfigResponse{config: testAutomationConfig}) + srv := serverMock(handleFunc) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL, GroupID: "1"}) + err := connection.ReadUpdateAutomationConfig(func(ac *AutomationConfig) error { + ac.AgentSSL = &AgentSSL{ + AutoPEMKeyFilePath: util.MergoDelete, + } + return nil + }, logger) + + assert.NoError(t, err) + assert.Equal(t, 1, counters.getHitCount) + assert.Equal(t, 0, counters.putHitCount) +} + +func TestRetriesOnWritingAutomationConfig(t *testing.T) { + logger := zap.NewNop().Sugar() + testAutomationConfig := getTestAutomationConfig() + successfulResponse := automationConfigResponse{config: testAutomationConfig} + errorResponse := automationConfigResponse{errorCode: 500, errorString: "testing"} + handleFunc, counters := automationConfig("1", errorResponse, errorResponse, successfulResponse) + srv := serverMock(handleFunc) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL, GroupID: "1"}) + err := connection.ReadUpdateAutomationConfig(func(ac *AutomationConfig) error { + return nil + }, logger) + + assert.NoError(t, err) + assert.Equal(t, 3, counters.getHitCount) +} + +func TestHTTPOmConnectionGetHTTPClientRace(t *testing.T) { + successfulResponse := automationConfigResponse{config: getTestAutomationConfig()} + errorResponse := automationConfigResponse{errorCode: 500, errorString: "testing"} + handleFunc, _ := automationConfig("1", errorResponse, errorResponse, successfulResponse) + srv := serverMock(handleFunc) + defer srv.Close() + + connection := NewOpsManagerConnection(&OMContext{BaseURL: srv.URL, GroupID: "1"}).(*HTTPOmConnection) + wg := sync.WaitGroup{} + + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + _, err := connection.getHTTPClient() + assert.NoError(t, err) + wg.Done() + }() + } + + wg.Wait() +} + +// ******************************* Mock HTTP Server methods ***************************************************** + +type handleFunc func(mux *http.ServeMux) + +type counters struct { + getHitCount int + putHitCount int + totalCount int +} + +func serverMock(handlers ...handleFunc) *httptest.Server { + handler := http.NewServeMux() + for _, h := range handlers { + h(handler) + } + + srv := httptest.NewServer(handler) + + return srv +} + +func projectsInOrganizationByName(orgId string, projects []*Project) handleFunc { + return func(mux *http.ServeMux) { + mux.HandleFunc(fmt.Sprintf("/api/public/v1.0/orgs/%s/groups", orgId), + func(w http.ResponseWriter, r *http.Request) { + found := false + for _, p := range projects { + if p.Name == r.URL.Query()["name"][0] { + found = true + } + } + if !found { + w.WriteHeader(http.StatusNotFound) + return + } + response := ProjectsResponse{Groups: projects} + data, _ := json.Marshal(response) + _, _ = w.Write(data) + }) + } +} + +func organizationsByName(organizations []*Organization) handleFunc { + return func(mux *http.ServeMux) { + mux.HandleFunc("/api/public/v1.0/orgs", + func(w http.ResponseWriter, r *http.Request) { + found := false + for _, p := range organizations { + if p.Name == r.URL.Query()["name"][0] { + found = true + } + } + if !found { + w.WriteHeader(http.StatusNotFound) + return + } + response := OrganizationsResponse{Organizations: organizations} + data, _ := json.Marshal(response) + _, _ = w.Write(data) + }) + } +} + +type automationConfigResponse struct { + config *AutomationConfig + errorCode int + errorString string +} + +func automationConfig(groupId string, responses ...automationConfigResponse) (handleFunc, *counters) { + counters := &counters{} + handle := func(mux *http.ServeMux) { + mux.HandleFunc(fmt.Sprintf("/api/public/v1.0/groups/%s/automationConfig", groupId), + func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "GET": + counters.getHitCount = counters.getHitCount + 1 + response := responses[counters.totalCount] + if response.config != nil { + data, _ := json.Marshal(response.config.Deployment) + _, _ = w.Write(data) + } else if response.errorCode != 0 { + http.Error(w, response.errorString, response.errorCode) + } + case "PUT": + counters.putHitCount = counters.putHitCount + 1 + w.WriteHeader(http.StatusOK) + } + counters.totalCount = counters.totalCount + 1 + }) + } + return handle, counters +} diff --git a/controllers/om/ompaginator.go b/controllers/om/ompaginator.go new file mode 100644 index 000000000..1159b82ec --- /dev/null +++ b/controllers/om/ompaginator.go @@ -0,0 +1,80 @@ +package om + +// Paginated is the general interface for a single page returned by Ops Manager api. +type Paginated interface { + HasNext() bool + Results() []interface{} + ItemsCount() int +} + +type OMPaginated struct { + TotalCount int `json:"totalCount"` + Links []*Link `json:"links,omitempty"` +} + +type Link struct { + Rel string `json:"rel"` +} + +// HasNext return true if there is next page (see 'ApiBaseResource.handlePaginationInternal' in mms code) +func (o OMPaginated) HasNext() bool { + for _, l := range o.Links { + if l.Rel == "next" { + return true + } + } + return false +} + +func (o OMPaginated) ItemsCount() int { + return o.TotalCount +} + +// PageReader is the function that reads a single page by its number +type PageReader func(pageNum int) (Paginated, error) + +// PageItemPredicate is the function that processes single item on the page and returns true if no further processing +// needs to be done (usually it's the search logic) +type PageItemPredicate func(interface{}) bool + +// TraversePages reads page after page using 'apiFunc' and applies the 'predicate' for each item on the page. +// Stops traversal when the 'predicate' returns true +// Note, that in OM 4.0 the max number of pages is 100, but in OM 4.1 and CM - 500. +// So we'll traverse 100000 (200 pages 500 items on each) records in Cloud Manager and 20000 records in OM 4.0 - I believe it's ok +// This won't be necessary if MMS-5638 is implemented or if we make 'orgId' configuration mandatory +func TraversePages(reader PageReader, predicate PageItemPredicate) (found bool, err error) { + // First we check the first page and get the number of items to calculate the max number of pages to traverse + paginated, e := reader(1) + if e != nil { + return false, e + } + for _, entity := range paginated.Results() { + if predicate(entity) { + return true, nil + } + } + if !paginated.HasNext() { + return false, nil + } + + // We take 100 as the denuminator here assuming it's the OM 4.0. If it's OM 4.1 or CM - then we'll stop earlier + // thanks to '!paginated.HasNext()' as they support pages of size 500 + pagesNum := (paginated.ItemsCount() / 100) + 1 + + // Note that we start from 2nd page as we've checked the 1st one above + for i := 2; i <= pagesNum; i++ { + paginated, e := reader(i) + if e != nil { + return false, e + } + for _, entity := range paginated.Results() { + if predicate(entity) { + return true, nil + } + } + if !paginated.HasNext() { + return false, nil + } + } + return false, nil +} diff --git a/controllers/om/ompaginator_test.go b/controllers/om/ompaginator_test.go new file mode 100644 index 000000000..2c8e65eb9 --- /dev/null +++ b/controllers/om/ompaginator_test.go @@ -0,0 +1,82 @@ +package om + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPagination_SinglePage(t *testing.T) { + found, err := TraversePages(singleOrganizationsPage, func(obj interface{}) bool { return obj.(*Organization).Name == "test" }) + assert.True(t, found) + assert.NoError(t, err) + + found, err = TraversePages(singleOrganizationsPage, func(obj interface{}) bool { return obj.(*Organization).Name == "fake" }) + assert.False(t, found) + assert.NoError(t, err) +} + +func TestPagination_MultiplePages(t *testing.T) { + found, err := TraversePages(multipleOrganizationsPage, func(obj interface{}) bool { return obj.(*Organization).Name == "test1220" }) + assert.True(t, found) + assert.NoError(t, err) + assert.Equal(t, 3, numberOfPagesTraversed) + + found, err = TraversePages(multipleOrganizationsPage, func(obj interface{}) bool { return obj.(*Organization).Name == "test1400" }) + assert.False(t, found) + assert.NoError(t, err) +} + +func TestPagination_Error(t *testing.T) { + _, err := TraversePages(func(pageNum int) (Paginated, error) { return nil, errors.New("Error!") }, + func(obj interface{}) bool { return obj.(*Organization).Name == "test1220" }) + assert.Errorf(t, err, "Error!") +} + +var singleOrganizationsPage = func(pageNum int) (Paginated, error) { + if pageNum == 1 { + // Note, that we don't specify 'next' attribute, so no extra pages will be requested + return &OrganizationsResponse{ + OMPaginated: OMPaginated{TotalCount: 1}, + Organizations: []*Organization{{ID: "1323", Name: "test"}}, + }, nil + } + return nil, errors.New("Not found!") +} + +var numberOfPagesTraversed = 0 + +var multipleOrganizationsPage = func(pageNum int) (Paginated, error) { + numberOfPagesTraversed++ + // page 1 + switch pageNum { + case 1: + return &OrganizationsResponse{ + OMPaginated: OMPaginated{TotalCount: 1300, Links: []*Link{{Rel: "next"}}}, + Organizations: generateOrganizations(0, 500), + }, nil + case 2: + return &OrganizationsResponse{ + OMPaginated: OMPaginated{TotalCount: 1300, Links: []*Link{{Rel: "next"}}}, + Organizations: generateOrganizations(500, 500), + }, nil + case 3: + return &OrganizationsResponse{ + OMPaginated: OMPaginated{TotalCount: 1300}, + Organizations: generateOrganizations(1000, 300), + }, nil + } + return nil, errors.New("Not found!") +} + +func generateOrganizations(startFrom, count int) []*Organization { + ans := make([]*Organization, count) + c := startFrom + for i := 0; i < count; i++ { + ans[i] = &Organization{ID: fmt.Sprintf("id%d", c), Name: fmt.Sprintf("test%d", c)} + c++ + } + return ans +} diff --git a/controllers/om/organization.go b/controllers/om/organization.go new file mode 100644 index 000000000..8911f13f7 --- /dev/null +++ b/controllers/om/organization.go @@ -0,0 +1,20 @@ +package om + +type OrganizationsResponse struct { + OMPaginated + Organizations []*Organization `json:"results"` +} + +func (o OrganizationsResponse) Results() []interface{} { + // Lack of covariance in Go... :( + ans := make([]interface{}, len(o.Organizations)) + for i, org := range o.Organizations { + ans[i] = org + } + return ans +} + +type Organization struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` +} diff --git a/controllers/om/process.go b/controllers/om/process.go new file mode 100644 index 000000000..0b7f6b20e --- /dev/null +++ b/controllers/om/process.go @@ -0,0 +1,580 @@ +package om + +import ( + "encoding/json" + "fmt" + "path" + "strings" + + "github.com/spf13/cast" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" +) + +// MongoType refers to the type of the Mongo process, `mongos` or `mongod`. +type MongoType string + +const ( + // ProcessTypeMongos defines a constant for the mongos process type + ProcessTypeMongos MongoType = "mongos" + + // ProcessTypeMongod defines a constant for the mongod process type. + ProcessTypeMongod MongoType = "mongod" +) + +/* +This is a class for all types of processes. +Note, that mongos types of processes don't have some fields (replication, storage etc.) but it's impossible to use a +separate types for different processes (mongos, mongod) with different methods due to limitation of Go embedding model. +So the code using this type must be careful and make sure the state is consistent. + +Dev notes: +- any new configurations must be "mirrored" in 'mergeFrom' method which merges the "operator owned" fields into +the process that was read from Ops Manager. +- the main principle used everywhere in 'om' code: the Operator overrides only the configurations it "owns" but leaves +the other properties unmodified. That's why structs are not used anywhere as they would result in possible overriding of +the whole elements which we don't want. Deal with data as with maps, create convenience methods (setters, getters, +ensuremap etc.) and make sure not to override anything unrelated. + +The resulting json for this type (example): + + { + "args2_6": { + "net": { + "port": 28002, + "ssl": { + "mode": "requireSSL", + "PEMKeyFile": "/mongodb-automation/server.pem" + "clusterAuthFile: "/mongodb-automation/clusterfile.pem" + } + }, + "security" { + "clusterAuthMode":"x509" + } + "replication": { + "replSetName": "blue" + }, + "storage": { + "dbPath": "/data/blue_2", + "wiredTiger": { + "engineConfig": { + "cacheSizeGB": 0.3 + } + } + }, + "systemLog": { + "destination": "file", + "path": "/data/blue_2/mongodb.log" + } + }, + "hostname": "AGENT_HOSTNAME", + "logRotate": { + "sizeThresholdMB": 1000, + "timeThresholdHrs": 24 + }, + "name": "blue_2", + "processType": "mongod", + "version": "3.0.1", + "authSchemaVersion": 3 + } +*/ + +// used to indicate standalones when a type is used as an identifier +type Standalone map[string]interface{} + +type Process map[string]interface{} + +func NewProcessFromInterface(i interface{}) Process { + return i.(map[string]interface{}) +} + +func NewMongosProcess(name, hostName, mongoDBImage string, forceEnterprise bool, additionalMongodConfig *mdbv1.AdditionalMongodConfig, spec mdbv1.DbSpec, certificateFilePath string, annotations map[string]string, fcv string) Process { + if additionalMongodConfig == nil { + additionalMongodConfig = mdbv1.NewEmptyAdditionalMongodConfig() + } + + architecture := architectures.GetArchitecture(annotations) + processVersion := architectures.GetMongoVersionForAutomationConfig(mongoDBImage, spec.GetMongoDBVersion(), forceEnterprise, architecture) + p := createProcess( + WithName(name), + WithHostname(hostName), + WithProcessType(ProcessTypeMongos), + WithAdditionalMongodConfig(*additionalMongodConfig), + WithResourceSpec(processVersion, fcv), + ) + + // default values for configurable values + p.SetLogPath(path.Join(util.PvcMountPathLogs, "/mongodb.log")) + if certificateFilePath == "" { + certificateFilePath = util.PEMKeyFilePathInContainer + } + + p.ConfigureTLS(getTLSMode(spec, *additionalMongodConfig), certificateFilePath) + + return p +} + +func NewMongodProcess(name, hostName, mongoDBImage string, forceEnterprise bool, additionalConfig *mdbv1.AdditionalMongodConfig, spec mdbv1.DbSpec, certificateFilePath string, annotations map[string]string, fcv string) Process { + if additionalConfig == nil { + additionalConfig = mdbv1.NewEmptyAdditionalMongodConfig() + } + + architecture := architectures.GetArchitecture(annotations) + processVersion := architectures.GetMongoVersionForAutomationConfig(mongoDBImage, spec.GetMongoDBVersion(), forceEnterprise, architecture) + p := createProcess( + WithName(name), + WithHostname(hostName), + WithProcessType(ProcessTypeMongod), + WithAdditionalMongodConfig(*additionalConfig), + WithResourceSpec(processVersion, fcv), + ) + + // default values for configurable values + p.SetDbPath("/data") + agentConfig := spec.GetAgentConfig() + if agentConfig.Mongod.SystemLog != nil { + p.SetLogPathFromCommunitySystemLog(agentConfig.Mongod.SystemLog) + } else { + // CLOUDP-33467: we put mongod logs to the same directory as AA/Monitoring/Backup ones to provide single mount point + // for all types of logs + p.SetLogPath(path.Join(util.PvcMountPathLogs, "mongodb.log")) + } + + if certificateFilePath == "" { + certificateFilePath = util.PEMKeyFilePathInContainer + } + p.ConfigureTLS(getTLSMode(spec, *additionalConfig), certificateFilePath) + + return p +} + +func getTLSMode(spec mdbv1.DbSpec, additionalMongodConfig mdbv1.AdditionalMongodConfig) tls.Mode { + if !spec.IsSecurityTLSConfigEnabled() { + return tls.Disabled + } + return tls.GetTLSModeFromMongodConfig(additionalMongodConfig.ToMap()) +} + +func (p Process) DeepCopy() (Process, error) { + return util.MapDeepCopy(p) +} + +// Name returns the name of the process. +func (p Process) Name() string { + return p["name"].(string) +} + +// HostName returns the hostname for this process. +func (p Process) HostName() string { + return p["hostname"].(string) +} + +// GetVotes returns the number of votes requested for the member using this process. +func (p Process) GetVotes() int { + if votes, ok := p["votes"]; ok { + return cast.ToInt(votes) + } + return 1 +} + +// GetPriority returns the requested priority for the member using this process. +func (p Process) GetPriority() float32 { + if priority, ok := p["priority"]; ok { + return cast.ToFloat32(priority) + } + return 1.0 +} + +func (p Process) GetLogRotate() map[string]interface{} { + if logRotate, ok := p["logRotate"]; ok { + return logRotate.(map[string]interface{}) + } + return make(map[string]interface{}) +} + +func (p Process) GetAuditLogRotate() map[string]interface{} { + if logRotate, ok := p["auditLogRotate"]; ok { + return logRotate.(map[string]interface{}) + } + return make(map[string]interface{}) +} + +// GetTags returns the requested tags for the member using this process. +func (p Process) GetTags() map[string]string { + if tags, ok := p["tags"]; ok { + tagMap, ok := tags.(map[string]interface{}) + if !ok { + return nil + } + result := make(map[string]string) + for k, v := range tagMap { + result[k] = cast.ToString(v) + } + return result + } + return nil +} + +// SetDbPath sets the DbPath for this process. +func (p Process) SetDbPath(dbPath string) Process { + util.ReadOrCreateMap(p.Args(), "storage")["dbPath"] = dbPath + return p +} + +// DbPath returns the DbPath for this process. +func (p Process) DbPath() string { + return maputil.ReadMapValueAsString(p.Args(), "storage", "dbPath") +} + +func (p Process) SetWiredTigerCache(cacheSizeGb float32) Process { + if p.ProcessType() != ProcessTypeMongod { + // WiredTigerCache can be set only for mongod processes + return p + } + storageMap := util.ReadOrCreateMap(p.Args(), "storage") + wiredTigerMap := util.ReadOrCreateMap(storageMap, "wiredTiger") + engineConfigMap := util.ReadOrCreateMap(wiredTigerMap, "engineConfig") + engineConfigMap["cacheSizeGB"] = cacheSizeGb + return p +} + +// WiredTigerCache returns wired tiger cache as pointer as it may be absent +func (p Process) WiredTigerCache() *float32 { + value := maputil.ReadMapValueAsInterface(p.Args(), "storage", "wiredTiger", "engineConfig", "cacheSizeGB") + if value == nil { + return nil + } + f := cast.ToFloat32(value) + return &f +} + +func (p Process) SetLogPath(logPath string) Process { + sysLogMap := util.ReadOrCreateMap(p.Args(), "systemLog") + sysLogMap["destination"] = "file" + sysLogMap["path"] = logPath + return p +} + +func (p Process) SetLogPathFromCommunitySystemLog(systemLog *automationconfig.SystemLog) Process { + sysLogMap := util.ReadOrCreateMap(p.Args(), "systemLog") + sysLogMap["destination"] = string(systemLog.Destination) + sysLogMap["path"] = systemLog.Path + sysLogMap["logAppend"] = systemLog.LogAppend + return p +} + +func (p Process) LogPath() string { + return maputil.ReadMapValueAsString(p.Args(), "systemLog", "path") +} + +func (p Process) LogRotateSizeThresholdMB() interface{} { + return maputil.ReadMapValueAsInterface(p, "logRotate", "sizeThresholdMB") +} + +// Args returns the "args" attribute in the form of a map, creates if it doesn't exist +func (p Process) Args() map[string]interface{} { + return util.ReadOrCreateMap(p, "args2_6") +} + +// Version of the process. This refers to the MongoDB server version. +func (p Process) Version() string { + return p["version"].(string) +} + +// ProcessType returns the type of process for the current process. +// It can be `mongos` or `mongod`. +func (p Process) ProcessType() MongoType { + switch v := p["processType"].(type) { + case string: + return MongoType(v) + case MongoType: + return v + default: + panic(fmt.Sprintf("Unexpected type of processType variable: %T", v)) + } +} + +// IsDisabled returns the "disabled" attribute. +func (p Process) IsDisabled() bool { + return p["disabled"].(bool) +} + +// SetDisabled sets the "disabled" attribute to `disabled`. +func (p Process) SetDisabled(disabled bool) { + p["disabled"] = disabled +} + +// EnsureNetConfig returns the Net configuration map ("net"), creates an empty map if it didn't exist +func (p Process) EnsureNetConfig() map[string]interface{} { + return util.ReadOrCreateMap(p.Args(), "net") +} + +// EnsureTLSConfig returns the TLS configuration map ("net.tls"), creates an empty map if it didn't exist. +// Use this method if you intend to make updates to the map returned +func (p Process) EnsureTLSConfig() map[string]interface{} { + netConfig := p.EnsureNetConfig() + return util.ReadOrCreateMap(netConfig, "tls") +} + +// SSLConfig returns the TLS configuration map ("net.tls") or an empty map if it doesn't exist. +// Use this method only to read values, not update +func (p Process) TLSConfig() map[string]interface{} { + netConfig := p.EnsureNetConfig() + if _, ok := netConfig["tls"]; ok { + return netConfig["tls"].(map[string]interface{}) + } + + return make(map[string]interface{}) +} + +func (p Process) EnsureSecurity() map[string]interface{} { + return util.ReadOrCreateMap(p.Args(), "security") +} + +// ConfigureClusterAuthMode sets the cluster auth mode for the process. +// Only accepted value for now is X509. +// internalClusterAuth is a parameter that overrides where the cert is located. +// If provided with an empty string, the operator will set it to +// a concatenation of the default mount path and the name of the process-pem +func (p Process) ConfigureClusterAuthMode(clusterAuthMode string, internalClusterPath string) Process { + if strings.ToUpper(clusterAuthMode) == util.X509 { // Ops Manager value is "x509" + // the individual key per pod will be podname-pem e.g. my-replica-set-0-pem + p.setClusterAuthMode("x509") + clusterFile := fmt.Sprintf("%s%s-pem", util.InternalClusterAuthMountPath, p.Name()) + if internalClusterPath != "" { + clusterFile = internalClusterPath + } + p.setClusterFile(clusterFile) + } + return p +} + +func (p Process) IsTLSEnabled() bool { + _, keyFile0 := p.TLSConfig()["PEMKeyFile"] + _, keyFile1 := p.TLSConfig()["certificateKeyFile"] + + return keyFile0 || keyFile1 +} + +func (p Process) HasInternalClusterAuthentication() bool { + return p.ClusterAuthMode() != "" +} + +func (p Process) FeatureCompatibilityVersion() string { + if p["featureCompatibilityVersion"] == nil { + return "" + } + return p["featureCompatibilityVersion"].(string) +} + +func (p Process) Alias() string { + if alias, ok := p["alias"].(string); ok { + return alias + } + + return "" +} + +// String +func (p Process) String() string { + return fmt.Sprintf("\"%s\" (hostName: %s, version: %s, args: %s)", p.Name(), p.HostName(), p.Version(), p.Args()) +} + +// ****************** These ones are private methods not exposed to other packages ************************************* + +// createProcess initializes a process. It's a common initialization done for both mongos and mongod processes +func createProcess(opts ...ProcessOption) Process { + process := Process{} + for _, opt := range opts { + opt(process) + } + return process +} + +type ProcessOption func(process Process) + +func WithResourceSpec(processVersion, fcv string) ProcessOption { + return func(process Process) { + process["version"] = processVersion + process["authSchemaVersion"] = CalculateAuthSchemaVersion() + process["featureCompatibilityVersion"] = fcv + } +} + +func WithName(name string) ProcessOption { + return func(process Process) { + process["name"] = name + } +} + +func WithHostname(hostname string) ProcessOption { + return func(process Process) { + process["hostname"] = hostname + } +} + +func WithProcessType(processType MongoType) ProcessOption { + return func(process Process) { + process["processType"] = processType + } +} + +func WithAdditionalMongodConfig(additionalConfig mdbv1.AdditionalMongodConfig) ProcessOption { + return func(process Process) { + // Applying the user-defined options if any + process["args2_6"] = additionalConfig.ToMap() + process.EnsureNetConfig()["port"] = additionalConfig.GetPortOrDefault() + } +} + +// ConfigureTLS enable TLS for this process. TLS will always be enabled after calling this. This function expects +// the value of "mode" to be an allowed ssl.mode from OM API perspective. +func (p Process) ConfigureTLS(mode tls.Mode, pemKeyFileLocation string) { + // Initializing SSL configuration if it's necessary + tlsConfig := p.EnsureTLSConfig() + tlsConfig["mode"] = string(mode) + + if mode == tls.Disabled { + // If these attribute exists, it needs to be removed + // PEMKeyFile is older + // certificateKeyFile is the current one + delete(tlsConfig, "certificateKeyFile") + delete(tlsConfig, "PEMKeyFile") + } else { + // PEMKeyFile is the legacy option found under net.ssl, deprecated since version 4.2 + // https://www.mongodb.com/docs/manual/reference/configuration-options/#mongodb-setting-net.ssl.PEMKeyFile + _, oldKeyInConfig := tlsConfig["PEMKeyFile"] + // certificateKeyFile is the current option under net.tls + // https://www.mongodb.com/docs/manual/reference/configuration-options/#mongodb-setting-net.tls.certificateKeyFile + _, newKeyInConfig := tlsConfig["certificateKeyFile"] + + // If both options are present in the TLS config we only want to keep the recent option. The problem + // can be encountered when migrating the operator from an older version which pushed an automation config + // containing the old key and the new operator is attempting to configure the new key. + if oldKeyInConfig == newKeyInConfig { + tlsConfig["certificateKeyFile"] = pemKeyFileLocation + delete(tlsConfig, "PEMKeyFile") + } else if newKeyInConfig { + tlsConfig["certificateKeyFile"] = pemKeyFileLocation + } else { + tlsConfig["PEMKeyFile"] = pemKeyFileLocation + } + } +} + +func CalculateAuthSchemaVersion() int { + return 5 +} + +// mergeFrom merges the Operator version of process ('operatorProcess') into OM one ('p'). +// Considers the type of process and rewrites only relevant fields +func (p Process) mergeFrom(operatorProcess Process, specArgs26, prevArgs26 map[string]interface{}) { + // Dev note: merging the maps overrides/add map keys+value but doesn't remove the existing ones + // If there are any keys that need to be removed explicitly (to ensure OM changes haven't sneaked through) + // this must be done manually + maputil.MergeMaps(p, operatorProcess) + + p["args2_6"] = maputil.RemoveFieldsBasedOnDesiredAndPrevious(p.Args(), specArgs26, prevArgs26) + + // Merge SSL configuration (update if it's specified - delete otherwise) + if mode, ok := operatorProcess.TLSConfig()["mode"]; ok { + tlsConfig := p.EnsureTLSConfig() + for key, value := range operatorProcess.TLSConfig() { + tlsConfig[key] = value + } + // PEMKeyFile is the legacy option found under net.ssl, deprecated since version 4.2 + // https://www.mongodb.com/docs/manual/reference/configuration-options/#mongodb-setting-net.ssl.PEMKeyFile + _, oldKeyInConfig := tlsConfig["PEMKeyFile"] + // certificateKeyFile is the current option under net.tls + // https://www.mongodb.com/docs/manual/reference/configuration-options/#mongodb-setting-net.tls.certificateKeyFile + _, newKeyInConfig := tlsConfig["certificateKeyFile"] + // If both options are present in the TLS config we only want to keep the recent option. + if oldKeyInConfig && newKeyInConfig { + delete(tlsConfig, "PEMKeyFile") + } + // if the mode is specified as disabled, providing "PEMKeyFile" is an invalid config + if mode == string(tls.Disabled) { + delete(tlsConfig, "PEMKeyFile") + delete(tlsConfig, "certificateKeyFile") + } + } else { + delete(p.EnsureNetConfig(), "tls") + } +} + +func (p Process) setName(name string) Process { + p["name"] = name + return p +} + +func (p Process) setClusterFile(filePath string) Process { + p.EnsureTLSConfig()["clusterFile"] = filePath + return p +} + +func (p Process) setClusterAuthMode(authMode string) Process { + p.EnsureSecurity()["clusterAuthMode"] = authMode + return p +} + +func (p Process) authSchemaVersion() int { + return p["authSchemaVersion"].(int) +} + +// These methods are ONLY FOR REPLICA SET members! +// external packages are not supposed to call this method directly as it should be called during replica set building +func (p Process) setReplicaSetName(rsName string) Process { + util.ReadOrCreateMap(p.Args(), "replication")["replSetName"] = rsName + return p +} + +func (p Process) replicaSetName() string { + return maputil.ReadMapValueAsString(p.Args(), "replication", "replSetName") +} + +func (p Process) security() map[string]interface{} { + args := p.Args() + if _, ok := args["security"]; ok { + return args["security"].(map[string]interface{}) + } + return make(map[string]interface{}) +} + +func (p Process) ClusterAuthMode() string { + if authMode, ok := p.security()["clusterAuthMode"]; ok { + return authMode.(string) + } + return "" +} + +// These methods are ONLY FOR CONFIG SERVER REPLICA SET members! +// external packages are not supposed to call this method directly as it should be called during sharded cluster merge +func (p Process) setClusterRoleConfigSrv() Process { + util.ReadOrCreateMap(p.Args(), "sharding")["clusterRole"] = "configsvr" + return p +} + +// These methods are ONLY FOR MONGOS types! +// external packages are not supposed to call this method directly as it should be called during sharded cluster building +func (p Process) setCluster(clusterName string) Process { + p["cluster"] = clusterName + return p +} + +func (p Process) cluster() string { + return p["cluster"].(string) +} + +func (p Process) json() string { + b, err := json.MarshalIndent(p, "", " ") + if err != nil { + fmt.Println("error:", err) + } + return string(b) +} diff --git a/controllers/om/process/om_process.go b/controllers/om/process/om_process.go new file mode 100644 index 000000000..be008e39e --- /dev/null +++ b/controllers/om/process/om_process.go @@ -0,0 +1,57 @@ +package process + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbmultiv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func CreateMongodProcessesWithLimit(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, limit int, fcv string) []om.Process { + hostnames, names := dns.GetDnsForStatefulSetReplicasSpecified(set, dbSpec.GetClusterDomain(), limit, dbSpec.GetExternalDomain()) + processes := make([]om.Process, len(hostnames)) + + certificateFileName := "" + if certificateHash, ok := set.Annotations[certs.CertHashAnnotationKey]; ok { + certificateFileName = fmt.Sprintf("%s/%s", util.TLSCertMountPath, certificateHash) + } + + for idx, hostname := range hostnames { + processes[idx] = om.NewMongodProcess(names[idx], hostname, mongoDBImage, forceEnterprise, dbSpec.GetAdditionalMongodConfig(), dbSpec, certificateFileName, set.Annotations, fcv) + } + + return processes +} + +// CreateMongodProcessesWithLimitMulti creates the process array for automationConfig based on MultiCluster CR spec +func CreateMongodProcessesWithLimitMulti(mongoDBImage string, forceEnterprise bool, mrs mdbmultiv1.MongoDBMultiCluster, certFileName string) ([]om.Process, error) { + hostnames := make([]string, 0) + clusterNums := make([]int, 0) + podNum := make([]int, 0) + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return nil, err + } + + for _, spec := range clusterSpecList { + agentHostNames := dns.GetMultiClusterProcessHostnames(mrs.Name, mrs.Namespace, mrs.ClusterNum(spec.ClusterName), spec.Members, mrs.Spec.GetClusterDomain(), mrs.Spec.GetExternalDomainForMemberCluster(spec.ClusterName)) + hostnames = append(hostnames, agentHostNames...) + for i := 0; i < len(agentHostNames); i++ { + clusterNums = append(clusterNums, mrs.ClusterNum(spec.ClusterName)) + podNum = append(podNum, i) + } + } + + processes := make([]om.Process, len(hostnames)) + for idx := range hostnames { + processes[idx] = om.NewMongodProcess(fmt.Sprintf("%s-%d-%d", mrs.Name, clusterNums[idx], podNum[idx]), hostnames[idx], mongoDBImage, forceEnterprise, mrs.Spec.GetAdditionalMongodConfig(), &mrs.Spec, certFileName, mrs.Annotations, mrs.CalculateFeatureCompatibilityVersion()) + } + + return processes, nil +} diff --git a/controllers/om/process_test.go b/controllers/om/process_test.go new file mode 100644 index 000000000..e3b97719d --- /dev/null +++ b/controllers/om/process_test.go @@ -0,0 +1,333 @@ +package om + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" +) + +func TestCreateMongodProcess(t *testing.T) { + mongoDBImage := "mongodb/mongodb-enterprise-server" + t.Run("Create AgentLoggingMongodConfig", func(t *testing.T) { + spec := defaultMongoDBVersioned("4.0.5") + process := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", mongoDBImage, false, spec.GetAdditionalMongodConfig(), spec, "", nil, "4.0") + + assert.Equal(t, "trinity", process.Name()) + assert.Equal(t, "trinity-0.trinity-svc.svc.cluster.local", process.HostName()) + assert.Equal(t, "4.0.5", process.Version()) + assert.Equal(t, "4.0", process.FeatureCompatibilityVersion()) + assert.Equal(t, "/data", process.DbPath()) + assert.Equal(t, "/var/log/mongodb-mms-automation/mongodb.log", process.LogPath()) + assert.Equal(t, 5, process.authSchemaVersion()) + assert.Equal(t, "", process.replicaSetName()) + assert.Equal(t, nil, process.LogRotateSizeThresholdMB()) + + expectedMap := map[string]interface{}{"port": int32(util.MongoDbDefaultPort), "tls": map[string]interface{}{ + "mode": "disabled", + }} + assert.Equal(t, expectedMap, process.EnsureNetConfig()) + }) + t.Run("Create with Mongodb options", func(t *testing.T) { + config := mdbv1.NewAdditionalMongodConfig("storage.engine", "inMemory"). + AddOption("setParameter.connPoolMaxConnsPerHost", 500). + AddOption("storage.dbPath", "/some/other/data") // this will be overridden + rs := mdbv1.NewReplicaSetBuilder().SetAdditionalConfig(config).Build() + + process := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", mongoDBImage, false, rs.Spec.AdditionalMongodConfig, rs.GetSpec(), "", nil, "") + + assert.Equal(t, "inMemory", maputil.ReadMapValueAsInterface(process.Args(), "storage", "engine")) + assert.Equal(t, 500, maputil.ReadMapValueAsInterface(process.Args(), "setParameter", "connPoolMaxConnsPerHost")) + assert.Equal(t, "/data", process.DbPath()) + }) +} + +func TestCreateMongodProcessStatic(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + mongoDBImage := "mongodb/mongodb-enterprise-server" + t.Run("Create AgentLoggingMongodConfig", func(t *testing.T) { + spec := defaultMongoDBVersioned("4.0.5") + process := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", mongoDBImage, false, spec.GetAdditionalMongodConfig(), spec, "", map[string]string{}, "4.0") + + assert.Equal(t, "trinity", process.Name()) + assert.Equal(t, "trinity-0.trinity-svc.svc.cluster.local", process.HostName()) + assert.Equal(t, "4.0.5-ent", process.Version()) + assert.Equal(t, "4.0", process.FeatureCompatibilityVersion()) + assert.Equal(t, "/data", process.DbPath()) + assert.Equal(t, "/var/log/mongodb-mms-automation/mongodb.log", process.LogPath()) + assert.Equal(t, 5, process.authSchemaVersion()) + assert.Equal(t, "", process.replicaSetName()) + + expectedMap := map[string]interface{}{"port": int32(util.MongoDbDefaultPort), "tls": map[string]interface{}{ + "mode": "disabled", + }} + assert.Equal(t, expectedMap, process.EnsureNetConfig()) + }) + t.Run("Create with Mongodb options", func(t *testing.T) { + config := mdbv1.NewAdditionalMongodConfig("storage.engine", "inMemory"). + AddOption("setParameter.connPoolMaxConnsPerHost", 500). + AddOption("storage.dbPath", "/some/other/data") // this will be overridden + rs := mdbv1.NewReplicaSetBuilder().SetAdditionalConfig(config).Build() + + process := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", mongoDBImage, false, rs.Spec.AdditionalMongodConfig, rs.GetSpec(), "", nil, "") + + assert.Equal(t, "inMemory", maputil.ReadMapValueAsInterface(process.Args(), "storage", "engine")) + assert.Equal(t, 500, maputil.ReadMapValueAsInterface(process.Args(), "setParameter", "connPoolMaxConnsPerHost")) + assert.Equal(t, "/data", process.DbPath()) + }) +} + +func TestConfigureSSL_Process(t *testing.T) { + process := Process{} + + process.ConfigureTLS(tls.Require, "pem-file0") + assert.Equal(t, map[string]interface{}{"mode": string(tls.Require), "certificateKeyFile": "pem-file0"}, process.TLSConfig()) + + process = Process{} + process.ConfigureTLS("", "pem-file1") + assert.Equal(t, map[string]interface{}{"mode": "", "certificateKeyFile": "pem-file1"}, process.TLSConfig()) + + process = Process{} + process.ConfigureTLS(tls.Disabled, "pem-file2") + assert.Equal(t, map[string]interface{}{"mode": string(tls.Disabled)}, process.TLSConfig()) +} + +func TestConfigureSSL_Process_CertificateKeyFile(t *testing.T) { + t.Run("When provided with certificateKeyFile attribute name, it is maintained", func(t *testing.T) { + process := Process{} + tlsConfig := process.EnsureTLSConfig() + tlsConfig["certificateKeyFile"] = "xxx" + process.ConfigureTLS(tls.Require, "pem-file0") + assert.Equal(t, map[string]interface{}{"mode": string(tls.Require), "certificateKeyFile": "pem-file0"}, process.TLSConfig()) + }) + + t.Run("A non-defined mode keeps the certificateKeyFile attribute name", func(t *testing.T) { + process := Process{} + tlsConfig := process.EnsureTLSConfig() + tlsConfig["certificateKeyFile"] = "xxx" + process.ConfigureTLS("", "pem-file1") + assert.Equal(t, map[string]interface{}{"mode": "", "certificateKeyFile": "pem-file1"}, process.TLSConfig()) + }) + + t.Run("If TLS is disabled, the certificateKeyFile attribute is deleted", func(t *testing.T) { + process := Process{} + tlsConfig := process.EnsureTLSConfig() + tlsConfig["certificateKeyFile"] = "xxx" + process.ConfigureTLS(tls.Disabled, "pem-file2") + assert.Equal(t, map[string]interface{}{"mode": string(tls.Disabled)}, process.TLSConfig()) + }) +} + +func TestTlsConfig(t *testing.T) { + process := Process{} + process.ConfigureTLS(tls.Require, "another-pem-file") + process.Args()["tls"] = map[string]interface{}{ + "mode": "requireTLS", + "PEMKeyFile": "another-pem-file", + } + + tlsConfig := process.TLSConfig() + assert.NotNil(t, tlsConfig) + assert.Equal(t, tlsConfig["mode"], "requireTLS") + assert.Equal(t, tlsConfig["certificateKeyFile"], "another-pem-file") +} + +func TestConfigureX509_Process(t *testing.T) { + mdb := &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + Version: "3.6.4", + Security: &mdbv1.Security{ + Authentication: &mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{util.X509}, + }, + }, + }, + }, + } + process := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, mdb.GetSpec(), "", nil, "") + + process.ConfigureClusterAuthMode("", "") // should not update fields + assert.NotContains(t, process.security(), "clusterAuthMode") + assert.NotContains(t, process.TLSConfig(), "clusterFile") + + process.ConfigureClusterAuthMode(util.X509, "") // should update fields if specified as x509 + assert.Equal(t, "x509", process.security()["clusterAuthMode"]) + assert.Equal(t, fmt.Sprintf("%s%s-pem", util.InternalClusterAuthMountPath, process.Name()), process.TLSConfig()["clusterFile"]) +} + +func TestCreateMongodProcess_SSL(t *testing.T) { + additionalConfig := mdbv1.NewAdditionalMongodConfig("net.ssl.mode", string(tls.Prefer)) + + mdb := mdbv1.NewStandaloneBuilder().SetVersion("3.6.4").SetFCVersion("3.6").SetAdditionalConfig(additionalConfig).Build() + process := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, additionalConfig, mdb.GetSpec(), "", nil, "") + assert.Equal(t, map[string]interface{}{"mode": string(tls.Disabled)}, process.TLSConfig()) + + mdb = mdbv1.NewStandaloneBuilder().SetVersion("3.6.4").SetFCVersion("3.6").SetAdditionalConfig(additionalConfig). + SetSecurityTLSEnabled().Build() + + process = NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, additionalConfig, mdb.GetSpec(), "", nil, "") + + assert.Equal(t, map[string]interface{}{ + "mode": string(tls.Prefer), + "certificateKeyFile": "/mongodb-automation/server.pem", + }, process.TLSConfig()) +} + +func TestCreateMongosProcess_SSL(t *testing.T) { + additionalConfig := mdbv1.NewAdditionalMongodConfig("net.ssl.mode", string(tls.Allow)) + mdb := mdbv1.NewStandaloneBuilder().SetVersion("3.6.4").SetFCVersion("3.6").SetAdditionalConfig(additionalConfig). + SetSecurityTLSEnabled().Build() + process := NewMongosProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, additionalConfig, mdb.GetSpec(), "", nil, "") + + assert.Equal(t, map[string]interface{}{"mode": string(tls.Allow), "certificateKeyFile": "/mongodb-automation/server.pem"}, process.TLSConfig()) +} + +func TestCreateMongodMongosProcess_TLSModeForDifferentSpecs(t *testing.T) { + assertTLSConfig := func(p Process) { + expectedMap := map[string]interface{}{ + "mode": string(tls.Allow), + "certificateKeyFile": "/mongodb-automation/server.pem", + } + assert.Equal(t, expectedMap, p.TLSConfig()) + } + + getSpec := func(builder *mdbv1.MongoDBBuilder) mdbv1.DbSpec { + return builder.SetSecurityTLSEnabled().Build().GetSpec() + } + + name := "name" + host := "host" + additionalConfig := mdbv1.NewAdditionalMongodConfig("net.tls.mode", string(tls.Allow)) + + // standalone spec + assertTLSConfig(NewMongodProcess(name, host, "fake-mongoDBImage", false, additionalConfig, getSpec(mdbv1.NewStandaloneBuilder()), "", nil, "")) + + // replica set spec + assertTLSConfig(NewMongodProcess(name, host, "fake-mongoDBImage", false, additionalConfig, getSpec(mdbv1.NewReplicaSetBuilder()), "", nil, "")) + + // sharded cluster spec + assertTLSConfig(NewMongosProcess(name, host, "fake-mongoDBImage", false, additionalConfig, getSpec(mdbv1.NewClusterBuilder()), "", nil, "")) + assertTLSConfig(NewMongodProcess(name, host, "fake-mongoDBImage", false, additionalConfig, getSpec(mdbv1.NewClusterBuilder()), "", nil, "")) +} + +// TestMergeMongodProcess_SSL verifies that merging for the process SSL settings keeps the Operator "owned" properties +// and doesn't overwrite the other Ops Manager initiated configuration +func TestMergeMongodProcess_SSL(t *testing.T) { + additionalConfig := mdbv1.NewAdditionalMongodConfig("net.ssl.mode", string(tls.Require)) + operatorMdb := mdbv1.NewStandaloneBuilder().SetVersion("3.6.4").SetFCVersion("3.6"). + SetAdditionalConfig(additionalConfig).SetSecurityTLSEnabled().Build() + + omMdb := mdbv1.NewStandaloneBuilder().SetVersion("3.6.4").SetFCVersion("3.6"). + SetAdditionalConfig(mdbv1.NewEmptyAdditionalMongodConfig()).Build() + + operatorProcess := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, operatorMdb.GetSpec(), "", nil, "") + omProcess := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, omMdb.GetSpec(), "", nil, "") + omProcess.EnsureTLSConfig()["mode"] = "allowTLS" // this will be overridden + omProcess.EnsureTLSConfig()["PEMKeyFile"] = "/var/mongodb/server.pem" // this will be overridden + omProcess.EnsureTLSConfig()["sslOnNormalPorts"] = "true" // this will be left as-is + omProcess.EnsureTLSConfig()["PEMKeyPassword"] = "qwerty" // this will be left as-is + + omProcess.mergeFrom(operatorProcess, nil, nil) + + expectedSSLConfig := map[string]interface{}{ + "mode": string(tls.Require), + "certificateKeyFile": "/mongodb-automation/server.pem", + "sslOnNormalPorts": "true", + "PEMKeyPassword": "qwerty", + } + assert.Equal(t, expectedSSLConfig, maputil.ReadMapValueAsInterface(omProcess, "args2_6", "net", "tls")) +} + +func TestMergeMongodProcess_MongodbOptions(t *testing.T) { + omMdb := mdbv1.NewStandaloneBuilder().SetAdditionalConfig( + mdbv1.NewAdditionalMongodConfig("storage.wiredTiger.engineConfig.cacheSizeGB", 3)).Build() + omProcess := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, omMdb.Spec.AdditionalMongodConfig, omMdb.GetSpec(), "", nil, "") + + operatorMdb := mdbv1.NewStandaloneBuilder().SetAdditionalConfig( + mdbv1.NewAdditionalMongodConfig("storage.wiredTiger.engineConfig.directoryForIndexes", "/some/dir")).Build() + operatorProcess := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, operatorMdb.Spec.AdditionalMongodConfig, operatorMdb.GetSpec(), "", nil, "") + + omProcess.mergeFrom(operatorProcess, nil, nil) + + expectedArgs := map[string]interface{}{ + "net": map[string]interface{}{ + "port": int32(27017), + "tls": map[string]interface{}{ + "mode": "disabled", + }, + }, + "storage": map[string]interface{}{ + "dbPath": "/data", + "wiredTiger": map[string]interface{}{ + "engineConfig": map[string]interface{}{ + "cacheSizeGB": 3, // This is the native OM configuration + "directoryForIndexes": "/some/dir", // This is the configuration set by MongoDB spec + }, + }, + }, + "systemLog": map[string]interface{}{ + "destination": "file", + "path": "/var/log/mongodb-mms-automation/mongodb.log", + }, + } + + assert.Equal(t, expectedArgs, omProcess.Args()) +} + +func TestMergeMongodProcess_AdditionalMongodConfig_CanBeRemoved(t *testing.T) { + prevAdditionalConfig := mdbv1.NewEmptyAdditionalMongodConfig() + prevAdditionalConfig.AddOption("storage.wiredTiger.engineConfig.cacheSizeGB", 3) + prevAdditionalConfig.AddOption("some.other.option", "value") + prevAdditionalConfig.AddOption("some.other.option2", "value2") + + omMdb := mdbv1.NewStandaloneBuilder().SetAdditionalConfig(prevAdditionalConfig).Build() + omProcess := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, omMdb.Spec.AdditionalMongodConfig, omMdb.GetSpec(), "", nil, "") + + specAdditionalConfig := mdbv1.NewEmptyAdditionalMongodConfig() + // we are changing the cacheSize to 4 + specAdditionalConfig.AddOption("storage.wiredTiger.engineConfig.cacheSizeGB", 4) + // here we are simulating removing "some.other.option2" by not specifying it. + specAdditionalConfig.AddOption("some.other.option", "value") + + operatorMdb := mdbv1.NewStandaloneBuilder().SetAdditionalConfig(specAdditionalConfig).Build() + operatorProcess := NewMongodProcess("trinity", "trinity-0.trinity-svc.svc.cluster.local", "fake-mongoDBImage", false, operatorMdb.Spec.AdditionalMongodConfig, operatorMdb.GetSpec(), "", nil, "") + + omProcess.mergeFrom(operatorProcess, specAdditionalConfig.ToMap(), prevAdditionalConfig.ToMap()) + + args := omProcess.Args() + + expectedArgs := map[string]interface{}{ + "net": map[string]interface{}{ + "port": int32(27017), + "tls": map[string]interface{}{ + "mode": "disabled", + }, + }, + "storage": map[string]interface{}{ + "dbPath": "/data", + "wiredTiger": map[string]interface{}{ + "engineConfig": map[string]interface{}{ + "cacheSizeGB": 4, + }, + }, + }, + "systemLog": map[string]interface{}{ + "destination": "file", + "path": "/var/log/mongodb-mms-automation/mongodb.log", + }, + "some": map[string]interface{}{ + "other": map[string]interface{}{ + "option": "value", + }, + }, + } + + assert.Equal(t, expectedArgs, args, "option2 should have been removed as it was not specified") +} diff --git a/controllers/om/replicaset.go b/controllers/om/replicaset.go new file mode 100644 index 000000000..dac3c4ff1 --- /dev/null +++ b/controllers/om/replicaset.go @@ -0,0 +1,331 @@ +package om + +import ( + "fmt" + "sort" + + "github.com/spf13/cast" + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +/* This corresponds to: + { + "_id": "blue", + "members": [ + { + "_id": 0, + "host": "blue_0" + }, + { + "_id": 1, + "host": "blue_1" + }, + { + "_id": 2, + "arbiterOnly": true, + "host": "blue_2", + "priority": 0 + } + ] +}*/ + +type ReplicaSet map[string]interface{} + +/* This corresponds to: + { + "_id": 0, + "host": "blue_0", + "priority": 0, + "slaveDelay": 0 + }*/ + +type ReplicaSetMember map[string]interface{} + +func NewReplicaSetFromInterface(i interface{}) ReplicaSet { + return i.(map[string]interface{}) +} + +func NewReplicaSetMemberFromInterface(i interface{}) ReplicaSetMember { + return i.(map[string]interface{}) +} + +func NewReplicaSet(name, version string) ReplicaSet { + ans := ReplicaSet{} + ans["members"] = make([]ReplicaSetMember, 0) + + // "protocolVersion" was a new field in 3.2+ Mongodb + var protocolVersion string + compare, err := util.CompareVersions(version, "3.2.0") + if err != nil { + zap.S().Warnf("Failed to parse version %s: %s", version, err) + } else if compare >= 0 { + protocolVersion = "1" + } + + initDefaultRs(ans, name, protocolVersion) + + return ans +} + +func (r ReplicaSet) Name() string { + return r["_id"].(string) +} + +func (r ReplicaSetMember) Name() string { + return r["host"].(string) +} + +func (r ReplicaSetMember) Id() int { + // Practice shows that the type of unmarshalled data can be even float64 (no floating point though) or int32 + return cast.ToInt(r["_id"]) +} + +func (r ReplicaSetMember) Votes() int { + return cast.ToInt(r["votes"]) +} + +func (r ReplicaSetMember) Priority() float32 { + return cast.ToFloat32(r["priority"]) +} + +func (r ReplicaSetMember) Tags() map[string]string { + return r["tags"].(map[string]string) +} + +/* Merges the other replica set to the current one. "otherRs" members have higher priority (as they are supposed + to be RS members managed by Kubernetes). + Returns the list of names of members which were removed as the result of merge (either they were added by mistake in OM + or we are scaling down) + + Example: + + Current RS: + + "members": [ + { + "_id": 0, + "host": "blue_0", + "arbiterOnly": true + }, + { + "_id": 1, + "host": "blue_1" + }] + + Other RS: + + "members": [ + { + "_id": 0, + "host": "green_0" + }, + { + "_id": 2, + "host": "green_2" + }] + + Merge result: + + "members": [ + { + "_id": 0, + "host": "green_0", + "arbiterOnly": true + }, + { + "_id": 2, + "host": "green_2" + }] +},*/ + +func (r ReplicaSet) String() string { + return fmt.Sprintf("\"%s\" (members: %v)", r.Name(), r.Members()) +} + +// ***************************************** Private methods *********************************************************** + +func initDefaultRs(set ReplicaSet, name string, protocolVersion string) { + if protocolVersion != "" { + // Automation Agent considers the cluster config with protocol version as string + set["protocolVersion"] = protocolVersion + } + set.setName(name) +} + +// Adding a member to the replicaset. The _id for the new member is calculated +// based on last existing member in the RS. +func (r ReplicaSet) addMember(process Process, id string, options automationconfig.MemberOptions) { + members := r.Members() + lastIndex := -1 + if len(members) > 0 { + lastIndex = members[len(members)-1].Id() + } + + rsMember := ReplicaSetMember{} + rsMember["_id"] = id + if id == "" { + rsMember["_id"] = lastIndex + 1 + } + rsMember["host"] = process.Name() + + // We always set this member to have vote (it will be set anyway on creation of deployment in OM), though this can + // be overriden by OM during merge and corrected in the end (as rs can have only 7 voting members) + rsMember.setVotes(options.GetVotes()).setPriority(options.GetPriority()).setTags(options.GetTags()) + r.setMembers(append(members, rsMember)) +} + +// mergeFrom merges "operatorRs" into "OM" one +func (r ReplicaSet) mergeFrom(operatorRs ReplicaSet) []string { + initDefaultRs(r, operatorRs.Name(), operatorRs.protocolVersion()) + + // technically we use "operatorMap" as the target map which will be used to update the members + // for the 'r' object + omMap := buildMapOfRsNodes(r) + operatorMap := buildMapOfRsNodes(operatorRs) + + // merge overlapping members into the operatorMap (overriding the 'host', + // 'horizons' and '_id' fields only) + for k, currentValue := range omMap { + if otherValue, ok := operatorMap[k]; ok { + currentValue["host"] = otherValue.Name() + currentValue["_id"] = otherValue.Id() + currentValue["votes"] = otherValue.Votes() + currentValue["priority"] = otherValue.Priority() + currentValue["tags"] = otherValue.Tags() + horizons := otherValue.getHorizonConfig() + if len(horizons) > 0 { + currentValue["horizons"] = horizons + } else { + delete(currentValue, "horizons") + } + operatorMap[k] = currentValue + } + } + + // find OM members that will be removed from RS. This can be either the result of scaling + // down or just OM added some members on its own + removedMembers := findDifference(omMap, operatorMap) + + // update replicaset back + replicas := make([]ReplicaSetMember, len(operatorMap)) + i := 0 + for _, v := range operatorMap { + replicas[i] = v + i++ + } + sort.Slice(replicas, func(i, j int) bool { + return replicas[i].Id() < replicas[j].Id() + }) + r.setMembers(replicas) + + return removedMembers +} + +// members returns all members of replica set. Note, that this should stay package-private as 'operator' package should +// not have direct access to members. +// The members returned are not copies and can be used direcly for mutations +func (r ReplicaSet) Members() []ReplicaSetMember { + switch v := r["members"].(type) { + case []ReplicaSetMember: + return v + case []interface{}: + ans := make([]ReplicaSetMember, len(v)) + for i, val := range v { + ans[i] = NewReplicaSetMemberFromInterface(val) + } + return ans + default: + panic("Unexpected type of members variable") + } +} + +func (r ReplicaSet) setName(name string) { + r["_id"] = name +} + +func (r ReplicaSet) setMembers(members []ReplicaSetMember) { + r["members"] = members +} + +func (r ReplicaSet) clearMembers() { + r["members"] = make([]ReplicaSetMember, 0) +} + +func (r ReplicaSet) findMemberByName(name string) *ReplicaSetMember { + members := r.Members() + for _, m := range members { + if m.Name() == name { + return &m + } + } + + return nil +} + +// mms uses string for this field to make it optional in json +func (r ReplicaSet) protocolVersion() string { + return r["protocolVersion"].(string) +} + +func (r ReplicaSetMember) getHorizonConfig() mdbv1.MongoDBHorizonConfig { + if horizons, okay := r["horizons"]; okay { + return horizons.(mdbv1.MongoDBHorizonConfig) + } + return mdbv1.MongoDBHorizonConfig{} +} + +func (r ReplicaSetMember) setHorizonConfig(horizonConfig mdbv1.MongoDBHorizonConfig) ReplicaSetMember { + // must not set empty horizon config + if len(horizonConfig) > 0 { + r["horizons"] = horizonConfig + } + + return r +} + +// Note, that setting vote to 0 without setting priority to the same value is not correct +func (r ReplicaSetMember) setVotes(votes int) ReplicaSetMember { + r["votes"] = votes + + return r +} + +func (r ReplicaSetMember) setPriority(priority float32) ReplicaSetMember { + r["priority"] = priority + + return r +} + +func (r ReplicaSetMember) setTags(tags map[string]string) ReplicaSetMember { + finalTags := make(map[string]string) + for k, v := range tags { + finalTags[k] = v + } + r["tags"] = finalTags + return r +} + +// Returns keys that exist in leftMap but don't exist in right one +func findDifference(leftMap map[string]ReplicaSetMember, rightMap map[string]ReplicaSetMember) []string { + ans := make([]string, 0) + for k := range leftMap { + if _, ok := rightMap[k]; !ok { + ans = append(ans, k) + } + } + return ans +} + +// Builds the map[]. This makes intersection easier +func buildMapOfRsNodes(rs ReplicaSet) map[string]ReplicaSetMember { + ans := make(map[string]ReplicaSetMember) + for _, r := range rs.Members() { + ans[r.Name()] = r + } + return ans +} diff --git a/controllers/om/replicaset/om_replicaset.go b/controllers/om/replicaset/om_replicaset.go new file mode 100644 index 000000000..42829639b --- /dev/null +++ b/controllers/om/replicaset/om_replicaset.go @@ -0,0 +1,101 @@ +package replicaset + +import ( + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/process" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" +) + +// BuildFromStatefulSet returns a replica set that can be set in the Automation Config +// based on the given StatefulSet and MongoDB resource. +func BuildFromStatefulSet(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, fcv string) om.ReplicaSetWithProcesses { + return BuildFromStatefulSetWithReplicas(mongoDBImage, forceEnterprise, set, dbSpec, int(*set.Spec.Replicas), fcv) +} + +// BuildFromStatefulSetWithReplicas returns a replica set that can be set in the Automation Config +// based on the given StatefulSet and MongoDB spec. The amount of members is set by the replicas +// parameter. +func BuildFromStatefulSetWithReplicas(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, dbSpec mdbv1.DbSpec, replicas int, fcv string) om.ReplicaSetWithProcesses { + members := process.CreateMongodProcessesWithLimit(mongoDBImage, forceEnterprise, set, dbSpec, replicas, fcv) + replicaSet := om.NewReplicaSet(set.Name, dbSpec.GetMongoDBVersion()) + rsWithProcesses := om.NewReplicaSetWithProcesses(replicaSet, members, dbSpec.GetMemberOptions()) + rsWithProcesses.SetHorizons(dbSpec.GetHorizonConfig()) + return rsWithProcesses +} + +// PrepareScaleDownFromMap performs additional steps necessary to make sure removed members are not primary (so no +// election happens and replica set is available) (see +// https://jira.mongodb.org/browse/HELP-3818?focusedCommentId=1548348 for more details) +// Note, that we are skipping setting nodes as "disabled" (but the code is commented to be able to revert this if +// needed) +func PrepareScaleDownFromMap(omClient om.Connection, rsMembers map[string][]string, processesToWaitForGoalState []string, log *zap.SugaredLogger) error { + processes := make([]string, 0) + for _, v := range rsMembers { + processes = append(processes, v...) + } + + // Stage 1. Set Votes and Priority to 0 + if len(rsMembers) > 0 && len(processes) > 0 { + err := omClient.ReadUpdateDeployment( + func(d om.Deployment) error { + for k, v := range rsMembers { + if err := d.MarkRsMembersUnvoted(k, v); err != nil { + log.Errorf("Problems scaling down some replica sets (were they changed in Ops Manager directly?): %s", err) + } + } + return nil + }, + log, + ) + if err != nil { + return xerrors.Errorf("unable to set votes, priority to 0 in Ops Manager, hosts: %v, err: %w", processes, err) + } + + if err := om.WaitForReadyState(omClient, processesToWaitForGoalState, false, log); err != nil { + return err + } + + log.Debugw("Marked replica set members as non-voting", "replica set with members", rsMembers) + } + + // TODO practice shows that automation agents can get stuck on setting db to "disabled" also it seems that this process + // works correctly without explicit disabling - feel free to remove this code after some time when it is clear + // that everything works correctly without disabling + + // Stage 2. Set disabled to true + //err = omClient.ReadUpdateDeployment( + // func(d om.Deployment) error { + // d.DisableProcesses(allProcesses) + // return nil + // }, + //) + // + //if err != nil { + // return errors.New(fmt.Sprintf("Unable to set disabled to true, hosts: %v, err: %w", allProcesses, err)) + //} + //log.Debugw("Disabled processes", "processes", allProcesses) + + log.Infow("Performed some preliminary steps to support scale down", "hosts", processes) + + return nil +} + +func PrepareScaleDownFromStatefulSet(omClient om.Connection, statefulSet appsv1.StatefulSet, rs *mdbv1.MongoDB, log *zap.SugaredLogger) error { + _, podNames := dns.GetDnsForStatefulSetReplicasSpecified(statefulSet, rs.Spec.GetClusterDomain(), rs.Status.Members, nil) + podNames = podNames[scale.ReplicasThisReconciliation(rs):rs.Status.Members] + + if len(podNames) != 1 { + return xerrors.Errorf("dev error: the number of members being scaled down was > 1, scaling more than one member at a time is not possible! %s", podNames) + } + + log.Debugw("Setting votes to 0 for members", "members", podNames) + return PrepareScaleDownFromMap(omClient, map[string][]string{rs.Name: podNames}, podNames, log) +} diff --git a/controllers/om/replicaset_test.go b/controllers/om/replicaset_test.go new file mode 100644 index 000000000..f2590d954 --- /dev/null +++ b/controllers/om/replicaset_test.go @@ -0,0 +1,86 @@ +package om + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +func makeMinimalRsWithProcesses() ReplicaSetWithProcesses { + replicaSetWithProcesses := NewReplicaSet("my-test-repl", "4.2.1") + mdb := mdbv1.MongoDB{Spec: mdbv1.MongoDbSpec{DbCommonSpec: mdbv1.DbCommonSpec{Version: "4.2.1"}}} + mdb.InitDefaults() + processes := make([]Process, 3) + memberOptions := make([]automationconfig.MemberOptions, 3) + for i := range processes { + proc := NewMongodProcess("my-test-repl-"+strconv.Itoa(i), "my-test-repl-"+strconv.Itoa(i), "fake-mongoDBImage", false, &mdbv1.AdditionalMongodConfig{}, &mdb.Spec, "", nil, "") + processes[i] = proc + replicaSetWithProcesses.addMember(proc, "", memberOptions[i]) + } + return NewReplicaSetWithProcesses(replicaSetWithProcesses, processes, memberOptions) +} + +// TestMergeHorizonsAdd checks that horizon configuration is appropriately +// added. +func TestMergeHorizonsAdd(t *testing.T) { + opsManagerRsWithProcesses := makeMinimalRsWithProcesses() + operatorRsWithProcesses := makeMinimalRsWithProcesses() + horizons := []mdbv1.MongoDBHorizonConfig{ + {"name1": "my-db.my-test.com:12345"}, + {"name1": "my-db.my-test.com:12346"}, + {"name1": "my-db.my-test.com:12347"}, + } + operatorRsWithProcesses.SetHorizons(horizons) + + opsManagerRsWithProcesses.Rs.mergeFrom(operatorRsWithProcesses.Rs) + for i, member := range opsManagerRsWithProcesses.Rs.Members() { + assert.Equal(t, horizons[i], member.getHorizonConfig()) + } +} + +// TestMergeHorizonsIgnore checks that old horizon configuration is removed +// when merged with a replica set with no horizon configuration. +func TestMergeHorizonsRemove(t *testing.T) { + opsManagerRsWithProcesses := makeMinimalRsWithProcesses() + horizons := []mdbv1.MongoDBHorizonConfig{ + {"name1": "my-db.my-test.com:12345"}, + {"name1": "my-db.my-test.com:12346"}, + {"name1": "my-db.my-test.com:12347"}, + } + opsManagerRsWithProcesses.SetHorizons(horizons) + operatorRsWithProcesses := makeMinimalRsWithProcesses() + + opsManagerRsWithProcesses.Rs.mergeFrom(operatorRsWithProcesses.Rs) + for _, member := range opsManagerRsWithProcesses.Rs.Members() { + assert.Equal(t, mdbv1.MongoDBHorizonConfig{}, member.getHorizonConfig()) + } +} + +// TestMergeHorizonsOverride checks that new horizon configuration overrides +// old horizon configuration. +func TestMergeHorizonsOverride(t *testing.T) { + opsManagerRsWithProcesses := makeMinimalRsWithProcesses() + horizonsOld := []mdbv1.MongoDBHorizonConfig{ + {"name1": "my-db.my-test.com:12345"}, + {"name1": "my-db.my-test.com:12346"}, + {"name1": "my-db.my-test.com:12347"}, + } + horizonsNew := []mdbv1.MongoDBHorizonConfig{ + {"name2": "my-db.my-test.com:12345"}, + {"name2": "my-db.my-test.com:12346"}, + {"name2": "my-db.my-test.com:12347"}, + } + opsManagerRsWithProcesses.SetHorizons(horizonsOld) + operatorRsWithProcesses := makeMinimalRsWithProcesses() + operatorRsWithProcesses.SetHorizons(horizonsNew) + + opsManagerRsWithProcesses.Rs.mergeFrom(operatorRsWithProcesses.Rs) + for i, member := range opsManagerRsWithProcesses.Rs.Members() { + assert.Equal(t, horizonsNew[i], member.getHorizonConfig()) + } +} diff --git a/controllers/om/shardedcluster.go b/controllers/om/shardedcluster.go new file mode 100644 index 000000000..64794d580 --- /dev/null +++ b/controllers/om/shardedcluster.go @@ -0,0 +1,258 @@ +package om + +import ( + "sort" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// Representation of one element in "sharding" array in OM json deployment: +/* +"sharding": [ + { + "shards": [ + { + "tags": [ + "gluetenfree" + ], + "_id": "electron_0", + "rs": "electron_0" + }, + { + "tags": [], + "_id": "electron_1", + "rs": "electron_1" + } + ], + "name": "electron", + "managedSharding": true, + "configServer": [], + "configServerReplica": "electroncsrs", + "collections": [ + { + "_id": "food.vegetables", + "dropped": false, + "key": [ + [ + "rand", + 1 + ] + ], + "unique": false + }, + { + "_id": "randomStuff.bigStuff", + "dropped": false, + "key": [ + [ + "x", + "hashed" + ] + ], + "unique": false + } + ], + "tags": [] + } + ] + } +*/ +type ShardedCluster map[string]interface{} + +type Shard map[string]interface{} + +func NewShardedClusterFromInterface(i interface{}) ShardedCluster { + return i.(map[string]interface{}) +} + +// NewShardedCluster builds a shard configuration with shards by replicasets names +func NewShardedCluster(name, configRsName string, replicaSets []ReplicaSetWithProcesses) ShardedCluster { + ans := ShardedCluster{} + ans.setName(name) + ans.setConfigServerRsName(configRsName) + + shards := make([]Shard, len(replicaSets)) + for k, v := range replicaSets { + s := newShard(v.Rs.Name()) + shards[k] = s + } + ans.setShards(shards) + return ans +} + +func (s ShardedCluster) Name() string { + return s["name"].(string) +} + +func (s ShardedCluster) ConfigServerRsName() string { + return s["configServerReplica"].(string) +} + +// ***************************************** Private methods *********************************************************** + +func newShard(name string) Shard { + s := Shard{} + s.setId(name) + s.setRs(name) + return s +} + +// mergeFrom merges the other (Kuberenetes owned) cluster configuration into OM one +func (s ShardedCluster) mergeFrom(operatorCluster ShardedCluster) []string { + s.setName(operatorCluster.Name()) + s.setConfigServerRsName(operatorCluster.ConfigServerRsName()) + + omMap := buildMapOfShards(s) + operatorMap := buildMapOfShards(operatorCluster) + + // merge overlapping members to the operatorMap + for k, currentValue := range omMap { + if otherValue, ok := operatorMap[k]; ok { + currentValue.mergeFrom(otherValue) + + operatorMap[k] = currentValue + } + } + + // find OM shards that will be removed from cluster. This can be either the result of shard cluster reconfiguration + // or just OM added some shards on its own + removedMembers := findDifferentKeys(omMap, operatorMap) + + // update cluster shards back + shards := make([]Shard, len(operatorMap)) + i := 0 + for _, v := range operatorMap { + shards[i] = v + i++ + } + sort.Slice(shards, func(i, j int) bool { + return shards[i].id() < shards[j].id() + }) + s.setShards(shards) + + return removedMembers +} + +// mergeFrom merges the operator shard into OM one. Only some fields are overriden, the others stay untouched +func (s Shard) mergeFrom(operatorShard Shard) { + s.setId(operatorShard.id()) + s.setRs(operatorShard.rs()) +} + +func (s ShardedCluster) shards() []Shard { + switch v := s["shards"].(type) { + case []Shard: + return v + case []interface{}: + ans := make([]Shard, len(v)) + for i, val := range v { + ans[i] = val.(map[string]interface{}) + } + return ans + default: + panic("Unexpected type of shards variable") + } +} + +func (s ShardedCluster) setConfigServerRsName(name string) { + s["configServerReplica"] = name +} + +func (s ShardedCluster) setName(name string) { + s["name"] = name +} + +func (s ShardedCluster) setShards(shards []Shard) { + s["shards"] = shards +} + +// draining returns the "draining" array which contains the names of replicasets for shards which are currently being +// removed. This is necessary for the AutomationAgent to keep the knowledge about shards to survive restarts (it's +// necessary to restart the mongod with the same 'shardSrv' option even if the shard is not in sharded cluster in +// Automation Config anymore) +func (s ShardedCluster) draining() []string { + if _, ok := s["draining"]; !ok { + return make([]string, 0) + } + + // When go unmarhals an empty list from Json, it becomes + // []interface{} and not []string, so we must check for + // that particular case. + if obj, ok := s["draining"].([]interface{}); ok { + var hostNames []string + for _, hn := range obj { + hostNames = append(hostNames, hn.(string)) + } + + return hostNames + } + + return s["draining"].([]string) +} + +func (s ShardedCluster) setDraining(rsNames []string) { + s["draining"] = rsNames +} + +func (s ShardedCluster) addToDraining(rsNames []string) { + // constructor is a better place to initialize the array, but we aim a better backward compatibility with OM 4.0 + // versions (which learnt about this field in 4.0.12) so doing lazy initialization + if _, ok := s["draining"]; !ok { + s.setDraining([]string{}) + } + for _, r := range rsNames { + if !stringutil.Contains(s.draining(), r) { + s["draining"] = append(s.draining(), r) + } + } +} + +func (s ShardedCluster) removeDraining() { + delete(s, "draining") +} + +// getAllReplicaSets returns all replica sets associated with sharded cluster +func (s ShardedCluster) getAllReplicaSets() []string { + var ans []string + for _, s := range s.shards() { + ans = append(ans, s.rs()) + } + ans = append(ans, s.ConfigServerRsName()) + return ans +} + +func (s Shard) id() string { + return s["_id"].(string) +} + +func (s Shard) setId(id string) { + s["_id"] = id +} + +func (s Shard) rs() string { + return s["rs"].(string) +} + +func (s Shard) setRs(rsName string) { + s["rs"] = rsName +} + +// Returns keys that exist in leftMap but don't exist in right one +func findDifferentKeys(leftMap map[string]Shard, rightMap map[string]Shard) []string { + ans := make([]string, 0) + for k := range leftMap { + if _, ok := rightMap[k]; !ok { + ans = append(ans, k) + } + } + return ans +} + +// Builds the map[]. This makes intersection easier +func buildMapOfShards(sh ShardedCluster) map[string]Shard { + ans := make(map[string]Shard) + for _, r := range sh.shards() { + ans[r.id()] = r + } + return ans +} diff --git a/controllers/om/test_utils.go b/controllers/om/test_utils.go new file mode 100644 index 000000000..db6618e01 --- /dev/null +++ b/controllers/om/test_utils.go @@ -0,0 +1,17 @@ +package om + +import ( + "fmt" + "os" + "path/filepath" +) + +func loadBytesFromTestData(name string) []byte { + // testdata is a special directory ignored by "go build" + path := filepath.Join("testdata", name) + bytes, err := os.ReadFile(path) + if err != nil { + fmt.Println(err) + } + return bytes +} diff --git a/controllers/om/testdata/automation_config.json b/controllers/om/testdata/automation_config.json new file mode 100644 index 000000000..e2e9041e8 --- /dev/null +++ b/controllers/om/testdata/automation_config.json @@ -0,0 +1,324 @@ +{ + "auth": { + "usersWanted": [ + { + "authenticationRestrictions": [], + "db": "testDb0", + "user": "testUser0", + "pwd": "somePassword0", + "roles": [ + { + "db": "admin", + "role": "backup" + }, + { + "db": "admin", + "role": "monitor" + }, + { + "db": "admin", + "role": "automation" + } + ], + "mechanisms": [], + "scramSha256Creds": { + "iterationCount": 15000, + "salt": "I570PanWIx1eNUTo7j4ROl2/zIqMsVd6CcIE+A==", + "serverKey": "M4/jskiMM0DpvG/qgMELWlfReqV2ZmwdU8+vJZ/4prc=", + "storedKey": "m1dXf5hHJk7EOAAyJBxfsZvFx1HwtTdda6pFPm0BlOE=" + } + }, + { + "authenticationRestrictions": [], + "db": "testDb1", + "user": "testUser1", + "pwd": "somePassword1", + "unknownFieldOne": "one", + "unknownFieldTwo": "two", + "roles": [ + { + "db": "admin", + "role": "backup" + }, + { + "db": "admin", + "role": "monitor" + }, + { + "db": "admin", + "role": "automation" + } + ], + "mechanisms": [], + "scramSha256Creds": { + "iterationCount": 15000, + "salt": "I570PanWIx1eNUTo7j4ROl2/zIqMsVd6CcIE+A==", + "serverKey": "M4/jskiMM0DpvG/qgMELWlfReqV2ZmwdU8+vJZ/4prc=", + "storedKey": "m1dXf5hHJk7EOAAyJBxfsZvFx1HwtTdda6pFPm0BlOE=" + } + }, + { + "authenticationRestrictions": [], + "db": "testDb2", + "user": "testUser2", + "pwd": "somePassword2", + "roles": [ + { + "db": "admin", + "role": "backup" + }, + { + "db": "admin", + "role": "monitor" + }, + { + "db": "admin", + "role": "automation" + } + ], + "mechanisms": [], + "scramSha256Creds": { + "iterationCount": 15000, + "salt": "I570PanWIx1eNUTo7j4ROl2/zIqMsVd6CcIE+A==", + "serverKey": "M4/jskiMM0DpvG/qgMELWlfReqV2ZmwdU8+vJZ/4prc=", + "storedKey": "m1dXf5hHJk7EOAAyJBxfsZvFx1HwtTdda6pFPm0BlOE=" + } + } + ], + "usersDeleted": [], + "disabled": true, + "authoritativeSet": false, + "autoAuthMechanisms": [], + "autoUser": "mms-automation", + "autoAuthRestrictions": [] + }, + "mongoDbVersions": [ + { + "builds": [ + { + "architecture": "amd64", + "bits": 64, + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "platform": "linux", + "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "ubuntu", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "15.04", + "minOsVersion": "14.04", + "platform": "linux", + "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1404-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "amazon", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "", + "minOsVersion": "2013.03", + "platform": "linux", + "url": "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-amazon-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "platform": "osx", + "url": "https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "platform": "windows", + "url": "https://fastdl.mongodb.org/win32/mongodb-win32-x86_64-3.2.0.zip" + }, + { + "architecture": "amd64", + "bits": 64, + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "platform": "windows", + "url": "https://fastdl.mongodb.org/win32/mongodb-win32-x86_64-2008plus-3.2.0.zip", + "win2008plus": true + } + ], + "name": "3.2.0" + }, + { + "builds": [ + { + "architecture": "amd64", + "bits": 64, + "flavor": "amazon", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "", + "minOsVersion": "2013.03", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-amzn64-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "rhel", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "6.0", + "minOsVersion": "5.7", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel57-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "rhel", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "7.0", + "minOsVersion": "6.2", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel62-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "rhel", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "8.0", + "minOsVersion": "7.0", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel70-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "suse", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "12", + "minOsVersion": "11", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-suse11-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "suse", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "13", + "minOsVersion": "12", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-suse12-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "ubuntu", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "13.04", + "minOsVersion": "12.04", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu1204-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "ubuntu", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "15.04", + "minOsVersion": "14.04", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu1404-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "flavor": "debian", + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "maxOsVersion": "8.0", + "minOsVersion": "7.1", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-debian71-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "modules": [ + "enterprise" + ], + "platform": "osx", + "url": "https://downloads.mongodb.com/osx/mongodb-osx-x86_64-enterprise-3.2.0.tgz" + }, + { + "architecture": "amd64", + "bits": 64, + "gitVersion": "45d947729a0315accb6d4f15a6b06be6d9c19fe7", + "modules": [ + "enterprise" + ], + "platform": "windows", + "url": "https://downloads.mongodb.com/win32/mongodb-win32-x86_64-enterprise-windows-64-3.2.0.zip", + "win2008plus": true, + "winVCRedistDll": "msvcr120.dll", + "winVCRedistOptions": [ + "/quiet", + "/norestart" + ], + "winVCRedistUrl": "http://download.microsoft.com/download/2/E/6/2E61CFA4-993B-4DD4-91DA-3737CD5CD6E3/vcredist_x64.exe", + "winVCRedistVersion": "12.0.21005.1" + } + ], + "name": "3.2.0-ent" + } + ], + "ldap": {}, + "processes": [], + "replicaSets": [], + "roles": [], + "monitoringVersions": [], + "backupVersions": [], + "mongosqlds": [], + "balancer": {}, + "indexConfigs": [], + "kerberos": { + "serviceName": "mongodb" + }, + "options": { + "downloadBase": "/var/lib/mongodb-mms-automation", + "downloadBaseWindows": "%SystemDrive%\\MMSAutomation\\versions" + }, + "tls": { + "CAFilePath": null, + "CAFilePathWindows": null, + "clientCertificateMode": "OPTIONAL" + }, + "version": null, + "sharding": [] +} diff --git a/controllers/om/testdata/backup_config.json b/controllers/om/testdata/backup_config.json new file mode 100644 index 000000000..427327cf7 --- /dev/null +++ b/controllers/om/testdata/backup_config.json @@ -0,0 +1,29 @@ +{ + "username": "some-username", + "sslPEMKeyFile" : "some-pem-key-file", + "name": "7.6.0.1059-1", + "logPath": "/var/log/mongodb-mms-automation/backup-agent.log", + "logPathWindows": "%SystemDrive%\\MMSAutomation\\log\\mongodb-mms-automation\\backup-agent.log", + "logRotate": { + "sizeThresholdMB": 1000, + "timeThresholdHrs": 24 + }, + "urls": { + "linux": { + "default": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.linux_x86_64.tar.gz", + "ppc64le_rhel7": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.rhel7_ppc64le.tar.gz", + "ppc64le_ubuntu1604": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.ubuntu1604_ppc64le.tar.gz", + "rhel7": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.rhel7_x86_64.tar.gz", + "s390x_rhel6": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.rhel6_s390x.tar.gz", + "s390x_rhel7": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.rhel7_s390x.tar.gz", + "s390x_suse12": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.suse12_s390x.tar.gz", + "s390x_ubuntu1804": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.ubuntu1804_s390x.tar.gz" + }, + "osx": { + "default": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.osx_x86_64.tar.gz" + }, + "windows": { + "default": "http://ec2-52-73-78-251.compute-1.amazonaws.com:9080/download/agent/backup/mongodb-mms-backup-agent-7.6.0.1059-1.windows_x86_64.msi" + } + } +} diff --git a/controllers/om/testdata/deployment_tls.json b/controllers/om/testdata/deployment_tls.json new file mode 100644 index 000000000..87839a6af --- /dev/null +++ b/controllers/om/testdata/deployment_tls.json @@ -0,0 +1,161 @@ +{ + "auth": { + "authoritativeSet": false, + "autoAuthMechanism": "MONGODB-CR", + "autoAuthMechanisms": [], + "autoAuthRestrictions": [], + "disabled": true, + "usersDeleted": [], + "usersWanted": [] + }, + "backupVersions": [ + { + "hostname": "test-tls-additional-domains-0.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "6.6.0.959-1" + }, + { + "hostname": "test-tls-additional-domains-1.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "6.6.0.959-1" + }, + { + "hostname": "test-tls-additional-domains-2.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "6.6.0.959-1" + } + ], + "balancer": {}, + "cpsModules": [], + "indexConfigs": [], + "kerberos": { + "serviceName": "mongodb" + }, + "ldap": {}, + "mongosqlds": [], + "mongots": [], + "monitoringVersions": [ + { + "hostname": "test-tls-additional-domains-0.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "6.4.0.433-1" + } + ], + "onlineArchiveModules": [], + "options": { + "downloadBase": "/var/lib/mongodb-mms-automation", + "downloadBaseWindows": "%SystemDrive%\\MMSAutomation\\versions" + }, + "processes": [ + { + "args2_6": { + "net": { + "port": 27017, + "tls": { + "PEMKeyFile": "/mongodb-automation/server.pem", + "mode": "requireSSL" + } + }, + "replication": { + "replSetName": "test-tls-additional-domains" + }, + "storage": { + "dbPath": "/data" + }, + "systemLog": { + "destination": "file", + "path": "/var/log/mongodb-mms-automation/mongodb.log" + } + }, + "authSchemaVersion": 5, + "featureCompatibilityVersion": "3.6", + "hostname": "test-tls-additional-domains-0.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "test-tls-additional-domains-0", + "processType": "mongod", + "version": "3.6.8" + }, + { + "args2_6": { + "net": { + "port": 27017, + "tls": { + "PEMKeyFile": "/mongodb-automation/server.pem", + "mode": "requireSSL" + } + }, + "replication": { + "replSetName": "test-tls-additional-domains" + }, + "storage": { + "dbPath": "/data" + }, + "systemLog": { + "destination": "file", + "path": "/var/log/mongodb-mms-automation/mongodb.log" + } + }, + "authSchemaVersion": 5, + "featureCompatibilityVersion": "3.6", + "hostname": "test-tls-additional-domains-1.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "test-tls-additional-domains-1", + "processType": "mongod", + "version": "3.6.8" + }, + { + "args2_6": { + "net": { + "port": 27017, + "tls": { + "PEMKeyFile": "/mongodb-automation/server.pem", + "mode": "requireSSL" + } + }, + "replication": { + "replSetName": "test-tls-additional-domains" + }, + "storage": { + "dbPath": "/data" + }, + "systemLog": { + "destination": "file", + "path": "/var/log/mongodb-mms-automation/mongodb.log" + } + }, + "authSchemaVersion": 5, + "featureCompatibilityVersion": "3.6", + "hostname": "test-tls-additional-domains-2.test-tls-additional-domains-svc.dev.svc.cluster.local", + "name": "test-tls-additional-domains-2", + "processType": "mongod", + "version": "3.6.8" + } + ], + "replicaSets": [ + { + "_id": "test-tls-additional-domains", + "members": [ + { + "_id": 0, + "host": "test-tls-additional-domains-0", + "priority": 1, + "votes": 1 + }, + { + "_id": 1, + "host": "test-tls-additional-domains-1", + "priority": 1, + "votes": 1 + }, + { + "_id": 2, + "host": "test-tls-additional-domains-2", + "priority": 1, + "votes": 1 + } + ], + "protocolVersion": "1" + } + ], + "roles": [], + "sharding": [], + "tls": { + "CAFilePath": "/mongodb-automation/ca.pem", + "clientCertificateMode": "OPTIONAL" + }, + "version": 138 +} diff --git a/controllers/om/testdata/monitoring_config.json b/controllers/om/testdata/monitoring_config.json new file mode 100644 index 000000000..cf6ff94cd --- /dev/null +++ b/controllers/om/testdata/monitoring_config.json @@ -0,0 +1,27 @@ +{ + "name": "6.6.2.466-1", + "logPath": "/var/log/mongodb-mms-automation/monitoring-agent.log", + "logPathWindows": "%SystemDrive%\\MMSAutomation\\log\\mongodb-mms-automation\\monitoring-agent.log", + "logRotate": { + "sizeThresholdMB": 1000, + "timeThresholdHrs": 24 + }, + "urls": { + "linux": { + "default": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.linux_x86_64.tar.gz", + "ppc64le_rhel7": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.rhel7_ppc64le.tar.gz", + "ppc64le_ubuntu1604": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.ubuntu1604_ppc64le.tar.gz", + "rhel7": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.rhel7_x86_64.tar.gz", + "s390x_rhel6": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.rhel6_s390x.tar.gz", + "s390x_rhel7": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.rhel7_s390x.tar.gz", + "s390x_suse12": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.suse12_s390x.tar.gz", + "s390x_ubuntu1804": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.ubuntu1804_s390x.tar.gz" + }, + "osx": { + "default": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.osx_x86_64.tar.gz" + }, + "windows": { + "default": "http://ec2-54-196-186-93.compute-1.amazonaws.com:9080/download/agent/monitoring/mongodb-mms-monitoring-agent-6.6.2.466-1.windows_x86_64.msi" + } + } +} \ No newline at end of file diff --git a/controllers/operator/agents/agents.go b/controllers/operator/agents/agents.go new file mode 100644 index 000000000..8e26ef346 --- /dev/null +++ b/controllers/operator/agents/agents.go @@ -0,0 +1,310 @@ +package agents + +import ( + "context" + "fmt" + "maps" + "slices" + "time" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +type SecretGetCreator interface { + secret.Getter + secret.Creator +} + +type retryParams struct { + waitSeconds int + retrials int +} + +const RollingChangeArgs = "RollingChangeArgs" + +// EnsureAgentKeySecretExists checks if the Secret with specified name (-group-secret) exists, otherwise tries to +// generate agent key using OM public API and create Secret containing this key. Generation of a key is expected to be +// a rare operation as the group creation api generates agent key already (so the only possible situation is when the group +// was created externally and agent key wasn't generated before). +// Returns agent key that was either generated or reused from parameter agentKey. +// We need to return the key, because in case it was generated here it has to be passed back on an agentKey argument when we're executing the +// function over multiple clusters. +func EnsureAgentKeySecretExists(ctx context.Context, secretClient secrets.SecretClient, agentKeyGenerator om.AgentKeyGenerator, namespace, agentKey, projectId, basePath string, log *zap.SugaredLogger) (string, error) { + secretName := ApiKeySecretName(projectId) + log = log.With("secret", secretName) + agentKeySecret, err := secretClient.ReadSecret(ctx, kube.ObjectKey(namespace, secretName), basePath) + if err != nil { + if !secrets.SecretNotExist(err) { + return "", xerrors.Errorf("error reading agent key secret: %w", err) + } + + if agentKey == "" { + log.Info("Generating agent key as current project doesn't have it") + + agentKey, err = agentKeyGenerator.GenerateAgentKey() + if err != nil { + return "", xerrors.Errorf("failed to generate agent key in OM: %w", err) + } + log.Info("Agent key was successfully generated") + } + + agentSecret := secret.Builder(). + SetField(util.OmAgentApiKey, agentKey). + SetNamespace(namespace). + SetName(secretName). + Build() + + if err := secretClient.PutSecret(ctx, agentSecret, basePath); err != nil { + return "", xerrors.Errorf("failed to create AgentKey secret: %w", err) + } + + log.Infof("Project agent key is saved for later usage") + return agentKey, nil + } + + return agentKeySecret[util.OmAgentApiKey], nil +} + +// ApiKeySecretName for a given ProjectID (`project`) returns the name of +// the secret associated with it. +func ApiKeySecretName(project string) string { + return fmt.Sprintf("%s-group-secret", project) +} + +// WaitForRsAgentsToRegister waits until all the agents associated with the given StatefulSet have registered with Ops Manager. +func WaitForRsAgentsToRegister(set appsv1.StatefulSet, members int, clusterName string, omConnection om.Connection, log *zap.SugaredLogger, rs *mdbv1.MongoDB) error { + hostnames, _ := dns.GetDnsForStatefulSetReplicasSpecified(set, clusterName, members, rs.Spec.DbCommonSpec.GetExternalDomain()) + + log = log.With("statefulset", set.Name) + + ok, msg := waitUntilRegistered(omConnection, log, retryParams{retrials: 5, waitSeconds: 3}, hostnames...) + if !ok { + return getAgentRegisterError(msg) + } + return nil +} + +// WaitForRsAgentsToRegisterSpecifiedHostnames waits for the specified agents to registry with Ops Manager. +func WaitForRsAgentsToRegisterSpecifiedHostnames(omConnection om.Connection, hostnames []string, log *zap.SugaredLogger) error { + ok, msg := waitUntilRegistered(omConnection, log, retryParams{retrials: 10, waitSeconds: 9}, hostnames...) + if !ok { + return getAgentRegisterError(msg) + } + return nil +} + +func getAgentRegisterError(errorMsg string) error { + return xerrors.New(fmt.Sprintf("some agents failed to register or the Operator is using the wrong host names for the pods. "+ + "Make sure the 'spec.clusterDomain' is set if it's different from the default Kubernetes cluster "+ + "name ('cluster.local'): %s", errorMsg)) +} + +const StaleProcessDuration = time.Minute * 2 + +// ProcessState represents the state of the mongodb process. +// Most importantly it contains the information whether the node is down (precisely whether the agent running next to mongod is actively reporting pings to OM), +// what is the last version of the automation config achieved and the step on which the agent is currently executing the plan. +type ProcessState struct { + Hostname string + LastAgentPing time.Time + GoalVersionAchieved int + Plan []string + ProcessName string +} + +// NewProcessState should be used to create new instances of ProcessState as it sets some reasonable default values. +// As ProcessState is combining the data from two sources, we don't have any guarantees that we'll have the information about the given hostname +// available from both sources, therefore we need to always assume some defaults. +func NewProcessState(hostname string) ProcessState { + return ProcessState{ + Hostname: hostname, + LastAgentPing: time.Time{}, + GoalVersionAchieved: -1, + Plan: nil, + } +} + +// IsStale returns true if this process is considered down, i.e. last ping of the agent is later than 2 minutes ago +// We use an in-the-middle value when considering the process to be down: +// - in waitForAgentsToRegister we use 1 min to consider the process "not registered" +// - Ops Manager is using 5 mins as a default for considering process as stale +func (p ProcessState) IsStale() bool { + return p.LastAgentPing.Add(StaleProcessDuration).Before(time.Now()) +} + +// MongoDBClusterStateInOM represents the state of the whole deployment from the Ops Manager's perspective by combining singnals about the processes from two sources: +// - from om.Connection.ReadAutomationAgents to get last ping of the agent (/groups//agents/AUTOMATION) +// - from om.Connection.ReadAutomationStatus to get the list of agent health statuses, AC version achieved, step of the agent's plan (/groups//automationStatus) +type MongoDBClusterStateInOM struct { + GoalVersion int + ProcessStateMap map[string]ProcessState +} + +// GetMongoDBClusterState executes requests to OM from the given omConnection to gather the current deployment state. +// It combines the data from the automation status and the list of automation agents. +func GetMongoDBClusterState(omConnection om.Connection) (MongoDBClusterStateInOM, error) { + var agentStatuses []om.AgentStatus + _, err := om.TraversePages( + omConnection.ReadAutomationAgents, + func(aa interface{}) bool { + agentStatuses = append(agentStatuses, aa.(om.AgentStatus)) + return false + }, + ) + if err != nil { + return MongoDBClusterStateInOM{}, xerrors.Errorf("error when reading automation agent pages: %v", err) + } + + automationStatus, err := omConnection.ReadAutomationStatus() + if err != nil { + return MongoDBClusterStateInOM{}, xerrors.Errorf("error reading automation status: %v", err) + } + + processStateMap, err := calculateProcessStateMap(automationStatus.Processes, agentStatuses) + if err != nil { + return MongoDBClusterStateInOM{}, err + } + + return MongoDBClusterStateInOM{ + GoalVersion: automationStatus.GoalVersion, + ProcessStateMap: processStateMap, + }, nil +} + +func (c *MongoDBClusterStateInOM) GetProcessState(hostname string) ProcessState { + if processState, ok := c.ProcessStateMap[hostname]; ok { + return processState + } + + return NewProcessState(hostname) +} + +func (c *MongoDBClusterStateInOM) GetProcesses() []ProcessState { + return slices.Collect(maps.Values(c.ProcessStateMap)) +} + +func (c *MongoDBClusterStateInOM) GetProcessesNotInGoalState() []ProcessState { + return slices.DeleteFunc(slices.Collect(maps.Values(c.ProcessStateMap)), func(processState ProcessState) bool { + return processState.GoalVersionAchieved >= c.GoalVersion + }) +} + +// calculateProcessStateMap combines information from ProcessStatuses and AgentStatuses returned by OpsManager +// and maps them to a unified data structure. +// +// The resulting ProcessState combines information from both agent and process status when refer to the same hostname. +// It is not guaranteed that we'll have the information from two sources, so in case one side is missing the defaults +// would be present as defined in NewProcessState. +// If multiple statuses exist for the same hostname, subsequent entries overwrite ones. +// Fields such as GoalVersionAchieved default to -1 if never set, and Plan defaults to nil. +// LastAgentPing defaults to the zero time if no AgentStatus entry is available. +func calculateProcessStateMap(processStatuses []om.ProcessStatus, agentStatuses []om.AgentStatus) (map[string]ProcessState, error) { + processStates := map[string]ProcessState{} + for _, agentStatus := range agentStatuses { + if agentStatus.TypeName != "AUTOMATION" { + return nil, xerrors.Errorf("encountered unexpected agent type in agent status type in %+v", agentStatus) + } + processState, ok := processStates[agentStatus.Hostname] + if !ok { + processState = NewProcessState(agentStatus.Hostname) + } + lastPing, err := time.Parse(time.RFC3339, agentStatus.LastConf) + if err != nil { + return nil, xerrors.Errorf("wrong format for lastConf field: expected UTC format but the value is %s, agentStatus=%+v: %v", agentStatus.LastConf, agentStatus, err) + } + processState.LastAgentPing = lastPing + + processStates[agentStatus.Hostname] = processState + } + + for _, processStatus := range processStatuses { + processState, ok := processStates[processStatus.Hostname] + if !ok { + processState = NewProcessState(processStatus.Hostname) + } + processState.GoalVersionAchieved = processStatus.LastGoalVersionAchieved + processState.ProcessName = processStatus.Name + processState.Plan = processStatus.Plan + processStates[processStatus.Hostname] = processState + } + + return processStates, nil +} + +func agentCheck(omConnection om.Connection, agentHostnames []string, log *zap.SugaredLogger) (string, bool) { + registeredHostnamesSet := map[string]struct{}{} + predicateFunc := func(aa interface{}) bool { + automationAgent := aa.(om.Status) + for _, hostname := range agentHostnames { + if automationAgent.IsRegistered(hostname, log) { + registeredHostnamesSet[hostname] = struct{}{} + if len(registeredHostnamesSet) == len(agentHostnames) { + return true + } + } + } + return false + } + + _, err := om.TraversePages( + omConnection.ReadAutomationAgents, + predicateFunc, + ) + if err != nil { + return fmt.Sprintf("Received error when reading automation agent pages: %v", err), false + } + + // convert to list of keys only for pretty printing in the error message + var registeredHostnamesList []string + for hostname := range registeredHostnamesSet { + registeredHostnamesList = append(registeredHostnamesList, hostname) + } + + var msg string + if len(registeredHostnamesList) == 0 { + return fmt.Sprintf("None of %d expected agents has registered with OM, expected hostnames: %+v", len(agentHostnames), agentHostnames), false + } else if len(registeredHostnamesList) == len(agentHostnames) { + return fmt.Sprintf("All of %d expected agents have registered with OM, hostnames: %+v", len(registeredHostnamesList), registeredHostnamesList), true + } else { + var missingHostnames []string + for _, expectedHostname := range agentHostnames { + if _, ok := registeredHostnamesSet[expectedHostname]; !ok { + missingHostnames = append(missingHostnames, expectedHostname) + } + } + msg = fmt.Sprintf("Only %d of %d expected agents have registered with OM, missing hostnames: %+v, registered hostnames in OM: %+v, expected hostnames: %+v", len(registeredHostnamesList), len(agentHostnames), missingHostnames, registeredHostnamesList, agentHostnames) + return msg, false + } +} + +// waitUntilRegistered waits until all agents with 'agentHostnames' are registered in OM. Note, that wait +// happens after retrial - this allows to skip waiting in case agents are already registered +func waitUntilRegistered(omConnection om.Connection, log *zap.SugaredLogger, r retryParams, agentHostnames ...string) (bool, string) { + if len(agentHostnames) == 0 { + log.Debugf("Not waiting for agents as the agentHostnames list is empty") + return true, "Not waiting for agents as the agentHostnames list is empty" + } + log.Infow("Waiting for agents to register with OM", "agent hosts", agentHostnames) + // environment variables are used only for tests + waitSeconds := env.ReadIntOrDefault(util.PodWaitSecondsEnv, r.waitSeconds) // nolint:forbidigo + retrials := env.ReadIntOrDefault(util.PodWaitRetriesEnv, r.retrials) // nolint:forbidigo + + agentsCheckFunc := func() (string, bool) { + return agentCheck(omConnection, agentHostnames, log) + } + + return util.DoAndRetry(agentsCheckFunc, log, retrials, waitSeconds) +} diff --git a/controllers/operator/agents/agents_test.go b/controllers/operator/agents/agents_test.go new file mode 100644 index 000000000..f493f594b --- /dev/null +++ b/controllers/operator/agents/agents_test.go @@ -0,0 +1,464 @@ +package agents + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" +) + +func TestCalculateProcessStateMap(t *testing.T) { + sampleTimeStamp := "2023-06-03T10:00:00Z" + testCases := []struct { + name string + processStatuses []om.ProcessStatus + agentStatuses []om.AgentStatus + expectError bool + expectedResult map[string]ProcessState + }{ + { + name: "Single valid agent and single valid process", + agentStatuses: []om.AgentStatus{ + { + Hostname: "host1", + TypeName: "AUTOMATION", + LastConf: "2023-01-02T15:04:05Z", + StateName: "SomeState", + }, + }, + processStatuses: []om.ProcessStatus{ + { + Hostname: "host1", + Name: "shard", + LastGoalVersionAchieved: 10, + Plan: []string{"step1", "step2"}, + }, + }, + expectError: false, + expectedResult: map[string]ProcessState{ + "host1": { + Hostname: "host1", + LastAgentPing: mustParseDate("2023-01-02T15:04:05Z"), + GoalVersionAchieved: 10, + ProcessName: "shard", + Plan: []string{"step1", "step2"}, + }, + }, + }, + { + name: "Multiple agents, single process (same host)", + agentStatuses: []om.AgentStatus{ + { + Hostname: "host1", + TypeName: "AUTOMATION", + LastConf: "2023-05-02T15:04:05Z", + }, + { + Hostname: "host1", // same host but repeated agent status + TypeName: "AUTOMATION", + LastConf: sampleTimeStamp, + }, + }, + processStatuses: []om.ProcessStatus{ + { + Hostname: "host1", + Name: "shard", + LastGoalVersionAchieved: 3, + Plan: []string{"planA"}, + }, + }, + expectError: false, + // LastConf from the second agent status should overwrite the LastAgentPing + expectedResult: map[string]ProcessState{ + "host1": { + Hostname: "host1", + LastAgentPing: mustParseDate(sampleTimeStamp), + GoalVersionAchieved: 3, + ProcessName: "shard", + Plan: []string{"planA"}, + }, + }, + }, + { + name: "Multiple agents, multiple processes", + agentStatuses: []om.AgentStatus{ + { + Hostname: "host1", + TypeName: "AUTOMATION", + LastConf: sampleTimeStamp, + }, + { + Hostname: "host1", + TypeName: "AUTOMATION", + LastConf: "2023-05-02T15:04:05Z", // Will overwrite above LastConf for host1 + }, + { + Hostname: "host2", + TypeName: "AUTOMATION", + LastConf: sampleTimeStamp, + }, + { + Hostname: "host3", // This host is not in process statuses + TypeName: "AUTOMATION", + LastConf: sampleTimeStamp, + }, + }, + processStatuses: []om.ProcessStatus{ + { + Hostname: "host1", + Name: "shard", + LastGoalVersionAchieved: 5, + Plan: []string{"planZ"}, + }, + { + Hostname: "host1", // These values will overwrite the ones above + Name: "mongos", + LastGoalVersionAchieved: 1, + Plan: []string{"planA, planB"}, + }, + { + Hostname: "host2", + Name: "configserver", + LastGoalVersionAchieved: 3, + Plan: []string{"planC"}, + }, + { + Hostname: "host4", // This host is not in agentStatuses + Name: "shard", + LastGoalVersionAchieved: 5, + Plan: []string{"planD"}, + }, + }, + expectError: false, + expectedResult: map[string]ProcessState{ + "host1": { + Hostname: "host1", + LastAgentPing: mustParseDate("2023-05-02T15:04:05Z"), + GoalVersionAchieved: 1, + ProcessName: "mongos", + Plan: []string{"planA, planB"}, + }, + "host2": { + Hostname: "host2", + LastAgentPing: mustParseDate(sampleTimeStamp), + GoalVersionAchieved: 3, + ProcessName: "configserver", + Plan: []string{"planC"}, + }, + "host3": { + Hostname: "host3", + LastAgentPing: mustParseDate(sampleTimeStamp), + GoalVersionAchieved: -1, + ProcessName: "", + Plan: nil, + }, + "host4": { + Hostname: "host4", + LastAgentPing: time.Time{}, + GoalVersionAchieved: 5, + ProcessName: "shard", + Plan: []string{"planD"}, + }, + }, + }, + { + name: "No overlapping values", + agentStatuses: []om.AgentStatus{ + { + Hostname: "host1", + TypeName: "AUTOMATION", + LastConf: sampleTimeStamp, + }, + { + Hostname: "host2", + TypeName: "AUTOMATION", + LastConf: "2023-05-02T15:04:05Z", + }, + }, + processStatuses: []om.ProcessStatus{ + { + Hostname: "host3", + Name: "configserver", + LastGoalVersionAchieved: 1, + Plan: []string{"planA"}, + }, + { + Hostname: "host4", + Name: "shard", + LastGoalVersionAchieved: 2, + Plan: []string{"planB"}, + }, + }, + expectError: false, + expectedResult: map[string]ProcessState{ + "host1": { + Hostname: "host1", + LastAgentPing: mustParseDate(sampleTimeStamp), + GoalVersionAchieved: -1, + ProcessName: "", + Plan: nil, + }, + "host2": { + Hostname: "host2", + LastAgentPing: mustParseDate("2023-05-02T15:04:05Z"), + GoalVersionAchieved: -1, + ProcessName: "", + Plan: nil, + }, + "host3": { + Hostname: "host3", + LastAgentPing: time.Time{}, + GoalVersionAchieved: 1, + ProcessName: "configserver", + Plan: []string{"planA"}, + }, + "host4": { + Hostname: "host4", + LastAgentPing: time.Time{}, + GoalVersionAchieved: 2, + ProcessName: "shard", + Plan: []string{"planB"}, + }, + }, + }, + { + name: "No agents, only processes", + agentStatuses: nil, + processStatuses: []om.ProcessStatus{ + { + Hostname: "host2", + Name: "mongos", + LastGoalVersionAchieved: 7, + Plan: []string{"stepX", "stepY"}, + }, + { + Hostname: "host3", + Name: "config", + LastGoalVersionAchieved: 1, + Plan: []string{"planC"}, + }, + }, + expectError: false, + expectedResult: map[string]ProcessState{ + "host2": { + Hostname: "host2", + LastAgentPing: time.Time{}, + GoalVersionAchieved: 7, + ProcessName: "mongos", + Plan: []string{"stepX", "stepY"}, + }, + "host3": { + Hostname: "host3", + LastAgentPing: time.Time{}, + GoalVersionAchieved: 1, + ProcessName: "config", + Plan: []string{"planC"}, + }, + }, + }, + { + name: "No processes, only agents", + processStatuses: nil, + agentStatuses: []om.AgentStatus{ + { + Hostname: "host4", + TypeName: "AUTOMATION", + LastConf: sampleTimeStamp, + }, + }, + expectError: false, + expectedResult: map[string]ProcessState{ + "host4": { + Hostname: "host4", + LastAgentPing: mustParseDate(sampleTimeStamp), + GoalVersionAchieved: -1, + Plan: nil, + ProcessName: "", + }, + }, + }, + { + name: "Agent with invalid TypeName", + agentStatuses: []om.AgentStatus{ + { + Hostname: "host5", + TypeName: "NOT_AUTOMATION", + LastConf: sampleTimeStamp, + }, + }, + processStatuses: nil, + expectError: true, + expectedResult: nil, + }, + { + name: "Agent with invalid LastConf format", + agentStatuses: []om.AgentStatus{ + { + Hostname: "host6", + TypeName: "AUTOMATION", + // Missing 'Z' or doesn't follow RFC3339 + LastConf: "2023-01-02 15:04:05", + }, + }, + processStatuses: nil, + expectError: true, + expectedResult: nil, + }, + { + name: "Empty slices", + agentStatuses: []om.AgentStatus{}, + processStatuses: []om.ProcessStatus{}, + expectError: false, + expectedResult: map[string]ProcessState{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := calculateProcessStateMap(tc.processStatuses, tc.agentStatuses) + if tc.expectError { + require.Error(t, err, "Expected an error but got none") + } else { + require.NoError(t, err, "Did not expect an error but got one") + require.Equal(t, tc.expectedResult, result) + } + }) + } +} + +// mustParseDate is a helper that must successfully parse an RFC3339 date. +func mustParseDate(value string) time.Time { + t, err := time.Parse(time.RFC3339, value) + if err != nil { + panic(err) + } + return t +} + +func TestGetClusterState(t *testing.T) { + testCases := []struct { + name string + // Mock ReadAutomationStatus func + mockAutomationStatus *om.AutomationStatus + mockAutomationStatusErr error + // Mock ReadAutomationAgents func + mockAgentStatusResponse om.Paginated + mockAgentStatusErr error + + expectErr bool + expectedGoalVersion int + expectedProcessStateMap map[string]ProcessState + }{ + { + name: "Happy path with one agent and one process", + mockAutomationStatus: &om.AutomationStatus{ + GoalVersion: 42, + Processes: []om.ProcessStatus{ + { + Hostname: "host1", + Name: "shard", + LastGoalVersionAchieved: 7, + Plan: []string{"step1", "step2"}, + }, + }, + }, + mockAutomationStatusErr: nil, + mockAgentStatusResponse: om.AutomationAgentStatusResponse{ + OMPaginated: om.OMPaginated{TotalCount: 1}, + AutomationAgents: []om.AgentStatus{ + { + Hostname: "host1", + TypeName: "AUTOMATION", + LastConf: "2024-01-01T00:00:00Z", + }, + }, + }, + mockAgentStatusErr: nil, + expectErr: false, + expectedGoalVersion: 42, + expectedProcessStateMap: map[string]ProcessState{ + "host1": { + Hostname: "host1", + ProcessName: "shard", + LastAgentPing: mustParseDate("2024-01-01T00:00:00Z"), + GoalVersionAchieved: 7, + Plan: []string{"step1", "step2"}, + }, + }, + }, + { + name: "Error when reading automation status", + mockAutomationStatus: nil, + mockAutomationStatusErr: errors.New("cannot read automation status"), + mockAgentStatusResponse: om.AutomationAgentStatusResponse{}, + mockAgentStatusErr: nil, + expectErr: true, + }, + { + name: "Error when reading agent pages", + mockAutomationStatus: &om.AutomationStatus{ + GoalVersion: 1, + Processes: nil, + }, + mockAutomationStatusErr: nil, + mockAgentStatusResponse: nil, // Not useful here + mockAgentStatusErr: errors.New("agent pages error"), + expectErr: true, + }, + { + name: "Invalid agent type triggers error in calculateProcessStateMap", + mockAutomationStatus: &om.AutomationStatus{ + GoalVersion: 10, + Processes: []om.ProcessStatus{ + { + Hostname: "host2", + Name: "shard", + LastGoalVersionAchieved: 2, + Plan: []string{"planA"}, + }, + }, + }, + mockAutomationStatusErr: nil, + mockAgentStatusResponse: om.AutomationAgentStatusResponse{ + OMPaginated: om.OMPaginated{TotalCount: 1}, + AutomationAgents: []om.AgentStatus{ + { + Hostname: "host2", + TypeName: "NOT_AUTOMATION", + LastConf: "2023-06-03T10:00:00Z", + }, + }, + }, + mockAgentStatusErr: nil, + expectErr: true, + }, + } + + // For each iteration, we create a mocked OM connection and override the default behaviour of status methods + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mockConn := &om.MockedOmConnection{} + + // Override the behavior of OM connection methods + mockConn.ReadAutomationStatusFunc = func() (*om.AutomationStatus, error) { + return tc.mockAutomationStatus, tc.mockAutomationStatusErr + } + mockConn.ReadAutomationAgentsFunc = func(_ int) (om.Paginated, error) { + return tc.mockAgentStatusResponse, tc.mockAgentStatusErr + } + + clusterState, err := GetMongoDBClusterState(mockConn) + if tc.expectErr { + require.Error(t, err, "Expected an error but got none") + return + } + require.NoError(t, err, "Did not expect an error but got one") + + require.Equal(t, tc.expectedGoalVersion, clusterState.GoalVersion) + require.Equal(t, tc.expectedProcessStateMap, clusterState.ProcessStateMap) + }) + } +} diff --git a/controllers/operator/agents/upgrade.go b/controllers/operator/agents/upgrade.go new file mode 100644 index 000000000..bcc69d2d3 --- /dev/null +++ b/controllers/operator/agents/upgrade.go @@ -0,0 +1,187 @@ +package agents + +import ( + "context" + "sync" + "time" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbmultiv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" +) + +var nextScheduledTime time.Time + +const pause = time.Hour * 24 + +var mux sync.Mutex + +func init() { + ScheduleUpgrade() +} + +// ClientSecret is a wrapper that joins a client and a secretClient. +type ClientSecret struct { + Client kubernetesClient.Client + SecretClient secrets.SecretClient +} + +// UpgradeAllIfNeeded performs the upgrade of agents for all the MongoDB resources registered in the system if necessary +// It's designed to be run "in background" - so must not break any existing reconciliations it's triggered from and +// so doesn't return errors +// Concurrency behavior: the mutex is used to: +// 1. ensure no separate routines invoke the upgrade in parallel +// 2. different reconciliations started in parallel (e.g. Operator has restarted) wait for the upgrade procedure to happen +// for all existing MongoDB resources before proceeding. This could be a critical thing when the major version OM upgrade +// happens and all existing MongoDBs are required to get agents upgraded (otherwise the "You need to upgrade the +// automation agent before publishing other changes" error happens for automation config pushes from the Operator) +func UpgradeAllIfNeeded(ctx context.Context, cs ClientSecret, omConnectionFactory om.ConnectionFactory, watchNamespace []string, isMulti bool) { + mux.Lock() + defer mux.Unlock() + + if !time.Now().After(nextScheduledTime) { + return + } + log := zap.S() + log.Info("Performing a regular upgrade of Agents for all the MongoDB resources in the cluster...") + + allMDBs, err := readAllMongoDBs(ctx, cs.Client, watchNamespace, isMulti) + if err != nil { + log.Errorf("Failed to read MongoDB resources to ensure Agents have the latest version: %s", err) + return + } + + err = doUpgrade(ctx, cs.Client, cs.SecretClient, omConnectionFactory, allMDBs) + if err != nil { + log.Errorf("Failed to perform upgrade of Agents: %s", err) + } + + log.Info("The upgrade of Agents for all the MongoDB resources in the cluster is finished.") + + nextScheduledTime = nextScheduledTime.Add(pause) +} + +// ScheduleUpgrade allows to reset the timer to Now() which makes sure the next MongoDB reconciliation will ensure +// all the watched agents are up-to-date. +// This is needed for major/minor OM upgrades as all dependent MongoDBs won't get reconciled with "You need to upgrade the +// automation agent before publishing other changes" +func ScheduleUpgrade() { + nextScheduledTime = time.Now() +} + +// NextScheduledUpgradeTime returns the next scheduled time. Mostly needed for testing. +func NextScheduledUpgradeTime() time.Time { + return nextScheduledTime +} + +type dbCommonWithNamespace struct { + objectKey types.NamespacedName + mdbv1.DbCommonSpec +} + +func doUpgrade(ctx context.Context, cmGetter configmap.Getter, secretGetter secrets.SecretClient, factory om.ConnectionFactory, mdbs []dbCommonWithNamespace) error { + for _, mdb := range mdbs { + log := zap.S().With(string(mdb.ResourceType), mdb.objectKey) + conn, err := connectToMongoDB(ctx, cmGetter, secretGetter, factory, mdb, log) + if err != nil { + log.Warnf("Failed to establish connection to Ops Manager to perform Agent upgrade: %s", err) + continue + } + + currentVersion := "" + if deployment, err := conn.ReadDeployment(); err == nil { + currentVersion = deployment.GetAgentVersion() + } + version, err := conn.UpgradeAgentsToLatest() + if err != nil { + log.Warnf("Failed to schedule Agent upgrade: %s, this could be due do ongoing Automation Config publishing in Ops Manager and will get fixed during next trial", err) + continue + } + if currentVersion != version && currentVersion != "" { + log.Debugf("Submitted the request to Ops Manager to upgrade the agents from %s to the latest version (%s)", currentVersion, version) + } + } + return nil +} + +// readAllMongoDBs returns a list of all the MongoDB resources found in the +// `watchNamespace` list. +// +// If the `watchNamespace` contains only the "" string, the MongoDB resources +// will be searched in every Namespace of the cluster. +func readAllMongoDBs(ctx context.Context, cl kubernetesClient.Client, watchNamespace []string, isMulti bool) ([]dbCommonWithNamespace, error) { + var namespaces []string + + // 1. Find which Namespaces to look for MongoDB resources + if len(watchNamespace) == 1 && watchNamespace[0] == "" { + namespaceList := corev1.NamespaceList{} + if err := cl.List(ctx, &namespaceList); err != nil { + return []dbCommonWithNamespace{}, err + } + for _, item := range namespaceList.Items { + namespaces = append(namespaces, item.Name) + } + } else { + namespaces = watchNamespace + } + + var mdbs []dbCommonWithNamespace + // 2. Find all MongoDBs in the namespaces + for _, ns := range namespaces { + if isMulti { + mongodbList := mdbmultiv1.MongoDBMultiClusterList{} + if err := cl.List(ctx, &mongodbList, client.InNamespace(ns)); err != nil { + return []dbCommonWithNamespace{}, err + } + for _, item := range mongodbList.Items { + mdbs = append(mdbs, dbCommonWithNamespace{ + objectKey: item.ObjectKey(), + DbCommonSpec: item.Spec.DbCommonSpec, + }) + } + + } else { + mongodbList := mdbv1.MongoDBList{} + if err := cl.List(ctx, &mongodbList, client.InNamespace(ns)); err != nil { + return []dbCommonWithNamespace{}, err + } + for _, item := range mongodbList.Items { + mdbs = append(mdbs, dbCommonWithNamespace{ + objectKey: item.ObjectKey(), + DbCommonSpec: item.Spec.DbCommonSpec, + }) + } + } + } + return mdbs, nil +} + +func connectToMongoDB(ctx context.Context, cmGetter configmap.Getter, secretGetter secrets.SecretClient, factory om.ConnectionFactory, mdb dbCommonWithNamespace, log *zap.SugaredLogger) (om.Connection, error) { + projectConfig, err := project.ReadProjectConfig(ctx, cmGetter, kube.ObjectKey(mdb.objectKey.Namespace, mdb.GetProject()), mdb.objectKey.Name) + if err != nil { + return nil, xerrors.Errorf("error reading Project Config: %w", err) + } + credsConfig, err := project.ReadCredentials(ctx, secretGetter, kube.ObjectKey(mdb.objectKey.Namespace, mdb.Credentials), log) + if err != nil { + return nil, xerrors.Errorf("error reading Credentials secret: %w", err) + } + + _, conn, err := project.ReadOrCreateProject(projectConfig, credsConfig, factory, log) + if err != nil { + return nil, xerrors.Errorf("error reading or creating project in Ops Manager: %w", err) + } + return conn, nil +} diff --git a/controllers/operator/appdbreplicaset_controller.go b/controllers/operator/appdbreplicaset_controller.go new file mode 100644 index 000000000..ca447d276 --- /dev/null +++ b/controllers/operator/appdbreplicaset_controller.go @@ -0,0 +1,2128 @@ +package operator + +import ( + "context" + "fmt" + "path" + "sort" + "strconv" + "strings" + + "github.com/hashicorp/go-multierror" + "github.com/stretchr/objx" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/agent" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/result" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + mdbcv1_controllers "github.com/mongodb/mongodb-kubernetes-operator/controllers" + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers/interfaces" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + mekoService "github.com/10gen/ops-manager-kubernetes/pkg/kube/service" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/placeholders" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/timeutil" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +type agentType string + +const ( + appdbCAFilePath = "/var/lib/mongodb-automation/secrets/ca/ca-pem" + appDBACConfigMapVersionField = "version" + + monitoring agentType = "MONITORING" + automation agentType = "AUTOMATION" + + // Used to note that for this particular case it is not necessary to pass + // the hash of the Prometheus certificate. This is to avoid having to + // calculate and pass the Prometheus Cert Hash when it is not needed. + UnusedPrometheusConfiguration string = "" + + // Used to convey to the operator to force reconfigure agent. At the moment + // it is used for DR in case of Multi-Cluster AppDB when after a cluster outage + // there is no primary in the AppDB deployment. + ForceReconfigureAnnotation = "mongodb.com/v1.forceReconfigure" + + ForcedReconfigureAlreadyPerformedAnnotation = "mongodb.com/v1.forceReconfigurePerformed" +) + +type CommonDeploymentState struct { + ClusterMapping map[string]int `json:"clusterMapping"` +} + +type AppDBDeploymentState struct { + CommonDeploymentState `json:",inline"` + LastAppliedMemberSpec map[string]int `json:"lastAppliedMemberSpec"` + LastAppliedMongoDBVersion string `json:"lastAppliedMongoDBVersion"` +} + +func NewAppDBDeploymentState() *AppDBDeploymentState { + return &AppDBDeploymentState{ + CommonDeploymentState: CommonDeploymentState{ClusterMapping: map[string]int{}}, + LastAppliedMemberSpec: map[string]int{}, + } +} + +// ReconcileAppDbReplicaSet reconciles a MongoDB with a type of ReplicaSet +type ReconcileAppDbReplicaSet struct { + *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + + centralClient kubernetesClient.Client + // ordered list of member clusters; order in this list is preserved across runs using memberClusterIndex + memberClusters []multicluster.MemberCluster + stateStore *StateStore[AppDBDeploymentState] + deploymentState *AppDBDeploymentState + + imageUrls images.ImageUrls + initAppdbVersion string +} + +func NewAppDBReplicaSetReconciler(ctx context.Context, imageUrls images.ImageUrls, initAppdbVersion string, appDBSpec omv1.AppDBSpec, commonController *ReconcileCommonController, omConnectionFactory om.ConnectionFactory, omAnnotations map[string]string, globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger) (*ReconcileAppDbReplicaSet, error) { + reconciler := &ReconcileAppDbReplicaSet{ + ReconcileCommonController: commonController, + omConnectionFactory: omConnectionFactory, + centralClient: commonController.client, + imageUrls: imageUrls, + initAppdbVersion: initAppdbVersion, + } + + if err := reconciler.initializeStateStore(ctx, appDBSpec, omAnnotations, log); err != nil { + return nil, xerrors.Errorf("failed to initialize appdb state store: %w", err) + } + + if err := reconciler.initializeMemberClusters(ctx, appDBSpec, globalMemberClustersMap, log); err != nil { + return nil, xerrors.Errorf("failed to initialize appdb replicaset controller: %w", err) + } + + return reconciler, nil +} + +// initializeStateStore initializes the deploymentState field by reading it from a state config map. +// In case there is no state config map, the new state map is created and saved after performing migration of the existing state data (see migrateToNewDeploymentState). +func (r *ReconcileAppDbReplicaSet) initializeStateStore(ctx context.Context, appDBSpec omv1.AppDBSpec, omAnnotations map[string]string, log *zap.SugaredLogger) error { + r.deploymentState = NewAppDBDeploymentState() + + r.stateStore = NewStateStore[AppDBDeploymentState](appDBSpec.GetNamespace(), appDBSpec.Name(), r.centralClient) + if state, err := r.stateStore.ReadState(ctx); err != nil { + if apiErrors.IsNotFound(err) { + // If the deployment state config map is missing, then it might be either: + // - fresh deployment + // - existing deployment, but it's a first reconcile on the operator version with the new deployment state + // - existing deployment, but for some reason the deployment state config map has been deleted + // In all cases, the deployment config map will be recreated from the state we're keeping and maintaining in + // the old place (in annotations, spec.status, config maps) in order to allow for the downgrade of the operator. + if err := r.migrateToNewDeploymentState(ctx, appDBSpec, omAnnotations); err != nil { + return err + } + // This will migrate the deployment state to the new structure and this branch of code won't be executed again. + // Here we don't use saveAppDBState wrapper, as we don't need to write the legacy state + if err := r.stateStore.WriteState(ctx, r.deploymentState, log); err != nil { + return err + } + } else { + return err + } + } else { + r.deploymentState = state + } + + return nil +} + +// initializeMemberClusters main goal is to initialise memberClusterList field with the ordered list of member clusters to iterate over. +// +// When in single-cluster topology it initializes memberClusterList with a dummy "central" cluster +// containing the number of members from appDBSpec.Members field. T +// Thanks to that all code in reconcile loop is always looping over member cluster. + +// For multi-cluster topology, this function maintains (updates or creates if doesn't exist yet) -cluster-mapping config map, to preserve +// mapping between clusterName from clusterSpecList and the assigned cluster index. +// For example, when user declares in CR: +// +// clusterSpecList: +// - clusterName: cluster-1 +// members: 1 +// - clusterName: cluster-2 +// members: 2 +// - clusterName: cluster-3 +// members: 3 +// +// The function will assign the following indexes when first deploying resources: +// - cluster-1, idx=0, members=1 (no index in map, get first next available index) +// - cluster-2: idx=1, members=2 (same as above) +// - cluster-3: idx=2, members=3 (same as above) +// +// Those indexes are crucial to maintain resources in member cluster and have to be preserved for the given cluster name. Cluster indexes are contained in +// statefulset names, process names, etc. +// +// If in the subsequent reconciliations the clusterSpecList is changed, this function guarantees that no matter what, assigned first cluster index will +// allways be preserved. +// For example, the user reorders clusterSpecList, removes cluster-1 and cluster-3 and adds two other cluster in random places: +// +// clusterSpecList: +// - clusterName: cluster-10 +// members: 10 +// - clusterName: cluster-2 +// members: 2 +// - clusterName: cluster-5 +// members: 5 +// +// initializeMemberClusters will then read existing cluster mapping from config map and create list of member clusters in the following order: +// - cluster-2, idx=1 as it was saved to map before +// - cluster-10, idx=3, assigns a new index that is the next available index (0,1,2 are taken) +// - cluster-5, idx=4, assigns a new index that is the next available index (0,1,2,3 are taken) +// +// On top of that, for all removed member clusters, if they previously contained more than one member (haven't been scaled to zero), +// the function will add them back with preserved indexes and saved previously member counts: +// In the end the function will contain the following list of member clusters to iterate on: +// - cluster-1, idx=0, members=1 (removed cluster, idx and previous members from map) +// - cluster-2, idx=1, members=2 (idx from map, members from clusterSpecList) +// - cluster-3, idx=2, members=3 (removed cluster, idx and previous members from map) +// - cluster-10, idx=3, members=10 (assigns a new index that is the next available index (0,1,2 are taken)) +// - cluster-5, idx=4, members=5 (assigns a new index that is the next available index (0,1,2,3 are taken)) +func (r *ReconcileAppDbReplicaSet) initializeMemberClusters(ctx context.Context, appDBSpec omv1.AppDBSpec, globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger) error { + if appDBSpec.IsMultiCluster() { + if len(globalMemberClustersMap) == 0 { + return xerrors.Errorf("member clusters have to be initialized for MultiCluster AppDB topology") + } + // here we access ClusterSpecList directly, as we have to check what's been defined in yaml + if len(appDBSpec.ClusterSpecList) == 0 { + return xerrors.Errorf("for appDBSpec.Topology = MultiCluster, clusterSpecList have to be non empty") + } + + r.updateMemberClusterMapping(appDBSpec) + + getLastAppliedMemberCountFunc := func(memberClusterName string) int { + return r.getLastAppliedMemberCount(appDBSpec, memberClusterName) + } + + clusterSpecList := appDBSpec.GetClusterSpecList() + r.memberClusters = createMemberClusterListFromClusterSpecList(clusterSpecList, globalMemberClustersMap, log, r.deploymentState.ClusterMapping, getLastAppliedMemberCountFunc, false) + + if err := r.saveAppDBState(ctx, appDBSpec, log); err != nil { + return err + } + } else { + // for SingleCluster member cluster list will contain one member which will be the central (default) cluster + r.memberClusters = []multicluster.MemberCluster{multicluster.GetLegacyCentralMemberCluster(appDBSpec.Members, 0, r.centralClient, r.SecretClient)} + } + + log.Debugf("Initialized member cluster list: %+v", util.Transform(r.memberClusters, func(m multicluster.MemberCluster) string { + return fmt.Sprintf("{Name: %s, Index: %d, Replicas: %d, Active: %t, Healthy: %t}", m.Name, m.Index, m.Replicas, m.Active, m.Healthy) + })) + + return nil +} + +// saveAppDBState is a wrapper method around WriteState, to ensure we keep updating the legacy Config Maps for downgrade +// compatibility +// This will write the legacy state to the cluster even for NEW deployments, created after upgrade of the operator. +// It is not incorrect and doesn't interfere with the logic, but it *could* be confusing for a user +// (this is also the case for OM controller) +func (r *ReconcileAppDbReplicaSet) saveAppDBState(ctx context.Context, spec omv1.AppDBSpec, log *zap.SugaredLogger) error { + if err := r.stateStore.WriteState(ctx, r.deploymentState, log); err != nil { + return err + } + if err := r.writeLegacyStateConfigMaps(ctx, spec, log); err != nil { + return err + } + return nil +} + +// writeLegacyStateConfigMaps converts the DeploymentState to legacy Config Maps and write them to the cluster +// LastAppliedMongoDBVersion is also part of the state, it is handled separately in the controller as it was an annotation +func (r *ReconcileAppDbReplicaSet) writeLegacyStateConfigMaps(ctx context.Context, spec omv1.AppDBSpec, log *zap.SugaredLogger) error { + // ClusterMapping ConfigMap + mappingConfigMapData := map[string]string{} + for k, v := range r.deploymentState.ClusterMapping { + mappingConfigMapData[k] = fmt.Sprintf("%d", v) + } + mappingConfigMap := configmap.Builder().SetName(spec.ClusterMappingConfigMapName()).SetNamespace(spec.Namespace).SetData(mappingConfigMapData).Build() + if err := configmap.CreateOrUpdate(ctx, r.centralClient, mappingConfigMap); err != nil { + return xerrors.Errorf("failed to update cluster mapping configmap %s: %w", spec.ClusterMappingConfigMapName(), err) + } + log.Debugf("Saving cluster mapping configmap %s: %v", spec.ClusterMappingConfigMapName(), mappingConfigMapData) + + // LastAppliedMemberSpec ConfigMap + specConfigMapData := map[string]string{} + for k, v := range r.deploymentState.LastAppliedMemberSpec { + specConfigMapData[k] = fmt.Sprintf("%d", v) + } + specConfigMap := configmap.Builder().SetName(spec.LastAppliedMemberSpecConfigMapName()).SetNamespace(spec.Namespace).SetData(specConfigMapData).Build() + if err := configmap.CreateOrUpdate(ctx, r.centralClient, specConfigMap); err != nil { + return xerrors.Errorf("failed to update last applied member spec configmap %s: %w", spec.LastAppliedMemberSpecConfigMapName(), err) + } + log.Debugf("Saving last applied member spec configmap %s: %v", spec.LastAppliedMemberSpecConfigMapName(), specConfigMapData) + + return nil +} + +func createMemberClusterListFromClusterSpecList(clusterSpecList mdbv1.ClusterSpecList, globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger, memberClusterMapping map[string]int, getLastAppliedMemberCountFunc func(memberClusterName string) int, legacyMemberCluster bool) []multicluster.MemberCluster { + var memberClusters []multicluster.MemberCluster + specClusterMap := map[string]struct{}{} + for _, clusterSpecItem := range clusterSpecList { + specClusterMap[clusterSpecItem.ClusterName] = struct{}{} + + var memberClusterKubeClient kubernetesClient.Client + var memberClusterSecretClient secrets.SecretClient + memberClusterClient, ok := globalMemberClustersMap[clusterSpecItem.ClusterName] + if !ok { + var clusterList []string + for m := range globalMemberClustersMap { + clusterList = append(clusterList, m) + } + log.Warnf("Member cluster %s specified in clusterSpecList is not found in the list of operator's member clusters: %+v. "+ + "Assuming the cluster is down. It will be ignored from reconciliation but its MongoDB processes will still be maintained in replicaset configuration.", clusterSpecItem.ClusterName, clusterList) + } else { + memberClusterKubeClient = kubernetesClient.NewClient(memberClusterClient) + memberClusterSecretClient = secrets.SecretClient{ + VaultClient: nil, // Vault is not supported yet on multi cluster + KubeClient: memberClusterKubeClient, + } + } + + memberClusters = append(memberClusters, multicluster.MemberCluster{ + Name: clusterSpecItem.ClusterName, + Index: memberClusterMapping[clusterSpecItem.ClusterName], + Client: memberClusterKubeClient, + SecretClient: memberClusterSecretClient, + Replicas: getLastAppliedMemberCountFunc(clusterSpecItem.ClusterName), + Active: true, + Healthy: memberClusterKubeClient != nil, + Legacy: legacyMemberCluster, + }) + } + + // add previous member clusters with last applied members. This is required for being able to scale down the appdb members one by one. + for previousMember := range memberClusterMapping { + // If the previous member is already present in the spec, skip it safely + if _, ok := specClusterMap[previousMember]; ok { + continue + } + + previousMemberReplicas := getLastAppliedMemberCountFunc(previousMember) + // If the previous member was already scaled down to 0 members, skip it safely + if previousMemberReplicas == 0 { + continue + } + + var memberClusterKubeClient kubernetesClient.Client + var memberClusterSecretClient secrets.SecretClient + memberClusterClient, ok := globalMemberClustersMap[previousMember] + if !ok { + var clusterList []string + for m := range globalMemberClustersMap { + clusterList = append(clusterList, m) + } + log.Warnf("Member cluster %s that has to be scaled to 0 replicas is not found in the list of operator's member clusters: %+v. "+ + "Assuming the cluster is down. It will be ignored from reconciliation but it's MongoDB processes will be scaled down to 0 in replicaset configuration.", previousMember, clusterList) + } else { + memberClusterKubeClient = kubernetesClient.NewClient(memberClusterClient) + memberClusterSecretClient = secrets.SecretClient{ + VaultClient: nil, // Vault is not supported yet on multi cluster + KubeClient: memberClusterKubeClient, + } + } + + memberClusters = append(memberClusters, multicluster.MemberCluster{ + Name: previousMember, + Index: memberClusterMapping[previousMember], + Client: memberClusterKubeClient, + SecretClient: memberClusterSecretClient, + Replicas: previousMemberReplicas, + Active: false, + Healthy: memberClusterKubeClient != nil, + }) + } + sort.Slice(memberClusters, func(i, j int) bool { + return memberClusters[i].Index < memberClusters[j].Index + }) + + return memberClusters +} + +func (r *ReconcileAppDbReplicaSet) getLastAppliedMemberCount(spec omv1.AppDBSpec, clusterName string) int { + if !spec.IsMultiCluster() { + panic(fmt.Errorf("the function cannot be used in SingleCluster topology)")) + } + specMapping := r.getLastAppliedMemberSpec(spec) + return specMapping[clusterName] +} + +func (r *ReconcileAppDbReplicaSet) getLastAppliedMemberSpec(spec omv1.AppDBSpec) map[string]int { + if !spec.IsMultiCluster() { + return nil + } + return r.deploymentState.LastAppliedMemberSpec +} + +func (r *ReconcileAppDbReplicaSet) getLegacyLastAppliedMemberSpec(ctx context.Context, spec omv1.AppDBSpec) (map[string]int, error) { + // read existing spec + existingSpec := map[string]int{} + existingConfigMap := corev1.ConfigMap{} + err := r.centralClient.Get(ctx, kube.ObjectKey(spec.Namespace, spec.LastAppliedMemberSpecConfigMapName()), &existingConfigMap) + if err != nil { + return nil, xerrors.Errorf("failed to read last applied member spec config map %s: %w", spec.LastAppliedMemberSpecConfigMapName(), err) + } else { + for clusterName, replicasStr := range existingConfigMap.Data { + replicas, err := strconv.Atoi(replicasStr) + if err != nil { + return nil, xerrors.Errorf("failed to read last applied member spec from config map %s (%+v): %w", spec.LastAppliedMemberSpecConfigMapName(), existingConfigMap.Data, err) + } + existingSpec[clusterName] = replicas + } + } + + return existingSpec, nil +} + +// getLegacyMemberClusterMapping is reading the cluster mapping from the old config map where it has been stored before introducing the deployment state config map. +func getLegacyMemberClusterMapping(ctx context.Context, namespace string, configMapName string, centralClient kubernetesClient.Client) (map[string]int, error) { + // read existing config map + existingMapping := map[string]int{} + existingConfigMap, err := centralClient.GetConfigMap(ctx, types.NamespacedName{Name: configMapName, Namespace: namespace}) + if err != nil { + return nil, xerrors.Errorf("failed to read cluster mapping config map %s: %w", configMapName, err) + } else { + for clusterName, indexStr := range existingConfigMap.Data { + index, err := strconv.Atoi(indexStr) + if err != nil { + return nil, xerrors.Errorf("failed to read cluster mapping indexes from config map %s (%+v): %w", configMapName, existingConfigMap.Data, err) + } + existingMapping[clusterName] = index + } + } + + return existingMapping, nil +} + +// updateMemberClusterMapping returns a map of member cluster name -> cluster index. +// Mapping is preserved in spec.ClusterMappingConfigMapName() config map. Config map is created if not exists. +// Subsequent executions will merge, update and store mappings from config map and from clusterSpecList and save back to config map. +func (r *ReconcileAppDbReplicaSet) updateMemberClusterMapping(spec omv1.AppDBSpec) { + if !spec.IsMultiCluster() { + return + } + + r.deploymentState.ClusterMapping = multicluster.AssignIndexesForMemberClusterNames(r.deploymentState.ClusterMapping, util.Transform(spec.GetClusterSpecList(), func(clusterSpecItem mdbv1.ClusterSpecItem) string { + return clusterSpecItem.ClusterName + })) +} + +// shouldReconcileAppDB returns a boolean indicating whether or not the reconciliation for this set of processes should occur. +func (r *ReconcileAppDbReplicaSet) shouldReconcileAppDB(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (bool, error) { + memberCluster := r.getMemberCluster(r.getNameOfFirstMemberCluster()) + currentAc, err := automationconfig.ReadFromSecret(ctx, memberCluster.Client, types.NamespacedName{ + Namespace: opsManager.GetNamespace(), + Name: opsManager.Spec.AppDB.AutomationConfigSecretName(), + }) + if err != nil { + return false, xerrors.Errorf("error reading AppDB Automation Config: %w", err) + } + + // there is no automation config yet,0 we can safely reconcile. + if currentAc.Processes == nil { + return true, nil + } + + desiredAc, err := r.buildAppDbAutomationConfig(ctx, opsManager, automation, UnusedPrometheusConfiguration, memberCluster.Name, log) + if err != nil { + return false, xerrors.Errorf("error building AppDB Automation Config: %w", err) + } + + currentProcessesAreDisabled := false + for _, p := range currentAc.Processes { + if p.Disabled { + currentProcessesAreDisabled = true + break + } + } + + desiredProcessesAreDisabled := false + for _, p := range desiredAc.Processes { + if p.Disabled { + desiredProcessesAreDisabled = true + break + } + } + + // skip the reconciliation as there are disabled processes, and we are not attempting to re-enable them. + if currentProcessesAreDisabled && desiredProcessesAreDisabled { + return false, nil + } + + return true, nil +} + +// ReconcileAppDB deploys the "headless" agent, and wait until it reaches the goal state +func (r *ReconcileAppDbReplicaSet) ReconcileAppDB(ctx context.Context, opsManager *omv1.MongoDBOpsManager) (res reconcile.Result, e error) { + rs := opsManager.Spec.AppDB + log := zap.S().With("ReplicaSet (AppDB)", kube.ObjectKey(opsManager.Namespace, rs.Name())) + + appDbStatusOption := status.NewOMPartOption(status.AppDb) + omStatusOption := status.NewOMPartOption(status.OpsManager) + + log.Info("AppDB ReplicaSet.Reconcile") + log.Infow("ReplicaSet.Spec", "spec", rs) + log.Infow("ReplicaSet.Status", "status", opsManager.Status.AppDbStatus) + + opsManagerUserPassword, err := r.ensureAppDbPassword(ctx, opsManager, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error ensuring Ops Manager user password: %w", err)), log, appDbStatusOption) + } + + // if any of the processes have been marked as disabled, we don't reconcile the AppDB. + // This could be the case if we want to disable a process to perform a manual backup of the AppDB. + shouldReconcile, err := r.shouldReconcileAppDB(ctx, opsManager, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error determining AppDB reconciliation state: %w", err)), log, appDbStatusOption) + } + if !shouldReconcile { + log.Info("Skipping reconciliation for AppDB because at least one of the processes has been disabled. To reconcile the AppDB all process need to be enabled in automation config") + return result.OK() + } + + podVars, err := r.tryConfigureMonitoringInOpsManager(ctx, opsManager, opsManagerUserPassword, log) + // it's possible that Ops Manager will not be available when we attempt to configure AppDB monitoring + // in Ops Manager. This is not a blocker to continue with the rest of the reconciliation. + if err != nil { + log.Errorf("Unable to configure monitoring of AppDB: %s, configuration will be attempted next reconciliation.", err) + + if podVars.ProjectID != "" { + // when there is an error, but projectID is configured, then that means OM has been configured before but might be down + // in that case, we need to ensure that all member clusters have all the secrets to be mounted properly + // newly added member clusters will not contain them otherwise until OM is recreated and running + if err := r.ensureProjectIDConfigMap(ctx, opsManager, podVars.ProjectID); err != nil { + // we ignore the error here and let reconciler continue + log.Warnf("ignoring ensureProjectIDConfigMap error: %v", err) + } + // OM connection is passed as nil as it's used only for generating agent api key. Here we have it already + if _, err := r.ensureAppDbAgentApiKey(ctx, opsManager, nil, podVars.ProjectID, log); err != nil { + // we ignore the error here and let reconciler continue + log.Warnf("ignoring ensureAppDbAgentApiKey error: %v", err) + } + } + + // errors returned from "tryConfigureMonitoringInOpsManager" could be either transient or persistent. Transient errors could be when the ops-manager pods + // are not ready and trying to connect to the ops-manager service timeout, a persistent error is when the "ops-manager-admin-key" is corrupted, in this case + // any API call to ops-manager will fail(including the configuration of AppDB monitoring), this error should be reflected to the user in the "OPSMANAGER" status. + if strings.Contains(err.Error(), "401 (Unauthorized)") { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("The admin-key secret might be corrupted: %w", err)), log, omStatusOption) + } + } + + appdbOpts := construct.AppDBStatefulSetOptions{ + InitAppDBImage: images.ContainerImage(r.imageUrls, util.InitAppdbImageUrlEnv, r.initAppdbVersion), + MongodbImage: images.GetOfficialImage(r.imageUrls, opsManager.Spec.AppDB.Version, opsManager.GetAnnotations()), + } + if architectures.IsRunningStaticArchitecture(opsManager.Annotations) { + if !rs.PodSpec.IsAgentImageOverridden() { + // Because OM is not available when starting AppDB, we read the version from the mapping + // We plan to change this in the future, but for the sake of simplicity we leave it that way for the moment + // It avoids unnecessary reconciles, race conditions... + agentVersion, err := r.getAgentVersion(nil, opsManager.Spec.Version, true, log) + if err != nil { + log.Errorf("Impossible to get agent version, please override the agent image by providing a pod template") + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Failed to get agent version: %w. Please use spec.statefulSet to supply proper Agent version", err)), log) + } + + appdbOpts.AgentImage = images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, agentVersion) + } + } else { + // instead of using a hard-coded monitoring version, we use the "newest" one based on the release.json. + // This ensures we need to care less about CVEs compared to the prior older hardcoded versions. + legacyMonitoringAgentVersion, err := r.getAgentVersion(nil, opsManager.Spec.Version, true, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error reading monitoring agent version: %w", err)), log, appDbStatusOption) + } + + appdbOpts.LegacyMonitoringAgentImage = images.ContainerImage(r.imageUrls, mcoConstruct.AgentImageEnv, legacyMonitoringAgentVersion) + + // AgentImageEnv contains the full container image uri e.g. quay.io/mongodb/mongodb-agent-ubi:107.0.0.8502-1 + // In non-static containers we don't ask OM for the correct version, therefore we just rely on the provided + // environment variable. + appdbOpts.AgentImage = r.imageUrls[mcoConstruct.AgentImageEnv] + } + + workflowStatus := r.ensureTLSSecretAndCreatePEMIfNeeded(ctx, opsManager, log) + if !workflowStatus.IsOK() { + return r.updateStatus(ctx, opsManager, workflowStatus, log, appDbStatusOption) + } + + if workflowStatus := r.replicateTLSCAConfigMap(ctx, opsManager, log); !workflowStatus.IsOK() { + return r.updateStatus(ctx, opsManager, workflowStatus, log, appDbStatusOption) + } + + if workflowStatus := r.replicateSSLMMSCAConfigMap(ctx, opsManager, &podVars, log); !workflowStatus.IsOK() { + return r.updateStatus(ctx, opsManager, workflowStatus, log, appDbStatusOption) + } + + var appdbSecretPath string + if r.VaultClient != nil { + appdbSecretPath = r.VaultClient.AppDBSecretPath() + } + tlsSecretName := opsManager.Spec.AppDB.GetSecurity().MemberCertificateSecretName(opsManager.Spec.AppDB.Name()) + certHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, opsManager.Namespace, tlsSecretName, appdbSecretPath, log) + + appdbOpts.CertHash = certHash + + var vaultConfig vault.VaultConfiguration + if r.VaultClient != nil { + vaultConfig = r.VaultClient.VaultConfig + } + appdbOpts.VaultConfig = vaultConfig + + prometheusCertHash, err := certs.EnsureTLSCertsForPrometheus(ctx, r.SecretClient, opsManager.GetNamespace(), opsManager.Spec.AppDB.Prometheus, certs.AppDB, log) + if err != nil { + // Do not fail on errors generating certs for Prometheus + log.Errorf("can't create a PEM-Format Secret for Prometheus certificates: %s", err) + } + appdbOpts.PrometheusTLSCertHash = prometheusCertHash + + allStatefulSetsExist, err := r.allStatefulSetsExist(ctx, opsManager, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("failed to check the state of all stateful sets: %w", err)), log, appDbStatusOption) + } + + publishAutomationConfigFirst := r.publishAutomationConfigFirst(opsManager, allStatefulSetsExist, log) + + workflowStatus = workflow.RunInGivenOrder(publishAutomationConfigFirst, + func() workflow.Status { + return r.deployAutomationConfigAndWaitForAgentsReachGoalState(ctx, log, opsManager, allStatefulSetsExist, appdbOpts) + }, + func() workflow.Status { + return r.deployStatefulSet(ctx, opsManager, log, podVars, appdbOpts) + }, + ) + + if !workflowStatus.IsOK() { + return r.updateStatus(ctx, opsManager, workflowStatus, log, appDbStatusOption) + } + + // We keep updating annotations for backward compatibility (e.g operator downgrade), so we write the + // lastAppliedMongoDBVersion both in the state and in annotations below + // here it doesn't matter for which cluster we'll generate the name - only AppDB's MongoDB version is used there, which is the same in all clusters + verionedImplForMemberCluster := opsManager.GetVersionedImplForMemberCluster(r.getMemberClusterIndex(r.getNameOfFirstMemberCluster())) + log.Debugf("Storing LastAppliedMongoDBVersion %s in annotations and deployment state", verionedImplForMemberCluster.GetMongoDBVersionForAnnotation()) + r.deploymentState.LastAppliedMongoDBVersion = verionedImplForMemberCluster.GetMongoDBVersionForAnnotation() + if err := annotations.UpdateLastAppliedMongoDBVersion(ctx, verionedImplForMemberCluster, r.centralClient); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Could not save current state as an annotation: %w", err)), log, omStatusOption) + } + + appDBScalers := []interfaces.MultiClusterReplicaSetScaler{} + achievedDesiredScaling := true + for _, member := range r.getAllMemberClusters() { + scaler := scalers.GetAppDBScaler(opsManager, member.Name, r.getMemberClusterIndex(member.Name), r.memberClusters) + appDBScalers = append(appDBScalers, scaler) + replicasThisReconcile := scale.ReplicasThisReconciliation(scaler) + specReplicas := opsManager.Spec.AppDB.GetMemberClusterSpecByName(member.Name).Members + if opsManager.Spec.AppDB.IsMultiCluster() && replicasThisReconcile != specReplicas { + achievedDesiredScaling = false + } + log.Debugf("Scaling status for memberCluster: %s, replicasThisReconcile=%d, specReplicas=%d, achievedDesiredScaling=%t", member.Name, replicasThisReconcile, specReplicas, achievedDesiredScaling) + } + + if err := r.saveAppDBState(ctx, opsManager.Spec.AppDB, log); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Could not save deployment state: %w", err)), log, omStatusOption) + } + + if podVars.ProjectID == "" { + // this doesn't requeue the reconciliation immediately, the calling OM controller + // requeues after Ops Manager has been fully configured. + log.Infof("Requeuing reconciliation to configure Monitoring in Ops Manager.") + + return r.updateStatus(ctx, opsManager, workflow.Pending("Enabling monitoring").Requeue(), log, appDbStatusOption, status.AppDBMemberOptions(appDBScalers...)) + } + + // We need to check for status compared to the spec because the scaler will report desired replicas to be different than what's present in the spec when the + // reconciler is not handling that specific cluster. + rsScalers := []scale.ReplicaSetScaler{} + for _, scaler := range appDBScalers { + rsScaler := scaler.(scale.ReplicaSetScaler) + rsScalers = append(rsScalers, rsScaler) + } + + if !achievedDesiredScaling || scale.AnyAreStillScaling(rsScalers...) { + return r.updateStatus(ctx, opsManager, workflow.Pending("Continuing scaling operation on AppDB %d", 1), log, appDbStatusOption, status.AppDBMemberOptions(appDBScalers...)) + } + + // set the annotation to AppDB that forced reconfigure is performed to indicate to customers + if opsManager.Annotations == nil { + opsManager.Annotations = map[string]string{} + } + + if val, ok := opsManager.Annotations[ForceReconfigureAnnotation]; ok && val == "true" { + annotationsToAdd := map[string]string{ForcedReconfigureAlreadyPerformedAnnotation: timeutil.Now()} + + err := annotations.SetAnnotations(ctx, opsManager, annotationsToAdd, r.client) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Failed to save force reconfigure annotation err: %s", err)), log, omStatusOption) + } + } + + log.Infof("Finished reconciliation for AppDB ReplicaSet!") + + return r.updateStatus(ctx, opsManager, workflow.OK(), log, appDbStatusOption, status.AppDBMemberOptions(appDBScalers...)) +} + +func (r *ReconcileAppDbReplicaSet) getNameOfFirstMemberCluster() string { + firstMemberClusterName := "" + for _, memberCluster := range r.GetHealthyMemberClusters() { + if memberCluster.Active { + firstMemberClusterName = memberCluster.Name + break + } + } + return firstMemberClusterName +} + +func (r *ReconcileAppDbReplicaSet) deployAutomationConfigAndWaitForAgentsReachGoalState(ctx context.Context, log *zap.SugaredLogger, opsManager *omv1.MongoDBOpsManager, allStatefulSetsExist bool, appdbOpts construct.AppDBStatefulSetOptions) workflow.Status { + configVersion, workflowStatus := r.deployAutomationConfigOnHealthyClusters(ctx, log, opsManager, appdbOpts) + if !workflowStatus.IsOK() { + return workflowStatus + } + if !allStatefulSetsExist { + log.Infof("Skipping waiting for all agents to reach the goal state because not all stateful sets are created yet.") + return workflow.OK() + } + // We have to separate automation config deployment from agent goal checks. + // Waiting for agents' goal state without updating config in other clusters could end up with a deadlock situation. + return r.allAgentsReachedGoalState(ctx, opsManager, configVersion, log) +} + +func (r *ReconcileAppDbReplicaSet) deployAutomationConfigOnHealthyClusters(ctx context.Context, log *zap.SugaredLogger, opsManager *omv1.MongoDBOpsManager, appdbOpts construct.AppDBStatefulSetOptions) (int, workflow.Status) { + configVersions := map[int]struct{}{} + for _, memberCluster := range r.GetHealthyMemberClusters() { + if configVersion, workflowStatus := r.deployAutomationConfig(ctx, opsManager, appdbOpts.PrometheusTLSCertHash, memberCluster.Name, log); !workflowStatus.IsOK() { + return 0, workflowStatus + } else { + log.Infof("Deployed Automation Config version: %d in cluster: %s", configVersion, memberCluster.Name) + configVersions[configVersion] = struct{}{} + } + } + + if len(configVersions) > 1 { + // automation config versions have diverged on different clusters, we need to align them. + // they potentially can diverge, because the version is determined at the time when the secret is published. + // We create ac with our builder and increment version, but then the config is compared with the one read from secret + // if they are equal (ignoring version), then the version from the secret is chosen. + // TODO CLOUDP-179139 + return 0, workflow.Failed(xerrors.Errorf("Automation config versions have diverged: %+v", configVersions)) + } + + // at this point there is exactly one "configVersion", so we just return it + for configVersion := range configVersions { + return configVersion, workflow.OK() + } + + // shouldn't happen because we should always have at least one member cluster + return 0, workflow.Failed(xerrors.Errorf("Failed to deploy automation configs")) +} + +func getAppDBPodService(appdb omv1.AppDBSpec, clusterNum int, podNum int) corev1.Service { + svcLabels := map[string]string{ + "statefulset.kubernetes.io/pod-name": appdb.GetPodName(clusterNum, podNum), + "controller": "mongodb-enterprise-operator", + } + labelSelectors := map[string]string{ + "statefulset.kubernetes.io/pod-name": appdb.GetPodName(clusterNum, podNum), + "controller": "mongodb-enterprise-operator", + } + additionalConfig := appdb.GetAdditionalMongodConfig() + port := additionalConfig.GetPortOrDefault() + svc := service.Builder(). + SetNamespace(appdb.Namespace). + SetSelector(labelSelectors). + SetLabels(svcLabels). + SetPublishNotReadyAddresses(true). + AddPort(&corev1.ServicePort{Port: port, Name: "mongodb"}). + Build() + return svc +} + +func getAppDBExternalService(appdb omv1.AppDBSpec, clusterIdx int, clusterName string, podIdx int) corev1.Service { + svc := getAppDBPodService(appdb, clusterIdx, podIdx) + svc.Name = appdb.GetExternalServiceName(clusterIdx, podIdx) + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + + externalAccessConfig := appdb.GetExternalAccessConfigurationForMemberCluster(clusterName) + if externalAccessConfig != nil { + // first we override with the Service spec from the root and then from a specific cluster. + if appdb.GetExternalAccessConfiguration() != nil { + globalOverrideSpecWrapper := appdb.ExternalAccessConfiguration.ExternalService.SpecWrapper + if globalOverrideSpecWrapper != nil { + svc.Spec = merge.ServiceSpec(svc.Spec, globalOverrideSpecWrapper.Spec) + } + svc.Annotations = merge.StringToStringMap(svc.Annotations, appdb.GetExternalAccessConfiguration().ExternalService.Annotations) + } + clusterLevelOverrideSpec := externalAccessConfig.ExternalService.SpecWrapper + additionalAnnotations := externalAccessConfig.ExternalService.Annotations + if clusterLevelOverrideSpec != nil { + svc.Spec = merge.ServiceSpec(svc.Spec, clusterLevelOverrideSpec.Spec) + } + svc.Annotations = merge.StringToStringMap(svc.Annotations, additionalAnnotations) + } + + return svc +} + +func getPlaceholderReplacer(appdb omv1.AppDBSpec, memberCluster multicluster.MemberCluster, podNum int) *placeholders.Replacer { + if appdb.IsMultiCluster() { + return create.GetMultiClusterMongoDBPlaceholderReplacer( + appdb.Name(), + appdb.Name(), + appdb.Namespace, + memberCluster.Name, + memberCluster.Index, + appdb.GetExternalDomainForMemberCluster(memberCluster.Name), + appdb.GetClusterDomain(), + podNum) + } + return create.GetSingleClusterMongoDBPlaceholderReplacer( + appdb.Name(), + appdb.Name(), + appdb.Namespace, + dns.GetServiceName(appdb.Name()), + appdb.GetExternalDomain(), + appdb.GetClusterDomain(), + podNum, + mdbv1.ReplicaSet) +} + +func (r *ReconcileAppDbReplicaSet) publishAutomationConfigFirst(opsManager *omv1.MongoDBOpsManager, allStatefulSetsExist bool, log *zap.SugaredLogger) bool { + // The only case when we push the StatefulSet first is when we are ensuring TLS for the already existing AppDB + // TODO this feels insufficient. Shouldn't we check if there is actual change in TLS settings requiring to push sts first? Now it will always publish sts first when TLS enabled + automationConfigFirst := !allStatefulSetsExist || !opsManager.Spec.AppDB.GetSecurity().IsTLSEnabled() + + if r.isChangingVersion(opsManager) { + log.Info("Version change in progress, the StatefulSet must be updated first") + automationConfigFirst = false + } + + // if we are performing a force reconfigure we should change the automation config first + if shouldPerformForcedReconfigure(opsManager.Annotations) { + automationConfigFirst = true + } + + return automationConfigFirst +} + +func (r *ReconcileAppDbReplicaSet) isChangingVersion(opsManager *omv1.MongoDBOpsManager) bool { + prevVersion := r.deploymentState.LastAppliedMongoDBVersion + return prevVersion != "" && prevVersion != opsManager.Spec.AppDB.Version +} + +func getDomain(service, namespace, clusterName string) string { + if clusterName == "" { + clusterName = "cluster.local" + } + return fmt.Sprintf("%s.%s.svc.%s", service, namespace, clusterName) +} + +// ensureTLSSecretAndCreatePEMIfNeeded checks that the needed TLS secrets are present, and creates the concatenated PEM if needed. +// This means that the secret referenced can either already contain a concatenation of certificate and private key +// or it can be of type kubernetes.io/tls. In this case the operator will read the tls.crt and tls.key entries, and it will +// generate a new secret containing their concatenation +func (r *ReconcileAppDbReplicaSet) ensureTLSSecretAndCreatePEMIfNeeded(ctx context.Context, om *omv1.MongoDBOpsManager, log *zap.SugaredLogger) workflow.Status { + rs := om.Spec.AppDB + if !rs.IsSecurityTLSConfigEnabled() { + return workflow.OK() + } + secretName := rs.Security.MemberCertificateSecretName(rs.Name()) + + needToCreatePEM := false + var err error + var secretData map[string][]byte + var s corev1.Secret + + if vault.IsVaultSecretBackend() { + needToCreatePEM = true + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.AppDBSecretPath(), om.Namespace, secretName) + secretData, err = r.VaultClient.ReadSecretBytes(path) + if err != nil { + return workflow.Failed(xerrors.Errorf("can't read current certificate secret from vault: %w", err)) + } + } else { + s, err = r.KubeClient.GetSecret(ctx, kube.ObjectKey(om.Namespace, secretName)) + if err != nil { + return workflow.Failed(xerrors.Errorf("can't read current certificate secret %s: %w", secretName, err)) + } + + // SecretTypeTLS is kubernetes.io/tls + // This is the standard way in K8S to have secrets that hold TLS certs + // And it is the one generated by cert manager + // These type of secrets contain tls.crt and tls.key entries + if s.Type == corev1.SecretTypeTLS { + needToCreatePEM = true + secretData = s.Data + } + } + + if needToCreatePEM { + var data string + for _, memberCluster := range r.GetHealthyMemberClusters() { + if om.Spec.AppDB.IsMultiCluster() { + data, err = certs.VerifyTLSSecretForStatefulSet(secretData, certs.AppDBMultiClusterReplicaSetConfig(om, scalers.GetAppDBScaler(om, memberCluster.Name, r.getMemberClusterIndex(memberCluster.Name), r.memberClusters))) + } else { + data, err = certs.VerifyTLSSecretForStatefulSet(secretData, certs.AppDBReplicaSetConfig(om)) + } + if err != nil { + return workflow.Failed(xerrors.Errorf("certificate for appdb is not valid: %w", err)) + } + } + + var appdbSecretPath string + if r.VaultClient != nil { + appdbSecretPath = r.VaultClient.AppDBSecretPath() + } + + secretHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, om.Namespace, secretName, appdbSecretPath, log) + + var errs error + for _, memberCluster := range r.GetHealthyMemberClusters() { + err = certs.CreateOrUpdatePEMSecretWithPreviousCert(ctx, memberCluster.SecretClient, kube.ObjectKey(om.Namespace, secretName), secretHash, data, nil, certs.AppDB) + if err != nil { + errs = multierror.Append(errs, xerrors.Errorf("can't create concatenated PEM certificate in cluster %s: %w", memberCluster.Name, err)) + continue + } + } + if errs != nil { + return workflow.Failed(errs) + } + } + + return workflow.OK() +} + +func (r *ReconcileAppDbReplicaSet) replicateTLSCAConfigMap(ctx context.Context, om *omv1.MongoDBOpsManager, log *zap.SugaredLogger) workflow.Status { + appDBSpec := om.Spec.AppDB + if !appDBSpec.IsMultiCluster() || !appDBSpec.IsSecurityTLSConfigEnabled() { + return workflow.OK() + } + + caConfigMapName := construct.CAConfigMapName(om.Spec.AppDB, log) + + cm, err := r.client.GetConfigMap(ctx, kube.ObjectKey(appDBSpec.Namespace, caConfigMapName)) + if err != nil { + return workflow.Failed(xerrors.Errorf("Expected CA ConfigMap not found on central cluster: %s", caConfigMapName)) + } + + for _, memberCluster := range r.GetHealthyMemberClusters() { + memberCm := configmap.Builder().SetName(caConfigMapName).SetNamespace(appDBSpec.Namespace).SetData(cm.Data).Build() + err = configmap.CreateOrUpdate(ctx, memberCluster.Client, memberCm) + + if err != nil && !apiErrors.IsAlreadyExists(err) { + return workflow.Failed(xerrors.Errorf("Failed to sync CA ConfigMap in cluster: %s, err: %w", memberCluster.Name, err)) + } + } + + return workflow.OK() +} + +func (r *ReconcileAppDbReplicaSet) replicateSSLMMSCAConfigMap(ctx context.Context, om *omv1.MongoDBOpsManager, podVars *env.PodEnvVars, log *zap.SugaredLogger) workflow.Status { + appDBSpec := om.Spec.AppDB + if !appDBSpec.IsMultiCluster() || !construct.ShouldMountSSLMMSCAConfigMap(podVars) { + log.Debug("Skipping replication of SSLMMSCAConfigMap.") + return workflow.OK() + } + + caConfigMapName := podVars.SSLMMSCAConfigMap + + cm, err := r.client.GetConfigMap(ctx, kube.ObjectKey(appDBSpec.Namespace, caConfigMapName)) + if err != nil { + return workflow.Failed(xerrors.Errorf("Expected SSLMMSCAConfigMap not found on central cluster: %s", caConfigMapName)) + } + + for _, memberCluster := range r.GetHealthyMemberClusters() { + memberCm := configmap.Builder().SetName(caConfigMapName).SetNamespace(appDBSpec.Namespace).SetData(cm.Data).Build() + err = configmap.CreateOrUpdate(ctx, memberCluster.Client, memberCm) + + if err != nil && !apiErrors.IsAlreadyExists(err) { + return workflow.Failed(xerrors.Errorf("Failed to sync SSLMMSCAConfigMap in cluster: %s, err: %w", memberCluster.Name, err)) + } + } + + return workflow.OK() +} + +// publishAutomationConfig publishes the automation config to the Secret if necessary. Note that it's done only +// if the automation config has changed - the version is incremented in this case. +// Method returns the version of the automation config. +// No optimistic concurrency control is done - there cannot be a concurrent reconciliation for the same Ops Manager +// object and the probability that the user will edit the config map manually in the same time is extremely low +// returns the version of AutomationConfig just published +func (r *ReconcileAppDbReplicaSet) publishAutomationConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, automationConfig automationconfig.AutomationConfig, secretName string, memberClusterName string) (int, error) { + ac, err := automationconfig.EnsureSecret(ctx, r.getMemberCluster(memberClusterName).SecretClient, kube.ObjectKey(opsManager.Namespace, secretName), nil, automationConfig) + if err != nil { + return -1, err + } + return ac.Version, err +} + +// getExistingAutomationConfig retrieves the existing automation config from the member clusters. +// This method retrieves the most recent automation config version to handle the case when adding a new cluster from scratch. +// This is required to avoid a situation where adding a new cluster assumes the automation is created from scratch. +func (r *ReconcileAppDbReplicaSet) getExistingAutomationConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, secretName string) (automationconfig.AutomationConfig, error) { + latestVersion := -1 + latestAc := automationconfig.AutomationConfig{} + for _, memberCluster := range r.GetHealthyMemberClusters() { + ac, err := automationconfig.ReadFromSecret(ctx, memberCluster.Client, types.NamespacedName{Name: secretName, Namespace: opsManager.Namespace}) + if err != nil { + return automationconfig.AutomationConfig{}, err + } + if ac.Version > latestVersion { + latestVersion = ac.Version + latestAc = ac + } + } + return latestAc, nil +} + +func (r *ReconcileAppDbReplicaSet) buildAppDbAutomationConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, acType agentType, prometheusCertHash string, memberClusterName string, log *zap.SugaredLogger) (automationconfig.AutomationConfig, error) { + rs := opsManager.Spec.AppDB + domain := getDomain(rs.ServiceName(), opsManager.Namespace, opsManager.Spec.GetClusterDomain()) + + auth := automationconfig.Auth{} + appDBConfigurable := omv1.AppDBConfigurable{AppDBSpec: rs, OpsManager: *opsManager} + + if err := scram.Enable(ctx, &auth, r.SecretClient, &appDBConfigurable); err != nil { + return automationconfig.AutomationConfig{}, err + } + + // the existing automation config is required as we compare it against what we build to determine + // if we need to increment the version. + secretName := rs.AutomationConfigSecretName() + if acType == monitoring { + secretName = rs.MonitoringAutomationConfigSecretName() + } + + existingAutomationConfig, err := r.getExistingAutomationConfig(ctx, opsManager, secretName) + if err != nil { + return automationconfig.AutomationConfig{}, err + } + + fcVersion := opsManager.CalculateFeatureCompatibilityVersion() + + tlsSecretName := opsManager.Spec.AppDB.GetSecurity().MemberCertificateSecretName(opsManager.Spec.AppDB.Name()) + var appdbSecretPath string + if r.VaultClient != nil { + appdbSecretPath = r.VaultClient.AppDBSecretPath() + } + certHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, opsManager.Namespace, tlsSecretName, appdbSecretPath, log) + + prometheusModification := automationconfig.NOOP() + if acType == automation { + // There are 2 agents running in the AppDB Pods, we will configure Prometheus + // only on the Automation Agent. + prometheusModification, err = buildPrometheusModification(ctx, r.SecretClient, opsManager, prometheusCertHash) + if err != nil { + log.Errorf("Could not enable Prometheus: %s", err) + } + + } + + processList := r.generateProcessList(opsManager) + existingAutomationMembers, nextId := getExistingAutomationReplicaSetMembers(existingAutomationConfig) + memberOptions := r.generateMemberOptions(opsManager, existingAutomationMembers) + replicasThisReconciliation := 0 + // we want to use all member clusters to maintain the same process list despite having some clusters down + for _, memberCluster := range r.getAllMemberClusters() { + replicasThisReconciliation += scale.ReplicasThisReconciliation(scalers.GetAppDBScaler(opsManager, memberCluster.Name, memberCluster.Index, r.memberClusters)) + } + + builder := automationconfig.NewBuilder(). + SetTopology(automationconfig.ReplicaSetTopology). + SetMemberOptions(memberOptions). + SetMembers(replicasThisReconciliation). + SetName(rs.Name()). + SetDomain(domain). + SetAuth(auth). + SetFCV(fcVersion). + AddVersions(existingAutomationConfig.Versions). + IsEnterprise(construct.IsEnterprise()). + SetMongoDBVersion(rs.GetMongoDBVersion()). + SetOptions(automationconfig.Options{DownloadBase: util.AgentDownloadsDir}). + SetPreviousAutomationConfig(existingAutomationConfig). + SetTLSConfig( + automationconfig.TLS{ + CAFilePath: appdbCAFilePath, + ClientCertificateMode: automationconfig.ClientCertificateModeOptional, + }). + AddProcessModification(func(i int, p *automationconfig.Process) { + p.Name = processList[i].Name + p.HostName = processList[i].HostName + + p.AuthSchemaVersion = om.CalculateAuthSchemaVersion() + p.Args26 = objx.New(rs.AdditionalMongodConfig.ToMap()) + p.SetPort(int(rs.AdditionalMongodConfig.GetPortOrDefault())) + p.SetReplicaSetName(rs.Name()) + p.SetStoragePath(automationconfig.DefaultMongoDBDataDir) + if rs.Security.IsTLSEnabled() { + + certFileName := certHash + if certFileName == "" { + certFileName = fmt.Sprintf("%s-pem", p.Name) + } + certFile := fmt.Sprintf("%s/certs/%s", util.SecretVolumeMountPath, certFileName) + + p.Args26.Set("net.tls.mode", string(tls.Require)) + + p.Args26.Set("net.tls.certificateKeyFile", certFile) + + } + systemLog := &automationconfig.SystemLog{ + Destination: automationconfig.File, + Path: path.Join(util.PvcMountPathLogs, "mongodb.log"), + } + + if opsManager.Spec.AppDB.AutomationAgent.SystemLog != nil { + systemLog = opsManager.Spec.AppDB.AutomationAgent.SystemLog + } + + // This setting takes precedence, above has been deprecated, and we should favor the one after mongod + if opsManager.Spec.AppDB.AutomationAgent.Mongod.SystemLog != nil { + systemLog = opsManager.Spec.AppDB.AutomationAgent.Mongod.SystemLog + } + + if acType == automation { + if opsManager.Spec.AppDB.AutomationAgent.Mongod.HasLoggingConfigured() { + automationconfig.ConfigureAgentConfiguration(systemLog, opsManager.Spec.AppDB.AutomationAgent.Mongod.LogRotate, opsManager.Spec.AppDB.AutomationAgent.Mongod.AuditLogRotate, p) + } else { + automationconfig.ConfigureAgentConfiguration(systemLog, opsManager.Spec.AppDB.AutomationAgent.LogRotate, opsManager.Spec.AppDB.AutomationAgent.Mongod.AuditLogRotate, p) + } + } + }). + AddModifications(func(automationConfig *automationconfig.AutomationConfig) { + if acType == monitoring { + addMonitoring(automationConfig, log, rs.GetSecurity().IsTLSEnabled()) + automationConfig.ReplicaSets = []automationconfig.ReplicaSet{} + automationConfig.Processes = []automationconfig.Process{} + } + setBaseUrlForAgents(automationConfig, opsManager.CentralURL()) + }). + AddModifications(func(automationConfig *automationconfig.AutomationConfig) { + if len(automationConfig.ReplicaSets) == 1 { + for idx, member := range automationConfig.ReplicaSets[0].Members { + if existingMember, ok := existingAutomationMembers[member.Host]; ok { + automationConfig.ReplicaSets[0].Members[idx].Id = existingMember.Id + } else { + automationConfig.ReplicaSets[0].Members[idx].Id = nextId + nextId = nextId + 1 + } + } + } + }). + AddModifications(prometheusModification) + + if opsManager.Spec.AppDB.IsMultiCluster() { + builder.SetDomain(fmt.Sprintf("%s.svc.%s", opsManager.Namespace, opsManager.Spec.GetClusterDomain())) + } + ac, err := builder.Build() + if err != nil { + return automationconfig.AutomationConfig{}, err + } + + if acType == automation && opsManager.Spec.AppDB.AutomationConfigOverride != nil { + acToMerge := mdbcv1_controllers.OverrideToAutomationConfig(*opsManager.Spec.AppDB.AutomationConfigOverride) + ac = merge.AutomationConfigs(ac, acToMerge) + } + + // this is for logging automation config, ignoring monitoring as it doesn't contain any processes) + if acType == automation { + processHostnames := util.Transform(ac.Processes, func(obj automationconfig.Process) string { + return obj.HostName + }) + + var replicaSetMembers []string + if len(ac.ReplicaSets) > 0 { + replicaSetMembers = util.Transform(ac.ReplicaSets[0].Members, func(member automationconfig.ReplicaSetMember) string { + return fmt.Sprintf("{Id=%d, Host=%s}", member.Id, member.Host) + }) + } + log.Debugf("Created automation config object (in-memory) for cluster=%s, total process count=%d, process hostnames=%+v, replicaset config=%+v", memberClusterName, replicasThisReconciliation, processHostnames, replicaSetMembers) + } + + // this is for force reconfigure. This sets "currentVersion: -1" in automation config + // when forceReconfig is triggered. + if acType == automation { + if shouldPerformForcedReconfigure(opsManager.Annotations) { + log.Debug("Performing forced reconfigure of AppDB") + builder.SetForceReconfigureToVersion(-1) + + ac, err = builder.Build() + if err != nil { + log.Errorf("failed to build AC: %w", err) + return ac, err + } + } + } + + return ac, nil +} + +// shouldPerformForcedReconfigure checks whether forced reconfigure of the automation config needs to be performed or not +// it checks this with the user provided annotation and if the operator has actually performed a force reconfigure already +func shouldPerformForcedReconfigure(annotations map[string]string) bool { + if val, ok := annotations[ForceReconfigureAnnotation]; ok { + if val == "true" { + if _, ok := annotations[ForcedReconfigureAlreadyPerformedAnnotation]; !ok { + return true + } + } + } + return false +} + +func getExistingAutomationReplicaSetMembers(automationConfig automationconfig.AutomationConfig) (map[string]automationconfig.ReplicaSetMember, int) { + nextId := 0 + existingMembers := map[string]automationconfig.ReplicaSetMember{} + if len(automationConfig.ReplicaSets) != 1 { + return existingMembers, nextId + } + for _, member := range automationConfig.ReplicaSets[0].Members { + existingMembers[member.Host] = member + if member.Id >= nextId { + nextId = member.Id + 1 + } + } + return existingMembers, nextId +} + +func (r *ReconcileAppDbReplicaSet) generateProcessHostnames(opsManager *omv1.MongoDBOpsManager) []string { + var hostnames []string + // We want all clusters to generate stable process list in case of some clusters being down. Process list cannot change regardless of the cluster health. + for _, memberCluster := range r.getAllMemberClusters() { + hostnames = append(hostnames, r.generateProcessHostnamesForCluster(opsManager, memberCluster)...) + } + + return hostnames +} + +func (r *ReconcileAppDbReplicaSet) generateProcessHostnamesForCluster(opsManager *omv1.MongoDBOpsManager, memberCluster multicluster.MemberCluster) []string { + members := scale.ReplicasThisReconciliation(scalers.GetAppDBScaler(opsManager, memberCluster.Name, r.getMemberClusterIndex(memberCluster.Name), r.memberClusters)) + + if opsManager.Spec.AppDB.IsMultiCluster() { + return dns.GetMultiClusterProcessHostnames(opsManager.Spec.AppDB.GetName(), opsManager.GetNamespace(), memberCluster.Index, members, opsManager.Spec.AppDB.GetClusterDomain(), opsManager.Spec.AppDB.GetExternalDomainForMemberCluster(memberCluster.Name)) + } + + hostnames, _ := dns.GetDNSNames(opsManager.Spec.AppDB.GetName(), opsManager.Spec.AppDB.ServiceName(), opsManager.GetNamespace(), opsManager.Spec.AppDB.GetClusterDomain(), members, opsManager.Spec.AppDB.GetExternalDomain()) + return hostnames +} + +func (r *ReconcileAppDbReplicaSet) generateProcessList(opsManager *omv1.MongoDBOpsManager) []automationconfig.Process { + var processList []automationconfig.Process + // We want all clusters to generate stable process list in case of some clusters being down. Process list cannot change regardless of the cluster health. + for _, memberCluster := range r.getAllMemberClusters() { + hostnames := r.generateProcessHostnamesForCluster(opsManager, memberCluster) + for idx, hostname := range hostnames { + process := automationconfig.Process{ + Name: fmt.Sprintf("%s-%d", opsManager.Spec.AppDB.NameForCluster(memberCluster.Index), idx), + HostName: hostname, + } + processList = append(processList, process) + } + } + return processList +} + +func (r *ReconcileAppDbReplicaSet) generateMemberOptions(opsManager *omv1.MongoDBOpsManager, previousMembers map[string]automationconfig.ReplicaSetMember) []automationconfig.MemberOptions { + var memberOptionsList []automationconfig.MemberOptions + for _, memberCluster := range r.getAllMemberClusters() { + hostnames := r.generateProcessHostnamesForCluster(opsManager, memberCluster) + memberConfig := make([]automationconfig.MemberOptions, 0) + if memberCluster.Active { + memberConfigForCluster := opsManager.Spec.AppDB.GetMemberClusterSpecByName(memberCluster.Name).MemberConfig + if memberConfigForCluster != nil { + memberConfig = append(memberConfig, memberConfigForCluster...) + } + } + for idx, hostname := range hostnames { + memberOptions := automationconfig.MemberOptions{} + if idx < len(memberConfig) { // There are member options configured in the spec + memberOptions.Votes = memberConfig[idx].Votes + memberOptions.Priority = memberConfig[idx].Priority + memberOptions.Tags = memberConfig[idx].Tags + } else { + // There are three cases we might not have memberOptions in spec: + // 1. user never specified member config in the spec + // 2. user scaled down members e.g. from 5 to 2 removing memberConfig elements at the same time + // 3. user removed whole clusterSpecItem from the list (removing cluster entirely) + // For 2. and 3. we should have those members in existing AC + if replicaSetMember, ok := previousMembers[hostname]; ok { + memberOptions.Votes = replicaSetMember.Votes + if replicaSetMember.Priority != nil { + memberOptions.Priority = ptr.To(fmt.Sprintf("%f", *replicaSetMember.Priority)) + } + memberOptions.Tags = replicaSetMember.Tags + + } else { + // If the member does not exist in the previous automation config, we populate the member options with defaults + memberOptions.Votes = ptr.To(1) + memberOptions.Priority = ptr.To("1.0") + } + } + memberOptionsList = append(memberOptionsList, memberOptions) + } + + } + return memberOptionsList +} + +// buildPrometheusModification returns a `Modification` function that will add a +// `prometheus` entry to the Automation Config if Prometheus has been enabled on +// the Application Database (`spec.applicationDatabase.Prometheus`). +func buildPrometheusModification(ctx context.Context, sClient secrets.SecretClient, om *omv1.MongoDBOpsManager, prometheusCertHash string) (automationconfig.Modification, error) { + if om.Spec.AppDB.Prometheus == nil { + return automationconfig.NOOP(), nil + } + + prom := om.Spec.AppDB.Prometheus + + var err error + var password string + prometheus := om.Spec.AppDB.Prometheus + + secretName := prometheus.PasswordSecretRef.Name + if vault.IsVaultSecretBackend() { + operatorSecretPath := sClient.VaultClient.OperatorSecretPath() + passwordString := fmt.Sprintf("%s/%s/%s", operatorSecretPath, om.GetNamespace(), secretName) + keyedPassword, err := sClient.VaultClient.ReadSecretString(passwordString) + if err != nil { + return automationconfig.NOOP(), err + } + + var ok bool + password, ok = keyedPassword[prometheus.GetPasswordKey()] + if !ok { + errMsg := fmt.Sprintf("Prometheus password %s not in Secret %s", prometheus.GetPasswordKey(), passwordString) + return automationconfig.NOOP(), xerrors.Errorf(errMsg) + } + } else { + secretNamespacedName := types.NamespacedName{Name: secretName, Namespace: om.Namespace} + password, err = secret.ReadKey(ctx, sClient, prometheus.GetPasswordKey(), secretNamespacedName) + if err != nil { + return automationconfig.NOOP(), err + } + } + + return func(config *automationconfig.AutomationConfig) { + promConfig := automationconfig.NewDefaultPrometheus(prom.Username) + + if prometheusCertHash != "" { + promConfig.TLSPemPath = util.SecretVolumeMountPathPrometheus + "/" + prometheusCertHash + promConfig.Scheme = "https" + } else { + promConfig.Scheme = "http" + } + + promConfig.Password = password + + if prom.Port > 0 { + promConfig.ListenAddress = fmt.Sprintf("%s:%d", mdbcv1_controllers.ListenAddress, prom.Port) + } + + if prom.MetricsPath != "" { + promConfig.MetricsPath = prom.MetricsPath + } + + config.Prometheus = &promConfig + }, nil +} + +// setBaseUrlForAgents will update the baseUrl for all backup and monitoring versions to the provided url. +func setBaseUrlForAgents(ac *automationconfig.AutomationConfig, url string) { + for i := range ac.MonitoringVersions { + ac.MonitoringVersions[i].BaseUrl = url + } + for i := range ac.BackupVersions { + ac.BackupVersions[i].BaseUrl = url + } +} + +func addMonitoring(ac *automationconfig.AutomationConfig, log *zap.SugaredLogger, tls bool) { + if len(ac.Processes) == 0 { + return + } + monitoringVersions := ac.MonitoringVersions + for _, p := range ac.Processes { + found := false + for _, m := range monitoringVersions { + if m.Hostname == p.HostName { + found = true + break + } + } + if !found { + monitoringVersion := automationconfig.MonitoringVersion{ + Hostname: p.HostName, + Name: om.MonitoringAgentDefaultVersion, + } + if tls { + additionalParams := map[string]string{ + "useSslForAllConnections": "true", + "sslTrustedServerCertificates": appdbCAFilePath, + } + pemKeyFile := p.Args26.Get("net.tls.certificateKeyFile") + if pemKeyFile != nil { + additionalParams["sslClientCertificate"] = pemKeyFile.String() + } + monitoringVersion.AdditionalParams = additionalParams + } + log.Debugw("Added monitoring agent configuration", "host", p.HostName, "tls", tls) + monitoringVersions = append(monitoringVersions, monitoringVersion) + } + } + ac.MonitoringVersions = monitoringVersions +} + +// registerAppDBHostsWithProject uses the Hosts API to add each process in the AppDB to the project +func (r *ReconcileAppDbReplicaSet) registerAppDBHostsWithProject(hostnames []string, conn om.Connection, opsManagerPassword string, log *zap.SugaredLogger) error { + getHostsResult, err := conn.GetHosts() + if err != nil { + return xerrors.Errorf("error fetching existing hosts: %w", err) + } + + hostMap := make(map[string]host.Host) + for _, host := range getHostsResult.Results { + hostMap[host.Hostname] = host + } + + for _, hostname := range hostnames { + appDbHost := host.Host{ + Port: util.MongoDbDefaultPort, + Username: util.OpsManagerMongoDBUserName, + Password: opsManagerPassword, + Hostname: hostname, + AuthMechanismName: "MONGODB_CR", + } + + if currentHost, ok := hostMap[hostname]; ok { + // Host is already on the list, we need to update it. + log.Debugf("Host %s is already registered with group %s", hostname, conn.GroupID()) + // Need to se the Id first + appDbHost.Id = currentHost.Id + + if err := conn.UpdateHost(appDbHost); err != nil { + return xerrors.Errorf("error updating appdb host %w", err) + } + } else { + // This is a new host. + log.Debugf("Registering AppDB host %s with project %s", hostname, conn.GroupID()) + if err := conn.AddHost(appDbHost); err != nil { + return xerrors.Errorf("*** error adding appdb host %w", err) + } + } + } + return nil +} + +// addPreferredHostnames will add the hostnames as preferred in Ops Manager +// Ops Manager does not check for duplicates, so we need to treat it here. +func (r *ReconcileAppDbReplicaSet) addPreferredHostnames(ctx context.Context, conn om.Connection, opsManager *omv1.MongoDBOpsManager, agentApiKey string, hostnames []string) error { + existingPreferredHostnames, err := conn.GetPreferredHostnames(agentApiKey) + if err != nil { + return err + } + + existingPreferredHostnamesMap := make(map[string]om.PreferredHostname) + for _, hostname := range existingPreferredHostnames { + existingPreferredHostnamesMap[hostname.Value] = hostname + } + + for _, hostname := range hostnames { + if _, ok := existingPreferredHostnamesMap[hostname]; !ok { + err := conn.AddPreferredHostname(agentApiKey, hostname, false) + if err != nil { + return err + } + } + } + return nil +} + +func (r *ReconcileAppDbReplicaSet) generatePasswordAndCreateSecret(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (string, error) { + // create the password + password, err := generate.RandomFixedLengthStringOfSize(12) + if err != nil { + return "", err + } + + passwordData := map[string]string{ + util.OpsManagerPasswordKey: password, + } + + secretObjectKey := kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName()) + + log.Infof("Creating mongodb-ops-manager password in secret/%s in namespace %s", secretObjectKey.Name, secretObjectKey.Namespace) + + appDbPasswordSecret := secret.Builder(). + SetName(secretObjectKey.Name). + SetNamespace(secretObjectKey.Namespace). + SetStringMapToData(passwordData). + SetOwnerReferences(kube.BaseOwnerReference(opsManager)). + Build() + + if err := r.CreateSecret(ctx, appDbPasswordSecret); err != nil { + return "", err + } + + return password, nil +} + +// ensureAppDbPassword will return the password that was specified by the user, or the auto generated password stored in +// the secret (generate it and store in secret otherwise) +func (r *ReconcileAppDbReplicaSet) ensureAppDbPassword(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (string, error) { + passwordRef := opsManager.Spec.AppDB.PasswordSecretKeyRef + if passwordRef != nil && passwordRef.Name != "" { // there is a secret specified for the Ops Manager user + if passwordRef.Key == "" { + passwordRef.Key = "password" + } + password, err := secret.ReadKey(ctx, r.SecretClient, passwordRef.Key, kube.ObjectKey(opsManager.Namespace, passwordRef.Name)) + if err != nil { + if secrets.SecretNotExist(err) { + log.Debugf("Generated AppDB password and storing in secret/%s", opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName()) + return r.generatePasswordAndCreateSecret(ctx, opsManager, log) + } + return "", err + } + + log.Debugf("Reading password from secret/%s", passwordRef.Name) + + // watch for any changes on the user provided password + r.resourceWatcher.AddWatchedResourceIfNotAdded( + passwordRef.Name, + opsManager.Namespace, + watch.Secret, + kube.ObjectKeyFromApiObject(opsManager), + ) + + // delete the auto generated password, we don't need it anymore. We can just generate a new one if + // the user password is deleted + log.Debugf("Deleting Operator managed password secret/%s from namespace %s", opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName(), opsManager.Namespace) + if err := r.DeleteSecret(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName())); err != nil && !secrets.SecretNotExist(err) { + return "", err + } + return password, nil + } + + // otherwise we'll ensure the auto generated password exists + secretObjectKey := kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName()) + appDbPasswordSecretStringData, err := secret.ReadStringData(ctx, r.SecretClient, secretObjectKey) + + if secrets.SecretNotExist(err) { + // create the password + if password, err := r.generatePasswordAndCreateSecret(ctx, opsManager, log); err != nil { + return "", err + } else { + log.Debugf("Using auto generated AppDB password stored in secret/%s", opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName()) + return password, nil + } + } else if err != nil { + // any other error + return "", err + } + log. + Debugf("Using auto generated AppDB password stored in secret/%s", opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName()) + return appDbPasswordSecretStringData[util.OpsManagerPasswordKey], nil +} + +// ensureAppDbAgentApiKey makes sure there is an agent API key for the AppDB automation agent +func (r *ReconcileAppDbReplicaSet) ensureAppDbAgentApiKey(ctx context.Context, opsManager *omv1.MongoDBOpsManager, conn om.Connection, projectID string, log *zap.SugaredLogger) (string, error) { + var appdbSecretPath string + if r.VaultClient != nil { + appdbSecretPath = r.VaultClient.AppDBSecretPath() + } + + agentKey := "" + for _, memberCluster := range r.GetHealthyMemberClusters() { + if agentKeyFromSecret, err := agents.EnsureAgentKeySecretExists(ctx, memberCluster.SecretClient, conn, opsManager.Namespace, agentKey, projectID, appdbSecretPath, log); err != nil { + return "", xerrors.Errorf("error ensuring agent key secret exists in cluster %s: %w", memberCluster.Name, err) + } else if agentKey == "" { + agentKey = agentKeyFromSecret + } + } + + return agentKey, nil +} + +// tryConfigureMonitoringInOpsManager attempts to configure monitoring in Ops Manager. This might not be possible if Ops Manager +// has not been created yet, if that is the case, an empty PodVars will be returned. +func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, opsManagerUserPassword string, log *zap.SugaredLogger) (env.PodEnvVars, error) { + var operatorVaultSecretPath string + if r.VaultClient != nil { + operatorVaultSecretPath = r.VaultClient.OperatorSecretPath() + } + + APIKeySecretName, err := opsManager.APIKeySecretName(ctx, r.SecretClient, operatorVaultSecretPath) + if err != nil { + return env.PodEnvVars{}, xerrors.Errorf("error getting opsManager secret name: %w", err) + } + + cred, err := project.ReadCredentials(ctx, r.SecretClient, kube.ObjectKey(operatorNamespace(), APIKeySecretName), log) + if err != nil { + log.Debugf("Ops Manager has not yet been created, not configuring monitoring: %s", err) + return env.PodEnvVars{}, nil + } + log.Debugf("Ensuring monitoring of AppDB is configured in Ops Manager") + + existingPodVars, err := r.readExistingPodVars(ctx, opsManager, log) + if client.IgnoreNotFound(err) != nil { + return env.PodEnvVars{}, xerrors.Errorf("error reading existing podVars: %w", err) + } + + projectConfig, err := opsManager.GetAppDBProjectConfig(ctx, r.SecretClient, r.client) + if err != nil { + return existingPodVars, xerrors.Errorf("error getting existing project config: %w", err) + } + + _, conn, err := project.ReadOrCreateProject(projectConfig, cred, r.omConnectionFactory, log) + if err != nil { + return existingPodVars, xerrors.Errorf("error reading/creating project: %w", err) + } + + // Configure Authentication Options. + opts := authentication.Options{ + AgentMechanism: util.SCRAM, + Mechanisms: []string{util.SCRAM}, + ClientCertificates: util.OptionalClientCertficates, + AutoUser: util.AutomationAgentUserName, + CAFilePath: util.CAFilePathInContainer, + } + err = authentication.Configure(conn, opts, false, log) + if err != nil { + log.Errorf("Could not set Automation Authentication options in Ops/Cloud Manager for the Application Database. "+ + "Application Database is always configured with authentication enabled, but this will not be "+ + "visible from Ops/Cloud Manager UI. %s", err) + } + + err = conn.ReadUpdateDeployment(func(d om.Deployment) error { + d.ConfigureTLS(opsManager.Spec.AppDB.GetSecurity(), util.CAFilePathInContainer) + return nil + }, log) + if err != nil { + log.Errorf("Could not set TLS configuration in Ops/Cloud Manager for the Application Database. "+ + "Application Database has been configured with TLS enabled, but this will not be "+ + "visible from Ops/Cloud Manager UI. %s", err) + } + + hostnames := r.generateProcessHostnames(opsManager) + if err != nil { + return existingPodVars, xerrors.Errorf("error getting current appdb statefulset hostnames: %w", err) + } + + if err := r.registerAppDBHostsWithProject(hostnames, conn, opsManagerUserPassword, log); err != nil { + return existingPodVars, xerrors.Errorf("error registering hosts with project: %w", err) + } + + agentApiKey, err := r.ensureAppDbAgentApiKey(ctx, opsManager, conn, conn.GroupID(), log) + if err != nil { + return existingPodVars, xerrors.Errorf("error ensuring AppDB agent api key: %w", err) + } + + if err := markAppDBAsBackingProject(conn, log); err != nil { + return existingPodVars, xerrors.Errorf("error marking project has backing db: %w", err) + } + + if err := r.ensureProjectIDConfigMap(ctx, opsManager, conn.GroupID()); err != nil { + return existingPodVars, xerrors.Errorf("error creating ConfigMap: %w", err) + } + + if err := r.addPreferredHostnames(ctx, conn, opsManager, agentApiKey, hostnames); err != nil { + return existingPodVars, xerrors.Errorf("error adding preferred hostnames: %w", err) + } + + return env.PodEnvVars{User: conn.PublicKey(), ProjectID: conn.GroupID(), SSLProjectConfig: env.SSLProjectConfig{ + SSLMMSCAConfigMap: opsManager.Spec.GetOpsManagerCA(), + }}, nil +} + +func (r *ReconcileAppDbReplicaSet) ensureProjectIDConfigMap(ctx context.Context, opsManager *omv1.MongoDBOpsManager, projectID string) error { + var errs error + for _, memberCluster := range r.GetHealthyMemberClusters() { + if err := r.ensureProjectIDConfigMapForCluster(ctx, opsManager, projectID, memberCluster.Client); err != nil { + errs = multierror.Append(errs, xerrors.Errorf("error creating ConfigMap in cluster %s: %w", memberCluster.Name, err)) + continue + } + } + + return errs +} + +func (r *ReconcileAppDbReplicaSet) ensureProjectIDConfigMapForCluster(ctx context.Context, opsManager *omv1.MongoDBOpsManager, projectID string, k8sClient kubernetesClient.Client) error { + cm := configmap.Builder(). + SetName(opsManager.Spec.AppDB.ProjectIDConfigMapName()). + SetNamespace(opsManager.Namespace). + SetDataField(util.AppDbProjectIdKey, projectID). + Build() + + // Saving the "backup" ConfigMap which contains the project id + if err := configmap.CreateOrUpdate(ctx, k8sClient, cm); err != nil { + return xerrors.Errorf("error creating ConfigMap: %w", err) + } + return nil +} + +// readExistingPodVars is a backup function which provides the required podVars for the AppDB +// in the case of Ops Manager not being reachable. An example of when this is used is: +// 1. The AppDB starts as normal +// 2. Ops Manager starts as normal +// 3. The AppDB password was configured mid-reconciliation +// 4. AppDB reconciles and attempts to configure monitoring, but this is not possible +// as OM cannot currently connect to the AppDB as it has not yet been provided the updated password. +// In such a case, we cannot read the groupId from OM, so we fall back to the ConfigMap we created +// before hand. This is required as with empty PodVars this would trigger an unintentional +// rolling restart of the AppDB. +func (r *ReconcileAppDbReplicaSet) readExistingPodVars(ctx context.Context, om *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (env.PodEnvVars, error) { + memberClient := r.getMemberCluster(r.getNameOfFirstMemberCluster()).Client + cm, err := memberClient.GetConfigMap(ctx, kube.ObjectKey(om.Namespace, om.Spec.AppDB.ProjectIDConfigMapName())) + if err != nil { + return env.PodEnvVars{}, err + } + var projectId string + if projectId = cm.Data[util.AppDbProjectIdKey]; projectId == "" { + return env.PodEnvVars{}, xerrors.Errorf("ConfigMap %s did not have the key %s", om.Spec.AppDB.ProjectIDConfigMapName(), util.AppDbProjectIdKey) + } + + var operatorVaultSecretPath string + if r.VaultClient != nil { + operatorVaultSecretPath = r.VaultClient.OperatorSecretPath() + } + APISecretName, err := om.APIKeySecretName(ctx, r.SecretClient, operatorVaultSecretPath) + if err != nil { + return env.PodEnvVars{}, xerrors.Errorf("error getting ops-manager API secret name: %w", err) + } + + cred, err := project.ReadCredentials(ctx, r.SecretClient, kube.ObjectKey(operatorNamespace(), APISecretName), log) + if err != nil { + return env.PodEnvVars{}, xerrors.Errorf("error reading credentials: %w", err) + } + + return env.PodEnvVars{ + User: cred.PublicAPIKey, + ProjectID: projectId, + SSLProjectConfig: env.SSLProjectConfig{ + SSLMMSCAConfigMap: om.Spec.GetOpsManagerCA(), + }, + }, nil +} + +func (r *ReconcileAppDbReplicaSet) publishACVersionAsConfigMap(ctx context.Context, cmName string, namespace string, version int, memberClusterName string) workflow.Status { + acVersionConfigMap := configmap.Builder(). + SetNamespace(namespace). + SetName(cmName). + SetDataField(appDBACConfigMapVersionField, fmt.Sprintf("%d", version)). + Build() + if err := configmap.CreateOrUpdate(ctx, r.getMemberCluster(memberClusterName).Client, acVersionConfigMap); err != nil { + return workflow.Failed(xerrors.Errorf("error creating automation config map in cluster %s: %w", memberClusterName, err)) + } + + return workflow.OK() +} + +// deployAutomationConfig updates the Automation Config secret if necessary and waits for the pods to fall to "not ready" +// In this case the next StatefulSet update will be safe as the rolling upgrade will wait for the pods to get ready +func (r *ReconcileAppDbReplicaSet) deployAutomationConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, prometheusCertHash string, memberClusterName string, log *zap.SugaredLogger) (int, workflow.Status) { + rs := opsManager.Spec.AppDB + + config, err := r.buildAppDbAutomationConfig(ctx, opsManager, automation, prometheusCertHash, memberClusterName, log) + if err != nil { + return 0, workflow.Failed(err) + } + var configVersion int + if configVersion, err = r.publishAutomationConfig(ctx, opsManager, config, rs.AutomationConfigSecretName(), memberClusterName); err != nil { + return 0, workflow.Failed(err) + } + + if workflowStatus := r.publishACVersionAsConfigMap(ctx, opsManager.Spec.AppDB.AutomationConfigConfigMapName(), opsManager.Namespace, configVersion, memberClusterName); !workflowStatus.IsOK() { + return 0, workflowStatus + } + + monitoringAc, err := r.buildAppDbAutomationConfig(ctx, opsManager, monitoring, UnusedPrometheusConfiguration, memberClusterName, log) + if err != nil { + return 0, workflow.Failed(err) + } + + if err := r.deployMonitoringAgentAutomationConfig(ctx, opsManager, memberClusterName, log); err != nil { + return 0, workflow.Failed(err) + } + + if workflowStatus := r.publishACVersionAsConfigMap(ctx, opsManager.Spec.AppDB.MonitoringAutomationConfigConfigMapName(), opsManager.Namespace, monitoringAc.Version, memberClusterName); !workflowStatus.IsOK() { + return 0, workflowStatus + } + + return configVersion, workflow.OK() +} + +// deployMonitoringAgentAutomationConfig deploys the monitoring agent's automation config. +func (r *ReconcileAppDbReplicaSet) deployMonitoringAgentAutomationConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, memberClusterName string, log *zap.SugaredLogger) error { + config, err := r.buildAppDbAutomationConfig(ctx, opsManager, monitoring, UnusedPrometheusConfiguration, memberClusterName, log) + if err != nil { + return err + } + if _, err = r.publishAutomationConfig(ctx, opsManager, config, opsManager.Spec.AppDB.MonitoringAutomationConfigSecretName(), memberClusterName); err != nil { + return err + } + return nil +} + +// GetAppDBUpdateStrategyType returns the update strategy type the AppDB Statefulset needs to be configured with. +// This depends on whether a version change is in progress. +func (r *ReconcileAppDbReplicaSet) GetAppDBUpdateStrategyType(om *omv1.MongoDBOpsManager) appsv1.StatefulSetUpdateStrategyType { + if !r.isChangingVersion(om) { + return appsv1.RollingUpdateStatefulSetStrategyType + } + return appsv1.OnDeleteStatefulSetStrategyType +} + +func (r *ReconcileAppDbReplicaSet) deployStatefulSet(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger, podVars env.PodEnvVars, appdbOpts construct.AppDBStatefulSetOptions) workflow.Status { + if err := r.createServices(ctx, opsManager, log); err != nil { + return workflow.Failed(err) + } + currentClusterSpecs := map[string]int{} + scalingFirstTime := false + // iterate over all clusters to scale even unhealthy ones + // currentClusterSpecs map is maintained for scaling therefore we need to update it here + for _, memberCluster := range r.getAllMemberClusters() { + scaler := scalers.GetAppDBScaler(opsManager, memberCluster.Name, r.getMemberClusterIndex(memberCluster.Name), r.memberClusters) + replicasThisReconciliation := scale.ReplicasThisReconciliation(scaler) + currentClusterSpecs[memberCluster.Name] = replicasThisReconciliation + + if !memberCluster.Healthy { + // do not proceed if this is unhealthy cluster + continue + } + + // in the case of an upgrade from the 1 to 3 container architecture, when the stateful set is updated before the agent automation config + // the monitoring agent automation config needs to exist for the volumes to mount correctly. + if err := r.deployMonitoringAgentAutomationConfig(ctx, opsManager, memberCluster.Name, log); err != nil { + return workflow.Failed(err) + } + + updateStrategy := r.GetAppDBUpdateStrategyType(opsManager) + + appDbSts, err := construct.AppDbStatefulSet(*opsManager, &podVars, appdbOpts, scaler, updateStrategy, log) + if err != nil { + return workflow.Failed(xerrors.Errorf("can't construct AppDB Statefulset: %w", err)) + } + + if workflowStatus := r.deployStatefulSetInMemberCluster(ctx, opsManager, appDbSts, memberCluster.Name, log); !workflowStatus.IsOK() { + return workflowStatus + } + + // we want to deploy all stateful sets the first time we're deploying stateful sets + if scaler.ScalingFirstTime() { + scalingFirstTime = true + continue + } + + if workflowStatus := getStatefulSetStatus(ctx, opsManager.Namespace, opsManager.Spec.AppDB.NameForCluster(memberCluster.Index), memberCluster.Client); !workflowStatus.IsOK() { + return workflowStatus + } + + if err := statefulset.ResetUpdateStrategy(ctx, opsManager.GetVersionedImplForMemberCluster(r.getMemberClusterIndex(memberCluster.Name)), memberCluster.Client); err != nil { + return workflow.Failed(xerrors.Errorf("can't reset AppDB StatefulSet UpdateStrategyType: %w", err)) + } + } + + // if this is the first time deployment, then we need to wait for all stateful sets to become ready after deploying all of them + if scalingFirstTime { + for _, memberCluster := range r.GetHealthyMemberClusters() { + if workflowStatus := getStatefulSetStatus(ctx, opsManager.Namespace, opsManager.Spec.AppDB.NameForCluster(memberCluster.Index), memberCluster.Client); !workflowStatus.IsOK() { + return workflowStatus + } + + if err := statefulset.ResetUpdateStrategy(ctx, opsManager.GetVersionedImplForMemberCluster(r.getMemberClusterIndex(memberCluster.Name)), memberCluster.Client); err != nil { + return workflow.Failed(xerrors.Errorf("can't reset AppDB StatefulSet UpdateStrategyType: %w", err)) + } + } + } + + for k, v := range currentClusterSpecs { + r.deploymentState.LastAppliedMemberSpec[k] = v + } + + return workflow.OK() +} + +// This method creates the following services: +// - external services for Single Cluster deployments +// - external services for Multi Cluster deployments +// - pod services for Multi Cluster deployments +// Note that this does not create any non-external services for Single Cluster deployments +// Those services are created by the method create.AppDBInKubernetes +func (r *ReconcileAppDbReplicaSet) createServices(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) error { + for _, memberCluster := range r.GetHealthyMemberClusters() { + clusterSpecItem := opsManager.Spec.AppDB.GetMemberClusterSpecByName(memberCluster.Name) + + for podIdx := 0; podIdx < clusterSpecItem.Members; podIdx++ { + + // Configures external service for both single and multi cluster deployments + // This will also delete external services if the externalAccess configuration is removed + if opsManager.Spec.AppDB.GetExternalAccessConfigurationForMemberCluster(memberCluster.Name) != nil { + svc := getAppDBExternalService(opsManager.Spec.AppDB, memberCluster.Index, memberCluster.Name, podIdx) + placeholderReplacer := getPlaceholderReplacer(opsManager.Spec.AppDB, memberCluster, podIdx) + + if processedAnnotations, replacedFlag, err := placeholderReplacer.ProcessMap(svc.Annotations); err != nil { + return xerrors.Errorf("failed to process annotations in external service %s in cluster %s: %w", svc.Name, memberCluster.Name, err) + } else if replacedFlag { + log.Debugf("Replaced placeholders in annotations in external service %s in cluster: %s. Annotations before: %+v, annotations after: %+v", svc.Name, memberCluster.Name, svc.Annotations, processedAnnotations) + svc.Annotations = processedAnnotations + } + + if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + } else { + svcName := opsManager.Spec.AppDB.GetExternalServiceName(memberCluster.Index, podIdx) + namespacedName := kube.ObjectKey(opsManager.Spec.AppDB.Namespace, svcName) + if err := mekoService.DeleteServiceIfItExists(ctx, memberCluster.Client, namespacedName); err != nil { + return xerrors.Errorf("failed to remove external service %s in cluster: %s, err: %w", svcName, memberCluster.Name, err) + } + } + + // Configures pod services for multi cluster deployments + if opsManager.Spec.AppDB.IsMultiCluster() && opsManager.Spec.AppDB.GetExternalDomainForMemberCluster(memberCluster.Name) == nil { + svc := getAppDBPodService(opsManager.Spec.AppDB, memberCluster.Index, podIdx) + svc.Name = dns.GetMultiServiceName(opsManager.Spec.AppDB.Name(), memberCluster.Index, podIdx) + err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create service: %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + } + } + } + + return nil +} + +// deployStatefulSetInMemberCluster updates the StatefulSet spec and returns its status (if it's ready or not) +func (r *ReconcileAppDbReplicaSet) deployStatefulSetInMemberCluster(ctx context.Context, opsManager *omv1.MongoDBOpsManager, appDbSts appsv1.StatefulSet, memberClusterName string, log *zap.SugaredLogger) workflow.Status { + workflowStatus := create.HandlePVCResize(ctx, r.getMemberCluster(memberClusterName).Client, &appDbSts, log) + if !workflowStatus.IsOK() { + return workflowStatus + } + + if workflow.ContainsPVCOption(workflowStatus.StatusOptions()) { + if _, err := r.updateStatus(ctx, opsManager, workflow.Pending(""), log, workflowStatus.StatusOptions()...); err != nil { + return workflow.Failed(xerrors.Errorf("error updating status: %w", err)) + } + } + + serviceSelectorLabel := opsManager.Spec.AppDB.HeadlessServiceSelectorAppLabel(r.getMemberCluster(memberClusterName).Index) + if err := create.AppDBInKubernetes(ctx, r.getMemberCluster(memberClusterName).Client, opsManager, appDbSts, serviceSelectorLabel, log); err != nil { + return workflow.Failed(err) + } + + return workflow.OK() +} + +func (r *ReconcileAppDbReplicaSet) allAgentsReachedGoalState(ctx context.Context, manager *omv1.MongoDBOpsManager, targetConfigVersion int, log *zap.SugaredLogger) workflow.Status { + for _, memberCluster := range r.GetHealthyMemberClusters() { + var workflowStatus workflow.Status + if manager.Spec.AppDB.IsMultiCluster() { + workflowStatus = r.allAgentsReachedGoalStateMultiCluster(ctx, manager, targetConfigVersion, memberCluster.Name, log) + } else { + workflowStatus = r.allAgentsReachedGoalStateSingleCluster(ctx, manager, targetConfigVersion, memberCluster.Name, log) + } + + if !workflowStatus.IsOK() { + return workflowStatus + } + } + + return workflow.OK() +} + +func (r *ReconcileAppDbReplicaSet) allAgentsReachedGoalStateMultiCluster(ctx context.Context, manager *omv1.MongoDBOpsManager, targetConfigVersion int, memberClusterName string, log *zap.SugaredLogger) workflow.Status { + memberClusterClient := r.getMemberCluster(memberClusterName).Client + set, err := memberClusterClient.GetStatefulSet(ctx, manager.AppDBStatefulSetObjectKey(r.getMemberClusterIndex(memberClusterName))) + if err != nil { + if apiErrors.IsNotFound(err) { + return workflow.OK() + } + return workflow.Failed(err) + } + + appDBSize := int(set.Status.Replicas) + goalState, err := agent.AllReachedGoalState(ctx, set, memberClusterClient, appDBSize, targetConfigVersion, log) + if err != nil { + return workflow.Failed(err) + } + if goalState { + return workflow.OK() + } + return workflow.Pending("Application Database Agents haven't reached Running state yet") +} + +// allAgentsReachedGoalState checks if all the AppDB Agents have reached the goal state. +func (r *ReconcileAppDbReplicaSet) allAgentsReachedGoalStateSingleCluster(ctx context.Context, manager *omv1.MongoDBOpsManager, targetConfigVersion int, memberClusterName string, log *zap.SugaredLogger) workflow.Status { + // We need to read the current StatefulSet to find the real number of pods - we cannot rely on OpsManager resource + set, err := r.client.GetStatefulSet(ctx, manager.AppDBStatefulSetObjectKey(r.getMemberClusterIndex(memberClusterName))) + if err != nil { + if apiErrors.IsNotFound(err) { + // If the StatefulSet could not be found, do not check agents during this reconcile. + // It means - we didn't deploy statefulset yet, and we should proceed. + return workflow.OK() + } + return workflow.Failed(err) + } + + appdbSize := int(set.Status.Replicas) + goalState, err := agent.AllReachedGoalState(ctx, set, r.client, appdbSize, targetConfigVersion, log) + if err != nil { + return workflow.Failed(err) + } + if goalState { + return workflow.OK() + } + return workflow.Pending("Application Database Agents haven't reached Running state yet") +} + +func (r *ReconcileAppDbReplicaSet) getAllMemberClusters() []multicluster.MemberCluster { + return r.memberClusters +} + +func (r *ReconcileAppDbReplicaSet) GetHealthyMemberClusters() []multicluster.MemberCluster { + var healthyMemberClusters []multicluster.MemberCluster + for i := 0; i < len(r.memberClusters); i++ { + if r.memberClusters[i].Healthy { + healthyMemberClusters = append(healthyMemberClusters, r.memberClusters[i]) + } + } + + return healthyMemberClusters +} + +func (r *ReconcileAppDbReplicaSet) getMemberCluster(name string) multicluster.MemberCluster { + for i := 0; i < len(r.memberClusters); i++ { + if r.memberClusters[i].Name == name { + return r.memberClusters[i] + } + } + + panic(xerrors.Errorf("member cluster %s not found", name)) +} + +func (r *ReconcileAppDbReplicaSet) getMemberClusterIndex(clusterName string) int { + return r.getMemberCluster(clusterName).Index +} + +func (r *ReconcileAppDbReplicaSet) getCurrentStatefulsetHostnames(opsManager *omv1.MongoDBOpsManager) []string { + return util.Transform(r.generateProcessList(opsManager), func(process automationconfig.Process) string { + return process.HostName + }) +} + +func (r *ReconcileAppDbReplicaSet) allStatefulSetsExist(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (bool, error) { + allStsExist := true + for _, memberCluster := range r.GetHealthyMemberClusters() { + stsName := opsManager.Spec.AppDB.NameForCluster(r.getMemberClusterIndex(memberCluster.Name)) + _, err := memberCluster.Client.GetStatefulSet(ctx, kube.ObjectKey(opsManager.Namespace, stsName)) + if err != nil { + if apiErrors.IsNotFound(err) { + // we do not return immediately here to check all clusters and also leave the information on other sts in the debug logs + log.Debugf("Statefulset %s/%s does not exist.", memberCluster.Name, stsName) + allStsExist = false + } else { + return false, err + } + } + } + + return allStsExist, nil +} + +// migrateToNewDeploymentState reads old config maps with the deployment state and writes them to the new deploymentState structure. +// This function is intended to be called only in the absence of the new deployment state config map. +// In this case, if the legacy config maps are also missing, then it means is a completely fresh deployments and this function does nothing. +func (r *ReconcileAppDbReplicaSet) migrateToNewDeploymentState(ctx context.Context, spec omv1.AppDBSpec, omAnnotations map[string]string) error { + if legacyMemberClusterMapping, err := getLegacyMemberClusterMapping(ctx, spec.Namespace, spec.ClusterMappingConfigMapName(), r.client); err != nil { + if !apiErrors.IsNotFound(err) && spec.IsMultiCluster() { + return err + } + } else { + r.deploymentState.ClusterMapping = legacyMemberClusterMapping + } + + if legacyLastAppliedMemberSpec, err := r.getLegacyLastAppliedMemberSpec(ctx, spec); err != nil { + if !apiErrors.IsNotFound(err) { + return err + } + } else { + r.deploymentState.LastAppliedMemberSpec = legacyLastAppliedMemberSpec + } + + if lastAppliedMongoDBVersion, found := omAnnotations[annotations.LastAppliedMongoDBVersion]; found { + r.deploymentState.LastAppliedMongoDBVersion = lastAppliedMongoDBVersion + } + + return nil +} + +// markAppDBAsBackingProject will configure the AppDB project to be read only. Errors are ignored +// if the OpsManager version does not support this feature. +func markAppDBAsBackingProject(conn om.Connection, log *zap.SugaredLogger) error { + log.Debugf("Configuring the project as a backing database project.") + err := conn.MarkProjectAsBackingDatabase(om.AppDBDatabaseType) + if err != nil { + if apiErr, ok := err.(*apierror.Error); ok { + opsManagerDoesNotSupportApi := apiErr.Status != nil && *apiErr.Status == 404 && apiErr.ErrorCode == "RESOURCE_NOT_FOUND" + if opsManagerDoesNotSupportApi { + msg := "This version of Ops Manager does not support the markAsBackingDatabase API." + if !conn.OpsManagerVersion().IsUnknown() { + msg += fmt.Sprintf(" Version=%s", conn.OpsManagerVersion()) + } + log.Debug(msg) + return nil + } + } + return err + } + return nil +} diff --git a/controllers/operator/appdbreplicaset_controller_multi_test.go b/controllers/operator/appdbreplicaset_controller_multi_test.go new file mode 100644 index 000000000..a9cc53d34 --- /dev/null +++ b/controllers/operator/appdbreplicaset_controller_multi_test.go @@ -0,0 +1,1772 @@ +package operator + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +const opsManagerUserPassword = "MBPYfkAj5ZM0l9uw6C7ggw" //nolint + +func TestAppDB_MultiCluster(t *testing.T) { + ctx := context.Background() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + clusters := []string{centralClusterName, memberClusterName, memberClusterName2} + + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 3, + }, + } + + builder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(clusterSpecItems). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster). + SetAppDBTLSConfig(mdbv1.TLSConfig{ + Enabled: true, + AdditionalCertificateDomains: nil, + CA: "appdb-ca", + }) + + opsManager := builder.Build() + appdb := opsManager.Spec.AppDB + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + // prepare CA config map in central cluster + caConfigMapName := createAppDbCAConfigMap(ctx, t, kubeClient, appdb) + tlsCertSecretName, tlsSecretPemHash := createAppDBTLSCert(ctx, t, kubeClient, appdb) + pemSecretName := tlsCertSecretName + "-pem" + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, zap.S(), omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + err = createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, opsManagerUserPassword) + assert.NoError(t, err) + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + // requeue is true to add monitoring + assert.True(t, reconcileResult.Requeue) + + centralClusterChecks := newClusterChecks(t, centralClusterName, -1, opsManager.Namespace, kubeClient) + // secrets and config maps created by the operator shouldn't be created in central cluster + centralClusterChecks.checkSecretNotFound(ctx, appdb.AutomationConfigSecretName()) + centralClusterChecks.checkConfigMapNotFound(ctx, appdb.AutomationConfigConfigMapName()) + centralClusterChecks.checkSecretNotFound(ctx, appdb.MonitoringAutomationConfigSecretName()) + centralClusterChecks.checkConfigMapNotFound(ctx, appdb.MonitoringAutomationConfigConfigMapName()) + centralClusterChecks.checkSecretNotFound(ctx, pemSecretName) + centralClusterChecks.checkTLSCAConfigMap(ctx, caConfigMapName) + centralClusterChecks.checkConfigMapNotFound(ctx, appdb.ProjectIDConfigMapName()) + + for clusterIdx, clusterSpecItem := range clusterSpecItems { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, opsManager.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + memberClusterChecks.checkAutomationConfigSecret(ctx, appdb.AutomationConfigSecretName()) + memberClusterChecks.checkAutomationConfigConfigMap(ctx, appdb.AutomationConfigConfigMapName()) + memberClusterChecks.checkAutomationConfigSecret(ctx, appdb.MonitoringAutomationConfigSecretName()) + memberClusterChecks.checkAutomationConfigConfigMap(ctx, appdb.MonitoringAutomationConfigConfigMapName()) + memberClusterChecks.checkTLSCAConfigMap(ctx, caConfigMapName) + // TLS secret should not be replicated, only PEM secret + memberClusterChecks.checkSecretNotFound(ctx, tlsCertSecretName) + memberClusterChecks.checkPEMSecret(ctx, pemSecretName, tlsSecretPemHash) + + memberClusterChecks.checkStatefulSet(ctx, opsManager.Spec.AppDB.NameForCluster(reconciler.getMemberClusterIndex(clusterSpecItem.ClusterName)), clusterSpecItem.Members) + memberClusterChecks.checkPerPodServices(ctx, opsManager.Spec.AppDB.NameForCluster(reconciler.getMemberClusterIndex(clusterSpecItem.ClusterName)), clusterSpecItem.Members) + } + + // OM API Key secret is required for enabling monitoring to OM + createOMAPIKeySecret(ctx, t, reconciler.SecretClient, opsManager) + + // reconcile to add monitoring + reconcileResult, err = reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + require.False(t, reconcileResult.Requeue) + + // monitoring here is configured, everything should be replicated + + // we create project id and agent key resources only in member clusters + centralClusterChecks.checkConfigMapNotFound(ctx, appdb.ProjectIDConfigMapName()) + agentAPIKey := "" + for clusterIdx, clusterSpecItem := range clusterSpecItems { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, opsManager.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + projectID := memberClusterChecks.checkProjectIDConfigMap(ctx, appdb.ProjectIDConfigMapName()) + agentAPIKeyFromSecret := memberClusterChecks.checkAgentAPIKeySecret(ctx, projectID) + assert.NotEmpty(t, agentAPIKeyFromSecret) + if agentAPIKey == "" { + // save the value to check if all member clusters contain the same value + agentAPIKey = agentAPIKeyFromSecret + } + assert.Equal(t, agentAPIKey, agentAPIKeyFromSecret) + } +} + +func agentAPIKeySecretName(projectID string) string { + return fmt.Sprintf("%s-group-secret", projectID) +} + +func TestAppDB_MultiCluster_AutomationConfig(t *testing.T) { + ctx := context.Background() + log := zap.S() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + clusters := []string{centralClusterName, memberClusterName, memberClusterName2, memberClusterName3} + + builder := DefaultOpsManagerBuilder(). + SetName("om"). + SetNamespace("ns"). + SetAppDBClusterSpecList(mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + }, + ). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + + opsManager := builder.Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + globalClusterMap := getFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + err := createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, opsManagerUserPassword) + assert.NoError(t, err) + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, globalClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + // requeue is true to add monitoring + assert.True(t, reconcileResult.Requeue) + + // OM API Key secret is required for enabling monitoring to OM + createOMAPIKeySecret(ctx, t, reconciler.SecretClient, opsManager) + + // reconcile to add monitoring + reconciler, err = newAppDbMultiReconciler(ctx, kubeClient, opsManager, globalClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + reconcileResult, err = reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + require.False(t, reconcileResult.Requeue) + + t.Run("check expected hostnames", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 1, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames := []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-1-0", + } + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalClusterMap, memberClusterName, clusterSpecItems, expectedHostnames, expectedProcessNames, 1, log) + }) + + t.Run("remove second cluster and add new one", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName3, + Members: 1, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames := []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-2-0", + } + + // 2 reconciles, remove 1 member from memberClusterName2 and add one from memberClusterName3 + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalClusterMap, memberClusterName, clusterSpecItems, expectedHostnames, expectedProcessNames, 2, log) + }) + + t.Run("add second cluster back to check indexes are preserved with different clusterSpecItem order", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName3, + Members: 1, + }, + { + ClusterName: memberClusterName2, + Members: 2, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-1-1-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames := []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-1-0", + "om-db-1-1", + "om-db-2-0", + } + + // 2 reconciles to add 2 members of memberClusterName2 + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalClusterMap, memberClusterName, clusterSpecItems, expectedHostnames, expectedProcessNames, 2, log) + }) + + t.Run("remove second cluster from global cluster to simulate full-cluster failure", func(t *testing.T) { + globalMemberClusterMapWithoutCluster2 := getFakeMultiClusterMapWithClusters([]string{memberClusterName, memberClusterName3}, omConnectionFactory) + // no changes to clusterSpecItems, nothing should be scaled, processes should be the same + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName3, + Members: 1, + }, + { + ClusterName: memberClusterName2, + Members: 2, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-1-1-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames := []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-1-0", + "om-db-1-1", + "om-db-2-0", + } + + // nothing to be scaled + reconcileAppDBOnceAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalMemberClusterMapWithoutCluster2, memberClusterName, clusterSpecItems, false, expectedHostnames, expectedProcessNames, log) + + // memberClusterName2 is removed + clusterSpecItems = mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName3, + Members: 1, + }, + } + + expectedHostnames = []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames = []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-1-0", + "om-db-2-0", + } + + // one process from memberClusterName2 should be removed + reconcileAppDBOnceAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalMemberClusterMapWithoutCluster2, memberClusterName, clusterSpecItems, true, expectedHostnames, expectedProcessNames, log) + + expectedHostnames = []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames = []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-2-0", + } + + // the last process from memberClusterName2 should be removed + // this should be final reconcile + reconcileAppDBOnceAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalMemberClusterMapWithoutCluster2, memberClusterName, clusterSpecItems, false, expectedHostnames, expectedProcessNames, log) + }) + t.Run("add second cluster back to check indexes are preserved with different clusterSpecItem order", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName3, + Members: 1, + }, + { + ClusterName: memberClusterName2, + Members: 2, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-1-1-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + } + + expectedProcessNames := []string{ + "om-db-0-0", + "om-db-0-1", + "om-db-1-0", + "om-db-1-1", + "om-db-2-0", + } + + // 2 reconciles to add 2 members of memberClusterName2 + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedProcesses(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, opsManager, globalClusterMap, memberClusterName, clusterSpecItems, expectedHostnames, expectedProcessNames, 2, log) + }) +} + +func assertExpectedHostnamesAndPreferred(t *testing.T, omConnection *om.MockedOmConnection, expectedHostnames []string) { + hosts, _ := omConnection.GetHosts() + assert.Equal(t, expectedHostnames, util.Transform(hosts.Results, func(obj host.Host) string { + return obj.Hostname + }), "the AppDB hosts should have been added") + + preferredHostnames, _ := omConnection.GetPreferredHostnames(omConnection.AgentAPIKey) + assert.Equal(t, expectedHostnames, util.Transform(preferredHostnames, func(obj om.PreferredHostname) string { + return obj.Value + }), "the AppDB preferred hostnames should have been added") +} + +func assertExpectedProcesses(ctx context.Context, t *testing.T, memberClusterName string, reconciler *ReconcileAppDbReplicaSet, opsManager *omv1.MongoDBOpsManager, expectedHostnames []string, expectedProcessNames []string) { + ac, err := automationconfig.ReadFromSecret(ctx, reconciler.getMemberCluster(memberClusterName).SecretClient, types.NamespacedName{ + Namespace: opsManager.GetNamespace(), + Name: opsManager.Spec.AppDB.AutomationConfigSecretName(), + }) + require.NoError(t, err) + + assert.Equal(t, expectedHostnames, util.Transform(ac.Processes, func(obj automationconfig.Process) string { + return obj.HostName + })) + assert.Equal(t, expectedProcessNames, util.Transform(ac.Processes, func(obj automationconfig.Process) string { + return obj.Name + })) + + assert.Equal(t, expectedHostnames, reconciler.getCurrentStatefulsetHostnames(opsManager)) +} + +func reconcileAppDBOnceAndCheckExpectedProcesses(ctx context.Context, t *testing.T, kubeClient client.Client, omConnectionFactoryFunc om.ConnectionFactory, opsManager *omv1.MongoDBOpsManager, memberClusterMap map[string]client.Client, memberClusterName string, clusterSpecItems mdbv1.ClusterSpecList, expectedRequeue bool, expectedHostnames []string, expectedProcessNames []string, log *zap.SugaredLogger) { + opsManager.Spec.AppDB.ClusterSpecList = clusterSpecItems + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, log, omConnectionFactoryFunc) + require.NoError(t, err) + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + + if expectedRequeue { + // we're expected to scale one by one for expectedReconciles count + require.Greater(t, reconcileResult.RequeueAfter, time.Duration(0)) + } else { + require.Equal(t, util.TWENTY_FOUR_HOURS, reconcileResult.RequeueAfter) + } + + assertExpectedProcesses(ctx, t, memberClusterName, reconciler, opsManager, expectedHostnames, expectedProcessNames) +} + +func reconcileAppDBForExpectedNumberOfTimes(ctx context.Context, t *testing.T, kubeClient client.Client, omConnectionFactoryFunc *om.ConnectionFactory, opsManager *omv1.MongoDBOpsManager, memberClusterMap map[string]client.Client, clusterSpecItems mdbv1.ClusterSpecList, expectedReconciles int, log *zap.SugaredLogger) *ReconcileAppDbReplicaSet { + opsManager.Spec.AppDB.ClusterSpecList = clusterSpecItems + + var reconciler *ReconcileAppDbReplicaSet + var err error + for i := 0; i < expectedReconciles; i++ { + reconciler, err = newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, log, *omConnectionFactoryFunc) + require.NoError(t, err) + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + + // when scaling only the last final reconcile will be without requeueAfter + if i < expectedReconciles-1 { + // we're expected to scale one by one for expectedReconciles count + require.Greater(t, reconcileResult.RequeueAfter, time.Duration(0), "failed in reconcile %d", i) + } else { + ok, _ := workflow.OK().ReconcileResult() + require.Equal(t, ok, reconcileResult, "failed in reconcile %d", i) + } + } + return reconciler +} + +func reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedProcesses(ctx context.Context, t *testing.T, kubeClient client.Client, omConnectionFactoryFunc om.ConnectionFactory, opsManager *omv1.MongoDBOpsManager, memberClusterMap map[string]client.Client, memberClusterName string, clusterSpecItems mdbv1.ClusterSpecList, expectedHostnames []string, expectedProcessNames []string, expectedReconciles int, log *zap.SugaredLogger) { + reconciler := reconcileAppDBForExpectedNumberOfTimes(ctx, t, kubeClient, &omConnectionFactoryFunc, opsManager, memberClusterMap, clusterSpecItems, expectedReconciles, log) + + assertExpectedProcesses(ctx, t, memberClusterName, reconciler, opsManager, expectedHostnames, expectedProcessNames) +} + +func reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedHostnames(ctx context.Context, t *testing.T, kubeClient client.Client, omConnectionFactoryFunc om.ConnectionFactory, omConnectionFactory *om.CachedOMConnectionFactory, opsManager *omv1.MongoDBOpsManager, memberClusterMap map[string]client.Client, clusterSpecItems mdbv1.ClusterSpecList, expectedHostnames []string, expectedReconciles int, log *zap.SugaredLogger) { + reconcileAppDBForExpectedNumberOfTimes(ctx, t, kubeClient, &omConnectionFactoryFunc, opsManager, memberClusterMap, clusterSpecItems, expectedReconciles, log) + + assertExpectedHostnamesAndPreferred(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection), expectedHostnames) +} + +func makeClusterSpecList(clusters ...string) mdbv1.ClusterSpecList { + var clusterSpecItems mdbv1.ClusterSpecList + for _, clusterName := range clusters { + clusterSpecItems = append(clusterSpecItems, mdbv1.ClusterSpecItem{ClusterName: clusterName, Members: 1}) + } + return clusterSpecItems +} + +func TestAppDB_MultiCluster_ClusterMapping(t *testing.T) { + ctx := context.Background() + log := zap.S() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName1 := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + memberClusterName4 := "member-cluster-4" + memberClusterName5 := "member-cluster-5" + clusters := []string{centralClusterName, memberClusterName1, memberClusterName2, memberClusterName3, memberClusterName4, memberClusterName5} + + builder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(makeClusterSpecList(memberClusterName1, memberClusterName2)). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + + opsManager := builder.Build() + appdb := opsManager.Spec.AppDB + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + // prepare CA config map in central cluster + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + t.Run("check mapping cm has been created", func(t *testing.T) { + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + }) + }) + + t.Run("deployment state config map should be recreated after deletion", func(t *testing.T) { + deleteDeploymentStateConfigMap(ctx, t, kubeClient, appdb) + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + }) + }) + + t.Run("config map is updated after adding new cluster", func(t *testing.T) { + appdb.ClusterSpecList = makeClusterSpecList(memberClusterName1, memberClusterName2, memberClusterName3) + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + }) + }) + + t.Run("mapping is preserved if cluster is removed", func(t *testing.T) { + appdb.ClusterSpecList = makeClusterSpecList(memberClusterName1, memberClusterName3) + + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + }) + }) + + t.Run("new cluster is assigned new index instead of the next one", func(t *testing.T) { + appdb.ClusterSpecList = makeClusterSpecList(memberClusterName1, memberClusterName3, memberClusterName4) + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + memberClusterName4: 3, + }) + }) + + t.Run("empty cluster spec list does not change mapping", func(t *testing.T) { + appdb.ClusterSpecList = mdbv1.ClusterSpecList{} + + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + memberClusterName4: 3, + }) + }) + + t.Run("new cluster alone will get new index", func(t *testing.T) { + appdb.ClusterSpecList = makeClusterSpecList(memberClusterName5) + + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + memberClusterName4: 3, + memberClusterName5: 4, + }) + }) + + t.Run("defining clusters again will get their old indexes, order doesn't matter", func(t *testing.T) { + appdb.ClusterSpecList = makeClusterSpecList(memberClusterName4, memberClusterName2, memberClusterName3, memberClusterName1) + + reconciler.updateMemberClusterMapping(appdb) + require.NoError(t, reconciler.stateStore.WriteState(ctx, reconciler.deploymentState, log)) + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + memberClusterName4: 3, + memberClusterName5: 4, + }) + }) +} + +func TestAppDB_MultiCluster_ClusterMappingMigrationToDeploymentState(t *testing.T) { + ctx := context.Background() + log := zap.S() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName1 := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + clusters := []string{centralClusterName, memberClusterName1, memberClusterName2, memberClusterName3} + + builder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(makeClusterSpecList(memberClusterName1, memberClusterName2)). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + + opsManager := builder.Build() + lastAppliedMongoDBVersion := "5.0" + opsManager.Annotations = map[string]string{annotations.LastAppliedMongoDBVersion: lastAppliedMongoDBVersion} + appdb := opsManager.Spec.AppDB + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + legacyCM := configmap.Builder(). + SetName(appdb.Name() + "-cluster-mapping"). + SetNamespace(appdb.Namespace). + SetData(map[string]string{ + memberClusterName3: "2", + memberClusterName1: "1", + memberClusterName2: "0", + }).Build() + require.NoError(t, kubeClient.Create(ctx, &legacyCM)) + + legacyLastAppliedSpecCM := configmap.Builder(). + SetName(appdb.Name() + "-member-spec"). + SetNamespace(appdb.Namespace). + SetData(map[string]string{ + memberClusterName3: "3", + memberClusterName1: "1", + memberClusterName2: "2", + }).Build() + require.NoError(t, kubeClient.Create(ctx, &legacyLastAppliedSpecCM)) + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + t.Run("check legacy cm should be migrated to the new deployment state", func(t *testing.T) { + checkClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 1, + memberClusterName2: 0, + memberClusterName3: 2, + }) + checkLastAppliedMemberSpec(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), map[string]int{ + memberClusterName1: 1, + memberClusterName3: 3, + memberClusterName2: 2, + }) + checkLastAppliedMongoDBVersion(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), lastAppliedMongoDBVersion) + }) +} + +// This test ensures that we update legacy Config Maps on top of the new Deployment State +func TestAppDB_MultiCluster_KeepUpdatingLegacyState(t *testing.T) { + ctx := context.Background() + log := zap.S() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName1 := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + clusters := []string{centralClusterName, memberClusterName1, memberClusterName2} + + expectedLastAppliedMongoDBVersion := "6.0.0" + builder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(makeClusterSpecList(memberClusterName1, memberClusterName2)). + SetAppDbMembers(1). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster). + SetAppDbVersion(expectedLastAppliedMongoDBVersion) + + opsManager := builder.Build() + appdb := opsManager.Spec.AppDB + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + + t.Run("check that legacy config maps are created based on deployment state", func(t *testing.T) { + expectedClusterMapping := map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + } + checkLegacyClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), expectedClusterMapping) + + expectedLastAppliedMemberSpec := map[string]int{ + memberClusterName1: 1, + memberClusterName2: 1, + } + checkLegacyLastAppliedMemberSpec(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), expectedLastAppliedMemberSpec) + checkLegacyLastAppliedMongoDBVersion(ctx, t, reconciler.centralClient, opsManager.Namespace, opsManager.GetName(), expectedLastAppliedMongoDBVersion) + }) + + // Update the cluster spec lists and perform new reconcile + opsManager.Spec.AppDB.ClusterSpecList = makeClusterSpecList(memberClusterName1, memberClusterName3) + reconciler, err = newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + + t.Run("check that legacy config maps are updated on reconcile", func(t *testing.T) { + // Cluster 2 is not in cluster spec list anymore, but the reconciler should keep it in the index map + expectedClusterMapping := map[string]int{ + memberClusterName1: 0, + memberClusterName2: 1, + memberClusterName3: 2, + } + checkLegacyClusterMapping(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), expectedClusterMapping) + + expectedLastAppliedMemberSpec := map[string]int{ + // After a full reconciliation, the final state would be [memberClusterName1: 1, memberClusterName3: 1], + // but we only run one loop here, and we can only modify one member at a time, so we only scaled down + // the replica on cluster 2, and end up with 1-0-0 + memberClusterName1: 1, + memberClusterName2: 0, + memberClusterName3: 0, + } + checkLegacyLastAppliedMemberSpec(ctx, t, reconciler.centralClient, appdb.Namespace, appdb.Name(), expectedLastAppliedMemberSpec) + checkLegacyLastAppliedMongoDBVersion(ctx, t, reconciler.centralClient, opsManager.Namespace, opsManager.GetName(), expectedLastAppliedMongoDBVersion) + }) +} + +func deleteDeploymentStateConfigMap(ctx context.Context, t *testing.T, kubeClient client.Client, appdb omv1.AppDBSpec) { + cm := corev1.ConfigMap{} + err := kubeClient.Get(ctx, kube.ObjectKey(appdb.Namespace, appdb.Name()+"-state"), &cm) + require.NoError(t, err) + err = kubeClient.Delete(ctx, &cm) + require.NoError(t, err) +} + +func readDeploymentState[T any](ctx context.Context, t *testing.T, c client.Client, namespace string, resourceName string) *T { + cm := corev1.ConfigMap{} + err := c.Get(ctx, kube.ObjectKey(namespace, resourceName+"-state"), &cm) + require.NoError(t, err) + + stateStruct := new(T) + require.NoError(t, json.Unmarshal([]byte(cm.Data["state"]), stateStruct)) + + return stateStruct +} + +func checkClusterMapping(ctx context.Context, t *testing.T, c client.Client, namespace string, resourceName string, expectedMapping map[string]int) { + deploymentState := readDeploymentState[AppDBDeploymentState](ctx, t, c, namespace, resourceName) + assert.Equal(t, expectedMapping, deploymentState.ClusterMapping) +} + +func checkLastAppliedMemberSpec(ctx context.Context, t *testing.T, c client.Client, namespace string, resourceName string, expectedMemberSpec map[string]int) { + deploymentState := readDeploymentState[AppDBDeploymentState](ctx, t, c, namespace, resourceName) + assert.Equal(t, expectedMemberSpec, deploymentState.LastAppliedMemberSpec) +} + +func checkLastAppliedMongoDBVersion(ctx context.Context, t *testing.T, c client.Client, namespace string, resourceName string, expectedVersion string) { + deploymentState := readDeploymentState[AppDBDeploymentState](ctx, t, c, namespace, resourceName) + assert.Equal(t, expectedVersion, deploymentState.LastAppliedMongoDBVersion) +} + +func checkLegacyClusterMapping(ctx context.Context, t *testing.T, client client.Client, namespace, appdbName string, expectedData map[string]int) { + cm := &corev1.ConfigMap{} + err := client.Get(ctx, types.NamespacedName{Name: appdbName + "-cluster-mapping", Namespace: namespace}, cm) + require.NoError(t, err) + for k, v := range expectedData { + assert.Equal(t, strconv.Itoa(v), cm.Data[k]) + } +} + +func checkLegacyLastAppliedMemberSpec(ctx context.Context, t *testing.T, client client.Client, namespace, appdbName string, expectedData map[string]int) { + cm := &corev1.ConfigMap{} + err := client.Get(ctx, types.NamespacedName{Name: appdbName + "-member-spec", Namespace: namespace}, cm) + require.NoError(t, err) + for k, v := range expectedData { + assert.Equal(t, strconv.Itoa(v), cm.Data[k]) + } +} + +func checkLegacyLastAppliedMongoDBVersion(ctx context.Context, t *testing.T, client client.Client, namespace, omName, expectedVersion string) { + opsManager := &omv1.MongoDBOpsManager{} + err := client.Get(ctx, types.NamespacedName{Name: omName, Namespace: namespace}, opsManager) + require.NoError(t, err) + assert.Equal(t, expectedVersion, opsManager.Annotations[annotations.LastAppliedMongoDBVersion]) +} + +func createOMAPIKeySecret(ctx context.Context, t *testing.T, secretClient secrets.SecretClient, opsManager *omv1.MongoDBOpsManager) { + APIKeySecretName, err := opsManager.APIKeySecretName(ctx, secretClient, "") + assert.NoError(t, err) + + data := map[string]string{ + util.OmPublicApiKey: "publicApiKey", + util.OmPrivateKey: "privateApiKey", + } + + apiKeySecret := secret.Builder(). + SetNamespace(operatorNamespace()). + SetName(APIKeySecretName). + SetStringMapToData(data). + Build() + + err = secretClient.CreateSecret(ctx, apiKeySecret) + require.NoError(t, err) +} + +func createAppDbCAConfigMap(ctx context.Context, t *testing.T, k8sClient client.Client, appDBSpec omv1.AppDBSpec) string { + cert, _ := createMockCertAndKeyBytes() + cm := configmap.Builder(). + SetName(appDBSpec.GetCAConfigMapName()). + SetNamespace(appDBSpec.Namespace). + SetDataField("ca-pem", string(cert)). + Build() + + err := k8sClient.Create(ctx, &cm) + require.NoError(t, err) + + return appDBSpec.GetCAConfigMapName() +} + +func createAppDBTLSCert(ctx context.Context, t *testing.T, k8sClient client.Client, appDBSpec omv1.AppDBSpec) (string, string) { + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: appDBSpec.GetTlsCertificatesSecretName(), + Namespace: appDBSpec.Namespace, + }, + Type: corev1.SecretTypeTLS, + } + + certs := map[string][]byte{} + certs["tls.crt"], certs["tls.key"] = createMockCertAndKeyBytes() + + tlsSecret.Data = certs + err := k8sClient.Create(ctx, tlsSecret) + require.NoError(t, err) + + pemHash := enterprisepem.ReadHashFromData(secrets.DataToStringData(tlsSecret.Data), zap.S()) + require.NotEmpty(t, pemHash) + + return tlsSecret.Name, pemHash +} + +func TestAppDB_MultiCluster_ReconcilerFailsWhenThereIsNoClusterListConfigured(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(mdbv1.ClusterSpecList{ + { + ClusterName: "a", + Members: 2, + }, + }). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + opsManager := builder.Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + _, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + assert.Error(t, err) +} + +func TestAppDBMultiClusterRemoveResources(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(mdbv1.ClusterSpecList{ + { + ClusterName: "a", + Members: 2, + }, + { + ClusterName: "b", + Members: 2, + }, + { + ClusterName: "c", + Members: 1, + }, + }). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + + opsManager := builder.Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + clusters = []string{"a", "b", "c"} + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters, omConnectionFactory) + reconciler, _, _ := defaultTestOmReconciler(ctx, t, nil, "", "", opsManager, memberClusterMap, omConnectionFactory) + + // create opsmanager reconciler + appDBReconciler, _ := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, zap.S(), omConnectionFactory.GetConnectionFunc) + + // initially requeued as monitoring needs to be configured + _, err := appDBReconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + + // check AppDB statefulset exists in cluster "a" and cluster "b" + for clusterIdx, clusterSpecItem := range opsManager.Spec.AppDB.ClusterSpecList { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, opsManager.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + + memberClusterChecks.checkStatefulSet(ctx, opsManager.Spec.AppDB.NameForCluster(appDBReconciler.getMemberClusterIndex(clusterSpecItem.ClusterName)), clusterSpecItem.Members) + } + + // delete the OM resource + reconciler.OnDelete(ctx, opsManager, zap.S()) + assert.Zero(t, len(reconciler.resourceWatcher.GetWatchedResources())) + + // assert STS objects in member cluster + for clusterIdx, clusterSpecItem := range opsManager.Spec.AppDB.ClusterSpecList { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, opsManager.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + + memberClusterChecks.checkStatefulSetDoesNotExist(ctx, opsManager.Spec.AppDB.NameForCluster(appDBReconciler.getMemberClusterIndex(clusterSpecItem.ClusterName))) + } +} + +func TestAppDBMultiClusterMonitoringHostnames(t *testing.T) { + ctx := context.Background() + log := zap.S() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + clusters := []string{centralClusterName, memberClusterName, memberClusterName2, memberClusterName3} + + builder := DefaultOpsManagerBuilder(). + SetName("om"). + SetNamespace("ns"). + SetAppDBClusterSpecList(mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 3, + }, + { + ClusterName: memberClusterName3, + Members: 2, + }, + }, + ). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + + opsManager := builder.Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + globalClusterMap := getFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, globalClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + hostnames := reconciler.generateProcessHostnames(opsManager) + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-1-1-svc.ns.svc.cluster.local", + "om-db-1-2-svc.ns.svc.cluster.local", + "om-db-2-0-svc.ns.svc.cluster.local", + "om-db-2-1-svc.ns.svc.cluster.local", + } + + assert.Equal(t, expectedHostnames, hostnames) + + /* Default external domain */ + + opsManager.Spec.AppDB.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("custom.domain"), + } + + hostnames = reconciler.generateProcessHostnames(opsManager) + + expectedHostnames = []string{ + "om-db-0-0.custom.domain", + "om-db-0-1.custom.domain", + "om-db-1-0.custom.domain", + "om-db-1-1.custom.domain", + "om-db-1-2.custom.domain", + "om-db-2-0.custom.domain", + "om-db-2-1.custom.domain", + } + + assert.Equal(t, expectedHostnames, hostnames) + + /* Per cluster external domain mixed with default domain */ + + opsManager.Spec.AppDB.ClusterSpecList[0].ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster0.domain"), + } + opsManager.Spec.AppDB.ClusterSpecList[2].ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + } + + hostnames = reconciler.generateProcessHostnames(opsManager) + + expectedHostnames = []string{ + "om-db-0-0.cluster0.domain", + "om-db-0-1.cluster0.domain", + "om-db-1-0.custom.domain", + "om-db-1-1.custom.domain", + "om-db-1-2.custom.domain", + "om-db-2-0.cluster2.domain", + "om-db-2-1.cluster2.domain", + } + + assert.Equal(t, expectedHostnames, hostnames) +} + +func TestAppDBMultiClusterTryConfigureMonitoring(t *testing.T) { + ctx := context.Background() + log := zap.S() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName1 := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + clusters := []string{centralClusterName, memberClusterName1, memberClusterName2, memberClusterName3} + + builder := DefaultOpsManagerBuilder(). + SetName("om"). + SetNamespace("ns"). + SetAppDBClusterSpecList(mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + Members: 2, + }, + }, + ). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + + opsManager := builder.Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + globalClusterMap := getAppDBFakeMultiClusterMapWithClusters(clusters[1:], omConnectionFactory) + + err := createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, opsManagerUserPassword) + assert.NoError(t, err) + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, globalClusterMap, log, omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + // requeue is true to add monitoring + assert.True(t, reconcileResult.Requeue) + + // OM API Key secret is required for enabling monitoring to OM + createOMAPIKeySecret(ctx, t, reconciler.SecretClient, opsManager) + + t.Run("check expected hostnames", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + Members: 2, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + } + + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedHostnames(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, omConnectionFactory, opsManager, globalClusterMap, clusterSpecItems, expectedHostnames, 1, log) + + // teardown + clearHostsAndPreferredHostnames(omConnectionFactory) + }) + t.Run("Add second cluster", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 3, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-1-1-svc.ns.svc.cluster.local", + "om-db-1-2-svc.ns.svc.cluster.local", + } + + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedHostnames(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, omConnectionFactory, opsManager, globalClusterMap, clusterSpecItems, expectedHostnames, 3, log) + + // teardown + clearHostsAndPreferredHostnames(omConnectionFactory) + }) + t.Run("Add default external domain", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 3, + }, + } + + opsManager.Spec.AppDB.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("custom.domain")} + + expectedHostnames := []string{ + "om-db-0-0.custom.domain", + "om-db-0-1.custom.domain", + "om-db-1-0.custom.domain", + "om-db-1-1.custom.domain", + "om-db-1-2.custom.domain", + } + + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedHostnames(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, omConnectionFactory, opsManager, globalClusterMap, clusterSpecItems, expectedHostnames, 3, log) + + // teardown + clearHostsAndPreferredHostnames(omConnectionFactory) + opsManager.Spec.AppDB.ExternalAccessConfiguration = nil + }) + t.Run("Add external domain for second cluster", func(t *testing.T) { + clearHostsAndPreferredHostnames(omConnectionFactory) + + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("cluster-1.domain")}, + }, + } + + expectedHostnames := []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + "om-db-1-0.cluster-1.domain", + "om-db-1-1.cluster-1.domain", + "om-db-1-2.cluster-1.domain", + } + + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedHostnames(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, omConnectionFactory, opsManager, globalClusterMap, clusterSpecItems, expectedHostnames, 3, log) + + // teardown + clearHostsAndPreferredHostnames(omConnectionFactory) + opsManager.Spec.AppDB.ExternalAccessConfiguration = nil + }) + t.Run("Add default external domain and custom for second cluster + ", func(t *testing.T) { + clusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + Members: 2, + }, + { + ClusterName: memberClusterName2, + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("cluster-1.domain")}, + }, + } + + opsManager.Spec.AppDB.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("custom.domain")} + + expectedHostnames := []string{ + "om-db-0-0.custom.domain", + "om-db-0-1.custom.domain", + "om-db-1-0.cluster-1.domain", + "om-db-1-1.cluster-1.domain", + "om-db-1-2.cluster-1.domain", + } + + reconcileAppDBForExpectedNumberOfTimesAndCheckExpectedHostnames(ctx, t, kubeClient, omConnectionFactory.GetConnectionFunc, omConnectionFactory, opsManager, globalClusterMap, clusterSpecItems, expectedHostnames, 3, log) + + // teardown + clearHostsAndPreferredHostnames(omConnectionFactory) + opsManager.Spec.AppDB.ExternalAccessConfiguration = nil + }) +} + +func clearHostsAndPreferredHostnames(omConnectionFactory *om.CachedOMConnectionFactory) { + // clear preferred hostnames so they can be populated during reconcile + if omConnection := omConnectionFactory.GetConnection(); omConnection != nil { + mockedOmConnection := omConnection.(*om.MockedOmConnection) + mockedOmConnection.PreferredHostnames = make([]om.PreferredHostname, 0) + mockedOmConnection.ClearHosts() + } +} + +func TestAppDBMultiClusterServiceCreation_WithExternalName(t *testing.T) { + memberClusterName1 := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + memberClusters := []string{memberClusterName1, memberClusterName2, memberClusterName3} + + tests := map[string]struct { + clusterSpecList mdbv1.ClusterSpecList + uniformExternalAccess *mdbv1.ExternalAccessConfiguration + additionalMongodConfig *mdbv1.AdditionalMongodConfig + result map[string]map[int]corev1.Service + }{ + "empty external access configured for single pod in first cluster": { + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{}, + Members: 1, + }, + }, + result: map[string]map[int]corev1.Service{ + memberClusterName1: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + }, + }, + }, + }, + "external access configured for two pods in first cluster": { + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalService: mdbv1.ExternalServiceConfiguration{ + SpecWrapper: &mdbv1.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + { + Name: "backup", + Port: 27018, + }, + { + Name: "testing2", + Port: 27019, + }, + }, + }, + }, + }, + }, + Members: 2, + }, + }, + result: map[string]map[int]corev1.Service{ + memberClusterName1: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + { + Name: "backup", + Port: 27018, + }, + { + Name: "testing2", + Port: 27019, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + }, + 1: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-1-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-1", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + { + Name: "backup", + Port: 27018, + }, + { + Name: "testing2", + Port: 27019, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-1", + }, + }, + }, + }, + }, + }, + "external domain configured for single pod in first cluster": { + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("custom.domain"), + }, + Members: 1, + }, + }, + result: map[string]map[int]corev1.Service{ + memberClusterName1: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + }, + }, + }, + }, + "non default port set in additional mongod config": { + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{}, + Members: 1, + }, + }, + additionalMongodConfig: mdbv1.NewAdditionalMongodConfig("net.port", 27027), + result: map[string]map[int]corev1.Service{ + memberClusterName1: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27027, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + }, + }, + }, + }, + "external service of NodePort type set in first cluster": { + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalService: mdbv1.ExternalServiceConfiguration{ + SpecWrapper: &mdbv1.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "NodePort", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + NodePort: 30003, + }, + }, + }, + }, + }, + }, + Members: 1, + }, + }, + result: map[string]map[int]corev1.Service{ + memberClusterName1: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "NodePort", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + NodePort: 30003, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + }, + }, + }, + }, + "service with annotations with placeholders": { + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalService: mdbv1.ExternalServiceConfiguration{ + Annotations: map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + create.PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + create.PlaceholderClusterName: "{clusterName}", + create.PlaceholderClusterIndex: "{clusterIndex}", + }, + SpecWrapper: &mdbv1.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + }, + }, + }, + }, + Members: 2, + }, + { + ClusterName: memberClusterName2, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalService: mdbv1.ExternalServiceConfiguration{ + Annotations: map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + create.PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + create.PlaceholderClusterName: "{clusterName}", + create.PlaceholderClusterIndex: "{clusterIndex}", + }, + SpecWrapper: &mdbv1.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + }, + }, + }, + ExternalDomain: ptr.To("custom.domain"), + }, + Members: 2, + }, + }, + uniformExternalAccess: &mdbv1.ExternalAccessConfiguration{ + ExternalService: mdbv1.ExternalServiceConfiguration{ + Annotations: map[string]string{ + "test-annotation": "test-placeholder-{podIndex}", + }, + }, + }, + result: map[string]map[int]corev1.Service{ + memberClusterName1: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + Annotations: map[string]string{ + "test-annotation": "test-placeholder-0", + create.PlaceholderPodIndex: "0", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db-0", + create.PlaceholderPodName: "test-om-db-0-0", + create.PlaceholderExternalServiceName: "test-om-db-0-0-svc-external", + create.PlaceholderMongodProcessDomain: "my-namespace.svc.cluster.local", + create.PlaceholderMongodProcessFQDN: "test-om-db-0-0-svc.my-namespace.svc.cluster.local", + create.PlaceholderClusterName: memberClusterName1, + create.PlaceholderClusterIndex: "0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-0", + }, + }, + }, + 1: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-1-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-1", + }, + Annotations: map[string]string{ + "test-annotation": "test-placeholder-1", + create.PlaceholderPodIndex: "1", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db-0", + create.PlaceholderPodName: "test-om-db-0-1", + create.PlaceholderExternalServiceName: "test-om-db-0-1-svc-external", + create.PlaceholderMongodProcessDomain: "my-namespace.svc.cluster.local", + create.PlaceholderMongodProcessFQDN: "test-om-db-0-1-svc.my-namespace.svc.cluster.local", + create.PlaceholderClusterName: memberClusterName1, + create.PlaceholderClusterIndex: "0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0-1", + }, + }, + }, + }, + memberClusterName2: { + 0: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-1-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1-0", + }, + Annotations: map[string]string{ + "test-annotation": "test-placeholder-0", + create.PlaceholderPodIndex: "0", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db-1", + create.PlaceholderPodName: "test-om-db-1-0", + create.PlaceholderExternalServiceName: "test-om-db-1-0-svc-external", + create.PlaceholderMongodProcessDomain: "custom.domain", + create.PlaceholderMongodProcessFQDN: "test-om-db-1-0.custom.domain", + create.PlaceholderClusterName: memberClusterName2, + create.PlaceholderClusterIndex: "1", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1-0", + }, + }, + }, + 1: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-1-1-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1-1", + }, + Annotations: map[string]string{ + "test-annotation": "test-placeholder-1", + create.PlaceholderPodIndex: "1", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db-1", + create.PlaceholderPodName: "test-om-db-1-1", + create.PlaceholderExternalServiceName: "test-om-db-1-1-svc-external", + create.PlaceholderMongodProcessDomain: "custom.domain", + create.PlaceholderMongodProcessFQDN: "test-om-db-1-1.custom.domain", + create.PlaceholderClusterName: memberClusterName2, + create.PlaceholderClusterIndex: "1", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1-1", + }, + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + opsManagerBuilder := DefaultOpsManagerBuilder(). + SetAppDBClusterSpecList(test.clusterSpecList). + SetAppDbMembers(0). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster). + SetAdditionalMongodbConfig(test.additionalMongodConfig) + + if test.uniformExternalAccess != nil { + opsManagerBuilder.SetAppDbExternalAccess(*test.uniformExternalAccess) + } + + opsManager := opsManagerBuilder.Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + memberClusterMap := getFakeMultiClusterMapWithClusters(memberClusters, omConnectionFactory) + + reconciler, err := newAppDbMultiReconciler(ctx, kubeClient, opsManager, memberClusterMap, zap.S(), omConnectionFactory.GetConnectionFunc) + require.NoError(t, err) + + err = createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, opsManagerUserPassword) + assert.NoError(t, err) + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + // requeue is true to add monitoring + assert.True(t, reconcileResult.Requeue) + + clusterSpecList := opsManager.Spec.AppDB.GetClusterSpecList() + for clusterIdx, item := range clusterSpecList { + clusterName := item.ClusterName + c := memberClusterMap[clusterName] + for podIdx := 0; podIdx < item.Members; podIdx++ { + svcName := dns.GetMultiExternalServiceName(opsManager.Spec.AppDB.GetName(), clusterIdx, podIdx) + svcNamespace := opsManager.Namespace + + svcList := corev1.ServiceList{} + err = c.List(ctx, &svcList) + assert.NoError(t, err) + + actualSvc := corev1.Service{} + + err = c.Get(ctx, kube.ObjectKey(svcNamespace, svcName), &actualSvc) + assert.NoError(t, err) + + expectedSvc := test.result[clusterName][podIdx] + assert.Equal(t, expectedSvc, actualSvc) + } + } + }) + } +} diff --git a/controllers/operator/appdbreplicaset_controller_test.go b/controllers/operator/appdbreplicaset_controller_test.go new file mode 100644 index 000000000..21a551dc3 --- /dev/null +++ b/controllers/operator/appdbreplicaset_controller_test.go @@ -0,0 +1,1446 @@ +package operator + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/agentVersionManagement" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func init() { + mock.InitDefaultEnvVariables() +} + +// getReleaseJsonPath searches for a specified target directory by traversing the directory tree backwards from the current working directory +func getReleaseJsonPath() (string, error) { + repositoryRootDirName := "ops-manager-kubernetes" + releaseFileName := "release.json" + + currentDir, err := os.Getwd() + if err != nil { + return "", err + } + for currentDir != "/" { + if strings.HasSuffix(currentDir, repositoryRootDirName) { + return filepath.Join(currentDir, releaseFileName), nil + } + currentDir = filepath.Dir(currentDir) + } + return currentDir, err +} + +// This approach ensures all test methods in this file have properly defined variables. +func TestMain(m *testing.M) { + path, _ := getReleaseJsonPath() + _ = os.Setenv(agentVersionManagement.MappingFilePathEnv, path) // nolint:forbidigo + defer func(key string) { + _ = os.Unsetenv(key) // nolint:forbidigo + }(agentVersionManagement.MappingFilePathEnv) + code := m.Run() + os.Exit(code) +} + +func TestMongoDB_ConnectionURL_DefaultCluster_AppDB(t *testing.T) { + opsManager := DefaultOpsManagerBuilder().Build() + appdb := opsManager.Spec.AppDB + + var cnx string + cnx = appdb.BuildConnectionURL("user", "passwd", connectionstring.SchemeMongoDB, nil, nil) + assert.Equal(t, "mongodb://user:passwd@test-om-db-0.test-om-db-svc.my-namespace.svc.cluster.local:27017,"+ + "test-om-db-1.test-om-db-svc.my-namespace.svc.cluster.local:27017,test-om-db-2.test-om-db-svc.my-namespace.svc.cluster.local:27017/"+ + "?authMechanism=SCRAM-SHA-256&authSource=admin&connectTimeoutMS=20000&replicaSet=test-om-db&serverSelectionTimeoutMS=20000", cnx) + + // Special symbols in the url + cnx = appdb.BuildConnectionURL("special/user#", "@passw!", connectionstring.SchemeMongoDB, nil, nil) + assert.Equal(t, "mongodb://special%2Fuser%23:%40passw%21@test-om-db-0.test-om-db-svc.my-namespace.svc.cluster.local:27017,"+ + "test-om-db-1.test-om-db-svc.my-namespace.svc.cluster.local:27017,test-om-db-2.test-om-db-svc.my-namespace.svc.cluster.local:27017/"+ + "?authMechanism=SCRAM-SHA-256&authSource=admin&connectTimeoutMS=20000&replicaSet=test-om-db&serverSelectionTimeoutMS=20000", cnx) + + // Connection parameters. The default one is overridden + cnx = appdb.BuildConnectionURL("user", "passwd", connectionstring.SchemeMongoDB, map[string]string{"connectTimeoutMS": "30000", "readPreference": "secondary"}, nil) + assert.Equal(t, "mongodb://user:passwd@test-om-db-0.test-om-db-svc.my-namespace.svc.cluster.local:27017,"+ + "test-om-db-1.test-om-db-svc.my-namespace.svc.cluster.local:27017,test-om-db-2.test-om-db-svc.my-namespace.svc.cluster.local:27017/"+ + "?authMechanism=SCRAM-SHA-256&authSource=admin&connectTimeoutMS=30000&readPreference=secondary&replicaSet=test-om-db&serverSelectionTimeoutMS=20000", + cnx) +} + +func TestMongoDB_ConnectionURL_OtherCluster_AppDB(t *testing.T) { + opsManager := DefaultOpsManagerBuilder().SetClusterDomain("my-cluster").Build() + appdb := opsManager.Spec.AppDB + + var cnx string + cnx = appdb.BuildConnectionURL("user", "passwd", connectionstring.SchemeMongoDB, nil, nil) + assert.Equal(t, "mongodb://user:passwd@test-om-db-0.test-om-db-svc.my-namespace.svc.my-cluster:27017,"+ + "test-om-db-1.test-om-db-svc.my-namespace.svc.my-cluster:27017,test-om-db-2.test-om-db-svc.my-namespace.svc.my-cluster:27017/"+ + "?authMechanism=SCRAM-SHA-256&authSource=admin&connectTimeoutMS=20000&replicaSet=test-om-db&serverSelectionTimeoutMS=20000", cnx) + + // Connection parameters. The default one is overridden + cnx = appdb.BuildConnectionURL("user", "passwd", connectionstring.SchemeMongoDB, map[string]string{"connectTimeoutMS": "30000", "readPreference": "secondary"}, nil) + assert.Equal(t, "mongodb://user:passwd@test-om-db-0.test-om-db-svc.my-namespace.svc.my-cluster:27017,"+ + "test-om-db-1.test-om-db-svc.my-namespace.svc.my-cluster:27017,test-om-db-2.test-om-db-svc.my-namespace.svc.my-cluster:27017/"+ + "?authMechanism=SCRAM-SHA-256&authSource=admin&connectTimeoutMS=30000&readPreference=secondary&replicaSet=test-om-db&serverSelectionTimeoutMS=20000", + cnx) +} + +// TestAutomationConfig_IsCreatedInSecret verifies that the automation config is created in a secret. +func TestAutomationConfig_IsCreatedInSecret(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + appdb := opsManager.Spec.AppDB + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + err = createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, "MBPYfkAj5ZM0l9uw6C7ggw") + assert.NoError(t, err) + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + + s := corev1.Secret{} + err = kubeClient.Get(ctx, kube.ObjectKey(opsManager.Namespace, appdb.AutomationConfigSecretName()), &s) + assert.NoError(t, err, "The Automation Config was created in a secret.") + assert.Contains(t, s.Data, automationconfig.ConfigKey) +} + +// TestPublishAutomationConfigCreate verifies that the automation config map is created if it doesn't exist +func TestPublishAutomationConfigCreate(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + appdb := opsManager.Spec.AppDB + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + kubeClient := mock.NewEmptyFakeClientWithInterceptor(omConnectionFactory) + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + automationConfig, err := buildAutomationConfigForAppDb(ctx, builder, kubeClient, omConnectionFactory.GetConnectionFunc, automation, zap.S()) + assert.NoError(t, err) + version, err := reconciler.publishAutomationConfig(ctx, opsManager, automationConfig, appdb.AutomationConfigSecretName(), multicluster.LegacyCentralClusterName) + assert.NoError(t, err) + assert.Equal(t, 1, version) + + monitoringAutomationConfig, err := buildAutomationConfigForAppDb(ctx, builder, kubeClient, omConnectionFactory.GetConnectionFunc, monitoring, zap.S()) + assert.NoError(t, err) + version, err = reconciler.publishAutomationConfig(ctx, opsManager, monitoringAutomationConfig, appdb.MonitoringAutomationConfigSecretName(), multicluster.LegacyCentralClusterName) + assert.NoError(t, err) + assert.Equal(t, 1, version) + + // verify the automation config secret for the automation agent + acSecret := readAutomationConfigSecret(ctx, t, kubeClient, opsManager) + checkDeploymentEqualToPublished(t, automationConfig, acSecret) + + // verify the automation config secret for the monitoring agent + acMonitoringSecret := readAutomationConfigMonitoringSecret(ctx, t, kubeClient, opsManager) + checkDeploymentEqualToPublished(t, monitoringAutomationConfig, acMonitoringSecret) + + assert.Len(t, mock.GetMapForObject(kubeClient, &corev1.Secret{}), 6) + _, err = kubeClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, appdb.GetOpsManagerUserPasswordSecretName())) + assert.NoError(t, err) + + _, err = kubeClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, appdb.GetAgentKeyfileSecretNamespacedName().Name)) + assert.NoError(t, err) + + _, err = kubeClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, appdb.GetAgentPasswordSecretNamespacedName().Name)) + assert.NoError(t, err) + + _, err = kubeClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, appdb.OpsManagerUserScramCredentialsName())) + assert.NoError(t, err) + + _, err = kubeClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, appdb.AutomationConfigSecretName())) + assert.NoError(t, err) + + _, err = kubeClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, appdb.MonitoringAutomationConfigSecretName())) + assert.NoError(t, err) + + // verifies Users and Roles are created + assert.Len(t, automationConfig.Auth.Users, 1) + + expectedRoles := []string{"readWriteAnyDatabase", "dbAdminAnyDatabase", "clusterMonitor", "backup", "restore", "hostManager"} + assert.Len(t, automationConfig.Auth.Users[0].Roles, len(expectedRoles)) + for idx, role := range expectedRoles { + assert.Equal(t, automationConfig.Auth.Users[0].Roles[idx], + automationconfig.Role{ + Role: role, + Database: "admin", + }) + } +} + +// TestPublishAutomationConfig_Update verifies that the automation config map is updated if it has changed +func TestPublishAutomationConfig_Update(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + appdb := opsManager.Spec.AppDB + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(opsManager) + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + // create + _ = createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, "MBPYfkAj5ZM0l9uw6C7ggw") + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + + ac, err := automationconfig.ReadFromSecret(ctx, reconciler.client, kube.ObjectKey(opsManager.Namespace, appdb.AutomationConfigSecretName())) + assert.NoError(t, err) + assert.Equal(t, 1, ac.Version) + + // publishing the config without updates should not result in API call + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + + ac, err = automationconfig.ReadFromSecret(ctx, reconciler.client, kube.ObjectKey(opsManager.Namespace, appdb.AutomationConfigSecretName())) + assert.NoError(t, err) + assert.Equal(t, 1, ac.Version) + + // publishing changed config will result in update + fcv := "4.4" + err = reconciler.client.Get(ctx, kube.ObjectKeyFromApiObject(opsManager), opsManager) + require.NoError(t, err) + + opsManager.Spec.AppDB.FeatureCompatibilityVersion = &fcv + err = kubeClient.Update(ctx, opsManager) + assert.NoError(t, err) + + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + + ac, err = automationconfig.ReadFromSecret(ctx, reconciler.client, kube.ObjectKey(opsManager.Namespace, appdb.AutomationConfigSecretName())) + assert.NoError(t, err) + assert.Equal(t, 2, ac.Version) +} + +// TestBuildAppDbAutomationConfig checks that the automation config is built correctly +func TestBuildAppDbAutomationConfig(t *testing.T) { + ctx := context.Background() + logRotateConfig := &automationconfig.CrdLogRotate{ + SizeThresholdMB: "1", + } + builder := DefaultOpsManagerBuilder(). + SetAppDbMembers(2). + SetAppDbVersion("4.2.11-ent"). + SetAppDbFeatureCompatibility("4.0"). + SetLogRotate(logRotateConfig). + SetSystemLog(&automationconfig.SystemLog{ + Destination: automationconfig.File, + Path: "/tmp/test", + }) + + om := builder.Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(om) + err := createOpsManagerUserPasswordSecret(ctx, kubeClient, om, "omPass") + assert.NoError(t, err) + + automationConfig, err := buildAutomationConfigForAppDb(ctx, builder, kubeClient, omConnectionFactory.GetConnectionFunc, automation, zap.S()) + assert.NoError(t, err) + monitoringAutomationConfig, err := buildAutomationConfigForAppDb(ctx, builder, kubeClient, omConnectionFactory.GetConnectionFunc, monitoring, zap.S()) + assert.NoError(t, err) + // processes + assert.Len(t, automationConfig.Processes, 2) + assert.Equal(t, "4.2.11-ent", automationConfig.Processes[0].Version) + assert.Equal(t, "test-om-db-0.test-om-db-svc.my-namespace.svc.cluster.local", automationConfig.Processes[0].HostName) + assert.Equal(t, "4.0", automationConfig.Processes[0].FeatureCompatibilityVersion) + assert.Equal(t, "4.2.11-ent", automationConfig.Processes[1].Version) + assert.Equal(t, "test-om-db-1.test-om-db-svc.my-namespace.svc.cluster.local", automationConfig.Processes[1].HostName) + assert.Equal(t, "4.0", automationConfig.Processes[1].FeatureCompatibilityVersion) + assert.Len(t, monitoringAutomationConfig.Processes, 0) + assert.Len(t, monitoringAutomationConfig.ReplicaSets, 0) + assert.Equal(t, automationconfig.ConvertCrdLogRotateToAC(logRotateConfig), automationConfig.Processes[0].LogRotate) + assert.Equal(t, "/tmp/test", automationConfig.Processes[0].Args26.Get("systemLog.path").String()) + assert.Equal(t, "file", automationConfig.Processes[0].Args26.Get("systemLog.destination").String()) + + // replicasets + assert.Len(t, automationConfig.ReplicaSets, 1) + db := builder.Build().Spec.AppDB + assert.Equal(t, db.Name(), automationConfig.ReplicaSets[0].Id) + + // monitoring agent has been configured + assert.Len(t, automationConfig.MonitoringVersions, 0) + + // backup agents have not been configured + assert.Len(t, automationConfig.BackupVersions, 0) + + // options + assert.Equal(t, automationconfig.Options{DownloadBase: util.AgentDownloadsDir}, automationConfig.Options) +} + +func TestRegisterAppDBHostsWithProject(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(om.NewDeployment())) + fakeClient := mock.NewEmptyFakeClientBuilder().Build() + fakeClient = interceptor.NewClient(fakeClient, interceptor.Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, true, false)(ctx, client, key, obj, opts...) + }, + }) + reconciler, err := newAppDbReconciler(ctx, fakeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + t.Run("Ensure all hosts are added", func(t *testing.T) { + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + + hostnames := reconciler.getCurrentStatefulsetHostnames(opsManager) + err = reconciler.registerAppDBHostsWithProject(hostnames, omConnectionFactory.GetConnection(), "password", zap.S()) + assert.NoError(t, err) + + hosts, _ := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetHosts() + assert.Len(t, hosts.Results, 3) + }) + + t.Run("Ensure hosts are added when scaled up", func(t *testing.T) { + opsManager.Spec.AppDB.Members = 5 + _, err = reconciler.ReconcileAppDB(ctx, opsManager) + + hostnames := reconciler.getCurrentStatefulsetHostnames(opsManager) + err = reconciler.registerAppDBHostsWithProject(hostnames, omConnectionFactory.GetConnection(), "password", zap.S()) + assert.NoError(t, err) + + hosts, _ := omConnectionFactory.GetConnection().GetHosts() + assert.Len(t, hosts.Results, 5) + }) +} + +func TestEnsureAppDbAgentApiKey(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + // we need to pre-initialize connection as we don't call full reconciler in this test and connection is never created by calling connection factory func + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(om.NewDeployment())) + fakeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory) + reconciler, err := newAppDbReconciler(ctx, fakeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + omConnectionFactory.GetConnection().(*om.MockedOmConnection).AgentAPIKey = "my-api-key" + _, err = reconciler.ensureAppDbAgentApiKey(ctx, opsManager, omConnectionFactory.GetConnection(), omConnectionFactory.GetConnection().GroupID(), zap.S()) + assert.NoError(t, err) + + secretName := agents.ApiKeySecretName(omConnectionFactory.GetConnection().GroupID()) + apiKey, err := secret.ReadKey(ctx, reconciler.client, util.OmAgentApiKey, kube.ObjectKey(opsManager.Namespace, secretName)) + assert.NoError(t, err) + assert.Equal(t, "my-api-key", apiKey) +} + +func TestTryConfigureMonitoringInOpsManager(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + appdbScaler := scalers.GetAppDBScaler(opsManager, multicluster.LegacyCentralClusterName, 0, nil) + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + // attempt configuring monitoring when there is no api key secret + podVars, err := reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S()) + assert.NoError(t, err) + + assert.Empty(t, podVars.ProjectID) + assert.Empty(t, podVars.User) + + opsManager.Spec.AppDB.Members = 5 + appDbSts, err := construct.AppDbStatefulSet(*opsManager, &podVars, construct.AppDBStatefulSetOptions{}, appdbScaler, v1.OnDeleteStatefulSetStrategyType, zap.S()) + assert.NoError(t, err) + + assert.Nil(t, findVolumeByName(appDbSts.Spec.Template.Spec.Volumes, construct.AgentAPIKeyVolumeName)) + + _ = kubeClient.Update(ctx, &appDbSts) + + data := map[string]string{ + util.OmPublicApiKey: "publicApiKey", + util.OmPrivateKey: "privateApiKey", + } + APIKeySecretName, err := opsManager.APIKeySecretName(ctx, secrets.SecretClient{KubeClient: kubeClient}, "") + assert.NoError(t, err) + + apiKeySecret := secret.Builder(). + SetNamespace(operatorNamespace()). + SetName(APIKeySecretName). + SetStringMapToData(data). + Build() + + err = reconciler.client.CreateSecret(ctx, apiKeySecret) + assert.NoError(t, err) + + // once the secret exists, monitoring should be fully configured + podVars, err = reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S()) + assert.NoError(t, err) + + assert.Equal(t, om.TestGroupID, podVars.ProjectID) + assert.Equal(t, "publicApiKey", podVars.User) + + expectedHostnames := []string{ + "test-om-db-0.test-om-db-svc.my-namespace.svc.cluster.local", + "test-om-db-1.test-om-db-svc.my-namespace.svc.cluster.local", + "test-om-db-2.test-om-db-svc.my-namespace.svc.cluster.local", + "test-om-db-3.test-om-db-svc.my-namespace.svc.cluster.local", + "test-om-db-4.test-om-db-svc.my-namespace.svc.cluster.local", + } + + assertExpectedHostnamesAndPreferred(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection), expectedHostnames) + + appDbSts, err = construct.AppDbStatefulSet(*opsManager, &podVars, construct.AppDBStatefulSetOptions{}, appdbScaler, v1.OnDeleteStatefulSetStrategyType, zap.S()) + assert.NoError(t, err) + + assert.NotNil(t, findVolumeByName(appDbSts.Spec.Template.Spec.Volumes, construct.AgentAPIKeyVolumeName)) +} + +// TestTryConfigureMonitoringInOpsManagerWithCustomTemplate runs different scenarios with activating monitoring and pod templates +func TestTryConfigureMonitoringInOpsManagerWithCustomTemplate(t *testing.T) { + builder := DefaultOpsManagerBuilder() + opsManager := builder.Build() + appdbScaler := scalers.GetAppDBScaler(opsManager, multicluster.LegacyCentralClusterName, 0, nil) + + opsManager.Spec.AppDB.PodSpec.PodTemplateWrapper = mdb.PodTemplateSpecWrapper{ + PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "mongodb-agent", + Image: "quay.io/mongodb/mongodb-agent-ubi:10", + }, + { + Name: "mongod", + Image: "quay.io/mongodb/mongodb:10", + }, + { + Name: "mongodb-agent-monitoring", + Image: "quay.io/mongodb/mongodb-agent-ubi:20", + }, + }, + }, + }, + } + + t.Run("do not override images while activating monitoring", func(t *testing.T) { + podVars := env.PodEnvVars{ProjectID: "something"} + appDbSts, err := construct.AppDbStatefulSet(*opsManager, &podVars, construct.AppDBStatefulSetOptions{}, appdbScaler, v1.OnDeleteStatefulSetStrategyType, zap.S()) + assert.NoError(t, err) + assert.NotNil(t, appDbSts) + + foundImages := 0 + for _, c := range appDbSts.Spec.Template.Spec.Containers { + if c.Name == "mongodb-agent" { + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:10", c.Image) + foundImages += 1 + } + if c.Name == "mongod" { + assert.Equal(t, "quay.io/mongodb/mongodb:10", c.Image) + foundImages += 1 + } + if c.Name == "mongodb-agent-monitoring" { + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:20", c.Image) + foundImages += 1 + } + } + + assert.Equal(t, 3, foundImages) + assert.Equal(t, 3, len(appDbSts.Spec.Template.Spec.Containers)) + }) + + t.Run("do not override images, but remove monitoring if not activated", func(t *testing.T) { + podVars := env.PodEnvVars{} + appDbSts, err := construct.AppDbStatefulSet(*opsManager, &podVars, construct.AppDBStatefulSetOptions{}, appdbScaler, v1.OnDeleteStatefulSetStrategyType, zap.S()) + assert.NoError(t, err) + assert.NotNil(t, appDbSts) + + foundImages := 0 + for _, c := range appDbSts.Spec.Template.Spec.Containers { + if c.Name == "mongodb-agent" { + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:10", c.Image) + foundImages += 1 + } + if c.Name == "mongod" { + assert.Equal(t, "quay.io/mongodb/mongodb:10", c.Image) + foundImages += 1 + } + if c.Name == "mongodb-agent-monitoring" { + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:20", c.Image) + foundImages += 1 + } + } + + assert.Equal(t, 2, foundImages) + assert.Equal(t, 2, len(appDbSts.Spec.Template.Spec.Containers)) + }) +} + +func TestTryConfigureMonitoringInOpsManagerWithExternalDomains(t *testing.T) { + ctx := context.Background() + opsManager := DefaultOpsManagerBuilder(). + SetAppDbExternalAccess(mdb.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("custom.domain"), + }).Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + appdbScaler := scalers.GetAppDBScaler(opsManager, multicluster.LegacyCentralClusterName, 0, nil) + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + // attempt configuring monitoring when there is no api key secret + podVars, err := reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S()) + assert.NoError(t, err) + + assert.Empty(t, podVars.ProjectID) + assert.Empty(t, podVars.User) + + opsManager.Spec.AppDB.Members = 5 + appDbSts, err := construct.AppDbStatefulSet(*opsManager, &podVars, construct.AppDBStatefulSetOptions{}, appdbScaler, v1.OnDeleteStatefulSetStrategyType, zap.S()) + assert.NoError(t, err) + + assert.Nil(t, findVolumeByName(appDbSts.Spec.Template.Spec.Volumes, construct.AgentAPIKeyVolumeName)) + + _ = kubeClient.Update(ctx, &appDbSts) + + data := map[string]string{ + util.OmPublicApiKey: "publicApiKey", + util.OmPrivateKey: "privateApiKey", + } + APIKeySecretName, err := opsManager.APIKeySecretName(ctx, secrets.SecretClient{KubeClient: kubeClient}, "") + assert.NoError(t, err) + + apiKeySecret := secret.Builder(). + SetNamespace(operatorNamespace()). + SetName(APIKeySecretName). + SetStringMapToData(data). + Build() + + err = reconciler.client.CreateSecret(ctx, apiKeySecret) + assert.NoError(t, err) + + // once the secret exists, monitoring should be fully configured + podVars, err = reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S()) + assert.NoError(t, err) + + assert.Equal(t, om.TestGroupID, podVars.ProjectID) + assert.Equal(t, "publicApiKey", podVars.User) + + expectedHostnames := []string{ + "test-om-db-0.custom.domain", + "test-om-db-1.custom.domain", + "test-om-db-2.custom.domain", + "test-om-db-3.custom.domain", + "test-om-db-4.custom.domain", + } + + assertExpectedHostnamesAndPreferred(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection), expectedHostnames) + + appDbSts, err = construct.AppDbStatefulSet(*opsManager, &podVars, construct.AppDBStatefulSetOptions{}, appdbScaler, v1.OnDeleteStatefulSetStrategyType, zap.S()) + assert.NoError(t, err) + + assert.NotNil(t, findVolumeByName(appDbSts.Spec.Template.Spec.Volumes, construct.AgentAPIKeyVolumeName)) +} + +func TestAppDBServiceCreation_WithExternalName(t *testing.T) { + tests := map[string]struct { + members int + externalAccess *mdb.ExternalAccessConfiguration + additionalMongodConfig *mdb.AdditionalMongodConfig + result map[int]corev1.Service + }{ + "empty external access configured for one pod": { + members: 1, + externalAccess: &mdb.ExternalAccessConfiguration{}, + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + }, + }, + "external access configured for two pods": { + members: 2, + externalAccess: &mdb.ExternalAccessConfiguration{ + ExternalService: mdb.ExternalServiceConfiguration{ + SpecWrapper: &mdb.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + { + Name: "backup", + Port: 27018, + }, + { + Name: "testing2", + Port: 27019, + }, + }, + }, + }, + }, + }, + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + { + Name: "backup", + Port: 27018, + }, + { + Name: "testing2", + Port: 27019, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + 1: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-1-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + { + Name: "backup", + Port: 27018, + }, + { + Name: "testing2", + Port: 27019, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1", + }, + }, + }, + }, + }, + "external domain configured for single pod in first cluster": { + members: 1, + externalAccess: &mdb.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.domain"), + }, + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + }, + }, + "non default port set in additional mongod config": { + members: 1, + externalAccess: &mdb.ExternalAccessConfiguration{}, + additionalMongodConfig: mdb.NewAdditionalMongodConfig("net.port", 27027), + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27027, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + }, + }, + "external service of NodePort type set in first cluster": { + members: 1, + externalAccess: &mdb.ExternalAccessConfiguration{ + ExternalService: mdb.ExternalServiceConfiguration{ + SpecWrapper: &mdb.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "NodePort", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + NodePort: 30003, + }, + }, + }, + }, + }, + }, + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "NodePort", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + NodePort: 30003, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + }, + }, + "service with annotations with placeholders": { + members: 2, + externalAccess: &mdb.ExternalAccessConfiguration{ + ExternalService: mdb.ExternalServiceConfiguration{ + Annotations: map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + create.PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + }, + SpecWrapper: &mdb.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + }, + }, + }, + }, + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + Annotations: map[string]string{ + create.PlaceholderPodIndex: "0", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db", + create.PlaceholderPodName: "test-om-db-0", + create.PlaceholderExternalServiceName: "test-om-db-0-svc-external", + create.PlaceholderMongodProcessDomain: "test-om-db-svc.my-namespace.svc.cluster.local", + create.PlaceholderMongodProcessFQDN: "test-om-db-0.test-om-db-svc.my-namespace.svc.cluster.local", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + 1: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-1-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1", + }, + Annotations: map[string]string{ + create.PlaceholderPodIndex: "1", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db", + create.PlaceholderPodName: "test-om-db-1", + create.PlaceholderExternalServiceName: "test-om-db-1-svc-external", + create.PlaceholderMongodProcessDomain: "test-om-db-svc.my-namespace.svc.cluster.local", + create.PlaceholderMongodProcessFQDN: "test-om-db-1.test-om-db-svc.my-namespace.svc.cluster.local", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1", + }, + }, + }, + }, + }, + "service with annotations with placeholders and external domain": { + members: 2, + externalAccess: &mdb.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("custom.domain"), + ExternalService: mdb.ExternalServiceConfiguration{ + Annotations: map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + create.PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + }, + SpecWrapper: &mdb.ServiceSpecWrapper{ + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + }, + }, + }, + }, + result: map[int]corev1.Service{ + 0: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-0-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + Annotations: map[string]string{ + create.PlaceholderPodIndex: "0", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db", + create.PlaceholderPodName: "test-om-db-0", + create.PlaceholderExternalServiceName: "test-om-db-0-svc-external", + create.PlaceholderMongodProcessDomain: "custom.domain", + create.PlaceholderMongodProcessFQDN: "test-om-db-0.custom.domain", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-0", + }, + }, + }, + 1: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-db-1-svc-external", + Namespace: "my-namespace", + ResourceVersion: "1", + Labels: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1", + }, + Annotations: map[string]string{ + create.PlaceholderPodIndex: "1", + create.PlaceholderNamespace: "my-namespace", + create.PlaceholderResourceName: "test-om-db", + create.PlaceholderStatefulSetName: "test-om-db", + create.PlaceholderPodName: "test-om-db-1", + create.PlaceholderExternalServiceName: "test-om-db-1-svc-external", + create.PlaceholderMongodProcessDomain: "custom.domain", + create.PlaceholderMongodProcessFQDN: "test-om-db-1.custom.domain", + }, + }, + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + Port: 27017, + }, + }, + Selector: map[string]string{ + construct.ControllerLabelName: util.OperatorName, + "statefulset.kubernetes.io/pod-name": "test-om-db-1", + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + opsManagerBuilder := DefaultOpsManagerBuilder(). + SetAppDbExternalAccess(*test.externalAccess). + SetAppDbMembers(test.members). + SetAdditionalMongodbConfig(test.additionalMongodConfig) + + opsManager := opsManagerBuilder.Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + err = createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, opsManagerUserPassword) + assert.NoError(t, err) + reconcileResult, err := reconciler.ReconcileAppDB(ctx, opsManager) + require.NoError(t, err) + + assert.Equal(t, time.Duration(10000000000), reconcileResult.RequeueAfter) + + for podIdx := 0; podIdx < opsManager.Spec.AppDB.Members; podIdx++ { + svcName := dns.GetExternalServiceName(opsManager.Spec.AppDB.GetName(), podIdx) + svcNamespace := opsManager.Namespace + + svcList := corev1.ServiceList{} + err = kubeClient.List(ctx, &svcList) + assert.NoError(t, err) + + actualSvc := corev1.Service{} + + err = kubeClient.Get(ctx, kube.ObjectKey(svcNamespace, svcName), &actualSvc) + assert.NoError(t, err) + + expectedSvc := test.result[podIdx] + assert.Equal(t, expectedSvc, actualSvc) + } + }) + } +} + +func findVolumeByName(volumes []corev1.Volume, name string) *corev1.Volume { + for i := 0; i < len(volumes); i++ { + if volumes[i].Name == name { + return &volumes[i] + } + } + + return nil +} + +func TestAppDBScaleUp_HappensIncrementally(t *testing.T) { + ctx := context.Background() + performAppDBScalingTest(ctx, t, 1, 5) +} + +func TestAppDBScaleDown_HappensIncrementally(t *testing.T) { + ctx := context.Background() + performAppDBScalingTest(ctx, t, 5, 1) +} + +func TestAppDBScaleUp_HappensIncrementally_FullOpsManagerReconcile(t *testing.T) { + ctx := context.Background() + + opsManager := DefaultOpsManagerBuilder(). + SetBackup(omv1.MongoDBOpsManagerBackup{Enabled: false}). + SetAppDbMembers(1). + SetVersion("7.0.0"). + Build() + omConnectionFactory := om.NewCachedOMConnectionFactory(om.NewEmptyMockedOmConnection) + omReconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", opsManager, nil, omConnectionFactory) + + checkOMReconciliationSuccessful(ctx, t, omReconciler, opsManager, client) + + err := client.Get(ctx, types.NamespacedName{Name: opsManager.Name, Namespace: opsManager.Namespace}, opsManager) + assert.NoError(t, err) + + opsManager.Spec.AppDB.Members = 3 + + err = client.Update(ctx, opsManager) + assert.NoError(t, err) + + checkOMReconciliationPending(ctx, t, omReconciler, opsManager) + + err = client.Get(ctx, types.NamespacedName{Name: opsManager.Name, Namespace: opsManager.Namespace}, opsManager) + assert.NoError(t, err) + + assert.Equal(t, 2, opsManager.Status.AppDbStatus.Members) + + res, err := omReconciler.Reconcile(ctx, requestFromObject(opsManager)) + assert.NoError(t, err) + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) + + err = client.Get(ctx, types.NamespacedName{Name: opsManager.Name, Namespace: opsManager.Namespace}, opsManager) + assert.NoError(t, err) + + assert.Equal(t, 3, opsManager.Status.AppDbStatus.Members) +} + +func TestAppDbPortIsConfigurable_WithAdditionalMongoConfig(t *testing.T) { + ctx := context.Background() + opsManager := DefaultOpsManagerBuilder(). + SetBackup(omv1.MongoDBOpsManagerBackup{Enabled: false}). + SetAppDbMembers(1). + SetAdditionalMongodbConfig(mdb.NewAdditionalMongodConfig("net.port", 30000)). + Build() + omConnectionFactory := om.NewCachedOMConnectionFactory(om.NewEmptyMockedOmConnection) + omReconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", opsManager, nil, omConnectionFactory) + + checkOMReconciliationSuccessful(ctx, t, omReconciler, opsManager, client) + + appdbSvc, err := client.GetService(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.ServiceName())) + assert.NoError(t, err) + assert.Equal(t, int32(30000), appdbSvc.Spec.Ports[0].Port) +} + +func TestAppDBSkipsReconciliation_IfAnyProcessesAreDisabled(t *testing.T) { + ctx := context.Background() + createReconcilerWithAllRequiredSecrets := func(opsManager *omv1.MongoDBOpsManager, createAutomationConfig bool) *ReconcileAppDbReplicaSet { + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + err := createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, "my-password") + assert.NoError(t, err) + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + reconciler.client = kubeClient + + // create a pre-existing automation config based on the resource provided. + // if the automation is not there, we will always want to reconcile. Otherwise, we may not reconcile + // based on whether or not there are disabled processes. + if createAutomationConfig { + ac, err := reconciler.buildAppDbAutomationConfig(ctx, opsManager, automation, UnusedPrometheusConfiguration, multicluster.LegacyCentralClusterName, zap.S()) + assert.NoError(t, err) + _, err = reconciler.publishAutomationConfig(ctx, opsManager, ac, opsManager.Spec.AppDB.AutomationConfigSecretName(), multicluster.LegacyCentralClusterName) + assert.NoError(t, err) + } + return reconciler + } + + t.Run("Reconciliation should happen if we are disabling a process", func(t *testing.T) { + // In this test, we create an OM + automation config (with no disabled processes), + // then update OM to have a disabled processes, and we assert that reconciliation should take place. + + omName := "test-om" + opsManager := DefaultOpsManagerBuilder().SetName(omName).Build() + + reconciler := createReconcilerWithAllRequiredSecrets(opsManager, true) + + opsManager = DefaultOpsManagerBuilder().SetName(omName).SetAppDBAutomationConfigOverride(mdbcv1.AutomationConfigOverride{ + Processes: []mdbcv1.OverrideProcess{ + { + // disable the process + Name: fmt.Sprintf("%s-db-0", omName), + Disabled: true, + }, + }, + }).Build() + + shouldReconcile, err := reconciler.shouldReconcileAppDB(ctx, opsManager, zap.S()) + assert.NoError(t, err) + assert.True(t, shouldReconcile) + }) + + t.Run("Reconciliation should not happen if a process is disabled", func(t *testing.T) { + // In this test, we create an OM with a disabled process, and assert that a reconciliation + // should not take place (since we are not changing a process back from disabled). + + omName := "test-om" + opsManager := DefaultOpsManagerBuilder().SetName(omName).SetAppDBAutomationConfigOverride(mdbcv1.AutomationConfigOverride{ + Processes: []mdbcv1.OverrideProcess{ + { + // disable the process + Name: fmt.Sprintf("%s-db-0", omName), + Disabled: true, + }, + }, + }).Build() + + reconciler := createReconcilerWithAllRequiredSecrets(opsManager, true) + + shouldReconcile, err := reconciler.shouldReconcileAppDB(ctx, opsManager, zap.S()) + assert.NoError(t, err) + assert.False(t, shouldReconcile) + }) + + t.Run("Reconciliation should happen if no automation config is present", func(t *testing.T) { + omName := "test-om" + opsManager := DefaultOpsManagerBuilder().SetName(omName).SetAppDBAutomationConfigOverride(mdbcv1.AutomationConfigOverride{ + Processes: []mdbcv1.OverrideProcess{ + { + // disable the process + Name: fmt.Sprintf("%s-db-0", omName), + Disabled: true, + }, + }, + }).Build() + + reconciler := createReconcilerWithAllRequiredSecrets(opsManager, false) + + shouldReconcile, err := reconciler.shouldReconcileAppDB(ctx, opsManager, zap.S()) + assert.NoError(t, err) + assert.True(t, shouldReconcile) + }) + + t.Run("Reconciliation should happen we are re-enabling a process", func(t *testing.T) { + omName := "test-om" + opsManager := DefaultOpsManagerBuilder().SetName(omName).SetAppDBAutomationConfigOverride(mdbcv1.AutomationConfigOverride{ + Processes: []mdbcv1.OverrideProcess{ + { + // disable the process + Name: fmt.Sprintf("%s-db-0", omName), + Disabled: true, + }, + }, + }).Build() + + reconciler := createReconcilerWithAllRequiredSecrets(opsManager, true) + + opsManager = DefaultOpsManagerBuilder().SetName(omName).Build() + + shouldReconcile, err := reconciler.shouldReconcileAppDB(ctx, opsManager, zap.S()) + assert.NoError(t, err) + assert.True(t, shouldReconcile) + }) +} + +// appDBStatefulSetLabelsAndServiceName returns extra fields that we have to manually set to the AppDB statefulset +// since we manually create it. Otherwise, the tests will fail as we try to update parts of the sts that we are not +// allowed to change +func appDBStatefulSetLabelsAndServiceName(omResourceName string) (map[string]string, string) { + appDbName := fmt.Sprintf("%s-db", omResourceName) + serviceName := fmt.Sprintf("%s-svc", appDbName) + labels := map[string]string{"app": serviceName, "controller": "mongodb-enterprise-operator", "pod-anti-affinity": appDbName} + return labels, serviceName +} + +func appDBStatefulSetVolumeClaimTemplates() []corev1.PersistentVolumeClaim { + resData, _ := resource.ParseQuantity("16G") + return []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{"storage": resData}, + }, + }, + }, + } +} + +func performAppDBScalingTest(ctx context.Context, t *testing.T, startingMembers, finalMembers int) { + builder := DefaultOpsManagerBuilder().SetAppDbMembers(startingMembers) + opsManager := builder.Build() + fakeClient, omConnectionFactory := mock.NewDefaultFakeClient() + reconciler := createRunningAppDB(ctx, t, startingMembers, fakeClient, opsManager, omConnectionFactory) + + // Scale the AppDB + opsManager.Spec.AppDB.Members = finalMembers + + if startingMembers < finalMembers { + for i := startingMembers; i < finalMembers-1; i++ { + err := fakeClient.Update(ctx, opsManager) + assert.NoError(t, err) + + res, err := reconciler.ReconcileAppDB(ctx, opsManager) + + assert.NoError(t, err) + assert.Equal(t, time.Duration(10000000000), res.RequeueAfter) + } + } else { + for i := startingMembers; i > finalMembers+1; i-- { + err := fakeClient.Update(ctx, opsManager) + assert.NoError(t, err) + + res, err := reconciler.ReconcileAppDB(ctx, opsManager) + + assert.NoError(t, err) + assert.Equal(t, time.Duration(10000000000), res.RequeueAfter) + } + } + + res, err := reconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) + + err = fakeClient.Get(ctx, types.NamespacedName{Name: opsManager.Name, Namespace: opsManager.Namespace}, opsManager) + assert.NoError(t, err) + + assert.Equal(t, finalMembers, opsManager.Status.AppDbStatus.Members) +} + +func buildAutomationConfigForAppDb(ctx context.Context, builder *omv1.OpsManagerBuilder, kubeClient client.Client, omConnectionFactoryFunc om.ConnectionFactory, acType agentType, log *zap.SugaredLogger) (automationconfig.AutomationConfig, error) { + opsManager := builder.Build() + + // Ensure the password exists for the Ops Manager User. The Ops Manager controller will have ensured this. + // We are ignoring this err on purpose since the secret might already exist. + _ = createOpsManagerUserPasswordSecret(ctx, kubeClient, opsManager, "my-password") + reconciler, err := newAppDbReconciler(ctx, kubeClient, opsManager, omConnectionFactoryFunc, zap.S()) + if err != nil { + return automationconfig.AutomationConfig{}, err + } + return reconciler.buildAppDbAutomationConfig(ctx, opsManager, acType, UnusedPrometheusConfiguration, multicluster.LegacyCentralClusterName, zap.S()) +} + +func checkDeploymentEqualToPublished(t *testing.T, expected automationconfig.AutomationConfig, s *corev1.Secret) { + actual, err := automationconfig.FromBytes(s.Data["cluster-config.json"]) + assert.NoError(t, err) + expectedBytes, err := json.Marshal(expected) + assert.NoError(t, err) + expectedAc := automationconfig.AutomationConfig{} + err = json.Unmarshal(expectedBytes, &expectedAc) + assert.NoError(t, err) + assert.Equal(t, expectedAc, actual) +} + +func newAppDbReconciler(ctx context.Context, c client.Client, opsManager *omv1.MongoDBOpsManager, omConnectionFactoryFunc om.ConnectionFactory, log *zap.SugaredLogger) (*ReconcileAppDbReplicaSet, error) { + commonController := NewReconcileCommonController(ctx, c) + return NewAppDBReplicaSetReconciler(ctx, nil, "", opsManager.Spec.AppDB, commonController, omConnectionFactoryFunc, opsManager.Annotations, nil, zap.S()) +} + +func newAppDbMultiReconciler(ctx context.Context, c client.Client, opsManager *omv1.MongoDBOpsManager, memberClusterMap map[string]client.Client, log *zap.SugaredLogger, omConnectionFactoryFunc om.ConnectionFactory) (*ReconcileAppDbReplicaSet, error) { + _ = c.Update(ctx, opsManager) + commonController := NewReconcileCommonController(ctx, c) + return NewAppDBReplicaSetReconciler(ctx, nil, "", opsManager.Spec.AppDB, commonController, omConnectionFactoryFunc, opsManager.Annotations, memberClusterMap, log) +} + +func TestChangingFCVAppDB(t *testing.T) { + ctx := context.Background() + builder := DefaultOpsManagerBuilder().SetAppDbMembers(3) + opsManager := builder.Build() + fakeClient, omConnectionFactory := mock.NewDefaultFakeClient() + reconciler := createRunningAppDB(ctx, t, 3, fakeClient, opsManager, omConnectionFactory) + + // Helper function to update and verify FCV + verifyFCV := func(version, expectedFCV string, fcvOverride *string, t *testing.T) { + if fcvOverride != nil { + opsManager.Spec.AppDB.FeatureCompatibilityVersion = fcvOverride + } + + opsManager.Spec.AppDB.Version = version + _ = fakeClient.Update(ctx, opsManager) + _, err := reconciler.ReconcileAppDB(ctx, opsManager) + assert.NoError(t, err) + assert.Equal(t, expectedFCV, opsManager.Status.AppDbStatus.FeatureCompatibilityVersion) + } + + testFCVsCases(t, verifyFCV) +} + +// createOpsManagerUserPasswordSecret creates the secret which holds the password that will be used for the Ops Manager user. +func createOpsManagerUserPasswordSecret(ctx context.Context, kubeClient client.Client, om *omv1.MongoDBOpsManager, password string) error { + sec := secret.Builder(). + SetName(om.Spec.AppDB.GetOpsManagerUserPasswordSecretName()). + SetNamespace(om.Namespace). + SetField("password", password). + Build() + return kubeClient.Create(ctx, &sec) +} + +func readAutomationConfigSecret(ctx context.Context, t *testing.T, kubeClient client.Client, opsManager *omv1.MongoDBOpsManager) *corev1.Secret { + s := &corev1.Secret{} + key := kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.AutomationConfigSecretName()) + assert.NoError(t, kubeClient.Get(ctx, key, s)) + return s +} + +func readAutomationConfigMonitoringSecret(ctx context.Context, t *testing.T, kubeClient client.Client, opsManager *omv1.MongoDBOpsManager) *corev1.Secret { + s := &corev1.Secret{} + key := kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.MonitoringAutomationConfigSecretName()) + assert.NoError(t, kubeClient.Get(ctx, key, s)) + return s +} + +func createRunningAppDB(ctx context.Context, t *testing.T, startingMembers int, fakeClient kubernetesClient.Client, opsManager *omv1.MongoDBOpsManager, omConnectionFactory *om.CachedOMConnectionFactory) *ReconcileAppDbReplicaSet { + err := createOpsManagerUserPasswordSecret(ctx, fakeClient, opsManager, "pass") + assert.NoError(t, err) + reconciler, err := newAppDbReconciler(ctx, fakeClient, opsManager, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + // create the apiKey and OM user + data := map[string]string{ + util.OmPublicApiKey: "publicApiKey", + util.OmPrivateKey: "privateApiKey", + } + + APIKeySecretName, err := opsManager.APIKeySecretName(ctx, secrets.SecretClient{KubeClient: fakeClient}, "") + assert.NoError(t, err) + + apiKeySecret := secret.Builder(). + SetNamespace(operatorNamespace()). + SetName(APIKeySecretName). + SetStringMapToData(data). + Build() + + err = reconciler.client.CreateSecret(ctx, apiKeySecret) + assert.NoError(t, err) + + err = fakeClient.Create(ctx, opsManager) + assert.NoError(t, err) + + matchLabels, serviceName := appDBStatefulSetLabelsAndServiceName(opsManager.Name) + // app db sts should exist before monitoring is configured + appDbSts, err := statefulset.NewBuilder(). + SetName(opsManager.Spec.AppDB.Name()). + SetNamespace(opsManager.Namespace). + SetMatchLabels(matchLabels). + SetServiceName(serviceName). + AddVolumeClaimTemplates(appDBStatefulSetVolumeClaimTemplates()). + SetReplicas(startingMembers). + Build() + + assert.NoError(t, err) + err = fakeClient.CreateStatefulSet(ctx, appDbSts) + assert.NoError(t, err) + + res, err := reconciler.ReconcileAppDB(ctx, opsManager) + + assert.NoError(t, err) + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) + return reconciler +} diff --git a/controllers/operator/authentication/authentication.go b/controllers/operator/authentication/authentication.go new file mode 100644 index 000000000..1785fadae --- /dev/null +++ b/controllers/operator/authentication/authentication.go @@ -0,0 +1,570 @@ +package authentication + +import ( + "crypto/sha1" //nolint //Part of the algorithm + "crypto/sha256" + "fmt" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// AuthResource is an interface that a resources that can have authentication enabled should implement. +type AuthResource interface { + GetName() string + GetNamespace() string + GetSecurity() *mdbv1.Security + IsLDAPEnabled() bool + GetLDAP(password, caContents string) *ldap.Ldap +} + +// Options contains all the required values that are required to configure authentication +// for a set of processes +type Options struct { + // Mechanisms is a list of strings coming from MongoDB.Spec.Security.Authentication.Modes, these strings + // are mapped to the corresponding mechanisms in the Automation Config + Mechanisms []string + + // ProcessNames is a list of the names of the processes which authentication will be configured for + ProcessNames []string + + // AuthoritativeSet maps directly to auth.authoritativeSet + AuthoritativeSet bool + + // AgentMechanism indicates which Agent Mechanism should be configured. This should be in the Operator format. + // I.e. X509, SCRAM and not MONGODB-X509 or SCRAM-SHA-256 + AgentMechanism string + + // ClientCertificates configures whether or not Client Certificates are required or optional. + // If X509 is the only mechanism, they must be Required, otherwise they should be Optional + // so it is possible to use other auth mechanisms without needing to provide client certs. + ClientCertificates string + + CAFilePath string + + // Use Agent Client Auth + AgentsShouldUseClientAuthentication bool + + UserOptions + + // Ldap is the LDAP configuration that will be passed to the Automation Config. + // Only required if LDAP is configured as an authentication mechanism + Ldap *ldap.Ldap + + AutoUser string + + AutoPwd string + + AutoLdapGroupDN string +} + +func Redact(o Options) Options { + if o.Ldap != nil && o.Ldap.BindQueryPassword != "" { + ldapCopy := *o.Ldap + o.Ldap = &ldapCopy + o.Ldap.BindQueryPassword = "" + } + return o +} + +// UserOptions is a struct that contains the different user names +// of the agents that should be added to the automation config. +type UserOptions struct { + AutomationSubject string +} + +// Configure will configure all the specified authentication Mechanisms. We need to ensure we wait for +// the agents to reach ready state after each operation as prematurely updating the automation config can cause the agents to get stuck. +func Configure(conn om.Connection, opts Options, isRecovering bool, log *zap.SugaredLogger) error { + log.Infow("ensuring correct deployment mechanisms", "ProcessNames", opts.ProcessNames, "Mechanisms", opts.Mechanisms) + + // In case we're recovering, we can push all changes at once, because the mechanism is triggered after 20min by default. + // Otherwise, we might unnecessarily enter this waiting loop 7 times, and waste >10 min + waitForReadyStateIfNeeded := func() error { + if isRecovering { + return nil + } + return om.WaitForReadyState(conn, opts.ProcessNames, false, log) + } + + // we need to make sure the desired authentication mechanism for the agent exists. If the desired agent + // authentication mechanism does not exist in auth.deploymentAuthMechanisms, it is an invalid config + if err := ensureDeploymentsMechanismsExist(conn, opts, log); err != nil { + return xerrors.Errorf("error ensuring deployment mechanisms: %w", err) + } + if err := waitForReadyStateIfNeeded(); err != nil { + return err + } + + // we make sure that the AuthoritativeSet options in the AC is correct + if err := ensureAuthoritativeSetIsConfigured(conn, opts.AuthoritativeSet, log); err != nil { + return xerrors.Errorf("error ensuring that authoritative set is configured: %w", err) + } + if err := waitForReadyStateIfNeeded(); err != nil { + return err + } + + // once we have made sure that the deployment authentication mechanism array contains the desired auth mechanism + // we can then configure the agent authentication. + if err := enableAgentAuthentication(conn, opts, log); err != nil { + return xerrors.Errorf("error enabling agent authentication: %w", err) + } + if err := waitForReadyStateIfNeeded(); err != nil { + return err + } + + // once we have successfully enabled auth for the agents, we need to remove mechanisms we don't need. + // this ensures we don't have mechanisms enabled that have not been configured. + if err := removeUnusedAuthenticationMechanisms(conn, opts, log); err != nil { + return xerrors.Errorf("error removing unused authentication mechanisms %w", err) + } + if err := waitForReadyStateIfNeeded(); err != nil { + return err + } + + // we remove any unrequired deployment auth mechanisms. This will generally be mechanisms + // that we are disabling. + if err := removeUnrequiredDeploymentMechanisms(conn, opts, log); err != nil { + return xerrors.Errorf("error removing unrequired deployment mechanisms: %w", err) + } + if err := waitForReadyStateIfNeeded(); err != nil { + return err + } + + // Adding a client certificate for agents + if err := addOrRemoveAgentClientCertificate(conn, opts, log); err != nil { + return xerrors.Errorf("error adding client certificates for the agents: %w", err) + } + if err := waitForReadyStateIfNeeded(); err != nil { + return err + } + + return nil +} + +// ConfigureScramCredentials creates both SCRAM-SHA-1 and SCRAM-SHA-256 credentials. This ensures +// that changes to the authentication settings on the MongoDB resources won't leave MongoDBUsers without +// the correct credentials. +func ConfigureScramCredentials(user *om.MongoDBUser, password string) error { + scram256Salt, err := GenerateSalt(sha256.New) + if err != nil { + return xerrors.Errorf("error generating scramSha256 salt: %w", err) + } + + scram1Salt, err := GenerateSalt(sha1.New) + if err != nil { + return xerrors.Errorf("error generating scramSha1 salt: %w", err) + } + + scram256Creds, err := ComputeScramShaCreds(user.Username, password, scram256Salt, ScramSha256) + if err != nil { + return xerrors.Errorf("error generating scramSha256 creds: %w", err) + } + scram1Creds, err := ComputeScramShaCreds(user.Username, password, scram1Salt, MongoDBCR) + if err != nil { + return xerrors.Errorf("error generating scramSha1Creds: %w", err) + } + user.ScramSha256Creds = scram256Creds + user.ScramSha1Creds = scram1Creds + return nil +} + +// Disable disables all authentication mechanisms, and waits for the agents to reach goal state. It is still required to provide +// automation agent username, password and keyfile contents to ensure a valid Automation Config. +func Disable(conn om.Connection, opts Options, deleteUsers bool, log *zap.SugaredLogger) error { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return xerrors.Errorf("error reading automation config: %w", err) + } + + // Disabling auth must be done in two steps, otherwise the agents might not be able to transition. + // From a Slack conversation with Agent team: + // "First disable with leaving credentials and mechanisms and users in place. Wait for goal state. Then remove the rest" + // "assume the agent is stateless. So if you remove the authentication information before it has transitioned then it won't be able to transition" + if ac.Auth.IsEnabled() { + log.Info("Disabling authentication") + + err := conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.Disabled = true + return nil + }, log) + if err != nil { + return xerrors.Errorf("error read/updating automation config: %w", err) + } + + if err := om.WaitForReadyState(conn, opts.ProcessNames, false, log); err != nil { + return xerrors.Errorf("error waiting for ready state: %w", err) + } + } + + err = conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + if err := ac.EnsureKeyFileContents(); err != nil { + return xerrors.Errorf("error ensuring keyfile contents: %w", err) + } + if _, err := ac.EnsurePassword(); err != nil { + return xerrors.Errorf("error ensuring agent password: %w", err) + } + + // we don't always want to delete the users. This can result in the agents getting stuck + // certain situations around auth transitions. + if deleteUsers { + ac.Auth.Users = []*om.MongoDBUser{} + } + ac.Auth.AutoAuthMechanisms = []string{} + ac.Auth.DeploymentAuthMechanisms = []string{} + ac.Auth.AutoUser = util.AutomationAgentName + ac.Auth.KeyFile = util.AutomationAgentKeyFilePathInContainer + ac.Auth.KeyFileWindows = util.AutomationAgentWindowsKeyFilePath + ac.Auth.AuthoritativeSet = opts.AuthoritativeSet + ac.AgentSSL.ClientCertificateMode = util.OptionalClientCertficates + ac.AgentSSL.AutoPEMKeyFilePath = util.MergoDelete + return nil + }, log) + if err != nil { + return xerrors.Errorf("error read/updating automation config: %w", err) + } + + // It is only required to update monitoring and backup agent configs in a 3 agent environment. + // we should eventually be able to remove this. + err = conn.ReadUpdateMonitoringAgentConfig(func(config *om.MonitoringAgentConfig) error { + config.DisableX509Authentication() + return nil + }, log) + if err != nil { + return xerrors.Errorf("error read/updating monitoring config: %w", err) + } + + err = conn.ReadUpdateBackupAgentConfig(func(config *om.BackupAgentConfig) error { + config.DisableX509Authentication() + return nil + }, log) + if err != nil { + return xerrors.Errorf("error read/updating backup agent config: %w", err) + } + + if err := om.WaitForReadyState(conn, opts.ProcessNames, false, log); err != nil { + return xerrors.Errorf("error waiting for ready state: %w", err) + } + + return nil +} + +func getMechanismName(mongodbResourceMode string, ac *om.AutomationConfig) MechanismName { + switch mongodbResourceMode { + case util.X509: + return MongoDBX509 + case util.LDAP: + return LDAPPlain + case util.SCRAMSHA1: + return ScramSha1 + case util.MONGODBCR: + return MongoDBCR + case util.SCRAMSHA256: + return ScramSha256 + case util.SCRAM: + // if we have already configured authentication and it has been set to MONGODB-CR/SCRAM-SHA-1 + // we can not transition. This needs to be done in the UI + + // if no authentication has been configured, the default value for "AutoAuthMechanism" is "MONGODB-CR" + // even if authentication is disabled, so we need to ensure that auth has been enabled. + if ac.Auth.AutoAuthMechanism == string(MongoDBCR) && ac.Auth.IsEnabled() { + return MongoDBCR + } + return ScramSha256 + } + // this should never be reached as validation of this string happens at the CR level + panic(fmt.Sprintf("unknown mechanism name %s", mongodbResourceMode)) +} + +// mechanism is an interface that needs to be implemented for any Ops Manager authentication mechanism +type Mechanism interface { + EnableAgentAuthentication(opts Options, log *zap.SugaredLogger) error + DisableAgentAuthentication(log *zap.SugaredLogger) error + EnableDeploymentAuthentication(opts Options) error + DisableDeploymentAuthentication() error + // IsAgentAuthenticationConfigured should not rely on util.MergoDelete since the method is always + // called directly after deserializing the response from OM which should not contain the util.MergoDelete value in any field. + IsAgentAuthenticationConfigured() bool + IsDeploymentAuthenticationConfigured() bool +} + +var ( + _ Mechanism = ConnectionScramSha{} + _ Mechanism = AutomationConfigScramSha{} + _ Mechanism = ConnectionX509{} + _ Mechanism = &ldapAuthMechanism{} +) + +// removeUnusedAuthenticationMechanisms removes authentication mechanism that were previously enabled, or were required +// as part of the transition process. +func removeUnusedAuthenticationMechanisms(conn om.Connection, opts Options, log *zap.SugaredLogger) error { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return xerrors.Errorf("error reading automation config: %w", err) + } + + automationConfigAuthMechanismNames := getMechanismNames(ac, opts.Mechanisms) + + unrequiredMechanisms := mechanismsToDisable(automationConfigAuthMechanismNames) + + log.Infow("configuring agent authentication mechanisms", "enabled", opts.AgentMechanism, "disabling", unrequiredMechanisms) + for _, mn := range unrequiredMechanisms { + m := fromName(mn, ac, conn, opts) + if m.IsAgentAuthenticationConfigured() { + log.Infof("disabling authentication mechanism %s", mn) + if err := m.DisableAgentAuthentication(log); err != nil { + return xerrors.Errorf("error disabling agent authentication: %w", err) + } + } else { + log.Infof("mechanism %s is already disabled", mn) + } + } + return nil +} + +// enableAgentAuthentication determines which agent authentication mechanism should be configured +// and enables it in Ops Manager +func enableAgentAuthentication(conn om.Connection, opts Options, log *zap.SugaredLogger) error { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return xerrors.Errorf("error reading automation config: %w", err) + } + + // we then configure the agent authentication for that type + agentAuthMechanism := getMechanismName(opts.AgentMechanism, ac) + if err := ensureAgentAuthenticationIsConfigured(conn, opts, ac, agentAuthMechanism, log); err != nil { + return xerrors.Errorf("error ensuring agent authentication is configured: %w", err) + } + + return nil +} + +// ensureAuthoritativeSetIsConfigured makes sure that the authoritativeSet options is correctly configured +// in Ops Manager +func ensureAuthoritativeSetIsConfigured(conn om.Connection, authoritativeSet bool, log *zap.SugaredLogger) error { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return xerrors.Errorf("error reading automation config: %w", err) + } + + if ac.Auth.AuthoritativeSet == authoritativeSet { + log.Debugf("Authoritative set %t is already configured", authoritativeSet) + return nil + } + + return conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.AuthoritativeSet = authoritativeSet + return nil + }, log) +} + +// ensureDeploymentsMechanismsExist makes sure that the corresponding deployment mechanisms which are required +// in order to enable the desired agent auth mechanisms are configured. +func ensureDeploymentsMechanismsExist(conn om.Connection, opts Options, log *zap.SugaredLogger) error { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return xerrors.Errorf("error reading automation config: %w", err) + } + + // "opts.Mechanisms" is the list of mechanism names passed through from the MongoDB resource. + // We need to convert this to the list of strings the automation config expects. + automationConfigMechanismNames := getMechanismNames(ac, opts.Mechanisms) + + log.Debugf("Automation config authentication mechanisms: %+v", automationConfigMechanismNames) + if err := ensureDeploymentMechanisms(conn, ac, automationConfigMechanismNames, opts, log); err != nil { + return xerrors.Errorf("error ensuring deployment mechanisms: %w", err) + } + + return nil +} + +// removeUnrequiredDeploymentMechanisms updates the given AutomationConfig struct to enable all the given +// authentication mechanisms. +func removeUnrequiredDeploymentMechanisms(conn om.Connection, opts Options, log *zap.SugaredLogger) error { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return xerrors.Errorf("error reading automation config: %w", err) + } + + // "opts.Mechanisms" is the list of mechanism names passed through from the MongoDB resource. + // We need to convert this to the list of strings the automation config expects. + automationConfigAuthMechanismNames := getMechanismNames(ac, opts.Mechanisms) + + toDisable := mechanismsToDisable(automationConfigAuthMechanismNames) + log.Infow("Removing unrequired deployment authentication mechanisms", "Mechanisms", toDisable) + if err := ensureDeploymentMechanismsAreDisabled(conn, ac, toDisable, opts, log); err != nil { + return xerrors.Errorf("error ensuring deployment mechanisms are disabled: %w", err) + } + + return nil +} + +// addOrRemoveAgentClientCertificate changes the automation config so it enables or disables +// client TLS authentication. +// This function will not change the automation config if x509 agent authentication has been +// enabled already (by the x509 auth package). +func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *zap.SugaredLogger) error { + // If x509 is not enabled but still Client Certificates are, this automation config update + // will add the required configuration. + return conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + if getMechanismName(opts.AgentMechanism, ac) == MongoDBX509 { + // If TLS client authentication is managed by x509, we won't disable or enable it + // in here. + return nil + } + + if opts.AgentsShouldUseClientAuthentication { + ac.AgentSSL = &om.AgentSSL{ + AutoPEMKeyFilePath: util.AutomationAgentPemFilePath, + CAFilePath: opts.CAFilePath, + ClientCertificateMode: opts.ClientCertificates, + } + } else { + ac.AgentSSL = &om.AgentSSL{ + AutoPEMKeyFilePath: util.MergoDelete, + ClientCertificateMode: util.OptionalClientCertficates, + } + } + return nil + }, log) +} + +func getMechanismNames(ac *om.AutomationConfig, mechanisms []string) []MechanismName { + automationConfigMechanismNames := make([]MechanismName, 0) + for _, m := range mechanisms { + automationConfigMechanismNames = append(automationConfigMechanismNames, getMechanismName(m, ac)) + } + return automationConfigMechanismNames +} + +// MechanismName corresponds to the string used in the automation config representing +// a particular type of authentication +type MechanismName string + +const ( + ScramSha256 MechanismName = "SCRAM-SHA-256" + ScramSha1 MechanismName = "SCRAM-SHA-1" + MongoDBX509 MechanismName = "MONGODB-X509" + LDAPPlain MechanismName = "PLAIN" + + // MongoDBCR is an umbrella term for SCRAM-SHA-1 and MONGODB-CR for legacy reasons, once MONGODB-CR + // is enabled, users can auth with SCRAM-SHA-1 credentials + MongoDBCR MechanismName = "MONGODB-CR" +) + +// supportedMechanisms returns a list of all the authentication mechanisms +// that can be configured by the Operator +func supportedMechanisms() []MechanismName { + return []MechanismName{ScramSha256, MongoDBCR, MongoDBX509, LDAPPlain} +} + +// fromName returns an implementation of mechanism from the string value +// used in the AutomationConfig. All supported fields are in supportedMechanisms +func fromName(name MechanismName, ac *om.AutomationConfig, conn om.Connection, opts Options) Mechanism { + switch name { + case MongoDBCR: + return NewConnectionCR(conn, ac) + case ScramSha1: + return NewConnectionScramSha1(conn, ac) + case ScramSha256: + return NewConnectionScramSha256(conn, ac) + case MongoDBX509: + return NewConnectionX509(conn, ac, opts) + case LDAPPlain: + return NewLdap(conn, ac, opts) + } + panic(xerrors.Errorf("unknown authentication mechanism %s. Supported mechanisms are %+v", name, supportedMechanisms())) +} + +// mechanismsToDisable returns a list of mechanisms which need to be disabled +// based on the currently supported authentication mechanisms and the desiredMechanisms +func mechanismsToDisable(desiredMechanisms []MechanismName) []MechanismName { + toDisable := make([]MechanismName, 0) + for _, m := range supportedMechanisms() { + if !containsMechanismName(desiredMechanisms, m) { + toDisable = append(toDisable, m) + } + } + return toDisable +} + +// ensureAgentAuthenticationIsConfigured will configure the agent authentication settings based on the desiredAgentAuthMechanism +func ensureAgentAuthenticationIsConfigured(conn om.Connection, opts Options, ac *om.AutomationConfig, desiredAgentAuthMechanismName MechanismName, log *zap.SugaredLogger) error { + m := fromName(desiredAgentAuthMechanismName, ac, conn, opts) + if m.IsAgentAuthenticationConfigured() { + log.Infof("Agent authentication mechanism %s is already configured", desiredAgentAuthMechanismName) + return nil + } + + log.Infof("Enabling %s agent authentication", desiredAgentAuthMechanismName) + return m.EnableAgentAuthentication(opts, log) +} + +// ensureDeploymentMechanisms configures the given AutomationConfig to allow deployments to +// authenticate using the specified mechanisms +func ensureDeploymentMechanisms(conn om.Connection, ac *om.AutomationConfig, desiredDeploymentAuthMechanisms []MechanismName, opts Options, log *zap.SugaredLogger) error { + allRequiredDeploymentMechanismsAreConfigured := true + for _, mn := range desiredDeploymentAuthMechanisms { + if !fromName(mn, ac, conn, opts).IsDeploymentAuthenticationConfigured() { + allRequiredDeploymentMechanismsAreConfigured = false + } else { + log.Debugf("Deployment mechanism %s is already configured", mn) + } + } + + if allRequiredDeploymentMechanismsAreConfigured { + log.Info("All required deployment authentication mechanisms are configured") + return nil + } + + return conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + for _, mechanismName := range desiredDeploymentAuthMechanisms { + log.Debugf("Enabling deployment mechanism %s", mechanismName) + if err := fromName(mechanismName, ac, conn, opts).EnableDeploymentAuthentication(opts); err != nil { + return xerrors.Errorf("error enabling deployment authentication: %w", err) + } + } + return nil + }, log) +} + +// ensureDeploymentMechanismsAreDisabled configures the given AutomationConfig to allow deployments to +// authenticate using the specified mechanisms +func ensureDeploymentMechanismsAreDisabled(conn om.Connection, ac *om.AutomationConfig, mechanismsToDisable []MechanismName, opts Options, log *zap.SugaredLogger) error { + allDeploymentMechanismsAreDisabled := true + for _, mn := range mechanismsToDisable { + if fromName(mn, ac, conn, opts).IsDeploymentAuthenticationConfigured() { + allDeploymentMechanismsAreDisabled = false + } + } + + if allDeploymentMechanismsAreDisabled { + log.Infof("Mechanisms %+v are all already disabled", mechanismsToDisable) + return nil + } + return conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + for _, mechanismName := range mechanismsToDisable { + log.Debugf("disabling deployment mechanism %s", mechanismName) + if err := fromName(mechanismName, ac, conn, opts).DisableDeploymentAuthentication(); err != nil { + return xerrors.Errorf("error disabling deployment authentication: %w", err) + } + } + return nil + }, log) +} + +// containsMechanismName returns true if there is at least one MechanismName in `slice` +// that is equal to `mn`. +func containsMechanismName(slice []MechanismName, mn MechanismName) bool { + for _, item := range slice { + if item == mn { + return true + } + } + return false +} diff --git a/controllers/operator/authentication/configure_authentication_test.go b/controllers/operator/authentication/configure_authentication_test.go new file mode 100644 index 000000000..c29fc7dcd --- /dev/null +++ b/controllers/operator/authentication/configure_authentication_test.go @@ -0,0 +1,238 @@ +package authentication + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +func TestConfigureScramSha256(t *testing.T) { + dep := om.NewDeployment() + conn := om.NewMockedOmConnection(dep) + + opts := Options{ + AuthoritativeSet: true, + ProcessNames: []string{"process-1", "process-2", "process-3"}, + Mechanisms: []string{"SCRAM"}, + AgentMechanism: "SCRAM", + } + + if err := Configure(conn, opts, false, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + t.Fatal(err) + } + + assertAuthenticationEnabled(t, ac.Auth) + assertAuthenticationMechanism(t, ac.Auth, "SCRAM-SHA-256") +} + +func TestConfigureX509(t *testing.T) { + dep := om.NewDeployment() + conn := om.NewMockedOmConnection(dep) + + opts := Options{ + AuthoritativeSet: true, + ProcessNames: []string{"process-1", "process-2", "process-3"}, + Mechanisms: []string{"X509"}, + AgentMechanism: "X509", + ClientCertificates: util.RequireClientCertificates, + UserOptions: UserOptions{ + AutomationSubject: validSubject("automation"), + }, + } + + if err := Configure(conn, opts, false, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + t.Fatal(err) + } + + assertAuthenticationEnabled(t, ac.Auth) + assertAuthenticationMechanism(t, ac.Auth, "MONGODB-X509") +} + +func TestConfigureScramSha1(t *testing.T) { + dep := om.NewDeployment() + conn := om.NewMockedOmConnection(dep) + + opts := Options{ + AuthoritativeSet: true, + ProcessNames: []string{"process-1", "process-2", "process-3"}, + Mechanisms: []string{"SCRAM-SHA-1"}, + AgentMechanism: "SCRAM-SHA-1", + } + + if err := Configure(conn, opts, false, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + assert.NoError(t, err) + + assertAuthenticationEnabled(t, ac.Auth) + assertAuthenticationMechanism(t, ac.Auth, "SCRAM-SHA-1") +} + +func TestConfigureMultipleAuthenticationMechanisms(t *testing.T) { + dep := om.NewDeployment() + conn := om.NewMockedOmConnection(dep) + + opts := Options{ + AuthoritativeSet: true, + ProcessNames: []string{"process-1", "process-2", "process-3"}, + Mechanisms: []string{"X509", "SCRAM"}, + AgentMechanism: "SCRAM", + UserOptions: UserOptions{ + AutomationSubject: validSubject("automation"), + }, + } + + if err := Configure(conn, opts, false, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + t.Fatal(err) + } + + assertAuthenticationEnabled(t, ac.Auth) + + assert.Contains(t, ac.Auth.AutoAuthMechanisms, "SCRAM-SHA-256") + + assert.Len(t, ac.Auth.DeploymentAuthMechanisms, 2) + assert.Len(t, ac.Auth.AutoAuthMechanisms, 1) + assert.Contains(t, ac.Auth.DeploymentAuthMechanisms, "SCRAM-SHA-256") + assert.Contains(t, ac.Auth.DeploymentAuthMechanisms, "MONGODB-X509") +} + +func TestDisableAuthentication(t *testing.T) { + dep := om.NewDeployment() + conn := om.NewMockedOmConnection(dep) + + // enable authentication + _ = conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.Enable() + return nil + }, zap.S()) + + if err := Disable(conn, Options{}, true, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + t.Fatal(err) + } + + assertAuthenticationDisabled(t, ac.Auth) +} + +func TestGetCorrectAuthMechanismFromVersion(t *testing.T) { + conn := om.NewMockedOmConnection(om.NewDeployment()) + ac, _ := conn.ReadAutomationConfig() + + mechanismNames := getMechanismNames(ac, []string{"X509"}) + + assert.Len(t, mechanismNames, 1) + assert.Contains(t, mechanismNames, MechanismName("MONGODB-X509")) + + mechanismNames = getMechanismNames(ac, []string{"SCRAM", "X509"}) + + assert.Contains(t, mechanismNames, MechanismName("SCRAM-SHA-256")) + assert.Contains(t, mechanismNames, MechanismName("MONGODB-X509")) + + // enable MONGODB-CR + ac.Auth.AutoAuthMechanism = "MONGODB-CR" + ac.Auth.Enable() + + mechanismNames = getMechanismNames(ac, []string{"SCRAM", "X509"}) + + assert.Contains(t, mechanismNames, MechanismName("MONGODB-CR")) + assert.Contains(t, mechanismNames, MechanismName("MONGODB-X509")) +} + +func assertAuthenticationEnabled(t *testing.T, auth *om.Auth) { + assertAuthenticationEnabledWithUsers(t, auth, 0) +} + +func assertAuthenticationEnabledWithUsers(t *testing.T, auth *om.Auth, numUsers int) { + assert.True(t, auth.AuthoritativeSet) + assert.False(t, auth.Disabled) + assert.NotEmpty(t, auth.Key) + assert.NotEmpty(t, auth.KeyFileWindows) + assert.NotEmpty(t, auth.KeyFile) + assert.Len(t, auth.Users, numUsers) + assert.True(t, noneNil(auth.Users)) +} + +func assertAuthenticationDisabled(t *testing.T, auth *om.Auth) { + assert.True(t, auth.Disabled) + assert.Empty(t, auth.DeploymentAuthMechanisms) + assert.Empty(t, auth.AutoAuthMechanisms) + assert.Equal(t, auth.AutoUser, util.AutomationAgentName) + assert.NotEmpty(t, auth.Key) + assert.NotEmpty(t, auth.AutoPwd) + assert.True(t, len(auth.Users) == 0 || allNil(auth.Users)) +} + +func assertAuthenticationMechanism(t *testing.T, auth *om.Auth, mechanism string) { + assert.Len(t, auth.DeploymentAuthMechanisms, 1) + assert.Len(t, auth.AutoAuthMechanisms, 1) + assert.Len(t, auth.Users, 0) + assert.Contains(t, auth.DeploymentAuthMechanisms, mechanism) + assert.Contains(t, auth.AutoAuthMechanisms, mechanism) +} + +func assertDeploymentMechanismsConfigured(t *testing.T, authMechanism Mechanism) { + _ = authMechanism.EnableDeploymentAuthentication(Options{CAFilePath: util.CAFilePathInContainer}) + assert.True(t, authMechanism.IsDeploymentAuthenticationConfigured()) +} + +func assertAgentAuthenticationDisabled(t *testing.T, authMechanism Mechanism, opts Options) { + _ = authMechanism.EnableAgentAuthentication(opts, zap.S()) + assert.True(t, authMechanism.IsAgentAuthenticationConfigured()) + + _ = authMechanism.DisableAgentAuthentication(zap.S()) + assert.False(t, authMechanism.IsAgentAuthenticationConfigured()) +} + +func noneNil(users []*om.MongoDBUser) bool { + for i := range users { + if users[i] == nil { + return false + } + } + return true +} + +func allNil(users []*om.MongoDBUser) bool { + for i := range users { + if users[i] != nil { + return false + } + } + return true +} + +func createConnectionAndAutomationConfig() (om.Connection, *om.AutomationConfig) { + conn := om.NewMockedOmConnection(om.NewDeployment()) + ac, _ := conn.ReadAutomationConfig() + return conn, ac +} diff --git a/controllers/operator/authentication/ldap.go b/controllers/operator/authentication/ldap.go new file mode 100644 index 000000000..80bbef5e4 --- /dev/null +++ b/controllers/operator/authentication/ldap.go @@ -0,0 +1,132 @@ +package authentication + +import ( + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +type ldapAuthMechanism struct { + AutomationConfig *om.AutomationConfig + Conn om.Connection + Options Options +} + +func NewLdap(conn om.Connection, ac *om.AutomationConfig, opts Options) Mechanism { + return &ldapAuthMechanism{ + AutomationConfig: ac, + Conn: conn, + Options: opts, + } +} + +func (l *ldapAuthMechanism) EnableAgentAuthentication(opts Options, log *zap.SugaredLogger) error { + log.Info("Configuring LDAP authentication") + err := l.Conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + if err := ac.EnsureKeyFileContents(); err != nil { + return err + } + auth := ac.Auth + auth.AutoPwd = opts.AutoPwd + auth.Disabled = false + auth.AuthoritativeSet = opts.AuthoritativeSet + auth.KeyFile = util.AutomationAgentKeyFilePathInContainer + auth.KeyFileWindows = util.AutomationAgentWindowsKeyFilePath + + auth.AutoUser = l.Options.AutomationSubject + auth.LdapGroupDN = opts.AutoLdapGroupDN + auth.AutoAuthMechanisms = []string{string(LDAPPlain)} + return nil + }, log) + if err != nil { + return err + } + + log.Info("Configuring backup agent user") + err = l.Conn.ReadUpdateBackupAgentConfig(func(config *om.BackupAgentConfig) error { + config.EnableLdapAuthentication(l.Options.AutomationSubject, opts.AutoPwd) + config.SetLdapGroupDN(opts.AutoLdapGroupDN) + return nil + }, log) + if err != nil { + return err + } + + log.Info("Configuring monitoring agent user") + return l.Conn.ReadUpdateMonitoringAgentConfig(func(config *om.MonitoringAgentConfig) error { + config.EnableLdapAuthentication(l.Options.AutomationSubject, opts.AutoPwd) + config.SetLdapGroupDN(opts.AutoLdapGroupDN) + return nil + }, log) +} + +func (l *ldapAuthMechanism) DisableAgentAuthentication(log *zap.SugaredLogger) error { + err := l.Conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + if stringutil.Contains(ac.Auth.AutoAuthMechanisms, string(LDAPPlain)) { + ac.Auth.AutoAuthMechanisms = stringutil.Remove(ac.Auth.AutoAuthMechanisms, string(LDAPPlain)) + } + return nil + }, log) + if err != nil { + return err + } + + err = l.Conn.ReadUpdateMonitoringAgentConfig(func(config *om.MonitoringAgentConfig) error { + config.DisableLdapAuthentication() + return nil + }, log) + if err != nil { + return err + } + + return l.Conn.ReadUpdateBackupAgentConfig(func(config *om.BackupAgentConfig) error { + config.DisableLdapAuthentication() + return nil + }, log) +} + +func (l *ldapAuthMechanism) EnableDeploymentAuthentication(opts Options) error { + ac := l.AutomationConfig + ac.Ldap = opts.Ldap + if !stringutil.Contains(ac.Auth.DeploymentAuthMechanisms, string(LDAPPlain)) { + ac.Auth.DeploymentAuthMechanisms = append(ac.Auth.DeploymentAuthMechanisms, string(LDAPPlain)) + } + + return nil +} + +func (l *ldapAuthMechanism) DisableDeploymentAuthentication() error { + ac := l.AutomationConfig + ac.Ldap = nil + ac.Auth.DeploymentAuthMechanisms = stringutil.Remove(ac.Auth.DeploymentAuthMechanisms, string(LDAPPlain)) + return nil +} + +func (l *ldapAuthMechanism) IsAgentAuthenticationConfigured() bool { + ac := l.AutomationConfig + if ac.Auth.Disabled { + return false + } + + if !stringutil.Contains(ac.Auth.AutoAuthMechanisms, string(LDAPPlain)) { + return false + } + + if ac.Auth.AutoUser == "" || ac.Auth.AutoPwd == "" { + return false + } + + return true +} + +func (l *ldapAuthMechanism) IsDeploymentAuthenticationConfigured() bool { + ac := l.AutomationConfig + return stringutil.Contains(ac.Auth.DeploymentAuthMechanisms, string(LDAPPlain)) && ldapObjectsEqual(ac.Ldap, l.Options.Ldap) +} + +func ldapObjectsEqual(lhs *ldap.Ldap, rhs *ldap.Ldap) bool { + return lhs != nil && rhs != nil && *lhs == *rhs +} diff --git a/controllers/operator/authentication/ldap_test.go b/controllers/operator/authentication/ldap_test.go new file mode 100644 index 000000000..9ff20eda5 --- /dev/null +++ b/controllers/operator/authentication/ldap_test.go @@ -0,0 +1,80 @@ +package authentication + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/ldap" +) + +func TestLdapDeploymentMechanism(t *testing.T) { + conn := om.NewMockedOmConnection(om.NewDeployment()) + ac, _ := conn.ReadAutomationConfig() + opts := Options{ + Ldap: &ldap.Ldap{ + BindMethod: "BindMethod", + BindQueryUser: "BindQueryUser", + Servers: "Servers", + }, + } + l := NewLdap(conn, ac, opts) + + if err := l.EnableDeploymentAuthentication(opts); err != nil { + t.Fatal(err) + } + assert.Contains(t, ac.Auth.DeploymentAuthMechanisms, string(LDAPPlain)) + assert.Equal(t, "BindQueryUser", ac.Ldap.BindQueryUser) + assert.Equal(t, "Servers", ac.Ldap.Servers) + assert.Equal(t, "BindMethod", ac.Ldap.BindMethod) + + if err := l.DisableDeploymentAuthentication(); err != nil { + t.Fatal(err) + } + + assert.NotContains(t, ac.Auth.DeploymentAuthMechanisms, string(LDAPPlain)) + assert.Nil(t, ac.Ldap) +} + +func TestLdapEnableAgentAuthentication(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + options := Options{ + AgentMechanism: "LDAP", + UserOptions: UserOptions{ + AutomationSubject: ("mms-automation"), + }, + } + + l := NewLdap(conn, ac, options) + + if err := l.EnableAgentAuthentication(Options{AuthoritativeSet: true, AutoPwd: "LDAPPassword."}, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, ac.Auth.AutoUser, options.AutomationSubject) + assert.Len(t, ac.Auth.AutoAuthMechanisms, 1) + assert.Contains(t, ac.Auth.AutoAuthMechanisms, string(LDAPPlain)) + assert.Equal(t, "LDAPPassword.", ac.Auth.AutoPwd) + assert.False(t, ac.Auth.Disabled) + + assert.True(t, ac.Auth.AuthoritativeSet) +} + +func TestLDAP_DisableAgentAuthentication(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + opts := Options{ + AutoPwd: "LDAPPassword.", + UserOptions: UserOptions{ + AutomationSubject: validSubject("automation"), + }, + } + ldap := NewLdap(conn, ac, opts) + assertAgentAuthenticationDisabled(t, ldap, opts) +} diff --git a/controllers/operator/authentication/pkix.go b/controllers/operator/authentication/pkix.go new file mode 100644 index 000000000..610d2c3f2 --- /dev/null +++ b/controllers/operator/authentication/pkix.go @@ -0,0 +1,267 @@ +package authentication + +import ( + "bytes" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/hex" + "encoding/pem" + "errors" + "strings" + "unicode/utf8" + + "golang.org/x/xerrors" +) + +type encodeState struct { + bytes.Buffer // accumulated output +} + +const ( + commonNameOID = "2.5.4.3" + serialNumberOID = "2.5.4.5" + countryNameOID = "2.5.4.6" + localityNameOID = "2.5.4.7" + stateOrProvinceNameOID = "2.5.4.8" + streetAddressOID = "2.5.4.9" + organizationNameOID = "2.5.4.10" + organizationUnitNameOID = "2.5.4.11" + postalCodeOID = "2.5.4.17" + useridOID = "0.9.2342.19200300.100.1.1" + domainComponentOID = "0.9.2342.19200300.100.1.25" + emailAddressOID = "1.2.840.113549.1.9.1" + subjectAltNameOID = "2.5.29.17" + businessCategoryOID = "2.5.4.15" +) + +var shortNamesByOID = map[string]string{ + commonNameOID: "CN", + serialNumberOID: "serialNumber", + countryNameOID: "C", + localityNameOID: "L", + stateOrProvinceNameOID: "ST", + streetAddressOID: "street", + organizationNameOID: "O", + organizationUnitNameOID: "OU", + postalCodeOID: "postalCode", + useridOID: "UID", + domainComponentOID: "DC", + emailAddressOID: "emailAddress", + subjectAltNameOID: "subjectAltName", + businessCategoryOID: "businessCategory", +} + +func GetCertificateSubject(certPEM string) (subject string, unknownOIDs []string, err error) { + rem := []byte(certPEM) + block, _ := pem.Decode(rem) + if block == nil { + return "", []string{}, xerrors.Errorf("no certificate found") + } + + x509Cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return "", []string{}, err + } + + var subjectRDN pkix.RDNSequence + if _, err := asn1.Unmarshal(x509Cert.RawSubject, &subjectRDN); err != nil { + return "", []string{}, err + } + + var enc encodeState + unknownOIDs, err = enc.writeDistinguishedName(subjectRDN) + if err != nil { + return "", []string{}, err + } + return enc.String(), unknownOIDs, nil +} + +func (enc *encodeState) writeDistinguishedName(subject pkix.RDNSequence) (allUnknownOIDs []string, err error) { + // Section 2.1. Converting the RDNSequence + // + // If the RDNSequence is an empty sequence, the result is the empty or + // zero length string. + // + // Otherwise, the output consists of the string encodings of each + // RelativeDistinguishedName in the RDNSequence (according to 2.2), + // starting with the last element of the sequence and moving backwards + // toward the first. + // + // The encodings of adjoining RelativeDistinguishedNames are separated + // by a comma character (',' ASCII 44). + + allUnknownOIDs = make([]string, 0) + + for i := len(subject) - 1; i >= 0; i-- { + if i < len(subject)-1 { + enc.WriteByte(',') + } + + unknownOIDs, err := enc.writeRelativeDistinguishedName(subject[i]) + if err != nil { + return []string{}, err + } + allUnknownOIDs = append(allUnknownOIDs, unknownOIDs...) + } + return allUnknownOIDs, nil +} + +func (enc *encodeState) writeRelativeDistinguishedName(rdn pkix.RelativeDistinguishedNameSET) (unknownOIDs []string, err error) { + if len(rdn) == 0 { + return []string{}, errors.New("expected RelativeDistinguishedNameSET to contain at least 1 attribute") + } + + // 2.2. Converting RelativeDistinguishedName + // + // When converting from an ASN.1 RelativeDistinguishedName to a string, + // the output consists of the string encodings of each + // AttributeTypeAndValue (according to 2.3), in any order. + // + // Where there is a multi-valued RDN, the outputs from adjoining + // AttributeTypeAndValues are separated by a plus ('+' ASCII 43) + // character. + + // TODO: This does not conform to the same order of attributes that OpenSSL uses + + unknownOIDs = make([]string, 0) + for i := 0; i < len(rdn); i++ { + if i > 0 { + enc.WriteByte('+') + } + + found, err := enc.writeAttributeTypeAndValue(rdn[i]) + if err != nil { + return []string{}, err + } + if !found { + unknownOIDs = append(unknownOIDs, rdn[i].Type.String()) + } + } + return unknownOIDs, nil +} + +func (enc *encodeState) getAttributeTypeShortName(attrType asn1.ObjectIdentifier) (shortName string, found bool) { + oid := attrType.String() + shortName, found = shortNamesByOID[oid] + if found { + return shortName, true + } + + return "", false +} + +func (enc *encodeState) writeAttributeTypeAndValue(atv pkix.AttributeTypeAndValue) (found bool, err error) { + // Section 2.3. Converting AttributeTypeAndValue + // + // The AttributeTypeAndValue is encoded as the string representation of + // the AttributeType, followed by an equals character ('=' ASCII 61), + // followed by the string representation of the AttributeValue. + + attrType, found := enc.getAttributeTypeShortName(atv.Type) + if err != nil { + // Technically, the AttributeType should be encoded using the dotted-decimal + // notation of its object identifier (OID); however, chances are good that + // it is exists in OpenSSL's list, so our authentication attempt would get + // rejected by the server anyway. + return false, err + } + + attrValue, ok := atv.Value.(string) + if !ok { + // Technically, the AttributeValue should be encoded as an octothorpe + // character ('#' ASCII 35) followed by the hexadecimal representation + // of the bytes from the BER encoding. However, there is no need to + // handle that case because all of the recognized attributes are of type + // IA5String, PrintableString, or UTF8String. + return false, xerrors.Errorf("value for attribute type `%v` was not a string: %v", atv.Type, atv.Value) + } + + if found { + enc.WriteString(attrType) + } else { + enc.WriteString(atv.Type.String()) + } + enc.WriteByte('=') + err = enc.writeEscapedAttributeValue(attrValue) + if err != nil { + return false, err + } + return found, nil +} + +func (enc *encodeState) writeEscapedAttributeValue(attrValue string) error { + // Section 2.4 Converting an AttributeValue from ASN.1 to a String + // + // If the UTF-8 string does not have any of the following characters + // which need escaping, then that string can be used as the string + // representation of the value. + // + // - a space or "#" character occurring at the beginning of the string + // - a space character occurring at the end of the string + // - one of the characters ",", "+", """, "\", "<", ">" or ";" + // + // If a character to be escaped is one of the list shown above, then it + // is prefixed by a backslash ('\' ASCII 92). + // + // Otherwise the character to be escaped is replaced by a backslash and + // two hex digits, which form a single byte in the code of the character. + + start := 0 + for i := 0; i < len(attrValue); { + b := attrValue[i] + // OpenSSL does not actually escape NUL as \00, as it only became required + // to do so in RFC 4514. Instead, the attribute value is terminated before + // the end of line character. For example, the inputs "CN=ab" and "CN=ab\x00c" + // both produce the same certificate request. This should mean that no null + // characters can appear in the UTF-8 string, so our handling of them here + // is irrelevant - but not incorrect. + if b < 0x20 || b == 0x7f || b >= utf8.RuneSelf { + if start < i { + enc.WriteString(attrValue[start:i]) + } + enc.WriteByte('\\') + hexDigits := hex.EncodeToString([]byte{b}) + // OpenSSL uses uppercase hexadecimal digits. + enc.WriteString(strings.ToUpper(hexDigits)) + i++ + start = i + continue + } + + switch b { + case ',', '+', '"', '\\', '<', '>', ';': + if start < i { + enc.WriteString(attrValue[start:i]) + } + enc.WriteByte('\\') + enc.WriteByte(b) + i++ + start = i + continue + } + + if i == 0 && (b == ' ' || b == '#') { + // OpenSSL only escapes the first space or hash character, not all leading ones. + enc.WriteByte('\\') + enc.WriteByte(b) + i++ + start = i + } else if i == len(attrValue)-1 && b == ' ' { + // OpenSSL only escapes the last space character, not all trailing ones. + if start < i { + enc.WriteString(attrValue[start:i]) + } + enc.WriteByte('\\') + enc.WriteByte(b) + i++ + start = i + } else { + i++ + } + } + if start < len(attrValue) { + enc.WriteString(attrValue[start:]) + } + return nil +} diff --git a/controllers/operator/authentication/scramsha.go b/controllers/operator/authentication/scramsha.go new file mode 100644 index 000000000..1cf2b5e4e --- /dev/null +++ b/controllers/operator/authentication/scramsha.go @@ -0,0 +1,158 @@ +package authentication + +import ( + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func NewConnectionScramSha256(conn om.Connection, ac *om.AutomationConfig) ConnectionScramSha { + return ConnectionScramSha{ + Conn: conn, + AutomationConfigScramSha: AutomationConfigScramSha{ + automationConfig: ac, + mechanismName: ScramSha256, + }, + } +} + +func NewConnectionCR(conn om.Connection, ac *om.AutomationConfig) ConnectionScramSha { + return ConnectionScramSha{ + Conn: conn, + AutomationConfigScramSha: AutomationConfigScramSha{ + automationConfig: ac, + mechanismName: MongoDBCR, + }, + } +} + +func NewConnectionScramSha1(conn om.Connection, ac *om.AutomationConfig) ConnectionScramSha { + return ConnectionScramSha{ + Conn: conn, + AutomationConfigScramSha: AutomationConfigScramSha{ + automationConfig: ac, + mechanismName: ScramSha1, + }, + } +} + +func NewAutomationConfigScramSha1(ac *om.AutomationConfig) AutomationConfigScramSha { + return AutomationConfigScramSha{ + automationConfig: ac, + mechanismName: MongoDBCR, + } +} + +func NewAutomationConfigScramSha256(ac *om.AutomationConfig) AutomationConfigScramSha { + return AutomationConfigScramSha{ + automationConfig: ac, + mechanismName: ScramSha256, + } +} + +// AutomationConfigScramSha applies all the changes required to configure SCRAM-SHA authentication +// directly to an AutomationConfig struct. This implementation does not communicate with Ops Manager in any way. +type AutomationConfigScramSha struct { + mechanismName MechanismName + automationConfig *om.AutomationConfig +} + +func (s AutomationConfigScramSha) EnableAgentAuthentication(opts Options, log *zap.SugaredLogger) error { + if err := configureScramAgentUsers(s.automationConfig, opts); err != nil { + return err + } + if err := s.automationConfig.EnsureKeyFileContents(); err != nil { + return err + } + + auth := s.automationConfig.Auth + auth.Disabled = false + auth.AuthoritativeSet = opts.AuthoritativeSet + auth.KeyFile = util.AutomationAgentKeyFilePathInContainer + auth.KeyFileWindows = util.AutomationAgentWindowsKeyFilePath + + // We can only have a single agent authentication mechanism specified at a given time + auth.AutoAuthMechanisms = []string{string(s.mechanismName)} + return nil +} + +func (s AutomationConfigScramSha) DisableAgentAuthentication(log *zap.SugaredLogger) error { + s.automationConfig.Auth.AutoAuthMechanisms = stringutil.Remove(s.automationConfig.Auth.AutoAuthMechanisms, string(s.mechanismName)) + return nil +} + +func (s AutomationConfigScramSha) DisableDeploymentAuthentication() error { + s.automationConfig.Auth.DeploymentAuthMechanisms = stringutil.Remove(s.automationConfig.Auth.DeploymentAuthMechanisms, string(s.mechanismName)) + return nil +} + +func (s AutomationConfigScramSha) EnableDeploymentAuthentication(Options) error { + auth := s.automationConfig.Auth + if !stringutil.Contains(auth.DeploymentAuthMechanisms, string(s.mechanismName)) { + auth.DeploymentAuthMechanisms = append(auth.DeploymentAuthMechanisms, string(s.mechanismName)) + } + return nil +} + +func (s AutomationConfigScramSha) IsAgentAuthenticationConfigured() bool { + ac := s.automationConfig + if ac.Auth.Disabled { + return false + } + + if !stringutil.Contains(ac.Auth.AutoAuthMechanisms, string(s.mechanismName)) { + return false + } + + if ac.Auth.AutoUser != util.AutomationAgentName || (ac.Auth.AutoPwd == "" || ac.Auth.AutoPwd == util.MergoDelete) { + return false + } + + if ac.Auth.Key == "" || ac.Auth.KeyFile == "" || ac.Auth.KeyFileWindows == "" { + return false + } + + return true +} + +func (s AutomationConfigScramSha) IsDeploymentAuthenticationConfigured() bool { + return stringutil.Contains(s.automationConfig.Auth.DeploymentAuthMechanisms, string(s.mechanismName)) +} + +// ConnectionScramSha is a wrapper around AutomationConfigScramSha which pulls the AutomationConfig +// from Ops Manager and sends back the AutomationConfig which has been configured for to enabled SCRAM-SHA +type ConnectionScramSha struct { + AutomationConfigScramSha + Conn om.Connection +} + +func (s ConnectionScramSha) EnableAgentAuthentication(opts Options, log *zap.SugaredLogger) error { + return s.Conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + s.automationConfig = ac + return s.AutomationConfigScramSha.EnableAgentAuthentication(opts, log) + }, log) +} + +func (s ConnectionScramSha) DisableAgentAuthentication(log *zap.SugaredLogger) error { + return s.Conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + s.automationConfig = ac + return s.AutomationConfigScramSha.DisableAgentAuthentication(log) + }, log) +} + +// configureScramAgentUsers makes sure that the given automation config always has the correct SCRAM-SHA users +func configureScramAgentUsers(ac *om.AutomationConfig, authOpts Options) error { + agentPassword, err := ac.EnsurePassword() + if err != nil { + return err + } + auth := ac.Auth + if auth.AutoUser == "" { + auth.AutoUser = authOpts.AutoUser + } + auth.AutoPwd = agentPassword + + return nil +} diff --git a/controllers/operator/authentication/scramsha_credentials.go b/controllers/operator/authentication/scramsha_credentials.go new file mode 100644 index 000000000..f5271f575 --- /dev/null +++ b/controllers/operator/authentication/scramsha_credentials.go @@ -0,0 +1,181 @@ +package authentication + +import ( + "crypto/hmac" + "crypto/sha1" //nolint //Part of the algorithm + "crypto/sha256" + "encoding/base64" + "hash" + + "github.com/xdg/stringprep" + "golang.org/x/xerrors" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/generate" +) + +const ( + clientKeyInput = "Client Key" // specified in RFC 5802 + serverKeyInput = "Server Key" // specified in RFC 5802 + + // using the default MongoDB values for the number of iterations depending on mechanism + scramSha1Iterations = 10000 + scramSha256Iterations = 15000 + + RFC5802MandatedSaltSize = 4 +) + +// The code in this file is largely adapted from the Automation Agent codebase. +// https://github.com/10gen/mms-automation/blob/c108e0319cc05c0d8719ceea91a0424a016db583/go_planner/src/com.tengen/cm/crypto/scram.go + +// ComputeScramShaCreds takes a plain text password and a specified mechanism name and generates +// the ScramShaCreds which will be embedded into a MongoDBUser. +func ComputeScramShaCreds(username, password string, salt []byte, name MechanismName) (*om.ScramShaCreds, error) { + var hashConstructor func() hash.Hash + iterations := 0 + if name == ScramSha256 { + hashConstructor = sha256.New + iterations = scramSha256Iterations + } else if name == MongoDBCR { + hashConstructor = sha1.New + iterations = scramSha1Iterations + + // MONGODB-CR/SCRAM-SHA-1 requires the hash of the password being passed computeScramCredentials + // instead of the plain text password. Generated the same was that Ops Manager does. + // See: https://github.com/10gen/mms/blob/a941f11a81fba4f85a9890eaf27605bd344af2a8/server/src/main/com/xgen/svc/mms/deployment/auth/AuthUser.java#L290 + password = util.MD5Hex(username + ":mongo:" + password) + } else { + return nil, xerrors.Errorf("unrecognized SCRAM-SHA format %s", name) + } + base64EncodedSalt := base64.StdEncoding.EncodeToString(salt) + return computeScramCredentials(hashConstructor, iterations, base64EncodedSalt, password) +} + +// GenerateSalt will create a salt for use with ComputeScramShaCreds based on the given hashConstructor. +// sha1.New should be used for MONGODB-CR/SCRAM-SHA-1 and sha256.New should be used for SCRAM-SHA-256 +func GenerateSalt(hashConstructor func() hash.Hash) ([]byte, error) { + saltSize := hashConstructor().Size() - RFC5802MandatedSaltSize + salt, err := generate.RandomFixedLengthStringOfSize(saltSize) + if err != nil { + return nil, err + } + return []byte(salt), nil +} + +func generateSaltedPassword(hashConstructor func() hash.Hash, password string, salt []byte, iterationCount int) ([]byte, error) { + preparedPassword, err := stringprep.SASLprep.Prepare(password) + if err != nil { + return nil, xerrors.Errorf("error SASLprep'ing password: %w", err) + } + + result, err := hmacIteration(hashConstructor, []byte(preparedPassword), salt, iterationCount) + if err != nil { + return nil, xerrors.Errorf("error running hmacIteration: %w", err) + } + return result, nil +} + +func hmacIteration(hashConstructor func() hash.Hash, input, salt []byte, iterationCount int) ([]byte, error) { + hashSize := hashConstructor().Size() + + // incorrect salt size will pass validation, but the credentials will be invalid. i.e. it will not + // be possible to auth with the password provided to create the credentials. + if len(salt) != hashSize-RFC5802MandatedSaltSize { + return nil, xerrors.Errorf("salt should have a size of %v bytes, but instead has a size of %v bytes", hashSize-RFC5802MandatedSaltSize, len(salt)) + } + + startKey := append(salt, 0, 0, 0, 1) + result := make([]byte, hashSize) + + hmacHash := hmac.New(hashConstructor, input) + if _, err := hmacHash.Write(startKey); err != nil { + return nil, xerrors.Errorf("error running hmacHash: %w", err) + } + + intermediateDigest := hmacHash.Sum(nil) + + copy(result, intermediateDigest) + + for i := 1; i < iterationCount; i++ { + hmacHash.Reset() + if _, err := hmacHash.Write(intermediateDigest); err != nil { + return nil, xerrors.Errorf("error running hmacHash: %w", err) + } + + intermediateDigest = hmacHash.Sum(nil) + + for i := 0; i < len(intermediateDigest); i++ { + result[i] ^= intermediateDigest[i] + } + } + + return result, nil +} + +func generateClientOrServerKey(hashConstructor func() hash.Hash, saltedPassword []byte, input string) ([]byte, error) { + hmacHash := hmac.New(hashConstructor, saltedPassword) + if _, err := hmacHash.Write([]byte(input)); err != nil { + return nil, xerrors.Errorf("error running hmacHash: %w", err) + } + + return hmacHash.Sum(nil), nil +} + +func generateStoredKey(hashConstructor func() hash.Hash, clientKey []byte) ([]byte, error) { + h := hashConstructor() + if _, err := h.Write(clientKey); err != nil { + return nil, xerrors.Errorf("error hashing: %w", err) + } + return h.Sum(nil), nil +} + +func generateSecrets(hashConstructor func() hash.Hash, password string, salt []byte, iterationCount int) (storedKey, serverKey []byte, err error) { + saltedPassword, err := generateSaltedPassword(hashConstructor, password, salt, iterationCount) + if err != nil { + return nil, nil, xerrors.Errorf("error generating salted password: %w", err) + } + + clientKey, err := generateClientOrServerKey(hashConstructor, saltedPassword, clientKeyInput) + if err != nil { + return nil, nil, xerrors.Errorf("error generating client key: %w", err) + } + + storedKey, err = generateStoredKey(hashConstructor, clientKey) + if err != nil { + return nil, nil, xerrors.Errorf("error generating stored key: %w", err) + } + + serverKey, err = generateClientOrServerKey(hashConstructor, saltedPassword, serverKeyInput) + if err != nil { + return nil, nil, xerrors.Errorf("error generating server key: %w", err) + } + + return storedKey, serverKey, err +} + +func generateB64EncodedSecrets(hashConstructor func() hash.Hash, password, b64EncodedSalt string, iterationCount int) (storedKey, serverKey string, err error) { + salt, err := base64.StdEncoding.DecodeString(b64EncodedSalt) + if err != nil { + return "", "", xerrors.Errorf("error decoding salt: %w", err) + } + + unencodedStoredKey, unencodedServerKey, err := generateSecrets(hashConstructor, password, salt, iterationCount) + if err != nil { + return "", "", xerrors.Errorf("error generating secrets: %w", err) + } + + storedKey = base64.StdEncoding.EncodeToString(unencodedStoredKey) + serverKey = base64.StdEncoding.EncodeToString(unencodedServerKey) + return storedKey, serverKey, nil +} + +// password should be encrypted in the case of SCRAM-SHA-1 and unencrypted in the case of SCRAM-SHA-256 +func computeScramCredentials(hashConstructor func() hash.Hash, iterationCount int, base64EncodedSalt string, password string) (*om.ScramShaCreds, error) { + storedKey, serverKey, err := generateB64EncodedSecrets(hashConstructor, password, base64EncodedSalt, iterationCount) + if err != nil { + return nil, xerrors.Errorf("error generating SCRAM-SHA keys: %w", err) + } + + return &om.ScramShaCreds{IterationCount: iterationCount, Salt: base64EncodedSalt, StoredKey: storedKey, ServerKey: serverKey}, nil +} diff --git a/controllers/operator/authentication/scramsha_credentials_test.go b/controllers/operator/authentication/scramsha_credentials_test.go new file mode 100644 index 000000000..9e569b480 --- /dev/null +++ b/controllers/operator/authentication/scramsha_credentials_test.go @@ -0,0 +1,25 @@ +package authentication + +import ( + "crypto/sha1" //nolint //Part of the algorithm + "hash" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestScramSha1SecretsMatch(t *testing.T) { + // these were taken from MongoDB. passwordHash is from authSchema + // 3. iterationCount, salt, storedKey, and serverKey are from + // authSchema 5 (after upgrading from authSchema 3) + assertSecretsMatch(t, sha1.New, "caeec61ba3b15b15b188d29e876514e8", 10, "S3cuk2Rnu/MlbewzxrmmVA==", "sYBa3XlSPKNrgjzhOuEuRlJY4dQ=", "zuAxRSQb3gZkbaB1IGlusK4jy1M=") + assertSecretsMatch(t, sha1.New, "4d9625b297999b3ca786d4a9622d04f1", 10, "kW9KbCQiCOll5Ljd44cjkQ==", "VJ8fFVHkPltibvT//mG/OWw44Hc=", "ceDRsgj9HezpZ4/vkZX8GZNNN50=") + assertSecretsMatch(t, sha1.New, "fd0a78e418dcef39f8c768222810b894", 10, "hhX6xsoID6FeWjXncuNgAg==", "TxgaZJ4cIn+S9EfTcc9IOEG7RGc=", "d6/qjwBs0qkPKfUAjSh5eemsySE=") +} + +func assertSecretsMatch(t *testing.T, hash func() hash.Hash, passwordHash string, iterationCount int, salt, storedKey, serverKey string) { + computedStoredKey, computedServerKey, err := generateB64EncodedSecrets(hash, passwordHash, salt, iterationCount) + assert.NoError(t, err) + assert.Equal(t, computedStoredKey, storedKey) + assert.Equal(t, computedServerKey, serverKey) +} diff --git a/controllers/operator/authentication/scramsha_test.go b/controllers/operator/authentication/scramsha_test.go new file mode 100644 index 000000000..a185335c7 --- /dev/null +++ b/controllers/operator/authentication/scramsha_test.go @@ -0,0 +1,75 @@ +package authentication + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestAgentsAuthentication(t *testing.T) { + type ConnectionFunction func(om.Connection, *om.AutomationConfig) Mechanism + type TestConfig struct { + connection ConnectionFunction + mechanismsUsed []MechanismName + } + tests := map[string]TestConfig{ + "SCRAM-SHA-1": { + connection: func(connection om.Connection, config *om.AutomationConfig) Mechanism { + return NewConnectionScramSha1(connection, config) + }, + mechanismsUsed: []MechanismName{ScramSha1}, + }, + "SCRAM-SHA-256": { + connection: func(connection om.Connection, config *om.AutomationConfig) Mechanism { + return NewConnectionScramSha256(connection, config) + }, + mechanismsUsed: []MechanismName{ScramSha256}, + }, + "CR": { + connection: func(connection om.Connection, config *om.AutomationConfig) Mechanism { + return NewConnectionCR(connection, config) + }, + mechanismsUsed: []MechanismName{MongoDBCR}, + }, + } + for testName, testConfig := range tests { + t.Run(testName, func(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + + s := testConfig.connection(conn, ac) + + err := s.EnableAgentAuthentication(Options{AuthoritativeSet: true}, zap.S()) + assert.NoError(t, err) + + err = s.EnableDeploymentAuthentication(Options{CAFilePath: util.CAFilePathInContainer}) + assert.NoError(t, err) + + ac, err = conn.ReadAutomationConfig() + assert.NoError(t, err) + + assertAuthenticationEnabled(t, ac.Auth) + assert.Equal(t, ac.Auth.AutoUser, util.AutomationAgentName) + assert.Len(t, ac.Auth.AutoAuthMechanisms, 1) + for _, mech := range testConfig.mechanismsUsed { + assert.Contains(t, ac.Auth.AutoAuthMechanisms, string(mech)) + } + assert.NotEmpty(t, ac.Auth.AutoPwd) + assert.True(t, s.IsAgentAuthenticationConfigured()) + assert.True(t, s.IsDeploymentAuthenticationConfigured()) + }) + } +} + +func TestScramSha1_DisableAgentAuthentication(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + assertAgentAuthenticationDisabled(t, NewConnectionScramSha1(conn, ac), Options{}) +} + +func TestScramSha256_DisableAgentAuthentication(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + assertAgentAuthenticationDisabled(t, NewConnectionScramSha256(conn, ac), Options{}) +} diff --git a/controllers/operator/authentication/x509.go b/controllers/operator/authentication/x509.go new file mode 100644 index 000000000..66cbe70a2 --- /dev/null +++ b/controllers/operator/authentication/x509.go @@ -0,0 +1,159 @@ +package authentication + +import ( + "regexp" + + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +const ExternalDB = "$external" + +func NewConnectionX509(conn om.Connection, ac *om.AutomationConfig, opts Options) ConnectionX509 { + return ConnectionX509{ + AutomationConfig: ac, + Conn: conn, + Options: opts, + } +} + +type ConnectionX509 struct { + AutomationConfig *om.AutomationConfig + Conn om.Connection + Options Options +} + +func (x ConnectionX509) EnableAgentAuthentication(opts Options, log *zap.SugaredLogger) error { + log.Info("Configuring x509 authentication") + err := x.Conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + if err := ac.EnsureKeyFileContents(); err != nil { + return err + } + auth := ac.Auth + auth.AutoPwd = util.MergoDelete + auth.Disabled = false + auth.AuthoritativeSet = opts.AuthoritativeSet + auth.KeyFile = util.AutomationAgentKeyFilePathInContainer + auth.KeyFileWindows = util.AutomationAgentWindowsKeyFilePath + ac.AgentSSL = &om.AgentSSL{ + AutoPEMKeyFilePath: util.AutomationAgentPemFilePath, + CAFilePath: opts.CAFilePath, + ClientCertificateMode: opts.ClientCertificates, + } + + auth.AutoUser = x.Options.AutomationSubject + auth.LdapGroupDN = opts.AutoLdapGroupDN + auth.AutoAuthMechanisms = []string{string(MongoDBX509)} + + return nil + }, log) + if err != nil { + return err + } + + log.Info("Configuring backup agent user") + err = x.Conn.ReadUpdateBackupAgentConfig(func(config *om.BackupAgentConfig) error { + config.EnableX509Authentication(opts.AutomationSubject) + config.SetLdapGroupDN(opts.AutoLdapGroupDN) + return nil + }, log) + if err != nil { + return err + } + + log.Info("Configuring monitoring agent user") + return x.Conn.ReadUpdateMonitoringAgentConfig(func(config *om.MonitoringAgentConfig) error { + config.EnableX509Authentication(opts.AutomationSubject) + config.SetLdapGroupDN(opts.AutoLdapGroupDN) + return nil + }, log) +} + +func (x ConnectionX509) DisableAgentAuthentication(log *zap.SugaredLogger) error { + err := x.Conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.AgentSSL = &om.AgentSSL{ + AutoPEMKeyFilePath: util.MergoDelete, + ClientCertificateMode: util.OptionalClientCertficates, + } + + if stringutil.Contains(ac.Auth.AutoAuthMechanisms, string(MongoDBX509)) { + ac.Auth.AutoAuthMechanisms = stringutil.Remove(ac.Auth.AutoAuthMechanisms, string(MongoDBX509)) + } + return nil + }, log) + if err != nil { + return err + } + err = x.Conn.ReadUpdateMonitoringAgentConfig(func(config *om.MonitoringAgentConfig) error { + config.DisableX509Authentication() + return nil + }, log) + if err != nil { + return err + } + + return x.Conn.ReadUpdateBackupAgentConfig(func(config *om.BackupAgentConfig) error { + config.DisableX509Authentication() + return nil + }, log) +} + +func (x ConnectionX509) EnableDeploymentAuthentication(opts Options) error { + ac := x.AutomationConfig + if !stringutil.Contains(ac.Auth.DeploymentAuthMechanisms, util.AutomationConfigX509Option) { + ac.Auth.DeploymentAuthMechanisms = append(ac.Auth.DeploymentAuthMechanisms, string(MongoDBX509)) + } + // AutomationConfig validation requires the CAFile path to be specified in the case of multiple auth + // mechanisms enabled. This is not required if only X509 is being configured + ac.AgentSSL.CAFilePath = opts.CAFilePath + return nil +} + +func (x ConnectionX509) DisableDeploymentAuthentication() error { + ac := x.AutomationConfig + ac.Auth.DeploymentAuthMechanisms = stringutil.Remove(ac.Auth.DeploymentAuthMechanisms, string(MongoDBX509)) + return nil +} + +func (x ConnectionX509) IsAgentAuthenticationConfigured() bool { + ac := x.AutomationConfig + if ac.Auth.Disabled { + return false + } + + if !stringutil.Contains(ac.Auth.AutoAuthMechanisms, string(MongoDBX509)) { + return false + } + + if !isValidX509Subject(ac.Auth.AutoUser) { + return false + } + + if ac.Auth.Key == "" || ac.Auth.KeyFile == "" || ac.Auth.KeyFileWindows == "" { + return false + } + + return true +} + +func (x ConnectionX509) IsDeploymentAuthenticationConfigured() bool { + return stringutil.Contains(x.AutomationConfig.Auth.DeploymentAuthMechanisms, string(MongoDBX509)) +} + +// isValidX509Subject checks the subject contains CommonName, Country and Organizational Unit, Location and State. +func isValidX509Subject(subject string) bool { + expected := []string{"CN", "C", "OU"} + for _, name := range expected { + matched, err := regexp.MatchString(name+`=\w+`, subject) + if err != nil { + continue + } + if !matched { + return false + } + } + return true +} diff --git a/controllers/operator/authentication/x509_test.go b/controllers/operator/authentication/x509_test.go new file mode 100644 index 000000000..5b5a69d88 --- /dev/null +++ b/controllers/operator/authentication/x509_test.go @@ -0,0 +1,64 @@ +package authentication + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestX509EnableAgentAuthentication(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + options := Options{ + AgentMechanism: "X509", + ClientCertificates: util.RequireClientCertificates, + UserOptions: UserOptions{ + AutomationSubject: validSubject("automation"), + }, + } + x := NewConnectionX509(conn, ac, options) + if err := x.EnableAgentAuthentication(Options{AuthoritativeSet: true}, zap.S()); err != nil { + t.Fatal(err) + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, ac.Auth.AutoUser, options.AutomationSubject) + assert.Len(t, ac.Auth.AutoAuthMechanisms, 1) + assert.Contains(t, ac.Auth.AutoAuthMechanisms, string(MongoDBX509)) + assert.Equal(t, ac.Auth.AutoPwd, util.MergoDelete) + assert.False(t, ac.Auth.Disabled) + assert.Len(t, ac.Auth.Users, 0) + + assert.True(t, ac.Auth.AuthoritativeSet) + assert.NotEmpty(t, ac.Auth.Key) + assert.NotEmpty(t, ac.Auth.KeyFileWindows) + assert.NotEmpty(t, ac.Auth.KeyFile) +} + +func TestX509_DisableAgentAuthentication(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + opts := Options{ + UserOptions: UserOptions{ + AutomationSubject: validSubject("automation"), + }, + } + x509 := NewConnectionX509(conn, ac, opts) + assertAgentAuthenticationDisabled(t, x509, opts) +} + +func TestX509_DeploymentConfigured(t *testing.T) { + conn, ac := createConnectionAndAutomationConfig() + assertDeploymentMechanismsConfigured(t, NewConnectionX509(conn, ac, Options{AgentMechanism: "SCRAM"})) + assert.Equal(t, ac.AgentSSL.CAFilePath, util.CAFilePathInContainer) +} + +func validSubject(o string) string { + return fmt.Sprintf("CN=mms-automation-agent,OU=MongoDB Kubernetes Operator,O=%s,L=NY,ST=NY,C=US", o) +} diff --git a/controllers/operator/authentication_test.go b/controllers/operator/authentication_test.go new file mode 100644 index 000000000..808610902 --- /dev/null +++ b/controllers/operator/authentication_test.go @@ -0,0 +1,876 @@ +package operator + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + certsv1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/deployment" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/test" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestX509CanBeEnabled_WhenThereAreOnlyTlsDeployments_ReplicaSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().EnableTLS().EnableX509().SetTLSCA("custom-ca").Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + + addKubernetesTlsResources(ctx, kubeClient, rs) + + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) +} + +func TestX509ClusterAuthentication_CanBeEnabled_IfX509AuthenticationIsEnabled_ReplicaSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().EnableTLS().EnableX509().SetTLSCA("custom-ca").Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + addKubernetesTlsResources(ctx, kubeClient, rs) + + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) +} + +func TestX509ClusterAuthentication_CanBeEnabled_IfX509AuthenticationIsEnabled_ShardedCluster(t *testing.T) { + ctx := context.Background() + scWithTls := test.DefaultClusterBuilder().EnableTLS().EnableX509().SetName("sc-with-tls").SetTLSCA("custom-ca").Build() + + reconciler, _, client, _, err := defaultClusterReconciler(ctx, nil, "", "", scWithTls, nil) + require.NoError(t, err) + addKubernetesTlsResources(ctx, client, scWithTls) + + checkReconcileSuccessful(ctx, t, reconciler, scWithTls, client) +} + +func TestX509CanBeEnabled_WhenThereAreOnlyTlsDeployments_ShardedCluster(t *testing.T) { + ctx := context.Background() + scWithTls := test.DefaultClusterBuilder().EnableTLS().EnableX509().SetName("sc-with-tls").SetTLSCA("custom-ca").Build() + + reconciler, _, client, _, err := defaultClusterReconciler(ctx, nil, "", "", scWithTls, nil) + require.NoError(t, err) + addKubernetesTlsResources(ctx, client, scWithTls) + + checkReconcileSuccessful(ctx, t, reconciler, scWithTls, client) +} + +func TestUpdateOmAuthentication_NoAuthenticationEnabled(t *testing.T) { + ctx := context.Background() + conn := om.NewMockedOmConnection(om.NewDeployment()) + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).Build() + processNames := []string{"my-rs-0", "my-rs-1", "my-rs-2"} + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + r.updateOmAuthentication(ctx, conn, processNames, rs, "", "", "", false, zap.S()) + + ac, _ := conn.ReadAutomationConfig() + + assert.True(t, ac.Auth.Disabled, "authentication was not specified to enabled, so it should remain disabled in Ops Manager") + assert.Len(t, ac.Auth.Users, 0) +} + +func TestUpdateOmAuthentication_EnableX509_TlsNotEnabled(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).Build() + // deployment with existing non-tls non-x509 replica set + conn := om.NewMockedOmConnection(deployment.CreateFromReplicaSet("fake-mongoDBImage", false, rs)) + + // configure X509 authentication & tls + rs.Spec.Security.Authentication.Modes = []mdbv1.AuthMode{"X509"} + rs.Spec.Security.Authentication.Enabled = true + rs.Spec.Security.TLSConfig.Enabled = true + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + status, isMultiStageReconciliation := r.updateOmAuthentication(ctx, conn, []string{"my-rs-0", "my-rs-1", "my-rs-2"}, rs, "", "", "", false, zap.S()) + + assert.True(t, status.IsOK(), "configuring both options at once should not result in a failed status") + assert.True(t, isMultiStageReconciliation, "configuring both tls and x509 at once should result in a multi stage reconciliation") +} + +func TestUpdateOmAuthentication_EnableX509_WithTlsAlreadyEnabled(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).EnableTLS().Build() + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(deployment.CreateFromReplicaSet("fake-mongoDBImage", false, rs))) + kubeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + status, isMultiStageReconciliation := r.updateOmAuthentication(ctx, omConnectionFactory.GetConnection(), []string{"my-rs-0", "my-rs-1", "my-rs-2"}, rs, "", "", "", false, zap.S()) + + assert.True(t, status.IsOK(), "configuring x509 when tls has already been enabled should not result in a failed status") + assert.False(t, isMultiStageReconciliation, "if tls is already enabled, we should be able to configure x509 is a single reconciliation") +} + +func TestUpdateOmAuthentication_AuthenticationIsNotConfigured_IfAuthIsNotSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).EnableTLS().SetAuthentication(nil).Build() + + rs.Spec.Security.Authentication = nil + + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(deployment.CreateFromReplicaSet("fake-mongoDBImage", false, rs))) + kubeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + status, _ := r.updateOmAuthentication(ctx, omConnectionFactory.GetConnection(), []string{"my-rs-0", "my-rs-1", "my-rs-2"}, rs, "", "", "", false, zap.S()) + assert.True(t, status.IsOK(), "no authentication should have been configured") + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + // authentication has not been touched + assert.True(t, ac.Auth.Disabled) + assert.Len(t, ac.Auth.Users, 0) + assert.Equal(t, "MONGODB-CR", ac.Auth.AutoAuthMechanism) +} + +func TestUpdateOmAuthentication_DoesNotDisableAuth_IfAuthIsNotSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder(). + EnableTLS(). + EnableAuth(). + EnableX509(). + SetTLSCA("custom-ca"). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + addKubernetesTlsResources(ctx, kubeClient, rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + // x509 auth has been enabled + assert.True(t, ac.Auth.IsEnabled()) + assert.Contains(t, ac.Auth.AutoAuthMechanism, authentication.MongoDBX509) + + rs.Spec.Security.Authentication = nil + + reconciler = newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + ac, _ = omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.True(t, ac.Auth.IsEnabled()) + assert.Contains(t, ac.Auth.AutoAuthMechanism, authentication.MongoDBX509) +} + +func TestCanConfigureAuthenticationDisabled_WithNoModes(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder(). + EnableTLS(). + SetTLSCA("custom-ca"). + SetAuthentication( + &mdbv1.Authentication{ + Enabled: false, + Modes: nil, + }). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + addKubernetesTlsResources(ctx, kubeClient, rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) +} + +func TestUpdateOmAuthentication_EnableX509_FromEmptyDeployment(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).EnableTLS().EnableAuth().EnableX509().Build() + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(om.NewDeployment())) + kubeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + createAgentCSRs(t, ctx, 1, r.client, certsv1.CertificateApproved) + + status, isMultiStageReconciliation := r.updateOmAuthentication(ctx, omConnectionFactory.GetConnection(), []string{"my-rs-0", "my-rs-1", "my-rs-2"}, rs, "", "", "", false, zap.S()) + assert.True(t, status.IsOK(), "configuring x509 and tls when there are no processes should not result in a failed status") + assert.False(t, isMultiStageReconciliation, "if we are enabling tls and x509 at once, this should be done in a single reconciliation") +} + +func TestX509AgentUserIsCorrectlyConfigured(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).EnableTLS().SetTLSCA("custom-ca").EnableAuth().EnableX509().Build() + x509User := DefaultMongoDBUserBuilder().SetDatabase(authentication.ExternalDB).SetMongoDBResourceName("my-rs").Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + memberClusterMap := getFakeMultiClusterMap(nil) + err := kubeClient.Create(ctx, x509User) + assert.NoError(t, err) + + // configure x509/tls resources + addKubernetesTlsResources(ctx, kubeClient, rs) + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + userReconciler := newMongoDBUserReconciler(ctx, kubeClient, omConnectionFactory.GetConnectionFunc, memberClusterMap) + + actual, err := userReconciler.Reconcile(ctx, requestFromObject(x509User)) + expected := reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS} + + assert.NoError(t, err) + assert.Equal(t, expected, actual) + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.Equal(t, ac.Auth.AutoUser, "CN=mms-automation-agent,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US") +} + +func TestScramAgentUserIsCorrectlyConfigured(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).EnableAuth().EnableSCRAM().Build() + scramUser := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + memberClusterMap := getFakeMultiClusterMap(nil) + err := kubeClient.Create(ctx, scramUser) + assert.NoError(t, err) + + userPassword := secret.Builder(). + SetNamespace(scramUser.Namespace). + SetName(scramUser.Spec.PasswordSecretKeyRef.Name). + SetField(scramUser.Spec.PasswordSecretKeyRef.Key, "password"). + Build() + + err = kubeClient.Create(ctx, &userPassword) + + assert.NoError(t, err) + + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + userReconciler := newMongoDBUserReconciler(ctx, kubeClient, omConnectionFactory.GetConnectionFunc, memberClusterMap) + + actual, err := userReconciler.Reconcile(ctx, requestFromObject(scramUser)) + expected := reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS} + + assert.NoError(t, err) + assert.Equal(t, expected, actual) + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.Equal(t, ac.Auth.AutoUser, util.AutomationAgentName) +} + +func TestScramAgentUser_IsNotOverridden(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs").SetMembers(3).EnableAuth().EnableSCRAM().Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + err := connection.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.AutoUser = "my-custom-agent-name" + return nil + }, nil) + if err != nil { + panic(err) + } + }) + + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + assert.Equal(t, "my-custom-agent-name", ac.Auth.AutoUser) +} + +func TestX509InternalClusterAuthentication_CanBeEnabledWithScram_ReplicaSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetName("my-rs"). + SetMembers(3). + EnableAuth(). + EnableSCRAM(). + EnableX509InternalClusterAuth(). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + addKubernetesTlsResources(ctx, r.client, rs) + + checkReconcileSuccessful(ctx, t, r, rs, kubeClient) + + dep, _ := omConnectionFactory.GetConnection().ReadDeployment() + for _, p := range dep.ProcessesCopy() { + assert.Equal(t, p.ClusterAuthMode(), "x509") + } +} + +func TestX509InternalClusterAuthentication_CanBeEnabledWithScram_ShardedCluster(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().SetName("my-sc"). + EnableAuth(). + EnableSCRAM(). + EnableX509InternalClusterAuth(). + Build() + + r, _, kubeClient, omConnectionFactory, _ := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + addKubernetesTlsResources(ctx, r.client, sc) + checkReconcileSuccessful(ctx, t, r, sc, kubeClient) + + dep, _ := omConnectionFactory.GetConnection().ReadDeployment() + for _, p := range dep.ProcessesCopy() { + assert.Equal(t, p.ClusterAuthMode(), "x509") + } +} + +func TestConfigureLdapDeploymentAuthentication_WithScramAgentAuthentication(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder(). + SetName("my-rs"). + SetMembers(3). + SetVersion("4.0.0-ent"). + EnableAuth(). + AgentAuthMode("SCRAM"). + EnableSCRAM(). + EnableLDAP(). + LDAP( + mdbv1.Ldap{ + BindQueryUser: "bindQueryUser", + Servers: []string{"server0:1234", "server1:9876"}, + BindQuerySecretRef: mdbv1.SecretRef{ + Name: "bind-query-password", + }, + TimeoutMS: 10000, + UserCacheInvalidationInterval: 60, + }, + ). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + data := map[string]string{ + "password": "LITZTOd6YiCV8j", + } + err := secret.CreateOrUpdate(ctx, r.client, secret.Builder(). + SetName("bind-query-password"). + SetNamespace(mock.TestNamespace). + SetStringMapToData(data). + Build(), + ) + assert.NoError(t, err) + checkReconcileSuccessful(ctx, t, r, rs, kubeClient) + + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + assert.Equal(t, "LITZTOd6YiCV8j", ac.Ldap.BindQueryPassword) + assert.Equal(t, "bindQueryUser", ac.Ldap.BindQueryUser) + assert.Equal(t, "server0:1234,server1:9876", ac.Ldap.Servers) + assert.Equal(t, 10000, ac.Ldap.TimeoutMS) + assert.Equal(t, 60, ac.Ldap.UserCacheInvalidationInterval) + assert.Contains(t, ac.Auth.DeploymentAuthMechanisms, "PLAIN") + assert.Contains(t, ac.Auth.DeploymentAuthMechanisms, "SCRAM-SHA-256") +} + +func TestConfigureLdapDeploymentAuthentication_WithCustomRole(t *testing.T) { + ctx := context.Background() + + customRoles := []mdbv1.MongoDbRole{ + { + Db: "admin", + Role: "customRole", + Roles: []mdbv1.InheritedRole{{Db: "Admin", Role: "inheritedrole"}}, + Privileges: []mdbv1.Privilege{}, + }, + } + + rs := DefaultReplicaSetBuilder(). + SetName("my-rs"). + SetMembers(3). + SetVersion("4.0.0-ent"). + EnableAuth(). + AgentAuthMode("SCRAM"). + EnableSCRAM(). + EnableLDAP(). + LDAP( + mdbv1.Ldap{ + BindQueryUser: "bindQueryUser", + Servers: []string{"server0:1234"}, + BindQuerySecretRef: mdbv1.SecretRef{ + Name: "bind-query-password", + }, + }, + ). + SetRoles(customRoles). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + data := map[string]string{ + "password": "LITZTOd6YiCV8j", + } + err := secret.CreateOrUpdate(ctx, r.client, secret.Builder(). + SetName("bind-query-password"). + SetNamespace(mock.TestNamespace). + SetStringMapToData(data). + Build(), + ) + assert.NoError(t, err) + checkReconcileSuccessful(ctx, t, r, rs, kubeClient) + + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + assert.Equal(t, "server0:1234", ac.Ldap.Servers) + + roles := ac.Deployment["roles"].([]mdbv1.MongoDbRole) + assert.Len(t, roles, 1) + assert.Equal(t, customRoles, roles) +} + +func TestConfigureLdapDeploymentAuthentication_WithAuthzQueryTemplate_AndUserToDnMapping(t *testing.T) { + ctx := context.Background() + + userMapping := `[ + { + match: "(.+)", + substitution: "uid={0},dc=example,dc=org" + } + ]` + authzTemplate := "{USER}?memberOf?base" + rs := DefaultReplicaSetBuilder(). + SetName("my-rs"). + SetMembers(3). + SetVersion("4.0.0-ent"). + EnableAuth(). + AgentAuthMode("SCRAM"). + EnableSCRAM(). + EnableLDAP(). + LDAP( + mdbv1.Ldap{ + BindQueryUser: "bindQueryUser", + Servers: []string{"server0:0000,server1:1111,server2:2222"}, + BindQuerySecretRef: mdbv1.SecretRef{ + Name: "bind-query-password", + }, + AuthzQueryTemplate: authzTemplate, + UserToDNMapping: userMapping, + }, + ). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + r := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + data := map[string]string{ + "password": "LITZTOd6YiCV8j", + } + err := secret.CreateOrUpdate(ctx, r.client, secret.Builder(). + SetName("bind-query-password"). + SetNamespace(mock.TestNamespace). + SetStringMapToData(data). + Build(), + ) + assert.NoError(t, err) + checkReconcileSuccessful(ctx, t, r, rs, kubeClient) + + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + assert.Equal(t, "server0:0000,server1:1111,server2:2222", ac.Ldap.Servers) + + assert.Equal(t, authzTemplate, ac.Ldap.AuthzQueryTemplate) + assert.Equal(t, userMapping, ac.Ldap.UserToDnMapping) +} + +// addKubernetesTlsResources ensures all the required TLS secrets exist for the given MongoDB resource +func addKubernetesTlsResources(ctx context.Context, client kubernetesClient.Client, mdb *mdbv1.MongoDB) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: mock.TestCredentialsSecretName, Namespace: mock.TestNamespace}, + Data: map[string][]byte{ + "publicApiKey": []byte("someapi"), + "user": []byte("someuser"), + }, + Type: corev1.SecretTypeTLS, + } + + _ = client.Create(ctx, secret) + switch mdb.Spec.ResourceType { + case mdbv1.ReplicaSet: + createReplicaSetTLSData(ctx, client, mdb) + case mdbv1.ShardedCluster: + createShardedClusterTLSData(ctx, client, mdb) + } +} + +// createMockCertAndKeyBytes generates a random key and certificate and returns +// them as bytes +func createMockCertAndKeyBytes(certOpts ...func(cert *x509.Certificate)) (cert, key []byte) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(err) + } + + privBytes, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + panic(err) + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + panic(err) + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"MongoDB"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), // cert expires in 10 years + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{"somehost.com"}, + } + + for _, opt := range certOpts { + opt(&template) + } + certBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + panic(err) + } + + certPemBytes := &bytes.Buffer{} + if err := pem.Encode(certPemBytes, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes}); err != nil { + panic(err) + } + + privPemBytes := &bytes.Buffer{} + if err := pem.Encode(privPemBytes, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil { + panic(err) + } + + return certPemBytes.Bytes(), privPemBytes.Bytes() +} + +// createReplicaSetTLSData creates and populates secrets required for a TLS enabled ReplicaSet +func createReplicaSetTLSData(ctx context.Context, client client.Client, mdb *mdbv1.MongoDB) { + // Lets create a secret with Certificates and private keys! + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-cert", mdb.Name), + Namespace: mock.TestNamespace, + }, + Type: corev1.SecretTypeTLS, + } + + certs := map[string][]byte{} + clientCerts := map[string][]byte{} + + certs["tls.crt"], certs["tls.key"] = createMockCertAndKeyBytes() + clientCerts["tls.crt"], clientCerts["tls.key"] = createMockCertAndKeyBytes() + secret.Data = certs + + _ = client.Create(ctx, secret) + + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-clusterfile", mdb.Name), Namespace: mock.TestNamespace}, + Data: clientCerts, + Type: corev1.SecretTypeTLS, + }) + + agentCerts := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-certs", + Namespace: mock.TestNamespace, + }, + Type: corev1.SecretTypeTLS, + } + + subjectModifier := func(cert *x509.Certificate) { + cert.Subject.OrganizationalUnit = []string{"cloud"} + cert.Subject.Locality = []string{"New York"} + cert.Subject.Province = []string{"New York"} + cert.Subject.Country = []string{"US"} + } + + agentCerts.Data = make(map[string][]byte) + agentCerts.Data["tls.crt"], agentCerts.Data["tls.key"] = createMockCertAndKeyBytes(subjectModifier, func(cert *x509.Certificate) { cert.Subject.CommonName = "mms-automation-agent" }) + _ = client.Create(ctx, agentCerts) +} + +// createShardedClusterTLSData creates and populates all the secrets needed for a TLS enabled Sharded +// Cluster with internal cluster authentication. Mongos, config server and all shards. +func createShardedClusterTLSData(ctx context.Context, client kubernetesClient.Client, mdb *mdbv1.MongoDB) { + // create the secrets for all the shards + for i := 0; i < mdb.Spec.ShardCount; i++ { + secretName := fmt.Sprintf("%s-%d-cert", mdb.Name, i) + shardData := make(map[string][]byte) + shardData["tls.crt"], shardData["tls.key"] = createMockCertAndKeyBytes() + + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: mock.TestNamespace}, + Data: shardData, + Type: corev1.SecretTypeTLS, + }) + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%d-clusterfile", mdb.Name, i), Namespace: mock.TestNamespace}, + Data: shardData, + Type: corev1.SecretTypeTLS, + }) + } + + // populate with the expected cert and key fields + mongosData := make(map[string][]byte) + mongosData["tls.crt"], mongosData["tls.key"] = createMockCertAndKeyBytes() + + // create the mongos secret + mongosSecretName := fmt.Sprintf("%s-mongos-cert", mdb.Name) + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: mongosSecretName, Namespace: mock.TestNamespace}, + Data: mongosData, + Type: corev1.SecretTypeTLS, + }) + + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-mongos-clusterfile", mdb.Name), Namespace: mock.TestNamespace}, + Data: mongosData, + Type: corev1.SecretTypeTLS, + }) + + // create secret for config server + configData := make(map[string][]byte) + configData["tls.crt"], configData["tls.key"] = createMockCertAndKeyBytes() + + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-config-cert", mdb.Name), Namespace: mock.TestNamespace}, + Data: configData, + Type: corev1.SecretTypeTLS, + }) + + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-config-clusterfile", mdb.Name), Namespace: mock.TestNamespace}, + Data: configData, + Type: corev1.SecretTypeTLS, + }) + agentCerts := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-certs", + Namespace: mock.TestNamespace, + }, + Type: corev1.SecretTypeTLS, + } + + subjectModifier := func(cert *x509.Certificate) { + cert.Subject.OrganizationalUnit = []string{"cloud"} + cert.Subject.Locality = []string{"New York"} + cert.Subject.Province = []string{"New York"} + cert.Subject.Country = []string{"US"} + } + + agentCerts.Data = make(map[string][]byte) + agentCerts.Data["tls.crt"], agentCerts.Data["tls.key"] = createMockCertAndKeyBytes(subjectModifier, func(cert *x509.Certificate) { cert.Subject.CommonName = "mms-automation-agent" }) + _ = client.Create(ctx, agentCerts) +} + +// createMultiClusterReplicaSetTLSData creates and populates secrets required for a TLS enabled MongoDBMultiCluster ReplicaSet. +func createMultiClusterReplicaSetTLSData(t *testing.T, ctx context.Context, client client.Client, mdbm *mdbmulti.MongoDBMultiCluster, caName string) { + // Create CA configmap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: caName, + Namespace: mock.TestNamespace, + }, + } + cm.Data = map[string]string{ + "ca-pem": "capublickey", + "mms-ca.crt": "capublickey", + } + err := client.Create(ctx, cm) + assert.NoError(t, err) + // Lets create a secret with Certificates and private keys! + secretName := fmt.Sprintf("%s-cert", mdbm.Name) + if mdbm.Spec.Security.CertificatesSecretsPrefix != "" { + secretName = fmt.Sprintf("%s-%s", mdbm.Spec.Security.CertificatesSecretsPrefix, secretName) + } + + secret := &corev1.Secret{ + Type: corev1.SecretTypeTLS, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: mock.TestNamespace, + }, + } + + certs := map[string][]byte{} + certs["tls.crt"], certs["tls.key"] = createMockCertAndKeyBytes() + + secret.Data = certs + // create cert in the central cluster, the operator would create the concatenated + // pem cert in the member clusters. + err = client.Create(ctx, secret) + assert.NoError(t, err) +} + +func TestInvalidPEM_SecretDoesNotContainKey(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder(). + EnableTLS(). + EnableAuth(). + EnableX509(). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + addKubernetesTlsResources(ctx, kubeClient, rs) + + // Replace the secret with an empty one + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-cert", rs.Name), + Namespace: mock.TestNamespace, + }, + Type: corev1.SecretTypeTLS, + } + + _ = kubeClient.Update(ctx, secret) + + err := certs.VerifyAndEnsureCertificatesForStatefulSet(ctx, reconciler.SecretClient, reconciler.SecretClient, fmt.Sprintf("%s-cert", rs.Name), certs.ReplicaSetConfig(*rs), nil) + assert.Equal(t, err.Error(), "the certificate is not complete\n") +} + +func Test_NoAdditionalDomainsPresent(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder(). + EnableTLS(). + EnableAuth(). + EnableX509(). + Build() + + // The default secret we create does not contain additional domains so it will not be valid for this RS + rs.Spec.Security.TLSConfig.AdditionalCertificateDomains = []string{"foo"} + + reconciler, _, client, _, err := defaultClusterReconciler(ctx, nil, "", "", rs, nil) + require.NoError(t, err) + addKubernetesTlsResources(ctx, client, rs) + + secret := &corev1.Secret{} + + _ = client.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-cert", rs.Name), Namespace: rs.Namespace}, secret) + + err = certs.VerifyAndEnsureCertificatesForStatefulSet(ctx, reconciler.SecretClient, reconciler.SecretClient, fmt.Sprintf("%s-cert", rs.Name), certs.ReplicaSetConfig(*rs), nil) + for i := 0; i < rs.Spec.Members; i++ { + expectedErrorMessage := fmt.Sprintf("domain %s-%d.foo is not contained in the list of DNSNames", rs.Name, i) + assert.Contains(t, err.Error(), expectedErrorMessage) + } +} + +func Test_NoExternalDomainPresent(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder(). + EnableTLS(). + EnableAuth(). + EnableX509(). + Build() + + rs.Spec.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("foo")} + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + reconciler := newReplicaSetReconciler(ctx, kubeClient, nil, "", "", false, omConnectionFactory.GetConnectionFunc) + addKubernetesTlsResources(ctx, kubeClient, rs) + + secret := &corev1.Secret{} + + _ = kubeClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-cert", rs.Name), Namespace: rs.Namespace}, secret) + + err := certs.VerifyAndEnsureCertificatesForStatefulSet(ctx, reconciler.SecretClient, reconciler.SecretClient, fmt.Sprintf("%s-cert", rs.Name), certs.ReplicaSetConfig(*rs), nil) + assert.Error(t, err) +} + +// createAgentCSRs creates all the agent CSRs needed for x509 at the specified condition type +func createAgentCSRs(t *testing.T, ctx context.Context, numAgents int, client kubernetesClient.Client, conditionType certsv1.RequestConditionType) { + if numAgents != 1 && numAgents != 3 { + return + } + // create the secret the agent certs will exist in + certAuto, _ := os.ReadFile("testdata/certificates/cert_auto") + + builder := secret.Builder(). + SetNamespace(mock.TestNamespace). + SetName(util.AgentSecretName). + SetField(util.AutomationAgentPemSecretKey, string(certAuto)) + + err := client.CreateSecret(ctx, builder.Build()) + assert.NoError(t, err) + + addCsrs(ctx, client, createCSR("mms-automation-agent", mock.TestNamespace, conditionType)) +} + +func addCsrs(ctx context.Context, client kubernetesClient.Client, csrs ...certsv1.CertificateSigningRequest) { + for _, csr := range csrs { + _ = client.Create(ctx, &csr) + } +} + +// createCSR creates a CSR object which can be set to either CertificateApproved or CertificateDenied +func createCSR(name, ns string, conditionType certsv1.RequestConditionType) certsv1.CertificateSigningRequest { + return certsv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s.%s", name, ns)}, + Spec: certsv1.CertificateSigningRequestSpec{ + Request: createMockCSRBytes(), + }, + Status: certsv1.CertificateSigningRequestStatus{ + Conditions: []certsv1.CertificateSigningRequestCondition{ + {Type: conditionType}, + }, + }, + } +} + +// createMockCSRBytes creates a new Certificate Signing Request, signed with a +// fresh private key +func createMockCSRBytes() []byte { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(err) + } + + template := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"MongoDB"}, + }, + DNSNames: []string{"somehost.com"}, + } + certRequestBytes, err := x509.CreateCertificateRequest(rand.Reader, &template, priv) + if err != nil { + panic(err) + } + + certRequestPemBytes := &bytes.Buffer{} + pemBlock := pem.Block{Type: "CERTIFICATE REQUEST", Bytes: certRequestBytes} + if err := pem.Encode(certRequestPemBytes, &pemBlock); err != nil { + panic(err) + } + + return certRequestPemBytes.Bytes() +} diff --git a/controllers/operator/backup_snapshot_schedule_test.go b/controllers/operator/backup_snapshot_schedule_test.go new file mode 100644 index 000000000..fc0443ef0 --- /dev/null +++ b/controllers/operator/backup_snapshot_schedule_test.go @@ -0,0 +1,157 @@ +package operator + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func backupSnapshotScheduleTests(mdb backup.ConfigReaderUpdater, client client.Client, reconciler reconcile.Reconciler, omConnectionFactory *om.CachedOMConnectionFactory, clusterID string) func(t *testing.T) { + ctx := context.Background() + return func(t *testing.T) { + t.Run("Backup schedule is not read and not updated if not specified in spec", testBackupScheduleNotReadAndNotUpdatedIfNotSpecifiedInSpec(ctx, mdb, client, reconciler, omConnectionFactory, clusterID)) + t.Run("Backup schedule is updated if specified in spec", testBackupScheduleIsUpdatedIfSpecifiedInSpec(ctx, mdb, client, reconciler, omConnectionFactory, clusterID)) + t.Run("Backup schedule is not updated if not changed", testBackupScheduleNotUpdatedIfNotChanged(ctx, mdb, client, reconciler, omConnectionFactory, clusterID)) + } +} + +func testBackupScheduleNotReadAndNotUpdatedIfNotSpecifiedInSpec(ctx context.Context, mdb backup.ConfigReaderUpdater, client client.Client, reconciler reconcile.Reconciler, omConnectionFactory *om.CachedOMConnectionFactory, clusterID string) func(t *testing.T) { + return func(t *testing.T) { + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + insertDefaultBackupSchedule(t, omConnectionFactory, clusterID) + + mdb.GetBackupSpec().SnapshotSchedule = nil + + err := client.Update(ctx, mdb) + assert.NoError(t, err) + + omConnectionFactory.GetConnection().(*om.MockedOmConnection).CleanHistory() + checkReconcile(ctx, t, reconciler, mdb) + omConnectionFactory.GetConnection().(*om.MockedOmConnection).CheckOperationsDidntHappen(t, reflect.ValueOf(omConnectionFactory.GetConnection().UpdateSnapshotSchedule)) + omConnectionFactory.GetConnection().(*om.MockedOmConnection).CheckOperationsDidntHappen(t, reflect.ValueOf(omConnectionFactory.GetConnection().ReadSnapshotSchedule)) + } +} + +func testBackupScheduleIsUpdatedIfSpecifiedInSpec(ctx context.Context, mdb backup.ConfigReaderUpdater, client client.Client, reconciler reconcile.Reconciler, omConnectionFactory *om.CachedOMConnectionFactory, clusterID string) func(t *testing.T) { + return func(t *testing.T) { + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + insertDefaultBackupSchedule(t, omConnectionFactory, clusterID) + + mdb.GetBackupSpec().SnapshotSchedule = &mdbv1.SnapshotSchedule{ + SnapshotIntervalHours: ptr.To(1), + SnapshotRetentionDays: ptr.To(2), + DailySnapshotRetentionDays: ptr.To(3), + WeeklySnapshotRetentionWeeks: ptr.To(4), + MonthlySnapshotRetentionMonths: ptr.To(5), + PointInTimeWindowHours: ptr.To(6), + ReferenceHourOfDay: ptr.To(7), + ReferenceMinuteOfHour: ptr.To(8), + FullIncrementalDayOfWeek: ptr.To("Sunday"), + ClusterCheckpointIntervalMin: ptr.To(9), + } + + err := client.Update(ctx, mdb) + require.NoError(t, err) + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + + checkReconcile(ctx, t, reconciler, mdb) + + snapshotSchedule, err := omConnectionFactory.GetConnection().ReadSnapshotSchedule(clusterID) + require.NoError(t, err) + assertSnapshotScheduleEqual(t, mdb.GetBackupSpec().SnapshotSchedule, snapshotSchedule) + } +} + +func testBackupScheduleNotUpdatedIfNotChanged(ctx context.Context, mdb backup.ConfigReaderUpdater, kubeClient client.Client, reconciler reconcile.Reconciler, omConnectionFactory *om.CachedOMConnectionFactory, clusterID string) func(t *testing.T) { + return func(t *testing.T) { + require.NoError(t, kubeClient.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + insertDefaultBackupSchedule(t, omConnectionFactory, clusterID) + + snapshotSchedule := &mdbv1.SnapshotSchedule{ + SnapshotIntervalHours: ptr.To(11), + SnapshotRetentionDays: ptr.To(12), + DailySnapshotRetentionDays: ptr.To(13), + WeeklySnapshotRetentionWeeks: ptr.To(14), + MonthlySnapshotRetentionMonths: ptr.To(15), + PointInTimeWindowHours: ptr.To(16), + ReferenceHourOfDay: ptr.To(17), + ReferenceMinuteOfHour: ptr.To(18), + FullIncrementalDayOfWeek: ptr.To("Thursday"), + ClusterCheckpointIntervalMin: ptr.To(19), + } + + mdb.GetBackupSpec().SnapshotSchedule = snapshotSchedule + + err := kubeClient.Update(ctx, mdb) + require.NoError(t, err) + + checkReconcile(ctx, t, reconciler, mdb) + require.NoError(t, kubeClient.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + + omSnapshotSchedule, err := omConnectionFactory.GetConnection().ReadSnapshotSchedule(clusterID) + require.NoError(t, err) + + assertSnapshotScheduleEqual(t, mdb.GetBackupSpec().SnapshotSchedule, omSnapshotSchedule) + + omConnectionFactory.GetConnection().(*om.MockedOmConnection).CleanHistory() + checkReconcile(ctx, t, reconciler, mdb) + require.NoError(t, kubeClient.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + + omConnectionFactory.GetConnection().(*om.MockedOmConnection).CheckOperationsDidntHappen(t, reflect.ValueOf(omConnectionFactory.GetConnection().UpdateSnapshotSchedule)) + + mdb.GetBackupSpec().SnapshotSchedule.FullIncrementalDayOfWeek = ptr.To("Monday") + err = kubeClient.Update(ctx, mdb) + require.NoError(t, err) + + checkReconcile(ctx, t, reconciler, mdb) + require.NoError(t, kubeClient.Get(ctx, kube.ObjectKeyFromApiObject(mdb), mdb)) + + omSnapshotSchedule, err = omConnectionFactory.GetConnection().ReadSnapshotSchedule(clusterID) + assert.NoError(t, err) + require.NotNil(t, omSnapshotSchedule) + require.NotNil(t, omSnapshotSchedule.FullIncrementalDayOfWeek) + assert.Equal(t, "Monday", *omSnapshotSchedule.FullIncrementalDayOfWeek) + } +} + +func insertDefaultBackupSchedule(t *testing.T, omConnectionFactory *om.CachedOMConnectionFactory, clusterID string) { + // insert default backup schedule + err := omConnectionFactory.GetConnection().UpdateSnapshotSchedule(clusterID, &backup.SnapshotSchedule{ + GroupID: om.TestGroupID, + ClusterID: clusterID, + }) + assert.NoError(t, err) +} + +func assertSnapshotScheduleEqual(t *testing.T, expected *mdbv1.SnapshotSchedule, actual *backup.SnapshotSchedule) { + assert.Equal(t, expected.SnapshotIntervalHours, actual.SnapshotIntervalHours) + assert.Equal(t, expected.SnapshotRetentionDays, actual.SnapshotRetentionDays) + assert.Equal(t, expected.DailySnapshotRetentionDays, actual.DailySnapshotRetentionDays) + assert.Equal(t, expected.WeeklySnapshotRetentionWeeks, actual.WeeklySnapshotRetentionWeeks) + assert.Equal(t, expected.MonthlySnapshotRetentionMonths, actual.MonthlySnapshotRetentionMonths) + assert.Equal(t, expected.PointInTimeWindowHours, actual.PointInTimeWindowHours) + assert.Equal(t, expected.ReferenceHourOfDay, actual.ReferenceHourOfDay) + assert.Equal(t, expected.ReferenceMinuteOfHour, actual.ReferenceMinuteOfHour) + assert.Equal(t, expected.FullIncrementalDayOfWeek, actual.FullIncrementalDayOfWeek) + assert.Equal(t, expected.ClusterCheckpointIntervalMin, actual.ClusterCheckpointIntervalMin) +} + +func checkReconcile(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, resource metav1.Object) { + result, e := reconciler.Reconcile(ctx, requestFromObject(resource)) + require.NoError(t, e) + require.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, result) +} diff --git a/controllers/operator/certs/cert_configurations.go b/controllers/operator/certs/cert_configurations.go new file mode 100644 index 000000000..684b7b1d7 --- /dev/null +++ b/controllers/operator/certs/cert_configurations.go @@ -0,0 +1,318 @@ +package certs + +import ( + "fmt" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers/interfaces" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +// X509CertConfigurator provides the methods required for ensuring the existence of X.509 certificates +// for encrypted communications in MongoDB resource +type X509CertConfigurator interface { + GetName() string + GetNamespace() string + GetDbCommonSpec() *mdbv1.DbCommonSpec + GetCertOptions() []Options + GetSecretReadClient() secrets.SecretClient + GetSecretWriteClient() secrets.SecretClient +} + +type ReplicaSetX509CertConfigurator struct { + *mdbv1.MongoDB + SecretClient secrets.SecretClient +} + +var _ X509CertConfigurator = &ReplicaSetX509CertConfigurator{} + +func (rs ReplicaSetX509CertConfigurator) GetCertOptions() []Options { + return []Options{ReplicaSetConfig(*rs.MongoDB)} +} + +func (rs ReplicaSetX509CertConfigurator) GetSecretReadClient() secrets.SecretClient { + return rs.SecretClient +} + +func (rs ReplicaSetX509CertConfigurator) GetSecretWriteClient() secrets.SecretClient { + return rs.SecretClient +} + +func (rs ReplicaSetX509CertConfigurator) GetDbCommonSpec() *mdbv1.DbCommonSpec { + return &rs.Spec.DbCommonSpec +} + +type ShardedSetX509CertConfigurator struct { + *mdbv1.MongoDB + MemberCluster multicluster.MemberCluster + SecretReadClient secrets.SecretClient + CertOptions []Options +} + +var _ X509CertConfigurator = ShardedSetX509CertConfigurator{} + +func (sc ShardedSetX509CertConfigurator) GetCertOptions() []Options { + return sc.CertOptions +} + +func (sc ShardedSetX509CertConfigurator) GetSecretReadClient() secrets.SecretClient { + return sc.SecretReadClient +} + +func (sc ShardedSetX509CertConfigurator) GetSecretWriteClient() secrets.SecretClient { + return sc.MemberCluster.SecretClient +} + +func (sc ShardedSetX509CertConfigurator) GetDbCommonSpec() *mdbv1.DbCommonSpec { + return &sc.Spec.DbCommonSpec +} + +type StandaloneX509CertConfigurator struct { + *mdbv1.MongoDB + SecretClient secrets.SecretClient +} + +var _ X509CertConfigurator = StandaloneX509CertConfigurator{} + +func (s StandaloneX509CertConfigurator) GetCertOptions() []Options { + return []Options{StandaloneConfig(*s.MongoDB)} +} + +func (s StandaloneX509CertConfigurator) GetSecretReadClient() secrets.SecretClient { + return s.SecretClient +} + +func (s StandaloneX509CertConfigurator) GetSecretWriteClient() secrets.SecretClient { + return s.SecretClient +} + +func (s StandaloneX509CertConfigurator) GetDbCommonSpec() *mdbv1.DbCommonSpec { + return &s.Spec.DbCommonSpec +} + +type MongoDBMultiX509CertConfigurator struct { + *mdbmulti.MongoDBMultiCluster + ClusterNum int + ClusterName string + Replicas int + SecretReadClient secrets.SecretClient + SecretWriteClient secrets.SecretClient +} + +var _ X509CertConfigurator = MongoDBMultiX509CertConfigurator{} + +func (mdbm MongoDBMultiX509CertConfigurator) GetCertOptions() []Options { + return []Options{MultiReplicaSetConfig(*mdbm.MongoDBMultiCluster, mdbm.ClusterNum, mdbm.ClusterName, mdbm.Replicas)} +} + +func (mdbm MongoDBMultiX509CertConfigurator) GetSecretReadClient() secrets.SecretClient { + return mdbm.SecretReadClient +} + +func (mdbm MongoDBMultiX509CertConfigurator) GetSecretWriteClient() secrets.SecretClient { + return mdbm.SecretWriteClient +} + +func (mdbm MongoDBMultiX509CertConfigurator) GetDbCommonSpec() *mdbv1.DbCommonSpec { + return &mdbm.Spec.DbCommonSpec +} + +type Options struct { + // CertSecretName is the name of the secret which contains the certs. + CertSecretName string + + // InternalClusterSecretName is the name of the secret which contains the certs of internal cluster auth. + InternalClusterSecretName string + // ResourceName is the name of the resource. + ResourceName string + // Replicas is the number of replicas. + Replicas int + // Namespace is the namespace the resource is in. + Namespace string + // ServiceName is the name of the service which is created for the resource. + ServiceName string + // ClusterDomain is the cluster domain for the resource + ClusterDomain string + additionalCertificateDomains []string + // External domain for external access (if enabled) + ExternalDomain *string + + // horizons is an array of MongoDBHorizonConfig which is used to determine any + // additional cert domains required. + horizons []mdbv1.MongoDBHorizonConfig + + Topology string + + OwnerReference []metav1.OwnerReference +} + +// StandaloneConfig returns a function which provides all of the configuration options required for the given Standalone. +func StandaloneConfig(mdb mdbv1.MongoDB) Options { + return Options{ + ResourceName: mdb.Name, + CertSecretName: GetCertNameWithPrefixOrDefault(*mdb.GetSecurity(), mdb.Name), + Namespace: mdb.Namespace, + ServiceName: mdb.ServiceName(), + Replicas: 1, + ClusterDomain: mdb.Spec.GetClusterDomain(), + additionalCertificateDomains: mdb.Spec.Security.TLSConfig.AdditionalCertificateDomains, + OwnerReference: mdb.GetOwnerReferences(), + ExternalDomain: mdb.Spec.DbCommonSpec.GetExternalDomain(), + } +} + +// ReplicaSetConfig returns a struct which provides all of the configuration options required for the given Replica Set. +func ReplicaSetConfig(mdb mdbv1.MongoDB) Options { + return Options{ + ResourceName: mdb.Name, + CertSecretName: mdb.GetSecurity().MemberCertificateSecretName(mdb.Name), + InternalClusterSecretName: mdb.GetSecurity().InternalClusterAuthSecretName(mdb.Name), + Namespace: mdb.Namespace, + Replicas: scale.ReplicasThisReconciliation(&mdb), + ServiceName: mdb.ServiceName(), + ClusterDomain: mdb.Spec.GetClusterDomain(), + additionalCertificateDomains: mdb.Spec.Security.TLSConfig.AdditionalCertificateDomains, + horizons: mdb.Spec.Connectivity.ReplicaSetHorizons, + OwnerReference: mdb.GetOwnerReferences(), + ExternalDomain: mdb.Spec.DbCommonSpec.GetExternalDomain(), + } +} + +func AppDBReplicaSetConfig(om *omv1.MongoDBOpsManager) Options { + mdb := om.Spec.AppDB + opts := Options{ + ResourceName: mdb.Name(), + CertSecretName: mdb.GetSecurity().MemberCertificateSecretName(mdb.Name()), + InternalClusterSecretName: mdb.GetSecurity().InternalClusterAuthSecretName(mdb.Name()), + Namespace: mdb.Namespace, + Replicas: scale.ReplicasThisReconciliation(scalers.NewAppDBSingleClusterScaler(om)), + ServiceName: mdb.ServiceName(), + ClusterDomain: mdb.ClusterDomain, + OwnerReference: om.GetOwnerReferences(), + } + + if mdb.GetSecurity().TLSConfig != nil { + opts.additionalCertificateDomains = append(opts.additionalCertificateDomains, mdb.GetSecurity().TLSConfig.AdditionalCertificateDomains...) + } + + return opts +} + +func AppDBMultiClusterReplicaSetConfig(om *omv1.MongoDBOpsManager, scaler interfaces.MultiClusterReplicaSetScaler) Options { + mdb := om.Spec.AppDB + opts := Options{ + ResourceName: mdb.NameForCluster(scaler.MemberClusterNum()), + CertSecretName: mdb.GetSecurity().MemberCertificateSecretName(mdb.Name()), + InternalClusterSecretName: mdb.GetSecurity().InternalClusterAuthSecretName(mdb.Name()), + Namespace: mdb.Namespace, + Replicas: scale.ReplicasThisReconciliation(scaler), + ClusterDomain: mdb.ClusterDomain, + OwnerReference: om.GetOwnerReferences(), + Topology: mdbv1.ClusterTopologyMultiCluster, + } + + if mdb.GetSecurity().TLSConfig != nil { + opts.additionalCertificateDomains = append(opts.additionalCertificateDomains, mdb.GetSecurity().TLSConfig.AdditionalCertificateDomains...) + } + + return opts +} + +// ShardConfig returns a struct which provides all the configuration options required for the given shard. +func ShardConfig(mdb mdbv1.MongoDB, shardNum int, externalDomain *string, scaler interfaces.MultiClusterReplicaSetScaler) Options { + resourceName := mdb.ShardRsName(shardNum) + if mdb.Spec.IsMultiCluster() { + resourceName = mdb.MultiShardRsName(scaler.MemberClusterNum(), shardNum) + } + + return Options{ + ResourceName: resourceName, + CertSecretName: mdb.GetSecurity().MemberCertificateSecretName(mdb.ShardRsName(shardNum)), + InternalClusterSecretName: mdb.GetSecurity().InternalClusterAuthSecretName(mdb.ShardRsName(shardNum)), + Namespace: mdb.Namespace, + Replicas: scale.ReplicasThisReconciliation(scaler), + ServiceName: mdb.ShardServiceName(), + ClusterDomain: mdb.Spec.GetClusterDomain(), + additionalCertificateDomains: mdb.Spec.Security.TLSConfig.AdditionalCertificateDomains, + OwnerReference: mdb.GetOwnerReferences(), + ExternalDomain: externalDomain, + Topology: mdb.Spec.GetTopology(), + } +} + +// MultiReplicaSetConfig returns a struct which provides all of the configuration required for a given MongoDB Multi Replicaset. +func MultiReplicaSetConfig(mdbm mdbmulti.MongoDBMultiCluster, clusterNum int, clusterName string, replicas int) Options { + return Options{ + ResourceName: mdbm.MultiStatefulsetName(clusterNum), + CertSecretName: mdbm.Spec.GetSecurity().MemberCertificateSecretName(mdbm.Name), + InternalClusterSecretName: mdbm.Spec.GetSecurity().InternalClusterAuthSecretName(mdbm.Name), + Namespace: mdbm.Namespace, + Replicas: replicas, + ClusterDomain: mdbm.Spec.GetClusterDomain(), + Topology: mdbv1.ClusterTopologyMultiCluster, + OwnerReference: mdbm.GetOwnerReferences(), + ExternalDomain: mdbm.Spec.GetExternalDomainForMemberCluster(clusterName), + } +} + +// MongosConfig returns a struct which provides all of the configuration options required for the given Mongos. +func MongosConfig(mdb mdbv1.MongoDB, externalDomain *string, scaler interfaces.MultiClusterReplicaSetScaler) Options { + resourceName := mdb.MongosRsName() + if mdb.Spec.IsMultiCluster() { + resourceName = mdb.MultiMongosRsName(scaler.MemberClusterNum()) + } + + return Options{ + ResourceName: resourceName, + CertSecretName: mdb.GetSecurity().MemberCertificateSecretName(mdb.MongosRsName()), + InternalClusterSecretName: mdb.GetSecurity().InternalClusterAuthSecretName(mdb.MongosRsName()), + Namespace: mdb.Namespace, + Replicas: scale.ReplicasThisReconciliation(scaler), + ServiceName: mdb.ServiceName(), + ClusterDomain: mdb.Spec.GetClusterDomain(), + additionalCertificateDomains: mdb.Spec.Security.TLSConfig.AdditionalCertificateDomains, + OwnerReference: mdb.GetOwnerReferences(), + ExternalDomain: externalDomain, + Topology: mdb.Spec.GetTopology(), + } +} + +// ConfigSrvConfig returns a struct which provides all of the configuration options required for the given ConfigServer. +func ConfigSrvConfig(mdb mdbv1.MongoDB, externalDomain *string, scaler interfaces.MultiClusterReplicaSetScaler) Options { + resourceName := mdb.ConfigRsName() + if mdb.Spec.IsMultiCluster() { + resourceName = mdb.MultiConfigRsName(scaler.MemberClusterNum()) + } + + return Options{ + ResourceName: resourceName, + CertSecretName: mdb.GetSecurity().MemberCertificateSecretName(mdb.ConfigRsName()), + InternalClusterSecretName: mdb.GetSecurity().InternalClusterAuthSecretName(mdb.ConfigRsName()), + Namespace: mdb.Namespace, + Replicas: scale.ReplicasThisReconciliation(scaler), + ServiceName: mdb.ConfigSrvServiceName(), + ClusterDomain: mdb.Spec.GetClusterDomain(), + additionalCertificateDomains: mdb.Spec.Security.TLSConfig.AdditionalCertificateDomains, + OwnerReference: mdb.GetOwnerReferences(), + ExternalDomain: externalDomain, + Topology: mdb.Spec.GetTopology(), + } +} + +// GetCertNameWithPrefixOrDefault returns the name of the cert that will store certificates for the given resource. +// this takes into account the tlsConfig.prefix option. +func GetCertNameWithPrefixOrDefault(ms mdbv1.Security, defaultName string) string { + if ms.CertificatesSecretsPrefix != "" { + return fmt.Sprintf("%s-%s-cert", ms.CertificatesSecretsPrefix, defaultName) + } + + return defaultName + "-cert" +} diff --git a/controllers/operator/certs/certificate_test.go b/controllers/operator/certs/certificate_test.go new file mode 100644 index 000000000..ad941a18b --- /dev/null +++ b/controllers/operator/certs/certificate_test.go @@ -0,0 +1,241 @@ +package certs + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/types" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestVerifyTLSSecretForStatefulSet(t *testing.T) { + //nolint + crt := `-----BEGIN CERTIFICATE----- +MIIFWzCCA0OgAwIBAgIRAJ/vHVZs6TGyhP/AIR+TAvMwDQYJKoZIhvcNAQELBQAw +cTELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcg +WW9yazEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UECwwHbW9uZ29kYjEYMBYGA1UE +AwwPd3d3Lm1vbmdvZGIuY29tMB4XDTIyMDQxMTA5MzkwM1oXDTIyMDQyMTA5Mzkw +M1owADCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANV64qjt1rmfuYAZ +Ly9rOogtQTiFNJqkfVCzaQ8ImztqxeXAwoNd97Jwm82xgpUR7Mrku0raexIhKD8R +zgOr870L7si4MXmZdkuxvQf+SASOsUuaBtc7epzOzfhV2F/2Rgg/RzZEPNLtF9KL +GWrS73q7cz2ZGbTmc6q6DLikQKVq1d/ufCo1ly0i/kc+Koxu4GdamHxyCysFRqIb +RoFdiRSHEYha65U2I+CERBie+LsxCJOK/uYWzP32XRGsFnbmPFscy5A5WMaeIpX/ +Bj8FShEBciwjiFrDktG3FeGd0JkVbUaa10i8xhAw0V66E/bPipKJiSNNxdMVeyhB +g5SQiCUCAwEAAaOCAV0wggFZMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD +AjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFM515VEbUh5N/Tl3GYUuanyfd4BO +MIIBBwYDVR0RAQH/BIH8MIH5ghFyZXBsaWNhLXNldC10bHMtMIIRcmVwbGljYS1z +ZXQtdGxzLTGCEXJlcGxpY2Etc2V0LXRscy0ygj5yZXBsaWNhLXNldC10bHMtMC5y +ZXBsaWNhLXNldC10bHMtc3ZjLnJhajI0Mi5zdmMuY2x1c3Rlci5sb2NhbII+cmVw +bGljYS1zZXQtdGxzLTEucmVwbGljYS1zZXQtdGxzLXN2Yy5yYWoyNDIuc3ZjLmNs +dXN0ZXIubG9jYWyCPnJlcGxpY2Etc2V0LXRscy0yLnJlcGxpY2Etc2V0LXRscy1z +dmMucmFqMjQyLnN2Yy5jbHVzdGVyLmxvY2FsMA0GCSqGSIb3DQEBCwUAA4ICAQCS +/0L4gwWJkfKoA/11EMY2PD+OdUtp0z3DXQH1M34iX8ZGWkej9W0FLDHeipVVKpUY +zoUJSlN2NGd+evSGiVG62z76fUu3ztsdVtL0/P4IILNowmj7KZ6AVfOWX/zYVKsy +EWoQp4dSged5HjvHGNgb7r78W5Ye7Bi6C6aLGBt1OSEA6aDr5HLoChxMFfVFw/gB +UFRZm6bnGRTIsb3NFHjw+CxzLGvf+I+RWzXoc9g5bNKCtlOBKSNao15fFp3YSYLK +9hsGe47sJH7WssPuxAELjJA8UOle3+pSl2mg2OhoYvF3YLTt2vMqOWh0emgEXiXk +eDzw0h6ZuucAXG8YkK0PAvNDHSnemyVtFmo5UUn8rPCRm3Ztxy5lAQlGSk5q1eKG +XxBrBf/eLEE6t6gkV17znQOyRgYgCsD90InLtoBK39dBgZcuDE+KJuY4/6a+GL0R +GgcqDSgjU2OgHXGDJM9GUQlMWOcIaxxEeuC3BYErZPlLg/URSWpfzxYWSxFUGk2p +uD7tgoEguEmih1e1Fc8nIlByx3zj+ifNkuwHytIvip3uvaiQZTEda9i9TMZ6sDeN +1pAwMVMKOGqYBR6ibZJ20vGiKMwBLCQOEqckZg5j2ULHx5LYbbF24uKpMTX8OAtQ +WyU44tyEDe5JUgwo3G/3/Hp6WJARpV1MVy3y5hHNaA== +-----END CERTIFICATE-----` + //nolint + key := `-----BEGIN RSA PRIVATE KEY----- +MIIEpgIBAAKCAQEA1XriqO3WuZ+5gBkvL2s6iC1BOIU0mqR9ULNpDwibO2rF5cDC +g133snCbzbGClRHsyuS7Stp7EiEoPxHOA6vzvQvuyLgxeZl2S7G9B/5IBI6xS5oG +1zt6nM7N+FXYX/ZGCD9HNkQ80u0X0osZatLvertzPZkZtOZzqroMuKRApWrV3+58 +KjWXLSL+Rz4qjG7gZ1qYfHILKwVGohtGgV2JFIcRiFrrlTYj4IREGJ74uzEIk4r+ +5hbM/fZdEawWduY8WxzLkDlYxp4ilf8GPwVKEQFyLCOIWsOS0bcV4Z3QmRVtRprX +SLzGEDDRXroT9s+KkomJI03F0xV7KEGDlJCIJQIDAQABAoIBAQCGltDru/cSVFb5 +IeeTt8DRNebWoXSGwomXJWVo6v4jOa/Gp/56H/YX89LmnbE8Fm75g7do+9F3npvn +F2yQ+AnU9/71YNsgVNY15rrMnU3+QZAZn+QMMh2dWuyUUlr2NSf17x8QYXkPahcI +0FWX+aCt+hwvi6SfXmMyEdYPWs6++jND8MpjrKJHoNObD22NloVw2JaT9msmYuI9 +j/JnTXbpxWSWbwmyqQOJub4kaTPANCRbTgmTersLk1Iu+Q+jCTtN8teRTmx+3kzy +mq8CdPBrME65yRorPZPGTdGwe2pWiLeug288DGKDvqusSv79piVY0wbK3mZYjX70 +MbEwzIfBAoGBAO56pfG9HwwP3ZS6OU22YK6oHY/moLsWYIkcA1qi9v+wQie/Rf+6 +tLEFNtQ94hw8kncUDPNv+FtnvpywL/5ZBF0JAAJqxr1tBQKtqvPFzA4qY53kE0Lz +fLLEA1rX4JeRQlDCRei8zGhgAYkWfSDKd6/DHqc0iTKMVMCh5D1BSqU5AoGBAOUq +DPv0uKCGi0s3eQTBgnJi0ivPt8RCmqZMi5Xss4y60sI7dZm5J7tIEZ9/4sqRSC6w +qQBWLwIVnVC3qUhubEXnfk+07mBVLohNTFeWW1Sb7WJp5NnUOeg+w2f5gvbfItzt +cY7l+rWkxuklKlq6AqfZfVk6aWoa8v4MkyUsqYZNAoGBAOAexcu9F+uHEZAPv4Do +UE50UmwFq7KHoivY9tH8a7L6XAHswYVHWz8uDkxC6DfvORrN7inuZfLJOhsZfdFE +qVQh/C9JWAN37IiK3CmDD3WUotAlI3D9UYjTq+95CGqJKlCpc3f5zwScjXTffLMP +dJHrBujO981YkuICg3SJ4vQJAoGBAONXrDnotaDK2TVteul07+x6jPZZw304diO0 +nGXHxPg//wYh5rDyNrBc9t69CEjdiDaJm59x4IC44LBLA+2PXmqbFXwNis6WsusV +hD8AMurlJcMUOqy/FhOI8GId7gbrprJ1/Mo+7VF2fr6c2D/ZePj7kpcKk7lnstjF +sNSYUjWhAoGBAKjzEsMga2Lonk5sHkVPkYm8Kl4PrfK4lj1U78to0tgcguLjGyRW +c4IFWO5/ibu3HV3eLMiJBmSR8iR5Iue9cCm782/tncuWLsCoDGUmCG1D2Q4iBx2J +EHE1zXjECRf87xR2aKbPUR+44bG/ILCbUypquP48GC+S6OqVF7utkXgF +-----END RSA PRIVATE KEY-----` + secretData := map[string][]byte{"tls.crt": []byte(crt), "tls.key": []byte(key)} + + _, err := VerifyTLSSecretForStatefulSet(secretData, Options{}) + assert.NoError(t, err) +} + +func TestVerifyTLSSecretForStatefulSetHorizonMemberDifference(t *testing.T) { + _, err := VerifyTLSSecretForStatefulSet(map[string][]byte{}, Options{ + Replicas: 4, + horizons: []mdbv1.MongoDBHorizonConfig{ + {"1": "a"}, + {"2": "b"}, + {"3": "c"}, + }, + }) + + assert.ErrorContains(t, err, "horizon configs") +} + +// This test uses mock hashes and certificate because CreatePemSecretClient does not verify the certificates. +// However, they are verified before and after calling this method. +func TestRotateCertificate(t *testing.T) { + ctx := context.Background() + rs := mdbv1.NewReplicaSetBuilder().SetSecurityTLSEnabled().Build() + + fakeClient, _ := mock.NewDefaultFakeClient(rs) + fakeSecretClient := secrets.SecretClient{ + VaultClient: nil, + KubeClient: fakeClient, + } + + secretNamespacedName := types.NamespacedName{ + Namespace: "test", + Name: "test-replica-set-cert", + } + + certificateKey1 := "mock_hash_1" + certificateValue1 := "mock_certificate_1" + certificate1 := map[string]string{certificateKey1: certificateValue1} + + certificateKey2 := "mock_hash_2" + certificateValue2 := "mock_certificate_2" + certificate2 := map[string]string{certificateKey2: certificateValue2} + + certificateKey3 := "mock_hash_3" + certificateValue3 := "mock_certificate_3" + + pemSecretNamespacedName := secretNamespacedName + pemSecretNamespacedName.Name = fmt.Sprintf("%s%s", secretNamespacedName.Name, OperatorGeneratedCertSuffix) + + t.Run("Case 1: Enabling TLS", func(t *testing.T) { + err := CreateOrUpdatePEMSecretWithPreviousCert(ctx, fakeSecretClient, secretNamespacedName, certificateKey1, certificateValue1, []v1.OwnerReference{}, Unused) + assert.NoError(t, err) + + pemSecret, _ := fakeSecretClient.ReadSecret(ctx, pemSecretNamespacedName, Unused) + expectedPem := map[string]string{ + util.LatestHashSecretKey: "mock_hash_1", + "mock_hash_1": "mock_certificate_1", + } + + assert.Equal(t, expectedPem, pemSecret) + }) + + t.Run("Case 2: Upgrading the operator, with no rotation", func(t *testing.T) { + existingPem := secret.Builder(). + SetStringMapToData(certificate1). + SetNamespace(pemSecretNamespacedName.Namespace). + SetName(pemSecretNamespacedName.Name). + Build() + + _ = fakeSecretClient.PutSecret(ctx, existingPem, Unused) + + err := CreateOrUpdatePEMSecretWithPreviousCert(ctx, fakeSecretClient, secretNamespacedName, certificateKey1, certificateValue1, []v1.OwnerReference{}, Unused) + assert.NoError(t, err) + + pemSecret, _ := fakeSecretClient.ReadSecret(ctx, pemSecretNamespacedName, Unused) + expectedPem := map[string]string{ + util.LatestHashSecretKey: "mock_hash_1", + "mock_hash_1": "mock_certificate_1", + } + + assert.Equal(t, expectedPem, pemSecret) + }) + + t.Run("Case 3: Upgrading the operator, with a rotation", func(t *testing.T) { + existingPem := secret.Builder(). + SetStringMapToData(certificate1). + SetNamespace(pemSecretNamespacedName.Namespace). + SetName(pemSecretNamespacedName.Name). + Build() + + _ = fakeSecretClient.PutSecret(ctx, existingPem, Unused) + + // The rotation happens here because CreateOrUpdatePEMSecretWithPreviousCert is called with certificate 2, but the existing pem secret references certificate 1. + err := CreateOrUpdatePEMSecretWithPreviousCert(ctx, fakeSecretClient, secretNamespacedName, certificateKey2, certificateValue2, []v1.OwnerReference{}, Unused) + assert.NoError(t, err) + + pemSecret, _ := fakeSecretClient.ReadSecret(ctx, pemSecretNamespacedName, Unused) + expectedPem := map[string]string{ + util.LatestHashSecretKey: "mock_hash_2", + util.PreviousHashSecretKey: "mock_hash_1", + "mock_hash_1": "mock_certificate_1", + "mock_hash_2": "mock_certificate_2", + } + + assert.Equal(t, expectedPem, pemSecret) + }) + + t.Run("Case 4: First TLS rotation", func(t *testing.T) { + existingPemData := certificate1 + existingPemData[util.LatestHashSecretKey] = "mock_hash_1" + existingPem := secret.Builder(). + SetStringMapToData(existingPemData). + SetNamespace(pemSecretNamespacedName.Namespace). + SetName(pemSecretNamespacedName.Name). + Build() + + _ = fakeSecretClient.PutSecret(ctx, existingPem, Unused) + + err := CreateOrUpdatePEMSecretWithPreviousCert(ctx, fakeSecretClient, secretNamespacedName, certificateKey2, certificateValue2, []v1.OwnerReference{}, Unused) + assert.NoError(t, err) + + pemSecret, _ := fakeSecretClient.ReadSecret(ctx, pemSecretNamespacedName, Unused) + expectedPem := map[string]string{ + util.LatestHashSecretKey: "mock_hash_2", + util.PreviousHashSecretKey: "mock_hash_1", + "mock_hash_1": "mock_certificate_1", + "mock_hash_2": "mock_certificate_2", + } + + assert.Equal(t, expectedPem, pemSecret) + }) + + t.Run("Case 5: Subsequent TLS rotations", func(t *testing.T) { + existingPemData := merge.StringToStringMap(certificate1, certificate2) + existingPemData[util.LatestHashSecretKey] = "mock_hash_2" + existingPemData[util.PreviousHashSecretKey] = "mock_hash_1" + existingPem := secret.Builder(). + SetStringMapToData(existingPemData). + SetNamespace(pemSecretNamespacedName.Namespace). + SetName(pemSecretNamespacedName.Name). + Build() + + _ = fakeSecretClient.PutSecret(ctx, existingPem, Unused) + + err := CreateOrUpdatePEMSecretWithPreviousCert(ctx, fakeSecretClient, secretNamespacedName, certificateKey3, certificateValue3, []v1.OwnerReference{}, Unused) + assert.NoError(t, err) + + pemSecret, _ := fakeSecretClient.ReadSecret(ctx, pemSecretNamespacedName, Unused) + expectedPem := map[string]string{ + util.LatestHashSecretKey: "mock_hash_3", + util.PreviousHashSecretKey: "mock_hash_2", + "mock_hash_3": "mock_certificate_3", + "mock_hash_2": "mock_certificate_2", + } + + assert.Equal(t, expectedPem, pemSecret) + }) +} diff --git a/controllers/operator/certs/certificates.go b/controllers/operator/certs/certificates.go new file mode 100644 index 000000000..c69a2aa87 --- /dev/null +++ b/controllers/operator/certs/certificates.go @@ -0,0 +1,453 @@ +package certs + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +type certDestination string + +const ( + OperatorGeneratedCertSuffix = "-pem" + CertHashAnnotationKey = "certHash" + + Unused = "unused" + Database = "database" + OpsManager = "opsmanager" + AppDB = "appdb" +) + +// CreateOrUpdatePEMSecretWithPreviousCert creates a PEM secret from the original secretName. +// Additionally, this method verifies if there already exists a PEM secret, and it will merge them to be able to keep the newest and the previous certificate. +func CreateOrUpdatePEMSecretWithPreviousCert(ctx context.Context, secretClient secrets.SecretClient, secretNamespacedName types.NamespacedName, certificateKey string, certificateValue string, ownerReferences []metav1.OwnerReference, podType certDestination) error { + path, err := getVaultBasePath(secretClient, podType) + if err != nil { + return err + } + + secretData, err := updateSecretDataWithPreviousCert(ctx, secretClient, getOperatorGeneratedSecret(secretNamespacedName), certificateKey, certificateValue, path) + if err != nil { + return err + } + + return CreateOrUpdatePEMSecret(ctx, secretClient, secretNamespacedName, secretData, ownerReferences, podType) +} + +// CreateOrUpdatePEMSecret creates a PEM secret from the original secretName. +func CreateOrUpdatePEMSecret(ctx context.Context, secretClient secrets.SecretClient, secretNamespacedName types.NamespacedName, secretData map[string]string, ownerReferences []metav1.OwnerReference, podType certDestination) error { + operatorGeneratedSecret := getOperatorGeneratedSecret(secretNamespacedName) + + path, err := getVaultBasePath(secretClient, podType) + if err != nil { + return err + } + + secretBuilder := secret.Builder(). + SetName(operatorGeneratedSecret.Name). + SetNamespace(operatorGeneratedSecret.Namespace). + SetStringMapToData(secretData). + SetOwnerReferences(ownerReferences) + + return secretClient.PutSecretIfChanged(ctx, secretBuilder.Build(), path) +} + +// updateSecretDataWithPreviousCert receives the new TLS certificate and returns the data for the new concatenated Pem Secret +// This method read the existing -pem secret and creates the secret data such that it keeps the previous TLS certificate +func updateSecretDataWithPreviousCert(ctx context.Context, secretClient secrets.SecretClient, operatorGeneratedSecret types.NamespacedName, certificateKey string, certificateValue string, basePath string) (map[string]string, error) { + newData := map[string]string{certificateKey: certificateValue} + newLatestHash := certificateKey + + newData[util.LatestHashSecretKey] = newLatestHash + + existingSecretData, err := secretClient.ReadSecret(ctx, operatorGeneratedSecret, basePath) + if secrets.SecretNotExist(err) { + // Case: creating the PEM secret the first time (example: enabling TLS) + return newData, nil + } else if err != nil { + return nil, err + } + + if oldLatestHash, ok := existingSecretData[util.LatestHashSecretKey]; ok { + if oldLatestHash == newLatestHash { + // Case: no new changes, the pem secrets have the annotations, no rotation happened + newData = existingSecretData + } else { + // Case: the pem secrets have the annotations, and a rotation happened + newData[util.PreviousHashSecretKey] = oldLatestHash + newData[oldLatestHash] = existingSecretData[oldLatestHash] + } + } else if len(existingSecretData) == 1 { + // Case: the operator is upgraded to 1.29, the pem secrets don't have the annotations + for hash, cert := range existingSecretData { + if hash != newLatestHash { + // Case: the operator is upgraded to 1.29, the pem secrets don't have the annotations, and a certificate rotation happened + newData[hash] = cert + newData[util.PreviousHashSecretKey] = hash + } + } + } + + return newData, nil +} + +// getVaultBasePath returns the path to secrets in the vault +func getVaultBasePath(secretClient secrets.SecretClient, podType certDestination) (string, error) { + var path string + if vault.IsVaultSecretBackend() && podType != Unused { + switch podType { + case Database: + path = secretClient.VaultClient.DatabaseSecretPath() + case OpsManager: + path = secretClient.VaultClient.OpsManagerSecretPath() + case AppDB: + path = secretClient.VaultClient.AppDBSecretPath() + default: + return "", xerrors.Errorf("unexpected pod type got: %s", podType) + } + } + return path, nil +} + +// getOperatorGeneratedSecret returns the namespaced name of the PEM secret the operator creates +func getOperatorGeneratedSecret(secretNamespacedName types.NamespacedName) types.NamespacedName { + operatorGeneratedSecret := secretNamespacedName + operatorGeneratedSecret.Name = fmt.Sprintf("%s%s", secretNamespacedName.Name, OperatorGeneratedCertSuffix) + return operatorGeneratedSecret +} + +// VerifyTLSSecretForStatefulSet verifies a `Secret`'s `StringData` is a valid +// certificate, considering the amount of members for a resource named on +// `opts`. +func VerifyTLSSecretForStatefulSet(secretData map[string][]byte, opts Options) (string, error) { + crt, key := secretData["tls.crt"], secretData["tls.key"] + + // add a line break to the end of certificate when performing concatenation + crtString := string(crt) + if !strings.HasSuffix(crtString, "\n") { + crtString = crtString + "\n" + } + crt = []byte(crtString) + + data := append(crt, key...) + + var additionalDomains []string + if len(opts.horizons) > 0 && len(opts.horizons) < opts.Replicas { + return "", xerrors.Errorf("less horizon configs than number for replicas this reconcile. Please make sure that "+ + "enough horizon configs are configured as members are until the scale down has finished. "+ + "Current number of replicas for this reconciliation: %d, number of horizons: %d", opts.Replicas, len(opts.horizons)) + } + for i := range getPodNames(opts) { + additionalDomains = append(additionalDomains, GetAdditionalCertDomainsForMember(opts, i)...) + } + if opts.ExternalDomain != nil { + additionalDomains = append(additionalDomains, "*."+*opts.ExternalDomain) + } + + if err := validatePemData(data, additionalDomains); err != nil { + return "", err + } + return string(data), nil +} + +// VerifyAndEnsureCertificatesForStatefulSet ensures that the provided certificates are correct. +// If the secret is of type kubernetes.io/tls, it creates a new secret containing the concatenation fo the tls.crt and tls.key fields +func VerifyAndEnsureCertificatesForStatefulSet(ctx context.Context, secretReadClient, secretWriteClient secrets.SecretClient, secretName string, opts Options, log *zap.SugaredLogger) error { + var err error + var secretData map[string][]byte + var s corev1.Secret + var databaseSecretPath string + + if vault.IsVaultSecretBackend() { + databaseSecretPath = secretReadClient.VaultClient.DatabaseSecretPath() + secretData, err = secretReadClient.VaultClient.ReadSecretBytes(fmt.Sprintf("%s/%s/%s", databaseSecretPath, opts.Namespace, secretName)) + if err != nil { + return err + } + + } else { + s, err = secretReadClient.KubeClient.GetSecret(ctx, kube.ObjectKey(opts.Namespace, secretName)) + if err != nil { + return err + } + + // SecretTypeTLS is kubernetes.io/tls + // This is the standard way in K8S to have secrets that hold TLS certs + // And it is the one generated by cert manager + // These type of secrets contain tls.crt and tls.key entries + if s.Type != corev1.SecretTypeTLS { + return xerrors.Errorf("The secret object '%s' is not of type kubernetes.io/tls: %s", secretName, s.Type) + } + secretData = s.Data + } + + data, err := VerifyTLSSecretForStatefulSet(secretData, opts) + if err != nil { + return err + } + + secretHash := enterprisepem.ReadHashFromSecret(ctx, secretReadClient, opts.Namespace, secretName, databaseSecretPath, log) + return CreateOrUpdatePEMSecretWithPreviousCert(ctx, secretWriteClient, kube.ObjectKey(opts.Namespace, secretName), secretHash, data, opts.OwnerReference, Database) +} + +// getPodNames returns the pod names based on the Cert Options provided. +func getPodNames(opts Options) []string { + _, podNames := dns.GetDNSNames(opts.ResourceName, opts.ServiceName, opts.Namespace, opts.ClusterDomain, opts.Replicas, nil) + return podNames +} + +func GetDNSNames(opts Options) (hostnames, podNames []string) { + return dns.GetDNSNames(opts.ResourceName, opts.ServiceName, opts.Namespace, opts.ClusterDomain, opts.Replicas, nil) +} + +// GetAdditionalCertDomainsForMember gets any additional domains that the +// certificate for the given member of the stateful set should be signed for. +func GetAdditionalCertDomainsForMember(opts Options, member int) (hostnames []string) { + _, podNames := GetDNSNames(opts) + for _, certDomain := range opts.additionalCertificateDomains { + hostnames = append(hostnames, podNames[member]+"."+certDomain) + } + if len(opts.horizons) > 0 { + // at this point len(ss.ReplicaSetHorizons) should be equal to the number + // of members in the replica set + for _, externalHost := range opts.horizons[member] { + // need to use the URL struct directly instead of url.Parse as + // Parse expects the URL to have a scheme. + hostURL := url.URL{Host: externalHost} + hostnames = append(hostnames, hostURL.Hostname()) + } + } + return hostnames +} + +func validatePemData(data []byte, additionalDomains []string) error { + pemFile := enterprisepem.NewFileFromData(data) + if !pemFile.IsComplete() { + return xerrors.Errorf("the certificate is not complete\n") + } + certs, err := pemFile.ParseCertificate() + if err != nil { + return xerrors.Errorf("can't parse certificate: %w\n", err) + } + + var errs error + // in case of using an intermediate certificate authority, the certificate + // data might contain all the certificate chain excluding the root-ca (in case of cert-manager). + // We need to iterate through the certificates in the chain and find the one at the bottom of the chain + // containing the additionalDomains which we're validating for. + for _, cert := range certs { + var err error + for _, domain := range additionalDomains { + if !stringutil.CheckCertificateAddresses(cert.DNSNames, domain) { + err = xerrors.Errorf("domain %s is not contained in the list of DNSNames %v\n", domain, cert.DNSNames) + errs = multierror.Append(errs, err) + } + } + if err == nil { + return nil + } + } + + return errs +} + +// validatePemSecret returns true if the given Secret contains a parsable certificate and contains all required domains. +func validatePemSecret(secret corev1.Secret, key string, additionalDomains []string) error { + data, ok := secret.Data[key] + if !ok { + return xerrors.Errorf("the secret %s does not contain the expected key %s\n", secret.Name, key) + } + + return validatePemData(data, additionalDomains) +} + +// ValidateCertificates verifies the Secret containing the certificates and the keys is valid. +func ValidateCertificates(ctx context.Context, secretGetter secret.Getter, name, namespace string, log *zap.SugaredLogger) error { + validateCertificates := func() (string, bool) { + byteData, err := secret.ReadByteData(ctx, secretGetter, kube.ObjectKey(namespace, name)) + if err == nil { + // Validate that the secret contains the keys, if it contains the certs. + for key, value := range byteData { + if key == util.LatestHashSecretKey || key == util.PreviousHashSecretKey { + continue + } + pemFile := enterprisepem.NewFileFromData(value) + if !pemFile.IsValid() { + return fmt.Sprintf("The Secret %s containing certificates is not valid. Entries must contain a certificate and a private key.", name), false + } + } + } + return "", true + } + + // we immediately create the certificate in a prior call, thus we need to retry to account for races. + if found, msg := util.DoAndRetry(validateCertificates, log, 10, 5); !found { + return xerrors.Errorf(msg) + } + return nil +} + +// VerifyAndEnsureClientCertificatesForAgentsAndTLSType ensures that agent certs are present and correct, and returns whether they are of the kubernetes.io/tls type. +// If the secret is of type kubernetes.io/tls, it creates a new secret containing the concatenation fo the tls.crt and tls.key fields +func VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx context.Context, secretReadClient, secretWriteClient secrets.SecretClient, secret types.NamespacedName) error { + needToCreatePEM := false + var secretData map[string][]byte + var s corev1.Secret + var err error + + if vault.IsVaultSecretBackend() { + needToCreatePEM = true + secretData, err = secretReadClient.VaultClient.ReadSecretBytes(fmt.Sprintf("%s/%s/%s", secretReadClient.VaultClient.DatabaseSecretPath(), secret.Namespace, secret.Name)) + if err != nil { + return err + } + } else { + s, err = secretReadClient.KubeClient.GetSecret(ctx, secret) + if err != nil { + return err + } + + if s.Type == corev1.SecretTypeTLS { + needToCreatePEM = true + secretData = s.Data + } + } + if needToCreatePEM { + data, err := VerifyTLSSecretForStatefulSet(secretData, Options{Replicas: 0}) + if err != nil { + return err + } + + dataMap := map[string]string{ + util.AutomationAgentPemSecretKey: data, + } + + return CreateOrUpdatePEMSecret(ctx, secretWriteClient, secret, dataMap, []metav1.OwnerReference{}, Database) + } + + return validatePemSecret(s, util.AutomationAgentPemSecretKey, nil) +} + +// EnsureSSLCertsForStatefulSet contains logic to ensure that all of the +// required SSL certs for a StatefulSet object exist. +func EnsureSSLCertsForStatefulSet(ctx context.Context, secretReadClient, secretWriteClient secrets.SecretClient, ms mdbv1.Security, opts Options, log *zap.SugaredLogger) workflow.Status { + if !ms.IsTLSEnabled() { + // if there's no SSL certs to generate, return + return workflow.OK() + } + + secretName := opts.CertSecretName + return ValidateSelfManagedSSLCertsForStatefulSet(ctx, secretReadClient, secretWriteClient, secretName, opts, log) +} + +// EnsureTLSCertsForPrometheus creates a new Secret with a Certificate in +// PEM-format. Returns the hash for the certificate in order to be used in +// AutomationConfig. +// +// For Prometheus we *only accept* certificates of type `corev1.SecretTypeTLS` +// so they always need to be concatenated into PEM-format. +func EnsureTLSCertsForPrometheus(ctx context.Context, secretClient secrets.SecretClient, namespace string, prom *mdbcv1.Prometheus, podType certDestination, log *zap.SugaredLogger) (string, error) { + if prom == nil || prom.TLSSecretRef.Name == "" { + return "", nil + } + + var secretData map[string][]byte + var err error + + var secretPath string + if vault.IsVaultSecretBackend() { + // TODO: This is calculated twice, can this be done better? + // This "calculation" is used in ReadHashFromSecret but calculated again in `CreateOrUpdatePEMSecretWithPreviousCert` + if podType == Database { + secretPath = secretClient.VaultClient.DatabaseSecretPath() + } else if podType == AppDB { + secretPath = secretClient.VaultClient.AppDBSecretPath() + } + + secretData, err = secretClient.VaultClient.ReadSecretBytes(fmt.Sprintf("%s/%s/%s", secretPath, namespace, prom.TLSSecretRef.Name)) + if err != nil { + return "", err + } + } else { + s, err := secretClient.KubeClient.GetSecret(ctx, kube.ObjectKey(namespace, prom.TLSSecretRef.Name)) + if err != nil { + return "", xerrors.Errorf("could not read Prometheus TLS certificate: %w", err) + } + + if s.Type != corev1.SecretTypeTLS { + return "", xerrors.Errorf("secret containing the Prometheus TLS certificate needs to be of type kubernetes.io/tls") + } + + secretData = s.Data + } + + // We only need VerifyTLSSecretForStatefulSet to return the concatenated + // tls.key and tls.crt as Strings, but to not divert from the existing code, + // I'm still calling it, but that can be definitely improved. + // + // Make VerifyTLSSecretForStatefulSet receive `s.Data` but only return if it + // has been verified to be valid or not (boolean return). + // + // Use another function to concatenate tls.key and tls.crt into a `string`, + // or make `CreateOrUpdatePEMSecretWithPreviousCert` able to receive a byte[] instead on its + // `data` parameter. + data, err := VerifyTLSSecretForStatefulSet(secretData, Options{Replicas: 0}) + if err != nil { + return "", err + } + + // ReadHashFromSecret will read the Secret once again from Kubernetes API, + // we can improve this function by providing the Secret Data contents, + // instead of `secretClient`. + secretHash := enterprisepem.ReadHashFromSecret(ctx, secretClient, namespace, prom.TLSSecretRef.Name, secretPath, log) + err = CreateOrUpdatePEMSecretWithPreviousCert(ctx, secretClient, kube.ObjectKey(namespace, prom.TLSSecretRef.Name), secretHash, data, []metav1.OwnerReference{}, podType) + if err != nil { + return "", xerrors.Errorf("error creating hashed Secret: %w", err) + } + + return secretHash, nil +} + +// ValidateSelfManagedSSLCertsForStatefulSet ensures that a stateful set using +// user-provided certificates has all of the relevant certificates in place. +func ValidateSelfManagedSSLCertsForStatefulSet(ctx context.Context, secretReadClient, secretWriteClient secrets.SecretClient, secretName string, opts Options, log *zap.SugaredLogger) workflow.Status { + // A "Certs" attribute has been provided + // This means that the customer has provided with a secret name they have + // already populated with the certs and keys for this deployment. + // Because of the async nature of Kubernetes, this object might not be ready yet, + // in which case, we'll keep reconciling until the object is created and is correct. + err := VerifyAndEnsureCertificatesForStatefulSet(ctx, secretReadClient, secretWriteClient, secretName, opts, log) + if err != nil { + return workflow.Failed(xerrors.Errorf("The secret object '%s' does not contain all the valid certificates needed: %w", secretName, err)) + } + + secretName = fmt.Sprintf("%s-pem", secretName) + + if err := ValidateCertificates(ctx, secretReadClient.KubeClient, secretName, opts.Namespace, log); err != nil { + return workflow.Failed(err) + } + + return workflow.OK() +} diff --git a/controllers/operator/clusterchecks_test.go b/controllers/operator/clusterchecks_test.go new file mode 100644 index 000000000..d53cc6916 --- /dev/null +++ b/controllers/operator/clusterchecks_test.go @@ -0,0 +1,286 @@ +package operator + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type clusterChecks struct { + t *testing.T + namespace string + clusterName string + clusterIndex int + kubeClient client.Client +} + +func newClusterChecks(t *testing.T, clusterName string, clusterIndex int, namespace string, kubeClient client.Client) *clusterChecks { + result := clusterChecks{ + t: t, + namespace: namespace, + clusterName: clusterName, + kubeClient: kubeClient, + clusterIndex: clusterIndex, + } + + return &result +} + +func (c *clusterChecks) checkAutomationConfigSecret(ctx context.Context, secretName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, secretName), &sec) + assert.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Contains(c.t, sec.Data, automationconfig.ConfigKey, "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkAgentAPIKeySecret(ctx context.Context, projectID string) string { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, agentAPIKeySecretName(projectID)), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, util.OmAgentApiKey, "clusterName: %s", c.clusterName) + return string(sec.Data[util.OmAgentApiKey]) +} + +func (c *clusterChecks) checkSecretNotFound(ctx context.Context, secretName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, secretName), &sec) + assert.Error(c.t, err, "clusterName: %s", c.clusterName) + assert.True(c.t, apiErrors.IsNotFound(err)) +} + +func (c *clusterChecks) checkConfigMapNotFound(ctx context.Context, configMapName string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + assert.Error(c.t, err, "clusterName: %s", c.clusterName) + assert.True(c.t, apiErrors.IsNotFound(err)) +} + +func (c *clusterChecks) checkPEMSecret(ctx context.Context, secretName string, pemHash string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, secretName), &sec) + assert.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Contains(c.t, sec.Data, pemHash, "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkAutomationConfigConfigMap(ctx context.Context, configMapName string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + assert.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Contains(c.t, cm.Data, appDBACConfigMapVersionField, "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkTLSCAConfigMap(ctx context.Context, configMapName string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + assert.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Contains(c.t, cm.Data, "ca-pem", "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkProjectIDConfigMap(ctx context.Context, configMapName string) string { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, cm.Data, util.AppDbProjectIdKey, "clusterName: %s", c.clusterName) + return cm.Data[util.AppDbProjectIdKey] +} + +func (c *clusterChecks) checkPerPodServices(ctx context.Context, statefulSetName string, expectedMembers int) { + for podIdx := 0; podIdx < expectedMembers; podIdx++ { + svc := corev1.Service{} + serviceName := fmt.Sprintf("%s-%d-svc", statefulSetName, podIdx) + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + + assert.Equal(c.t, map[string]string{ + "controller": "mongodb-enterprise-operator", + "statefulset.kubernetes.io/pod-name": fmt.Sprintf("%s-%d", statefulSetName, podIdx), + }, + svc.Spec.Selector) + } +} + +func (c *clusterChecks) checkPerPodServicesDontExist(ctx context.Context, statefulSetName string, expectedMembers int) { + for podIdx := 0; podIdx < expectedMembers; podIdx++ { + svc := corev1.Service{} + serviceName := fmt.Sprintf("%s-%d-svc", statefulSetName, podIdx) + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.True(c.t, apiErrors.IsNotFound(err)) + } +} + +func (c *clusterChecks) checkExternalServices(ctx context.Context, statefulSetName string, expectedMembers int) { + for podIdx := 0; podIdx < expectedMembers; podIdx++ { + svc := corev1.Service{} + serviceName := fmt.Sprintf("%s-%d-svc-external", statefulSetName, podIdx) + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Subset(c.t, svc.Spec.Selector, map[string]string{ + "controller": "mongodb-enterprise-operator", + "statefulset.kubernetes.io/pod-name": fmt.Sprintf("%s-%d", statefulSetName, podIdx), + }) + } +} + +func (c *clusterChecks) checkExternalServicesDontExist(ctx context.Context, statefulSetName string, expectedMembers int) { + for podIdx := 0; podIdx < expectedMembers; podIdx++ { + svc := corev1.Service{} + serviceName := fmt.Sprintf("%s-%d-svc-external", statefulSetName, podIdx) + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.True(c.t, apiErrors.IsNotFound(err)) + } +} + +func (c *clusterChecks) checkInternalServices(ctx context.Context, statefulSetName string) { + svc := corev1.Service{} + // the statefulSetName already contains the clusterNumber + serviceName := fmt.Sprintf("%s-svc", statefulSetName) + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + + assert.Equal(c.t, map[string]string{ + "controller": "mongodb-enterprise-operator", + "app": serviceName, + }, + svc.Spec.Selector) +} + +func (c *clusterChecks) checkServiceExists(ctx context.Context, serviceName string) { + svc := corev1.Service{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkServiceAnnotations(ctx context.Context, statefulSetName string, expectedMembers int, sc *mdbv1.MongoDB, clusterName string, clusterIdx int, externalDomain string) { + for podIdx := 0; podIdx < expectedMembers; podIdx++ { + svc := corev1.Service{} + serviceName := fmt.Sprintf("%s-%d-svc-external", statefulSetName, podIdx) + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, serviceName), &svc) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + podName := fmt.Sprintf("%s-%d", statefulSetName, podIdx) + + useExternalDomain := sc.Spec.GetExternalDomain() != nil + if !useExternalDomain { + if strings.HasSuffix(statefulSetName, "-mongos") { + useExternalDomain = sc.Spec.MongosSpec.ClusterSpecList.GetExternalDomainForMemberCluster(clusterName) != nil + } else if strings.HasSuffix(statefulSetName, "-config") { + useExternalDomain = sc.Spec.ConfigSrvSpec.ClusterSpecList.GetExternalDomainForMemberCluster(clusterName) != nil + } else { + useExternalDomain = sc.Spec.ShardSpec.ClusterSpecList.GetExternalDomainForMemberCluster(clusterName) != nil + } + } + + expectedAnnotations := map[string]string{ + create.PlaceholderPodIndex: fmt.Sprintf("%d", podIdx), + create.PlaceholderNamespace: sc.Namespace, + create.PlaceholderResourceName: sc.Name, + create.PlaceholderStatefulSetName: statefulSetName, + create.PlaceholderPodName: podName, + create.PlaceholderExternalServiceName: fmt.Sprintf("%s-svc-external", podName), + } + if sc.Spec.IsMultiCluster() { + expectedAnnotations[create.PlaceholderClusterName] = clusterName + expectedAnnotations[create.PlaceholderClusterIndex] = fmt.Sprintf("%d", clusterIdx) + } + if strings.HasSuffix(statefulSetName, "-mongos") { + expectedAnnotations[create.PlaceholderMongosProcessDomain] = externalDomain + if useExternalDomain { + expectedAnnotations[create.PlaceholderMongosProcessFQDN] = fmt.Sprintf("%s.%s", podName, externalDomain) + } else { + expectedAnnotations[create.PlaceholderMongosProcessFQDN] = fmt.Sprintf("%s-svc.%s", podName, externalDomain) + } + } else { + expectedAnnotations[create.PlaceholderMongodProcessDomain] = externalDomain + if useExternalDomain { + expectedAnnotations[create.PlaceholderMongodProcessFQDN] = fmt.Sprintf("%s.%s", podName, externalDomain) + } else { + expectedAnnotations[create.PlaceholderMongodProcessFQDN] = fmt.Sprintf("%s-svc.%s", podName, externalDomain) + } + + } + assert.Equal(c.t, expectedAnnotations, svc.Annotations) + } +} + +func (c *clusterChecks) checkStatefulSet(ctx context.Context, statefulSetName string, expectedMembers int) { + sts := appsv1.StatefulSet{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, statefulSetName), &sts) + require.NoError(c.t, err, "clusterName: %s stsName: %s", c.clusterName, statefulSetName) + require.Equal(c.t, expectedMembers, int(*sts.Spec.Replicas)) + require.Equal(c.t, statefulSetName, sts.Name) +} + +func (c *clusterChecks) checkStatefulSetDoesNotExist(ctx context.Context, statefulSetName string) { + sts := appsv1.StatefulSet{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, statefulSetName), &sts) + require.True(c.t, apiErrors.IsNotFound(err)) +} + +func (c *clusterChecks) checkAgentCertsSecret(ctx context.Context, certificatesSecretsPrefix string, resourceName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, fmt.Sprintf("%s-%s-%s-pem", certificatesSecretsPrefix, resourceName, util.AgentSecretName)), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, util.AutomationAgentPemSecretKey, "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkMongosCertsSecret(ctx context.Context, certificatesSecretsPrefix string, resourceName string, shouldExist bool) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, fmt.Sprintf("%s-%s-%s-pem", certificatesSecretsPrefix, resourceName, "mongos-cert")), &sec) + c.assertErrNotFound(err, shouldExist) +} + +func (c *clusterChecks) checkConfigSrvCertsSecret(ctx context.Context, certificatesSecretsPrefix string, resourceName string, shouldExist bool) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, fmt.Sprintf("%s-%s-%s-pem", certificatesSecretsPrefix, resourceName, "config-cert")), &sec) + c.assertErrNotFound(err, shouldExist) +} + +func (c *clusterChecks) checkInternalClusterCertSecret(ctx context.Context, certificatesSecretsPrefix string, resourceName string, shardIdx int, shouldExist bool) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, fmt.Sprintf("%s-%s-%d-%s-pem", certificatesSecretsPrefix, resourceName, shardIdx, util.ClusterFileName)), &sec) + c.assertErrNotFound(err, shouldExist) +} + +func (c *clusterChecks) checkMemberCertSecret(ctx context.Context, certificatesSecretsPrefix string, resourceName string, shardIdx int, shouldExist bool) { + sec := corev1.Secret{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, fmt.Sprintf("%s-%s-%d-cert-pem", certificatesSecretsPrefix, resourceName, shardIdx)), &sec) + c.assertErrNotFound(err, shouldExist) +} + +func (c *clusterChecks) assertErrNotFound(err error, shouldExist bool) { + if shouldExist { + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + } else { + require.Error(c.t, err, "clusterName: %s", c.clusterName) + require.True(c.t, apiErrors.IsNotFound(err)) + } +} + +func (c *clusterChecks) checkMMSCAConfigMap(ctx context.Context, configMapName string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + assert.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Contains(c.t, cm.Data, util.CaCertMMS, "clusterName: %s", c.clusterName) +} + +func (c *clusterChecks) checkHostnameOverrideConfigMap(ctx context.Context, configMapName string, expectedPodNameToHostnameMap map[string]string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Equal(c.t, expectedPodNameToHostnameMap, cm.Data) +} diff --git a/controllers/operator/common_controller.go b/controllers/operator/common_controller.go new file mode 100644 index 000000000..679494b1e --- /dev/null +++ b/controllers/operator/common_controller.go @@ -0,0 +1,1029 @@ +package operator + +import ( + "context" + "encoding/json" + "encoding/pem" + "fmt" + "reflect" + "strings" + "time" + + "github.com/blang/semver" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/inspect" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/agentVersionManagement" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/passwordhash" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func automationConfigFirstMsg(resourceType string, valueToSet string) string { + return fmt.Sprintf("About to set `%s` to %s. automationConfig needs to be updated first", resourceType, valueToSet) +} + +type patchValue struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value"` +} + +// ReconcileCommonController is the "parent" controller that is included into each specific controller and allows +// to reuse the common functionality +type ReconcileCommonController struct { + client kubernetesClient.Client + secrets.SecretClient + + resourceWatcher *watch.ResourceWatcher +} + +func NewReconcileCommonController(ctx context.Context, client client.Client) *ReconcileCommonController { + newClient := kubernetesClient.NewClient(client) + var vaultClient *vault.VaultClient + + if vault.IsVaultSecretBackend() { + var err error + // creates the in-cluster config, we cannot use the controller-runtime manager client + // since the manager hasn't been started yet. Using it will cause error "the cache is not started, can not read objects" + config, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + vaultClient, err = vault.InitVaultClient(ctx, clientset) + if err != nil { + panic(fmt.Sprintf("Can not initialize vault client: %s", err)) + } + if err := vaultClient.Login(); err != nil { + panic(xerrors.Errorf("unable to log in with vault client: %w", err)) + } + } + return &ReconcileCommonController{ + client: newClient, + SecretClient: secrets.SecretClient{ + VaultClient: vaultClient, + KubeClient: newClient, + }, + resourceWatcher: watch.NewResourceWatcher(), + } +} + +func ensureRoles(roles []mdbv1.MongoDbRole, conn om.Connection, log *zap.SugaredLogger) workflow.Status { + d, err := conn.ReadDeployment() + if err != nil { + return workflow.Failed(err) + } + dRoles := d.GetRoles() + if reflect.DeepEqual(dRoles, roles) { + return workflow.OK() + } + // HELP-20798: the agent deals correctly with a null value for + // privileges only when creating a role, not when updating + // we work around it by explicitly passing empty array + for i, role := range roles { + if role.Privileges == nil { + roles[i].Privileges = []mdbv1.Privilege{} + } + } + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + d.SetRoles(roles) + return nil + }, + log, + ) + if err != nil { + return workflow.Failed(err) + } + return workflow.OK() +} + +// updateStatus updates the status for the CR using patch operation. Note, that the resource status is mutated and +// it's important to pass resource by pointer to all methods which invoke current 'updateStatus'. +func (r *ReconcileCommonController) updateStatus(ctx context.Context, reconciledResource v1.CustomResourceReadWriter, st workflow.Status, log *zap.SugaredLogger, statusOptions ...status.Option) (reconcile.Result, error) { + mergedOptions := append(statusOptions, st.StatusOptions()...) + log.Debugf("Updating status: phase=%v, options=%+v", st.Phase(), mergedOptions) + reconciledResource.UpdateStatus(st.Phase(), mergedOptions...) + if err := r.patchUpdateStatus(ctx, reconciledResource, statusOptions...); err != nil { + log.Errorf("Error updating status to %s: %s", st.Phase(), err) + return reconcile.Result{}, err + } + return st.ReconcileResult() +} + +type WatcherResource interface { + ObjectKey() client.ObjectKey + GetSecurity() *mdbv1.Security + GetConnectionSpec() *mdbv1.ConnectionSpec +} + +// SetupCommonWatchers is the common shared method for all controller to watch the following resources: +// - OM related cm and secret +// - TLS related secrets, if enabled, this includes x509 internal authentication secrets +// +// Note: everything is watched under the same namespace as the objectKey +// in case getSecretNames func is nil, we will default to common mechanism to get the secret names +// TODO: unify the watcher setup with the secret creation/mounting code in database creation +func (r *ReconcileCommonController) SetupCommonWatchers(watcherResource WatcherResource, getTLSSecretNames func() []string, getInternalAuthSecretNames func() []string, resourceNameForSecret string) { + // We remove all watched resources + objectToReconcile := watcherResource.ObjectKey() + r.resourceWatcher.RemoveDependentWatchedResources(objectToReconcile) + + // And then add the ones we care about + connectionSpec := watcherResource.GetConnectionSpec() + if connectionSpec != nil { + r.resourceWatcher.RegisterWatchedMongodbResources(objectToReconcile, connectionSpec.GetProject(), connectionSpec.Credentials) + } + + security := watcherResource.GetSecurity() + // And TLS if needed + if security.IsTLSEnabled() { + var secretNames []string + if getTLSSecretNames != nil { + secretNames = getTLSSecretNames() + } else { + secretNames = []string{security.MemberCertificateSecretName(resourceNameForSecret)} + if security.ShouldUseX509("") { + secretNames = append(secretNames, security.AgentClientCertificateSecretName(resourceNameForSecret).Name) + } + } + r.resourceWatcher.RegisterWatchedTLSResources(objectToReconcile, security.TLSConfig.CA, secretNames) + } + + if security.GetInternalClusterAuthenticationMode() == util.X509 { + var secretNames []string + if getInternalAuthSecretNames != nil { + secretNames = getInternalAuthSecretNames() + } else { + secretNames = []string{security.InternalClusterAuthSecretName(resourceNameForSecret)} + } + for _, secretName := range secretNames { + r.resourceWatcher.AddWatchedResourceIfNotAdded(secretName, objectToReconcile.Namespace, watch.Secret, objectToReconcile) + } + } +} + +// We fetch a fresh version in case any modifications have been made. +// Note, that this method enforces update ONLY to the status, so the reconciliation events happening because of this +// can be filtered out by 'controller.shouldReconcile' +// The "jsonPatch" merge allows to update only status field +func (r *ReconcileCommonController) patchUpdateStatus(ctx context.Context, resource v1.CustomResourceReadWriter, options ...status.Option) error { + payload := []patchValue{{ + Op: "replace", + Path: resource.GetStatusPath(options...), + // in most cases this will be "/status", but for each of the different Ops Manager components + // this will be different + Value: resource.GetStatus(options...), + }} + + data, err := json.Marshal(payload) + if err != nil { + return err + } + + patch := client.RawPatch(types.JSONPatchType, data) + err = r.client.Status().Patch(ctx, resource, patch) + + if err != nil && apiErrors.IsInvalid(err) { + zap.S().Debug("The Status subresource might not exist yet, creating empty subresource") + if err := r.ensureStatusSubresourceExists(ctx, resource, options...); err != nil { + zap.S().Debug("Error from ensuring status subresource: %s", err) + return err + } + return r.client.Status().Patch(ctx, resource, patch) + } + + return nil +} + +type emptyPayload struct{} + +// ensureStatusSubresourceExists ensures that the status subresource section we are trying to write to exists. +// if we just try and patch the full path directly, the subresource sections are not recursively created, so +// we need to ensure that the actual object we're trying to write to exists, otherwise we will get errors. +func (r *ReconcileCommonController) ensureStatusSubresourceExists(ctx context.Context, resource v1.CustomResourceReadWriter, options ...status.Option) error { + fullPath := resource.GetStatusPath(options...) + parts := strings.Split(fullPath, "/") + + if strings.HasPrefix(fullPath, "/") { + parts = parts[1:] + } + + var path []string + for _, part := range parts { + pathStr := "/" + strings.Join(path, "/") + path = append(path, part) + emptyPatchPayload := []patchValue{{ + Op: "add", + Path: pathStr, + Value: emptyPayload{}, + }} + data, err := json.Marshal(emptyPatchPayload) + if err != nil { + return err + } + patch := client.RawPatch(types.JSONPatchType, data) + if err := r.client.Status().Patch(ctx, resource, patch); err != nil && !apiErrors.IsInvalid(err) { + return err + } + } + return nil +} + +// getResource populates the provided runtime.Object with some additional error handling +func (r *ReconcileCommonController) getResource(ctx context.Context, request reconcile.Request, resource v1.CustomResourceReadWriter, log *zap.SugaredLogger) (reconcile.Result, error) { + err := r.client.Get(ctx, request.NamespacedName, resource) + if err != nil { + if apiErrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Return and don't requeue + log.Debugf("Object %s doesn't exist, was it deleted after reconcile request?", request.NamespacedName) + return reconcile.Result{}, err + } + // Error reading the object - requeue the request. + log.Errorf("Failed to query object %s: %s", request.NamespacedName, err) + return reconcile.Result{RequeueAfter: 10 * time.Second}, err + } + return reconcile.Result{}, nil +} + +// prepareResourceForReconciliation finds the object being reconciled. Returns the reconcile result and any error that +// occurred. +func (r *ReconcileCommonController) prepareResourceForReconciliation(ctx context.Context, request reconcile.Request, resource v1.CustomResourceReadWriter, log *zap.SugaredLogger) (reconcile.Result, error) { + if result, err := r.getResource(ctx, request, resource, log); err != nil { + return result, err + } + + // Reset warnings so that they are not stale, will populate accurate warnings in reconciliation + resource.SetWarnings([]status.Warning{}) + + return reconcile.Result{}, nil +} + +// checkIfHasExcessProcesses will check if the project has excess processes. +// Also, it removes the tag ExternallyManaged from the project in this case as +// the user may need to clean the resources from OM UI if they move the +// resource to another project (as recommended by the migration instructions). +func checkIfHasExcessProcesses(conn om.Connection, resourceName string, log *zap.SugaredLogger) workflow.Status { + deployment, err := conn.ReadDeployment() + if err != nil { + return workflow.Failed(err) + } + excessProcesses := deployment.GetNumberOfExcessProcesses(resourceName) + if excessProcesses == 0 { + // cluster is empty or this resource is the only one living on it + return workflow.OK() + } + // remove tags if multiple clusters in project + groupWithTags := &om.Project{ + Name: conn.GroupName(), + OrgID: conn.OrgID(), + ID: conn.GroupID(), + Tags: []string{}, + } + _, err = conn.UpdateProject(groupWithTags) + if err != nil { + log.Warnw("could not remove externally managed tag from Ops Manager group", "error", err) + } + + return workflow.Pending("cannot have more than 1 MongoDB Cluster per project (see https://docs.mongodb.com/kubernetes-operator/stable/tutorial/migrate-to-single-resource/)") +} + +// validateInternalClusterCertsAndCheckTLSType verifies that all the x509 internal cluster certs exist and return whether they are built following the kubernetes.io/tls secret type (tls.crt/tls.key entries). +// TODO: this is almost the same as certs.EnsureSSLCertsForStatefulSet, we should centralize the functionality +func (r *ReconcileCommonController) validateInternalClusterCertsAndCheckTLSType(ctx context.Context, configurator certs.X509CertConfigurator, opts certs.Options, log *zap.SugaredLogger) error { + secretName := opts.InternalClusterSecretName + + err := certs.VerifyAndEnsureCertificatesForStatefulSet(ctx, configurator.GetSecretReadClient(), configurator.GetSecretWriteClient(), secretName, opts, log) + if err != nil { + return xerrors.Errorf("the secret object '%s' does not contain all the certificates needed: %w", secretName, err) + } + + secretName = fmt.Sprintf("%s%s", secretName, certs.OperatorGeneratedCertSuffix) + + // Validates that the secret is valid + if err := certs.ValidateCertificates(ctx, r.client, secretName, opts.Namespace, log); err != nil { + return err + } + return nil +} + +// ensureBackupConfigurationAndUpdateStatus configures backup in Ops Manager based on the MongoDB resources spec +func (r *ReconcileCommonController) ensureBackupConfigurationAndUpdateStatus(ctx context.Context, conn om.Connection, mdb backup.ConfigReaderUpdater, secretsReader secrets.SecretClient, log *zap.SugaredLogger) workflow.Status { + statusOpt, opts := backup.EnsureBackupConfigurationInOpsManager(ctx, mdb, secretsReader, conn.GroupID(), conn, conn, conn, log) + if len(opts) > 0 { + if _, err := r.updateStatus(ctx, mdb, statusOpt, log, opts...); err != nil { + return workflow.Failed(err) + } + } + return statusOpt +} + +// validateMongoDBResource performs validation on the MongoDBResource +func validateMongoDBResource(mdb *mdbv1.MongoDB, conn om.Connection) workflow.Status { + ac, err := conn.ReadAutomationConfig() + if err != nil { + return workflow.Failed(err) + } + + if status := validateScram(mdb, ac); !status.IsOK() { + return status + } + + return workflow.OK() +} + +func ensureSupportedOpsManagerVersion(conn om.Connection) workflow.Status { + omVersionString := conn.OpsManagerVersion() + if !omVersionString.IsCloudManager() { + omVersion, err := omVersionString.Semver() + if err != nil { + return workflow.Failed(xerrors.Errorf("Failed when trying to parse Ops Manager version")) + } + if omVersion.LT(semver.MustParse(oldestSupportedOpsManagerVersion)) { + return workflow.Unsupported("This MongoDB ReplicaSet is managed by Ops Manager version %s, which is not supported by this version of the operator. Please upgrade it to a version >=%s", omVersion, oldestSupportedOpsManagerVersion) + } + } + return workflow.OK() +} + +// scaleStatefulSet sets the number of replicas for a StatefulSet and returns a reference of the updated resource. +func (r *ReconcileCommonController) scaleStatefulSet(ctx context.Context, namespace, name string, replicas int32, client kubernetesClient.Client) (appsv1.StatefulSet, error) { + if set, err := client.GetStatefulSet(ctx, kube.ObjectKey(namespace, name)); err != nil { + return set, err + } else { + set.Spec.Replicas = &replicas + return client.UpdateStatefulSet(ctx, set) + } +} + +// getStatefulSetStatus returns the workflow.Status based on the status of the StatefulSet. +// If the StatefulSet is not ready the request will be retried in 3 seconds (instead of the default 10 seconds) +// allowing to reach "ready" status sooner +func getStatefulSetStatus(ctx context.Context, namespace, name string, client kubernetesClient.Client) workflow.Status { + set, err := client.GetStatefulSet(ctx, kube.ObjectKey(namespace, name)) + i := 0 + + // Sometimes it is possible that the StatefulSet which has just been created + // returns a not found error when getting it too soon afterwards. + for apiErrors.IsNotFound(err) && i < 10 { + i++ + zap.S().Debugf("StatefulSet was not found: %s, attempt %d", err, i) + time.Sleep(time.Second * 1) + set, err = client.GetStatefulSet(ctx, kube.ObjectKey(namespace, name)) + } + + if err != nil { + return workflow.Failed(err) + } + + if statefulSetState := inspect.StatefulSet(set); !statefulSetState.IsReady() { + return workflow. + Pending("%s", statefulSetState.GetMessage()). + WithResourcesNotReady(statefulSetState.GetResourcesNotReadyStatus()). + WithRetry(3) + } else { + zap.S().Debugf("StatefulSet %s/%s is ready on check attempt #%d, state: %+v: ", namespace, name, i, statefulSetState) + } + + return workflow.OK() +} + +// validateScram ensures that the SCRAM configuration is valid for the MongoDBResource +func validateScram(mdb *mdbv1.MongoDB, ac *om.AutomationConfig) workflow.Status { + specVersion, err := semver.Make(util.StripEnt(mdb.Spec.GetMongoDBVersion())) + if err != nil { + return workflow.Failed(err) + } + + scram256IsAlreadyEnabled := stringutil.Contains(ac.Auth.DeploymentAuthMechanisms, string(authentication.ScramSha256)) + attemptingToDowngradeMongoDBVersion := ac.Deployment.MinimumMajorVersion() >= 4 && specVersion.Major < 4 + isDowngradingFromScramSha256ToScramSha1 := attemptingToDowngradeMongoDBVersion && stringutil.Contains(mdb.Spec.Security.Authentication.GetModes(), "SCRAM") && scram256IsAlreadyEnabled + + if isDowngradingFromScramSha256ToScramSha1 { + return workflow.Invalid("Unable to downgrade to SCRAM-SHA-1 when SCRAM-SHA-256 has been enabled") + } + + return workflow.OK() +} + +// Use the first "CERTIFICATE" block found in the PEM file. +func getSubjectFromCertificate(cert string) (string, error) { + block, rest := pem.Decode([]byte(cert)) + if block != nil && block.Type == "CERTIFICATE" { + subjects, _, err := authentication.GetCertificateSubject(cert) + if err != nil { + return "", err + } + return subjects, nil + } + if len(rest) > 0 { + subjects, _, err := authentication.GetCertificateSubject(string(rest)) + if err != nil { + return "", err + } + return subjects, nil + } + return "", xerrors.Errorf("unable to extract the subject line from the provided certificate") +} + +// updateOmAuthentication examines the state of Ops Manager and the desired state of the MongoDB resource and +// enables/disables authentication. If the authentication can't be fully configured, a boolean value indicating that +// an additional reconciliation needs to be queued up to fully make the authentication changes is returned. +// Note: updateOmAuthentication needs to be called before reconciling other auth related settings. +func (r *ReconcileCommonController) updateOmAuthentication(ctx context.Context, conn om.Connection, processNames []string, ar authentication.AuthResource, agentCertSecretName string, caFilepath string, clusterFilePath string, isRecovering bool, log *zap.SugaredLogger) (status workflow.Status, multiStageReconciliation bool) { + // don't touch authentication settings if resource has not been configured with them + if ar.GetSecurity() == nil || ar.GetSecurity().Authentication == nil { + return workflow.OK(), false + } + + ac, err := conn.ReadAutomationConfig() + if err != nil { + return workflow.Failed(err), false + } + + // if we have changed the internal cluster auth, we need to update the ac first + authenticationMode := ar.GetSecurity().GetInternalClusterAuthenticationMode() + err = r.setupInternalClusterAuthIfItHasChanged(conn, processNames, authenticationMode, clusterFilePath, isRecovering) + if err != nil { + return workflow.Failed(err), false + } + + // we need to wait for all agents to be ready before configuring any authentication settings + if err := om.WaitForReadyState(conn, processNames, isRecovering, log); err != nil { + return workflow.Failed(err), false + } + + clientCerts := util.OptionalClientCertficates + if ar.GetSecurity().RequiresClientTLSAuthentication() { + clientCerts = util.RequireClientCertificates + } + + scramAgentUserName := util.AutomationAgentUserName + // only use the default name if there is not already a configured username + if ac.Auth.AutoUser != "" && ac.Auth.AutoUser != scramAgentUserName { + scramAgentUserName = ac.Auth.AutoUser + } + + authOpts := authentication.Options{ + Mechanisms: mdbv1.ConvertAuthModesToStrings(ar.GetSecurity().Authentication.Modes), + ProcessNames: processNames, + AuthoritativeSet: !ar.GetSecurity().Authentication.IgnoreUnknownUsers, + AgentMechanism: ar.GetSecurity().GetAgentMechanism(ac.Auth.AutoAuthMechanism), + ClientCertificates: clientCerts, + AutoUser: scramAgentUserName, + AutoLdapGroupDN: ar.GetSecurity().Authentication.Agents.AutomationLdapGroupDN, + CAFilePath: caFilepath, + } + var databaseSecretPath string + if r.VaultClient != nil { + databaseSecretPath = r.VaultClient.DatabaseSecretPath() + } + if ar.IsLDAPEnabled() { + bindUserPassword, err := r.ReadSecretKey(ctx, kube.ObjectKey(ar.GetNamespace(), ar.GetSecurity().Authentication.Ldap.BindQuerySecretRef.Name), databaseSecretPath, "password") + if err != nil { + return workflow.Failed(xerrors.Errorf("error reading bind user password: %w", err)), false + } + + caContents := "" + ca := ar.GetSecurity().Authentication.Ldap.CAConfigMapRef + if ca != nil { + log.Debugf("Sending CA file to Pods via AutomationConfig: %s/%s/%s", ar.GetNamespace(), ca.Name, ca.Key) + caContents, err = configmap.ReadKey(ctx, r.client, ca.Key, types.NamespacedName{Name: ca.Name, Namespace: ar.GetNamespace()}) + if err != nil { + return workflow.Failed(xerrors.Errorf("error reading CA configmap: %w", err)), false + } + } + + authOpts.Ldap = ar.GetLDAP(bindUserPassword, caContents) + } + + log.Debugf("Using authentication options %+v", authentication.Redact(authOpts)) + + agentSecretSelector := ar.GetSecurity().AgentClientCertificateSecretName(ar.GetName()) + if agentCertSecretName != "" { + agentSecretSelector.Name = agentCertSecretName + } + wantToEnableAuthentication := ar.GetSecurity().Authentication.Enabled + if wantToEnableAuthentication && canConfigureAuthentication(ac, ar.GetSecurity().Authentication.GetModes(), log) { + log.Info("Configuring authentication for MongoDB resource") + + if ar.GetSecurity().ShouldUseX509(ac.Auth.AutoAuthMechanism) || ar.GetSecurity().ShouldUseClientCertificates() { + agentSecret := &corev1.Secret{} + if err := r.client.Get(ctx, kube.ObjectKey(ar.GetNamespace(), agentSecretSelector.Name), agentSecret); client.IgnoreNotFound(err) != nil { + return workflow.Failed(err), false + } + // If the agent secret is of type TLS, we can find the certificate under the standard key, + // otherwise the concatenated PEM secret would contain the certificate and keys under the selector's + // Key identifying the agent. + // In single cluster workloads this path is working with the concatenated PEM secret, + // + // Important: In multi cluster it is working with the TLS secret in the central cluster, hence below selector update. + if agentSecret.Type == corev1.SecretTypeTLS { + agentSecretSelector.Key = corev1.TLSCertKey + } + + authOpts, err = r.configureAgentSubjects(ctx, ar.GetNamespace(), agentSecretSelector, authOpts, log) + if err != nil { + return workflow.Failed(xerrors.Errorf("error configuring agent subjects: %w", err)), false + } + authOpts.AgentsShouldUseClientAuthentication = ar.GetSecurity().ShouldUseClientCertificates() + } + if ar.GetSecurity().ShouldUseLDAP(ac.Auth.AutoAuthMechanism) { + secretRef := ar.GetSecurity().Authentication.Agents.AutomationPasswordSecretRef + autoConfigPassword, err := r.ReadSecretKey(ctx, kube.ObjectKey(ar.GetNamespace(), secretRef.Name), databaseSecretPath, secretRef.Key) + if err != nil { + return workflow.Failed(xerrors.Errorf("error reading automation agent password: %w", err)), false + } + + authOpts.AutoPwd = autoConfigPassword + userOpts := authentication.UserOptions{} + agentName := ar.GetSecurity().Authentication.Agents.AutomationUserName + userOpts.AutomationSubject = agentName + authOpts.UserOptions = userOpts + } + + if err := authentication.Configure(conn, authOpts, isRecovering, log); err != nil { + return workflow.Failed(err), false + } + } else if wantToEnableAuthentication { + // The MongoDB resource has been configured with a type of authentication + // but the current state in Ops Manager does not allow a direct transition. This will require + // an additional reconciliation after a partial update to Ops Manager. + log.Debug("Attempting to enable authentication, but Ops Manager state will not allow this") + return workflow.OK(), true + } else { + agentSecret := &corev1.Secret{} + if err := r.client.Get(ctx, kube.ObjectKey(ar.GetNamespace(), agentSecretSelector.Name), agentSecret); client.IgnoreNotFound(err) != nil { + return workflow.Failed(err), false + } + + if agentSecret.Type == corev1.SecretTypeTLS { + agentSecretSelector.Name = fmt.Sprintf("%s%s", agentSecretSelector.Name, certs.OperatorGeneratedCertSuffix) + } + + // Should not fail if the Secret object with agent certs is not found. + // It will only exist on x509 client auth enabled deployments. + userOpts, err := r.readAgentSubjectsFromSecret(ctx, ar.GetNamespace(), agentSecretSelector, log) + err = client.IgnoreNotFound(err) + if err != nil { + return workflow.Failed(err), true + } + + authOpts.UserOptions = userOpts + if err := authentication.Disable(conn, authOpts, false, log); err != nil { + return workflow.Failed(err), false + } + } + return workflow.OK(), false +} + +// configureAgentSubjects returns a new authentication.Options which has configured the Subject lines for the automation agent. +// The Ops Manager user names for these agents will be configured based on the contents of the secret. +func (r *ReconcileCommonController) configureAgentSubjects(ctx context.Context, namespace string, secretKeySelector corev1.SecretKeySelector, authOpts authentication.Options, log *zap.SugaredLogger) (authentication.Options, error) { + userOpts, err := r.readAgentSubjectsFromSecret(ctx, namespace, secretKeySelector, log) + if err != nil { + return authentication.Options{}, xerrors.Errorf("error reading agent subjects from secret: %w", err) + } + authOpts.UserOptions = userOpts + return authOpts, nil +} + +func (r *ReconcileCommonController) readAgentSubjectsFromSecret(ctx context.Context, namespace string, secretKeySelector corev1.SecretKeySelector, log *zap.SugaredLogger) (authentication.UserOptions, error) { + userOpts := authentication.UserOptions{} + + var databaseSecretPath string + if r.VaultClient != nil { + databaseSecretPath = r.VaultClient.DatabaseSecretPath() + } + agentCerts, err := r.ReadSecret(ctx, kube.ObjectKey(namespace, secretKeySelector.Name), databaseSecretPath) + if err != nil { + return userOpts, err + } + + automationAgentCert, ok := agentCerts[secretKeySelector.Key] + if !ok { + return userOpts, xerrors.Errorf("could not find certificate with name %s", secretKeySelector.Key) + } + + automationAgentSubject, err := getSubjectFromCertificate(automationAgentCert) + if err != nil { + return userOpts, xerrors.Errorf("error extracting automation agent subject is not present %w", err) + } + + log.Debugf("Automation certificate subject is %s", automationAgentSubject) + + return authentication.UserOptions{ + AutomationSubject: automationAgentSubject, + }, nil +} + +func (r *ReconcileCommonController) clearProjectAuthenticationSettings(ctx context.Context, conn om.Connection, mdb *mdbv1.MongoDB, processNames []string, log *zap.SugaredLogger) error { + secretKeySelector := mdb.Spec.Security.AgentClientCertificateSecretName(mdb.Name) + agentSecret := &corev1.Secret{} + if err := r.client.Get(ctx, kube.ObjectKey(mdb.Namespace, secretKeySelector.Name), agentSecret); client.IgnoreNotFound(err) != nil { + return nil + } + + if agentSecret.Type == corev1.SecretTypeTLS { + secretKeySelector.Name = fmt.Sprintf("%s%s", secretKeySelector.Name, certs.OperatorGeneratedCertSuffix) + } + + userOpts, err := r.readAgentSubjectsFromSecret(ctx, mdb.Namespace, secretKeySelector, log) + err = client.IgnoreNotFound(err) + if err != nil { + return err + } + log.Infof("Disabling authentication for project: %s", conn.GroupName()) + disableOpts := authentication.Options{ + ProcessNames: processNames, + UserOptions: userOpts, + } + + return authentication.Disable(conn, disableOpts, true, log) +} + +// ensureX509SecretAndCheckTLSType checks if the secrets containing the certificates are present and whether the certificate are of kubernetes.io/tls type. +func (r *ReconcileCommonController) ensureX509SecretAndCheckTLSType(ctx context.Context, configurator certs.X509CertConfigurator, currentAuthMechanism string, log *zap.SugaredLogger) workflow.Status { + security := configurator.GetDbCommonSpec().GetSecurity() + authSpec := security.Authentication + if authSpec == nil || !security.Authentication.Enabled { + return workflow.OK() + } + + if security.ShouldUseX509(currentAuthMechanism) || security.ShouldUseClientCertificates() { + if !security.IsTLSEnabled() { + return workflow.Failed(xerrors.Errorf("Authentication mode for project is x509 but this MDB resource is not TLS enabled")) + } + agentSecretName := security.AgentClientCertificateSecretName(configurator.GetName()).Name + err := certs.VerifyAndEnsureClientCertificatesForAgentsAndTLSType(ctx, configurator.GetSecretReadClient(), configurator.GetSecretWriteClient(), kube.ObjectKey(configurator.GetNamespace(), agentSecretName)) + if err != nil { + return workflow.Failed(err) + } + } + + if security.GetInternalClusterAuthenticationMode() == util.X509 { + errors := make([]error, 0) + for _, certOption := range configurator.GetCertOptions() { + err := r.validateInternalClusterCertsAndCheckTLSType(ctx, configurator, certOption, log) + if err != nil { + errors = append(errors, err) + } + } + if len(errors) > 0 { + return workflow.Failed(xerrors.Errorf("failed ensuring internal cluster authentication certs %w", errors[0])) + } + } + + return workflow.OK() +} + +// setupInternalClusterAuthIfItHasChanged enables internal cluster auth if possible in case the path has changed and did exist before. +func (r *ReconcileCommonController) setupInternalClusterAuthIfItHasChanged(conn om.Connection, names []string, clusterAuth string, filePath string, isRecovering bool) error { + if filePath == "" { + return nil + } + err := conn.ReadUpdateDeployment(func(deployment om.Deployment) error { + deployment.SetInternalClusterFilePathOnlyIfItThePathHasChanged(names, filePath, clusterAuth, isRecovering) + return nil + }, zap.S()) + return err +} + +// getAgentVersion handles the common logic for error handling and instance initialisation +// when retrieving the agent version from a controller +func (r *ReconcileCommonController) getAgentVersion(conn om.Connection, omVersion string, isAppDB bool, log *zap.SugaredLogger) (string, error) { + m, err := agentVersionManagement.GetAgentVersionManager() + if err != nil || m == nil { + return "", xerrors.Errorf("not able to init agentVersionManager: %w", err) + } + + if agentVersion, err := m.GetAgentVersion(conn, omVersion, isAppDB); err != nil { + log.Errorf("Failed to get the agent version from the Agent Version manager: %s", err) + return "", err + } else { + log.Debugf("Using agent version %s", agentVersion) + currentOperatorVersion := versionutil.StaticContainersOperatorVersion() + log.Debugf("Using Operator version: %s", currentOperatorVersion) + return agentVersion + "_" + currentOperatorVersion, nil + } +} + +// isPrometheusSupported checks if Prometheus integration can be enabled. +// +// Prometheus is only enabled in Cloud Manager and Ops Manager 5.9 (6.0) and above. +func isPrometheusSupported(conn om.Connection) bool { + if conn.OpsManagerVersion().IsCloudManager() { + return true + } + + omVersion, err := conn.OpsManagerVersion().Semver() + return err == nil && omVersion.GTE(semver.MustParse("5.9.0")) +} + +// UpdatePrometheus configures Prometheus on the Deployment for this resource. +func UpdatePrometheus(ctx context.Context, d *om.Deployment, conn om.Connection, prometheus *mdbcv1.Prometheus, sClient secrets.SecretClient, namespace string, certName string, log *zap.SugaredLogger) error { + if prometheus == nil { + return nil + } + + if !isPrometheusSupported(conn) { + log.Info("Prometheus can't be enabled, Prometheus is not supported in this version of Ops Manager") + return nil + } + + var err error + var password string + + secretName := prometheus.PasswordSecretRef.Name + if vault.IsVaultSecretBackend() { + operatorSecretPath := sClient.VaultClient.OperatorSecretPath() + passwordString := fmt.Sprintf("%s/%s/%s", operatorSecretPath, namespace, secretName) + keyedPassword, err := sClient.VaultClient.ReadSecretString(passwordString) + if err != nil { + log.Infof("Prometheus can't be enabled, %s", err) + return err + } + + var ok bool + password, ok = keyedPassword[prometheus.GetPasswordKey()] + if !ok { + errMsg := fmt.Sprintf("Prometheus password %s not in Secret %s", prometheus.GetPasswordKey(), passwordString) + log.Info(errMsg) + return xerrors.Errorf(errMsg) + } + } else { + secretNamespacedName := types.NamespacedName{Name: secretName, Namespace: namespace} + password, err = secret.ReadKey(ctx, sClient, prometheus.GetPasswordKey(), secretNamespacedName) + if err != nil { + log.Infof("Prometheus can't be enabled, %s", err) + return err + } + } + + hash, salt := passwordhash.GenerateHashAndSaltForPassword(password) + d.ConfigurePrometheus(prometheus, hash, salt, certName) + + return nil +} + +// canConfigureAuthentication determines if based on the existing state of Ops Manager +// it is possible to configure the authentication mechanisms specified by the given MongoDB resource +// during this reconciliation. This function may return a different value on the next reconciliation +// if the state of Ops Manager has been changed. +func canConfigureAuthentication(ac *om.AutomationConfig, authenticationModes []string, log *zap.SugaredLogger) bool { + attemptingToEnableX509 := !stringutil.Contains(ac.Auth.DeploymentAuthMechanisms, util.AutomationConfigX509Option) && stringutil.Contains(authenticationModes, util.X509) + canEnableX509InOpsManager := ac.Deployment.AllProcessesAreTLSEnabled() || ac.Deployment.NumberOfProcesses() == 0 + + log.Debugw("canConfigureAuthentication", + "attemptingToEnableX509", attemptingToEnableX509, + "deploymentAuthMechanisms", ac.Auth.DeploymentAuthMechanisms, + "modes", authenticationModes, + "canEnableX509InOpsManager", canEnableX509InOpsManager, + "allProcessesAreTLSEnabled", ac.Deployment.AllProcessesAreTLSEnabled(), + "numberOfProcesses", ac.Deployment.NumberOfProcesses()) + + if attemptingToEnableX509 { + return canEnableX509InOpsManager + } + + // x509 is the only mechanism with restrictions determined based on Ops Manager state + return true +} + +// newPodVars initializes a PodEnvVars instance based on the values of the provided Ops Manager connection, project config +// and connection spec +func newPodVars(conn om.Connection, projectConfig mdbv1.ProjectConfig, logLevel mdbv1.LogLevel) *env.PodEnvVars { + podVars := &env.PodEnvVars{} + podVars.BaseURL = conn.BaseURL() + podVars.ProjectID = conn.GroupID() + podVars.User = conn.PublicKey() + podVars.LogLevel = string(logLevel) + podVars.SSLProjectConfig = projectConfig.SSLProjectConfig + return podVars +} + +func getVolumeFromStatefulSet(sts appsv1.StatefulSet, name string) (corev1.Volume, error) { + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Name == name { + return v, nil + } + } + return corev1.Volume{}, xerrors.Errorf("can't find volume %s in list of volumes: %v", name, sts.Spec.Template.Spec.Volumes) +} + +// wasTLSSecretMounted checks whether TLS was previously enabled by looking at the state of the volumeMounts of the pod. +func wasTLSSecretMounted(ctx context.Context, secretGetter secret.Getter, currentSts appsv1.StatefulSet, mdb mdbv1.MongoDB, log *zap.SugaredLogger) bool { + tlsVolume, err := getVolumeFromStatefulSet(currentSts, util.SecretVolumeName) + if err != nil { + return false + } + + // With the new design, the volume is always mounted + // But it is marked with optional. + // + // TLS was enabled if the secret it refers to is present + + secretName := tlsVolume.Secret.SecretName + exists, err := secret.Exists(ctx, secretGetter, types.NamespacedName{ + Namespace: mdb.Namespace, + Name: secretName, + }, + ) + if err != nil { + log.Warnf("can't determine whether the TLS certificate secret exists or not: %s. Will assume it doesn't", err) + return false + } + log.Debugf("checking if secret %s exists: %v", secretName, exists) + + return exists +} + +// wasCAConfigMapMounted checks whether the CA ConfigMap by looking at the state of the volumeMounts of the pod. +func wasCAConfigMapMounted(ctx context.Context, configMapGetter configmap.Getter, currentSts appsv1.StatefulSet, mdb mdbv1.MongoDB, log *zap.SugaredLogger) bool { + caVolume, err := getVolumeFromStatefulSet(currentSts, util.ConfigMapVolumeCAMountPath) + if err != nil { + return false + } + + // With the new design, the volume is always mounted + // But it is marked with optional. + // + // The configMap was mounted if the configMap it refers to is present + + cmName := caVolume.ConfigMap.Name + exists, err := configmap.Exists(ctx, configMapGetter, types.NamespacedName{ + Namespace: mdb.Namespace, + Name: cmName, + }, + ) + if err != nil { + log.Warnf("can't determine whether the TLS ConfigMap exists or not: %s. Will assume it doesn't", err) + return false + } + log.Debugf("checking if ConfigMap %s exists: %v", cmName, exists) + + return exists +} + +type ConfigMapStatefulSetSecretGetter interface { + statefulset.Getter + secret.Getter + configmap.Getter +} + +// publishAutomationConfigFirst will check if the Published State of the StatefulSet backed MongoDB Deployments +// needs to be updated first. In the case of unmounting certs, for instance, the certs should be not +// required anymore before we unmount them, or the automation-agent and readiness probe will never +// reach goal state. +func publishAutomationConfigFirst(ctx context.Context, getter ConfigMapStatefulSetSecretGetter, mdb mdbv1.MongoDB, lastSpec *mdbv1.MongoDbSpec, configFunc func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions, log *zap.SugaredLogger) bool { + opts := configFunc(mdb) + + namespacedName := kube.ObjectKey(mdb.Namespace, opts.GetStatefulSetName()) + currentSts, err := getter.GetStatefulSet(ctx, namespacedName) + if err != nil { + if apiErrors.IsNotFound(err) { + // No need to publish state as this is a new StatefulSet + log.Debugf("New StatefulSet %s", namespacedName) + return false + } + + log.Debugw(fmt.Sprintf("Error getting StatefulSet %s", namespacedName), "error", err) + return false + } + + databaseContainer := container.GetByName(util.DatabaseContainerName, currentSts.Spec.Template.Spec.Containers) + volumeMounts := databaseContainer.VolumeMounts + + if !mdb.Spec.Security.IsTLSEnabled() && wasTLSSecretMounted(ctx, getter, currentSts, mdb, log) { + log.Debug(automationConfigFirstMsg("security.tls.enabled", "false")) + return true + } + + if mdb.Spec.Security.TLSConfig.CA == "" && wasCAConfigMapMounted(ctx, getter, currentSts, mdb, log) { + log.Debug(automationConfigFirstMsg("security.tls.CA", "empty")) + return true + + } + + if opts.PodVars.SSLMMSCAConfigMap == "" && statefulset.VolumeMountWithNameExists(volumeMounts, construct.CaCertName) { + log.Debug(automationConfigFirstMsg("SSLMMSCAConfigMap", "empty")) + return true + } + + if mdb.Spec.Security.GetAgentMechanism(opts.CurrentAgentAuthMode) != util.X509 && statefulset.VolumeMountWithNameExists(volumeMounts, util.AgentSecretName) { + log.Debug(automationConfigFirstMsg("project.AuthMode", "empty")) + return true + } + + if opts.Replicas < int(*currentSts.Spec.Replicas) { + log.Debug("Scaling down operation. automationConfig needs to be updated first") + return true + } + + if architectures.IsRunningStaticArchitecture(mdb.GetAnnotations()) { + if mdb.Spec.IsInChangeVersion(lastSpec) { + return true + } + } + + return false +} + +// completionMessage is just a general message printed in the logs after mongodb resource is created/updated +func completionMessage(url, projectID string) string { + return fmt.Sprintf("Please check the link %s/v2/%s to see the status of the deployment", url, projectID) +} + +// getAnnotationsForResource returns all of the annotations that should be applied to the resource +// at the end of the reconciliation. The additional mongod options must be manually +// set as the wrapper type we use prevents a regular `json.Marshal` from working in this case due to +// the `json "-"` tag. +func getAnnotationsForResource(mdb *mdbv1.MongoDB) (map[string]string, error) { + finalAnnotations := make(map[string]string) + specBytes, err := json.Marshal(mdb.Spec) + if err != nil { + return nil, err + } + finalAnnotations[util.LastAchievedSpec] = string(specBytes) + return finalAnnotations, nil +} + +type PrometheusConfiguration struct { + prometheus *mdbcv1.Prometheus + conn om.Connection + secretsClient secrets.SecretClient + namespace string + prometheusCertHash string +} + +func ReconcileReplicaSetAC(ctx context.Context, d om.Deployment, spec mdbv1.DbCommonSpec, lastMongodConfig map[string]interface{}, resourceName string, rs om.ReplicaSetWithProcesses, caFilePath string, internalClusterPath string, pc *PrometheusConfiguration, log *zap.SugaredLogger) error { + // it is not possible to disable internal cluster authentication once enabled + if d.ExistingProcessesHaveInternalClusterAuthentication(rs.Processes) && spec.Security.GetInternalClusterAuthenticationMode() == "" { + return xerrors.Errorf("cannot disable x509 internal cluster authentication") + } + + excessProcesses := d.GetNumberOfExcessProcesses(resourceName) + if excessProcesses > 0 { + return xerrors.Errorf("cannot have more than 1 MongoDB Cluster per project (see https://docs.mongodb.com/kubernetes-operator/stable/tutorial/migrate-to-single-resource/)") + } + + d.MergeReplicaSet(rs, spec.GetAdditionalMongodConfig().ToMap(), lastMongodConfig, log) + d.AddMonitoringAndBackup(log, spec.GetSecurity().IsTLSEnabled(), caFilePath) + d.ConfigureTLS(spec.GetSecurity(), caFilePath) + d.ConfigureInternalClusterAuthentication(rs.GetProcessNames(), spec.GetSecurity().GetInternalClusterAuthenticationMode(), internalClusterPath) + + // if we don't set up a prometheus connection, then we don't want to set up prometheus for instance because we do not support it yet. + if pc != nil { + // At this point, we won't bubble-up the error we got from this + // function, we don't want to fail the MongoDB resource because + // Prometheus can't be enabled. + _ = UpdatePrometheus(ctx, &d, pc.conn, pc.prometheus, pc.secretsClient, pc.namespace, pc.prometheusCertHash, log) + } + + return nil +} + +func ReconcileLogRotateSetting(conn om.Connection, agentConfig mdbv1.AgentConfig, log *zap.SugaredLogger) (workflow.Status, error) { + if err := conn.ReadUpdateAgentsLogRotation(agentConfig, log); err != nil { + return workflow.Failed(err), err + } + return workflow.OK(), nil +} diff --git a/controllers/operator/common_controller_test.go b/controllers/operator/common_controller_test.go new file mode 100644 index 000000000..608866258 --- /dev/null +++ b/controllers/operator/common_controller_test.go @@ -0,0 +1,596 @@ +package operator + +import ( + "context" + "fmt" + "os" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/deployment" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/agentVersionManagement" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +const OperatorNamespace = "operatorNs" + +func init() { + util.OperatorVersion = "9.9.9-test" + _ = os.Setenv(util.CurrentNamespace, OperatorNamespace) // nolint:forbidigo +} + +func TestEnsureTagAdded(t *testing.T) { + ctx := context.Background() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + controller := NewReconcileCommonController(ctx, kubeClient) + mockOm, _ := prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + + // normal tag + err := connection.EnsureTagAdded(mockOm, mockOm.FindGroup(om.TestGroupName), "myTag", zap.S()) + assert.NoError(t, err) + + // long tag + err = connection.EnsureTagAdded(mockOm, mockOm.FindGroup(om.TestGroupName), "LOOKATTHISTRINGTHATISTOOLONGFORTHEFIELD", zap.S()) + assert.NoError(t, err) + + expected := []string{"EXTERNALLY_MANAGED_BY_KUBERNETES", "MY-NAMESPACE", "MYTAG", "LOOKATTHISTRINGTHATISTOOLONGFORT"} + assert.Equal(t, expected, mockOm.FindGroup(om.TestGroupName).Tags) +} + +func TestEnsureTagAddedDuplicates(t *testing.T) { + ctx := context.Background() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + opsManagerController := NewReconcileCommonController(ctx, kubeClient) + + mockOm, _ := prepareConnection(ctx, opsManagerController, omConnectionFactory.GetConnectionFunc, t) + err := connection.EnsureTagAdded(mockOm, mockOm.FindGroup(om.TestGroupName), "MYTAG", zap.S()) + assert.NoError(t, err) + err = connection.EnsureTagAdded(mockOm, mockOm.FindGroup(om.TestGroupName), "MYTAG", zap.S()) + assert.NoError(t, err) + err = connection.EnsureTagAdded(mockOm, mockOm.FindGroup(om.TestGroupName), "MYOTHERTAG", zap.S()) + assert.NoError(t, err) + expected := []string{"EXTERNALLY_MANAGED_BY_KUBERNETES", "MY-NAMESPACE", "MYTAG", "MYOTHERTAG"} + assert.Equal(t, expected, mockOm.FindGroup(om.TestGroupName).Tags) +} + +// TestPrepareOmConnection_FindExistingGroup finds existing group when org ID is specified, no new Project or Organization +// is created +func TestPrepareOmConnection_FindExistingGroup(t *testing.T) { + ctx := context.Background() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + kubeClient := mock.NewEmptyFakeClientWithInterceptor(omConnectionFactory, []client.Object{ + mock.GetProjectConfigMap(mock.TestProjectConfigMapName, om.TestGroupName, om.TestOrgID), + mock.GetCredentialsSecret(om.TestUser, om.TestApiKey), + }...) + omConnectionFactory.SetPostCreateHook(func(c om.Connection) { + // Important: the organization for the group has a different name ("foo") then group (om.TestGroupName). + // So it won't work for cases when the group "was created before" by Operator + c.(*om.MockedOmConnection).OrganizationsWithGroups = map[*om.Organization][]*om.Project{{ID: om.TestOrgID, Name: "foo"}: {{Name: om.TestGroupName, ID: "existing-group-id", OrgID: om.TestOrgID}}} + }) + + controller := NewReconcileCommonController(ctx, kubeClient) + mockOm, _ := prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + assert.Equal(t, "existing-group-id", mockOm.GroupID()) + // No new group was created + assert.Len(t, mockOm.OrganizationsWithGroups, 1) + + mockOm.CheckOrderOfOperations(t, reflect.ValueOf(mockOm.ReadOrganization), reflect.ValueOf(mockOm.ReadProjectsInOrganizationByName)) + mockOm.CheckOperationsDidntHappen(t, reflect.ValueOf(mockOm.ReadOrganizations), reflect.ValueOf(mockOm.CreateProject), reflect.ValueOf(mockOm.ReadProjectsInOrganization)) +} + +// TestPrepareOmConnection_DuplicatedGroups verifies that if there are groups with the same name but in different organization +// then the new group is created +func TestPrepareOmConnection_DuplicatedGroups(t *testing.T) { + ctx := context.Background() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + kubeClient := mock.NewEmptyFakeClientWithInterceptor(omConnectionFactory, []client.Object{ + mock.GetProjectConfigMap(mock.TestProjectConfigMapName, om.TestGroupName, ""), + mock.GetCredentialsSecret(om.TestUser, om.TestApiKey), + }...) + omConnectionFactory.SetPostCreateHook(func(c om.Connection) { + // Important: the organization for the group has a different name ("foo") then group (om.TestGroupName). + // So it won't work for cases when the group "was created before" by Operator + c.(*om.MockedOmConnection).OrganizationsWithGroups = map[*om.Organization][]*om.Project{{ID: om.TestOrgID, Name: "foo"}: {{Name: om.TestGroupName, ID: "existing-group-id", OrgID: om.TestOrgID}}} + }) + + // The only difference from TestPrepareOmConnection_FindExistingGroup above is that the config map contains only project name + // but no org ID (see newMockedKubeApi()) + controller := NewReconcileCommonController(ctx, kubeClient) + + mockOm, _ := prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + assert.Equal(t, om.TestGroupID, mockOm.GroupID()) + mockOm.CheckGroupInOrganization(t, om.TestGroupName, om.TestGroupName) + // New group and organization will be created in addition to existing ones + assert.Len(t, mockOm.OrganizationsWithGroups, 2) + + mockOm.CheckOrderOfOperations(t, reflect.ValueOf(mockOm.ReadOrganizationsByName), reflect.ValueOf(mockOm.CreateProject)) + mockOm.CheckOperationsDidntHappen(t, reflect.ValueOf(mockOm.ReadOrganizations), + reflect.ValueOf(mockOm.ReadProjectsInOrganization), reflect.ValueOf(mockOm.ReadProjectsInOrganizationByName)) +} + +// TestPrepareOmConnection_CreateGroup checks that if the group doesn't exist in OM - it is created +func TestPrepareOmConnection_CreateGroup(t *testing.T) { + ctx := context.Background() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + omConnectionFactory.SetPostCreateHook(func(c om.Connection) { + c.(*om.MockedOmConnection).OrganizationsWithGroups = map[*om.Organization][]*om.Project{} + }) + + controller := NewReconcileCommonController(ctx, kubeClient) + + mockOm, vars := prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + + assert.Equal(t, om.TestGroupID, vars.ProjectID) + assert.Equal(t, om.TestGroupID, mockOm.GroupID()) + mockOm.CheckGroupInOrganization(t, om.TestGroupName, om.TestGroupName) + assert.Len(t, mockOm.OrganizationsWithGroups, 1) + assert.Contains(t, mockOm.FindGroup(om.TestGroupName).Tags, strings.ToUpper(mock.TestNamespace)) + + mockOm.CheckOrderOfOperations(t, reflect.ValueOf(mockOm.ReadOrganizationsByName), reflect.ValueOf(mockOm.CreateProject)) + mockOm.CheckOperationsDidntHappen(t, reflect.ValueOf(mockOm.ReadProjectsInOrganization)) +} + +// TestPrepareOmConnection_CreateGroupFixTags fixes tags if they are not set for existing group +func TestPrepareOmConnection_CreateGroupFixTags(t *testing.T) { + ctx := context.Background() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + omConnectionFactory.SetPostCreateHook(func(c om.Connection) { + c.(*om.MockedOmConnection).OrganizationsWithGroups = map[*om.Organization][]*om.Project{ + { + ID: om.TestOrgID, + Name: om.TestGroupName, + }: { + { + Name: om.TestGroupName, + ID: "123", + AgentAPIKey: "12345abcd", + OrgID: om.TestOrgID, + }, + }, + } + }) + + controller := NewReconcileCommonController(ctx, kubeClient) + + mockOm, _ := prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + assert.Contains(t, mockOm.FindGroup(om.TestGroupName).Tags, strings.ToUpper(mock.TestNamespace)) + + mockOm.CheckOrderOfOperations(t, reflect.ValueOf(mockOm.UpdateProject)) +} + +func readAgentApiKeyForProject(ctx context.Context, client kubernetesClient.Client, namespace, agentKeySecretName string) (string, error) { + secret, err := client.GetSecret(ctx, kube.ObjectKey(namespace, agentKeySecretName)) + if err != nil { + return "", err + } + + key, ok := secret.Data[util.OmAgentApiKey] + if !ok { + return "", xerrors.Errorf("Could not find key \"%s\" in secret %s", util.OmAgentApiKey, agentKeySecretName) + } + + return strings.TrimSuffix(string(key), "\n"), nil +} + +// TestPrepareOmConnection_PrepareAgentKeys checks that agent key is generated and put to secret +func TestPrepareOmConnection_PrepareAgentKeys(t *testing.T) { + ctx := context.Background() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + controller := NewReconcileCommonController(ctx, kubeClient) + + prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + key, e := readAgentApiKeyForProject(ctx, controller.client, mock.TestNamespace, agents.ApiKeySecretName(om.TestGroupID)) + + assert.NoError(t, e) + // Unfortunately the key read is not equal to om.TestAgentKey - it's just some set of bytes. + // This is reproduced only in mocked tests - the production is fine (the key is real string) + // I assume that it's because when setting the secret data we use 'StringData' but read it back as + // 'Data' which is binary. May be real kubernetes api reads data as string and updates + assert.NotNil(t, key) +} + +// TestUpdateStatus_Patched makes sure that 'ReconcileCommonController.updateStatus()' changes only status for current +// object in Kubernetes and leaves spec unchanged +func TestUpdateStatus_Patched(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().Build() + kubeClient, _ := mock.NewDefaultFakeClient(rs) + controller := NewReconcileCommonController(ctx, kubeClient) + reconciledObject := rs.DeepCopy() + // The current reconciled object "has diverged" from the one in API server + reconciledObject.Spec.Version = "10.0.0" + _, err := controller.updateStatus(ctx, reconciledObject, workflow.Pending("Waiting for secret..."), zap.S()) + assert.NoError(t, err) + + // Verifying that the resource in API server still has the correct spec + currentRs := mdbv1.MongoDB{} + assert.NoError(t, kubeClient.Get(ctx, rs.ObjectKey(), ¤tRs)) + + // The spec hasn't changed - only status + assert.Equal(t, rs.Spec, currentRs.Spec) + assert.Equal(t, status.PhasePending, currentRs.Status.Phase) + assert.Equal(t, "Waiting for secret...", currentRs.Status.Message) +} + +func TestReadSubjectFromJustCertificate(t *testing.T) { + ctx := context.Background() + assertSubjectFromFileSucceeds(ctx, t, "CN=mms-automation-agent,OU=MongoDB Kubernetes Operator,O=mms-automation-agent,L=NY,ST=NY,C=US", "testdata/certificates/just_certificate") +} + +func TestReadSubjectFromCertificateThenKey(t *testing.T) { + ctx := context.Background() + assertSubjectFromFileSucceeds(ctx, t, "CN=mms-automation-agent,OU=MongoDB Kubernetes Operator,O=mms-automation-agent,L=NY,ST=NY,C=US", "testdata/certificates/certificate_then_key") +} + +func TestReadSubjectFromKeyThenCertificate(t *testing.T) { + ctx := context.Background() + assertSubjectFromFileSucceeds(ctx, t, "CN=mms-automation-agent,OU=MongoDB Kubernetes Operator,O=mms-automation-agent,L=NY,ST=NY,C=US", "testdata/certificates/key_then_certificate") +} + +func TestReadSubjectFromCertInStrictlyRFC2253(t *testing.T) { + ctx := context.Background() + assertSubjectFromFileSucceeds(ctx, t, "CN=mms-agent-cert,O=MongoDB-agent,OU=TSE,L=New Delhi,ST=New Delhi,C=IN", "testdata/certificates/cert_rfc2253") +} + +func TestReadSubjectNoCertificate(t *testing.T) { + assertSubjectFromFileFails(t, "testdata/certificates/just_key") +} + +func TestDontSendNilPrivileges(t *testing.T) { + ctx := context.Background() + customRole := mdbv1.MongoDbRole{ + Role: "foo", + AuthenticationRestrictions: []mdbv1.AuthenticationRestriction{}, + Db: "admin", + Roles: []mdbv1.InheritedRole{{ + Db: "admin", + Role: "readWriteAnyDatabase", + }}, + } + assert.Nil(t, customRole.Privileges) + rs := DefaultReplicaSetBuilder().SetRoles([]mdbv1.MongoDbRole{customRole}).Build() + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient() + controller := NewReconcileCommonController(ctx, kubeClient) + mockOm, _ := prepareConnection(ctx, controller, omConnectionFactory.GetConnectionFunc, t) + ensureRoles(rs.Spec.Security.Roles, mockOm, &zap.SugaredLogger{}) + ac, err := mockOm.ReadAutomationConfig() + assert.NoError(t, err) + roles, ok := ac.Deployment["roles"].([]mdbv1.MongoDbRole) + assert.True(t, ok) + assert.NotNil(t, roles[0].Privileges) +} + +func TestSecretWatcherWithAllResources(t *testing.T) { + ctx := context.Background() + caName := "custom-ca" + rs := DefaultReplicaSetBuilder().EnableTLS().EnableX509().SetTLSCA(caName).Build() + rs.Spec.Security.Authentication.InternalCluster = "X509" + kubeClient, _ := mock.NewDefaultFakeClient(rs) + controller := NewReconcileCommonController(ctx, kubeClient) + + controller.SetupCommonWatchers(rs, nil, nil, rs.Name) + + // TODO: unify the watcher setup with the secret creation/mounting code in database creation + memberCert := rs.GetSecurity().MemberCertificateSecretName(rs.Name) + internalAuthCert := rs.GetSecurity().InternalClusterAuthSecretName(rs.Name) + agentCert := rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, caName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, rs.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, memberCert)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, internalAuthCert)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, agentCert)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + } + + assert.Equal(t, expected, controller.resourceWatcher.GetWatchedResources()) +} + +func TestSecretWatcherWithSelfProvidedTLSSecretNames(t *testing.T) { + ctx := context.Background() + caName := "custom-ca" + + rs := DefaultReplicaSetBuilder().EnableTLS().EnableX509().SetTLSCA(caName).Build() + kubeClient, _ := mock.NewDefaultFakeClient(rs) + controller := NewReconcileCommonController(ctx, kubeClient) + + controller.SetupCommonWatchers(rs, func() []string { + return []string{"a-secret"} + }, nil, rs.Name) + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, caName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, rs.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, "a-secret")}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + } + + assert.Equal(t, expected, controller.resourceWatcher.GetWatchedResources()) +} + +func assertSubjectFromFileFails(t *testing.T, filePath string) { + assertSubjectFromFile(t, "", filePath, false) +} + +func assertSubjectFromFileSucceeds(ctx context.Context, t *testing.T, expectedSubject, filePath string) { + assertSubjectFromFile(t, expectedSubject, filePath, true) +} + +func assertSubjectFromFile(t *testing.T, expectedSubject, filePath string, passes bool) { + data, err := os.ReadFile(filePath) + assert.NoError(t, err) + subject, err := getSubjectFromCertificate(string(data)) + if passes { + assert.NoError(t, err) + } else { + assert.Error(t, err) + } + assert.Equal(t, expectedSubject, subject) +} + +func prepareConnection(ctx context.Context, controller *ReconcileCommonController, omConnectionFunc om.ConnectionFactory, t *testing.T) (*om.MockedOmConnection, *env.PodEnvVars) { + projectConfig, err := project.ReadProjectConfig(ctx, controller.client, kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName), "mdb-name") + assert.NoError(t, err) + credsConfig, err := project.ReadCredentials(ctx, controller.SecretClient, kube.ObjectKey(mock.TestNamespace, mock.TestCredentialsSecretName), &zap.SugaredLogger{}) + assert.NoError(t, err) + + conn, _, e := connection.PrepareOpsManagerConnection(ctx, controller.SecretClient, projectConfig, credsConfig, omConnectionFunc, mock.TestNamespace, zap.S()) + mockOm := conn.(*om.MockedOmConnection) + assert.NoError(t, e) + return mockOm, newPodVars(conn, projectConfig, mdbv1.Warn) +} + +func requestFromObject(object metav1.Object) reconcile.Request { + return reconcile.Request{NamespacedName: mock.ObjectKeyFromApiObject(object)} +} + +func testConnectionSpec() mdbv1.ConnectionSpec { + return mdbv1.ConnectionSpec{ + SharedConnectionSpec: mdbv1.SharedConnectionSpec{ + OpsManagerConfig: &mdbv1.PrivateCloudConfig{ + ConfigMapRef: mdbv1.ConfigMapRef{ + Name: mock.TestProjectConfigMapName, + }, + }, + }, + Credentials: mock.TestCredentialsSecretName, + } +} + +func checkReconcileSuccessful(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, object *mdbv1.MongoDB, client client.Client) { + err := client.Update(ctx, object) + require.NoError(t, err) + + result, err := reconciler.Reconcile(ctx, requestFromObject(object)) + require.NoError(t, err) + require.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, result) + + // also need to make sure the object status is updated to successful + assert.NoError(t, client.Get(ctx, mock.ObjectKeyFromApiObject(object), object)) + assert.Equal(t, status.PhaseRunning, object.Status.Phase) + + expectedLink := deployment.Link(om.TestURL, om.TestGroupID) + + // fields common to all resource types + assert.Equal(t, object.Spec.Version, object.Status.Version) + assert.Equal(t, expectedLink, object.Status.Link) + assert.NotNil(t, object.Status.LastTransition) + assert.NotEqual(t, object.Status.LastTransition, "") + + assert.Equal(t, object.GetGeneration(), object.Status.ObservedGeneration) + + switch object.Spec.ResourceType { + case mdbv1.ReplicaSet: + assert.Equal(t, object.Spec.Members, object.Status.Members) + case mdbv1.ShardedCluster: + if object.Spec.IsMultiCluster() { + assert.Equal(t, object.Spec.ShardCount, object.Status.ShardCount) + } else { + assert.Equal(t, object.Spec.ConfigServerCount, object.Status.ConfigServerCount) + assert.Equal(t, object.Spec.MongosCount, object.Status.MongosCount) + assert.Equal(t, object.Spec.MongodsPerShardCount, object.Status.MongodsPerShardCount) + assert.Equal(t, object.Spec.ShardCount, object.Status.ShardCount) + } + } + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(object), object)) +} + +func checkOMReconciliationSuccessful(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, om *omv1.MongoDBOpsManager, client client.Client) { + res, err := reconciler.Reconcile(ctx, requestFromObject(om)) + expected := reconcile.Result{Requeue: true} + assert.Equal(t, expected, res) + assert.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(om)) + expected = reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS} + assert.Equal(t, expected, res) + assert.NoError(t, err) + + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(om), om)) +} + +func checkOMReconciliationInvalid(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, om *omv1.MongoDBOpsManager, client client.Client) { + res, err := reconciler.Reconcile(ctx, requestFromObject(om)) + expected, _ := workflow.OK().Requeue().ReconcileResult() + assert.Equal(t, expected, res) + assert.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(om)) + expected, _ = workflow.Invalid("doesn't matter").ReconcileResult() + assert.Equal(t, expected, res) + assert.NoError(t, err) + + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(om), om)) +} + +func checkOMReconciliationPending(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, om *omv1.MongoDBOpsManager) { + res, err := reconciler.Reconcile(ctx, requestFromObject(om)) + assert.NoError(t, err) + assert.True(t, res.Requeue || res.RequeueAfter == time.Duration(10000000000)) +} + +func checkReconcileFailed(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, object *mdbv1.MongoDB, expectedRetry bool, expectedErrorMessage string, client client.Client) { + failedResult := reconcile.Result{} + if expectedRetry { + failedResult.RequeueAfter = 10 * time.Second + } + result, e := reconciler.Reconcile(ctx, requestFromObject(object)) + assert.Nil(t, e, "When retrying, error should be nil") + assert.Equal(t, failedResult, result) + + // also need to make sure the object status is updated to failed + assert.NoError(t, client.Get(ctx, mock.ObjectKeyFromApiObject(object), object)) + assert.Equal(t, status.PhaseFailed, object.Status.Phase) + assert.Contains(t, object.Status.Message, expectedErrorMessage) +} + +func checkReconcilePending(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, object *mdbv1.MongoDB, expectedErrorMessage string, client client.Client, requeueAfter time.Duration) { + failedResult := reconcile.Result{RequeueAfter: requeueAfter * time.Second} + result, e := reconciler.Reconcile(ctx, requestFromObject(object)) + assert.Nil(t, e, "When pending, error should be nil") + assert.Equal(t, failedResult, result) + + // also need to make sure the object status is pending + assert.NoError(t, client.Get(ctx, mock.ObjectKeyFromApiObject(object), object)) + assert.Equal(t, status.PhasePending, object.Status.Phase) + assert.Contains(t, object.Status.Message, expectedErrorMessage) +} + +func getWatch(namespace string, resourceName string, t watch.Type) watch.Object { + configSecret := watch.Object{ + ResourceType: t, + Resource: types.NamespacedName{ + Namespace: namespace, + Name: resourceName, + }, + } + return configSecret +} + +type testReconciliationResources struct { + Resource *mdbv1.MongoDB + ReconcilerFactory func(rs *mdbv1.MongoDB) (reconcile.Reconciler, kubernetesClient.Client) +} + +// agentVersionMappingTest is a helper function to verify that the version mapping mechanism works correctly in controllers +// in case retrieving the version fails, the user should have the possibility to override the image in the pod specs +func agentVersionMappingTest(ctx context.Context, t *testing.T, defaultResource testReconciliationResources, overridenResource testReconciliationResources) { + nonExistingPath := "/foo/bar/foo" + + t.Run("Static architecture, version retrieving fails, image is overriden, reconciliation should succeed", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + t.Setenv(agentVersionManagement.MappingFilePathEnv, nonExistingPath) + overridenReconciler, overridenClient := overridenResource.ReconcilerFactory(overridenResource.Resource) + checkReconcileSuccessful(ctx, t, overridenReconciler, overridenResource.Resource, overridenClient) + }) + + t.Run("Static architecture, version retrieving fails, image is not overriden, reconciliation should fail", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + t.Setenv(agentVersionManagement.MappingFilePathEnv, nonExistingPath) + defaultReconciler, defaultClient := defaultResource.ReconcilerFactory(defaultResource.Resource) + checkReconcileFailed(ctx, t, defaultReconciler, defaultResource.Resource, true, "", defaultClient) + }) + + t.Run("Static architecture, version retrieving succeeds, image is not overriden, reconciliation should succeed", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + defaultReconciler, defaultClient := defaultResource.ReconcilerFactory(defaultResource.Resource) + checkReconcileSuccessful(ctx, t, defaultReconciler, defaultResource.Resource, defaultClient) + }) + + t.Run("Non-Static architecture, version retrieving fails, image is not overriden, reconciliation should succeed", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.NonStatic)) + t.Setenv(agentVersionManagement.MappingFilePathEnv, nonExistingPath) + defaultReconciler, defaultClient := defaultResource.ReconcilerFactory(defaultResource.Resource) + checkReconcileSuccessful(ctx, t, defaultReconciler, defaultResource.Resource, defaultClient) + }) +} + +func testConcurrentReconciles(ctx context.Context, t *testing.T, client client.Client, reconciler reconcile.Reconciler, objects ...client.Object) { + for _, object := range objects { + err := mock.CreateOrUpdate(ctx, client, object) + require.NoError(t, err) + } + + // Let's have one reconcile first, such that we have the same object reconciles multiple times + _, err := reconciler.Reconcile(ctx, requestFromObject(objects[0])) + require.NoError(t, err) + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(objects[0]), objects[0])) + + var wg sync.WaitGroup + for _, object := range objects { + wg.Add(1) + go func() { + result, err := reconciler.Reconcile(ctx, requestFromObject(object)) + assert.NoError(t, err) + + // Reconcile again if it's OpsManager, it has to configure AppDB in second run + if _, ok := object.(*omv1.MongoDBOpsManager); ok { + result, err = reconciler.Reconcile(ctx, requestFromObject(object)) + assert.NoError(t, err) + } + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, result) + + // Idempotent reconcile that does not mutate anything, but makes sure we have better code coverage + result, err = reconciler.Reconcile(ctx, requestFromObject(object)) + assert.NoError(t, err) + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, result) + + wg.Done() + }() + } + wg.Wait() +} + +func testFCVsCases(t *testing.T, verifyFCV func(version string, expectedFCV string, fcvOverride *string, t *testing.T)) { + // Define the test cases in order. They need to be in order! + testCases := []struct { + version string + expectedFCV string + fcvOverride *string + }{ + {"4.0.0", "4.0", nil}, + {"5.0.0", "4.0", nil}, + {"5.0.0", "4.0", nil}, + {"6.0.0", "6.0", nil}, + {"7.0.0", "7.0", ptr.To("AlwaysMatchVersion")}, + {"8.0.0", "8.0", nil}, + {"7.0.0", "7.0", nil}, + } + + // Iterate through the test cases in order + for _, tc := range testCases { + t.Run(fmt.Sprintf("Version=%s", tc.version), func(t *testing.T) { + verifyFCV(tc.version, tc.expectedFCV, tc.fcvOverride, t) + }) + } +} diff --git a/controllers/operator/connection/opsmanager_connection.go b/controllers/operator/connection/opsmanager_connection.go new file mode 100644 index 000000000..59c69641d --- /dev/null +++ b/controllers/operator/connection/opsmanager_connection.go @@ -0,0 +1,76 @@ +package connection + +import ( + "context" + "fmt" + "strings" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func PrepareOpsManagerConnection(ctx context.Context, client secrets.SecretClient, projectConfig mdbv1.ProjectConfig, credentials mdbv1.Credentials, connectionFunc om.ConnectionFactory, namespace string, log *zap.SugaredLogger) (om.Connection, string, error) { + omProject, conn, err := project.ReadOrCreateProject(projectConfig, credentials, connectionFunc, log) + if err != nil { + return nil, "", xerrors.Errorf("error reading or creating project in Ops Manager: %w", err) + } + + omVersion := conn.OpsManagerVersion() + if omVersion.VersionString != "" { // older versions of Ops Manager will not include the version in the header + log.Infof("Using Ops Manager version %s", omVersion) + } + + // adds the namespace as a tag to the Ops Manager project + if err = EnsureTagAdded(conn, omProject, namespace, log); err != nil { + return nil, "", err + } + + // adds the externally_managed tag if feature controls is not available. + if !controlledfeature.ShouldUseFeatureControls(conn.OpsManagerVersion()) { + if err = EnsureTagAdded(conn, omProject, util.OmGroupExternallyManagedTag, log); err != nil { + return nil, "", err + } + } + + var databaseSecretPath string + if client.VaultClient != nil { + databaseSecretPath = client.VaultClient.DatabaseSecretPath() + } + if agentAPIKey, err := agents.EnsureAgentKeySecretExists(ctx, client, conn, namespace, omProject.AgentAPIKey, conn.GroupID(), databaseSecretPath, log); err != nil { + return nil, "", err + } else { + return conn, agentAPIKey, err + } +} + +// EnsureTagAdded makes sure that the given project has the provided tag +func EnsureTagAdded(conn om.Connection, project *om.Project, tag string, log *zap.SugaredLogger) error { + // must truncate the tag to at most 32 characters and capitalise as + // these are Ops Manager requirements + + sanitisedTag := strings.ToUpper(fmt.Sprintf("%.32s", tag)) + alreadyHasTag := stringutil.Contains(project.Tags, sanitisedTag) + if alreadyHasTag { + return nil + } + + project.Tags = append(project.Tags, sanitisedTag) + + log.Infow("Updating group tags", "newTags", project.Tags) + _, err := conn.UpdateProject(project) + if err != nil { + log.Warnf("Failed to update tags for project: %s", err) + } else { + log.Info("Project tags are fixed") + } + return err +} diff --git a/controllers/operator/connectionstring/connectionstring.go b/controllers/operator/connectionstring/connectionstring.go new file mode 100644 index 000000000..da966768c --- /dev/null +++ b/controllers/operator/connectionstring/connectionstring.go @@ -0,0 +1,226 @@ +// Presents a builder to programmatically build a MongoDB connection string. +// +// We are waiting for a more consistent solution to this, based on a +// ConnString structure. +// +// https://jira.mongodb.org/browse/GODRIVER-2226 + +package connectionstring + +import ( + "fmt" + "net/url" + "sort" + "strings" + + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +type ConnectionStringBuilder interface { + BuildConnectionString(userName, password string, scheme Scheme, connectionParams map[string]string) string +} + +// Scheme states the connection string format. +// https://docs.mongodb.com/manual/reference/connection-string/#connection-string-formats +type Scheme string + +const ( + SchemeMongoDB Scheme = "mongodb" + SchemeMongoDBSRV Scheme = "mongodb+srv" +) + +// builder is a struct that we'll use to build a connection string. +type builder struct { + name string + namespace string + + username string + password string + replicas int + port int32 + service string + version string + + authenticationModes []string + clusterDomain string + externalDomain *string + isReplicaSet bool + isTLSEnabled bool + + multiClusterHosts []string + + scheme Scheme + connectionParams map[string]string +} + +func (b *builder) SetName(name string) *builder { + b.name = name + return b +} + +func (b *builder) SetNamespace(namespace string) *builder { + b.namespace = namespace + return b +} + +func (b *builder) SetUsername(username string) *builder { + b.username = username + return b +} + +func (b *builder) SetPassword(password string) *builder { + b.password = password + return b +} + +func (b *builder) SetReplicas(replicas int) *builder { + b.replicas = replicas + return b +} + +func (b *builder) SetService(service string) *builder { + b.service = service + return b +} + +func (b *builder) SetPort(port int32) *builder { + b.port = port + return b +} + +func (b *builder) SetVersion(version string) *builder { + b.version = version + return b +} + +func (b *builder) SetAuthenticationModes(authenticationModes []string) *builder { + b.authenticationModes = authenticationModes + return b +} + +func (b *builder) SetClusterDomain(clusterDomain string) *builder { + b.clusterDomain = clusterDomain + return b +} + +func (b *builder) SetExternalDomain(externalDomain *string) *builder { + b.externalDomain = externalDomain + return b +} + +func (b *builder) SetIsReplicaSet(isReplicaSet bool) *builder { + b.isReplicaSet = isReplicaSet + return b +} + +func (b *builder) SetIsTLSEnabled(isTLSEnabled bool) *builder { + b.isTLSEnabled = isTLSEnabled + return b +} + +func (b *builder) SetMultiClusterHosts(multiClusterHosts []string) *builder { + b.multiClusterHosts = multiClusterHosts + return b +} + +func (b *builder) SetScheme(scheme Scheme) *builder { + b.scheme = scheme + return b +} + +func (b *builder) SetConnectionParams(cParams map[string]string) *builder { + for key, value := range cParams { + b.connectionParams[key] = value + } + return b +} + +// Build builds a new connection string from the builder. +func (b *builder) Build() string { + var userAuth string + if stringutil.Contains(b.authenticationModes, util.SCRAM) && + b.username != "" && b.password != "" { + + userAuth = fmt.Sprintf("%s:%s@", url.QueryEscape(b.username), url.QueryEscape(b.password)) + } + + var uri string + if b.scheme == SchemeMongoDBSRV { + uri = fmt.Sprintf("mongodb+srv://%s", userAuth) + uri += fmt.Sprintf("%s.%s.svc.%s", b.service, b.namespace, b.clusterDomain) + } else { + uri = fmt.Sprintf("mongodb://%s", userAuth) + var hostnames []string + if len(b.multiClusterHosts) > 0 { + hostnames = b.multiClusterHosts + } else { + hostnames, _ = dns.GetDNSNames(b.name, b.service, b.namespace, b.clusterDomain, b.replicas, b.externalDomain) + for i, h := range hostnames { + hostnames[i] = fmt.Sprintf("%s:%d", h, b.port) + } + } + uri += strings.Join(hostnames, ",") + } + + connectionParams := make(map[string]string) + if b.isReplicaSet { + connectionParams["replicaSet"] = b.name + } + if b.isTLSEnabled { + connectionParams["ssl"] = "true" + } + + authSource, authMechanism := authSourceAndMechanism(b.authenticationModes, b.version) + if authSource != "" && authMechanism != "" { + connectionParams["authSource"] = authSource + connectionParams["authMechanism"] = authMechanism + } + + // Merge received (b.connectionParams) on top of local (connectionParams) + // Make sure that received parameters have priority. + for k, v := range b.connectionParams { + connectionParams[k] = v + } + var keys []string + for k := range connectionParams { + keys = append(keys, k) + } + uri += "/?" + + // sorting parameters to make a url stable + sort.Strings(keys) + for _, k := range keys { + uri += fmt.Sprintf("%s=%s&", k, connectionParams[k]) + } + return strings.TrimSuffix(uri, "&") +} + +func Builder() *builder { + return &builder{ + port: util.MongoDbDefaultPort, + connectionParams: map[string]string{"connectTimeoutMS": "20000", "serverSelectionTimeoutMS": "20000"}, + } +} + +// authSourceAndMechanism returns AuthSource and AuthMechanism. +func authSourceAndMechanism(authenticationModes []string, version string) (string, string) { + var authSource string + var authMechanism string + if stringutil.Contains(authenticationModes, util.SCRAM) { + authSource = util.DefaultUserDatabase + + comparison, err := util.CompareVersions(version, util.MinimumScramSha256MdbVersion) + if err != nil { + return "", "" + } + if comparison < 0 { + authMechanism = "SCRAM-SHA-1" + } else { + authMechanism = "SCRAM-SHA-256" + } + } + + return authSource, authMechanism +} diff --git a/controllers/operator/construct/appdb_construction.go b/controllers/operator/construct/appdb_construction.go new file mode 100644 index 000000000..7d9039c0e --- /dev/null +++ b/controllers/operator/construct/appdb_construction.go @@ -0,0 +1,662 @@ +package construct + +import ( + "fmt" + "os" + "path" + "strconv" + + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers/interfaces" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +const ( + appDBServiceAccount = "mongodb-enterprise-appdb" + InitAppDbContainerName = "mongodb-enterprise-init-appdb" + // AppDB environment variable names + InitAppdbVersionEnv = "INIT_APPDB_VERSION" + podNamespaceEnv = "POD_NAMESPACE" + automationConfigMapEnv = "AUTOMATION_CONFIG_MAP" + headlessAgentEnv = "HEADLESS_AGENT" + clusterDomainEnv = "CLUSTER_DOMAIN" + monitoringAgentContainerName = "mongodb-agent-monitoring" + // Since the Monitoring Agent is created based on Agent's Pod spec (we modfy it using addMonitoringContainer), + // We can not reuse "tmp" here - this name is already taken and could lead to a clash. It's better to + // come up with a unique name here. + tmpSubpathName = "mongodb-agent-monitoring-tmp" + + monitoringAgentHealthStatusFilePathValue = "/var/log/mongodb-mms-automation/healthstatus/monitoring-agent-health-status.json" +) + +type AppDBStatefulSetOptions struct { + VaultConfig vault.VaultConfiguration + CertHash string + + InitAppDBImage string + MongodbImage string + AgentImage string + LegacyMonitoringAgentImage string + + PrometheusTLSCertHash string +} + +func getMonitoringAgentLogOptions(spec om.AppDBSpec) string { + return fmt.Sprintf(" -logFile=/var/log/mongodb-mms-automation/monitoring-agent.log -maxLogFileDurationHrs=%d -logLevel=%s", spec.GetAgentMaxLogFileDurationHours(), spec.GetAgentLogLevel()) +} + +// getContainerIndexByName returns the index of a container with the given name in a slice of containers. +// It returns -1 if it doesn't exist +func getContainerIndexByName(containers []corev1.Container, name string) int { + for i, container := range containers { + if container.Name == name { + return i + } + } + return -1 +} + +// removeContainerByName removes the container with the given name from the input slice, if it exists. +func removeContainerByName(containers []corev1.Container, name string) []corev1.Container { + index := getContainerIndexByName(containers, name) + if index == -1 { + return containers + } + return append(containers[:index], containers[index+1:]...) +} + +// appDbLabels returns a statefulset modification which adds labels that are specific to the appDB. +func appDbLabels(opsManager *om.MongoDBOpsManager, memberClusterNum int) statefulset.Modification { + podLabels := map[string]string{ + appLabelKey: opsManager.Spec.AppDB.HeadlessServiceSelectorAppLabel(memberClusterNum), + ControllerLabelName: util.OperatorName, + PodAntiAffinityLabelKey: opsManager.Spec.AppDB.NameForCluster(memberClusterNum), + } + return statefulset.Apply( + statefulset.WithLabels(opsManager.Labels), + statefulset.WithMatchLabels(podLabels), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + podtemplatespec.WithPodLabels(podLabels), + ), + ), + ) +} + +// appDbPodSpec return the podtemplatespec modification required for the AppDB statefulset. +func appDbPodSpec(initContainerImage string, om om.MongoDBOpsManager) podtemplatespec.Modification { + // The following sets almost the exact same values for the containers + // But with the addition of a default memory request for the mongod one + appdbPodSpec := NewDefaultPodSpecWrapper(*om.Spec.AppDB.PodSpec) + mongoPodSpec := *appdbPodSpec + mongoPodSpec.Default.MemoryRequests = util.DefaultMemoryAppDB + mongoPodTemplateFunc := podtemplatespec.WithContainer( + construct.MongodbName, + container.WithResourceRequirements(buildRequirementsFromPodSpec(mongoPodSpec)), + ) + automationPodTemplateFunc := podtemplatespec.WithContainer( + construct.AgentName, + container.WithResourceRequirements(buildRequirementsFromPodSpec(*appdbPodSpec)), + ) + + initUpdateFunc := podtemplatespec.NOOP() + if !architectures.IsRunningStaticArchitecture(om.Annotations) { + // appdb will have a single init container, + // all the necessary binaries will be copied into the various + // volumes of different containers. + initUpdateFunc = func(templateSpec *corev1.PodTemplateSpec) { + templateSpec.Spec.InitContainers = []corev1.Container{} + scriptsVolumeMount := statefulset.CreateVolumeMount("agent-scripts", "/opt/scripts", statefulset.WithReadOnly(false)) + hooksVolumeMount := statefulset.CreateVolumeMount("hooks", "/hooks", statefulset.WithReadOnly(false)) + podtemplatespec.WithInitContainer(InitAppDbContainerName, buildAppDBInitContainer(initContainerImage, []corev1.VolumeMount{scriptsVolumeMount, hooksVolumeMount}))(templateSpec) + } + } + + return podtemplatespec.Apply( + mongoPodTemplateFunc, + automationPodTemplateFunc, + initUpdateFunc, + ) +} + +// buildAppDBInitContainer builds the container specification for mongodb-enterprise-init-appdb image. +func buildAppDBInitContainer(initContainerImageURL string, volumeMounts []corev1.VolumeMount) container.Modification { + _, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + return container.Apply( + container.WithName(InitAppDbContainerName), + container.WithImage(initContainerImageURL), + container.WithCommand([]string{"/bin/sh", "-c", ` + +# the agent requires the readiness probe +cp /probes/readinessprobe /opt/scripts/readinessprobe + +# the mongod requires the version upgrade hook +cp /probes/version-upgrade-hook /hooks/version-upgrade +`}), + container.WithVolumeMounts(volumeMounts), + configureContainerSecurityContext, + ) +} + +// getTLSVolumesAndVolumeMounts returns the slices of volumes and volume-mounts +// that the AppDB STS needs for TLS resources. +func getTLSVolumesAndVolumeMounts(appDb om.AppDBSpec, podVars *env.PodEnvVars, log *zap.SugaredLogger) ([]corev1.Volume, []corev1.VolumeMount) { + if log == nil { + log = zap.S() + } + var volumesToAdd []corev1.Volume + var volumeMounts []corev1.VolumeMount + + if ShouldMountSSLMMSCAConfigMap(podVars) { + // This volume wil contain the OM CA + caCertVolume := statefulset.CreateVolumeFromConfigMap(CaCertName, podVars.SSLMMSCAConfigMap) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: caCertMountPath, + Name: caCertVolume.Name, + ReadOnly: true, + }) + volumesToAdd = append(volumesToAdd, caCertVolume) + } + + secretName := appDb.GetSecurity().MemberCertificateSecretName(appDb.Name()) + + secretName += certs.OperatorGeneratedCertSuffix + optionalSecretFunc := func(v *corev1.Volume) { v.Secret.Optional = util.BooleanRef(true) } + optionalConfigMapFunc := func(v *corev1.Volume) { v.ConfigMap.Optional = util.BooleanRef(true) } + + if !vault.IsVaultSecretBackend() { + secretVolume := statefulset.CreateVolumeFromSecret(util.SecretVolumeName, secretName, optionalSecretFunc) + + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: util.SecretVolumeMountPath + "/certs", + Name: secretVolume.Name, + ReadOnly: true, + }) + volumesToAdd = append(volumesToAdd, secretVolume) + } + caName := CAConfigMapName(appDb, log) + + caVolume := statefulset.CreateVolumeFromConfigMap(tls.ConfigMapVolumeCAName, caName, optionalConfigMapFunc) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: util.ConfigMapVolumeCAMountPath, + Name: caVolume.Name, + ReadOnly: true, + }) + volumesToAdd = append(volumesToAdd, caVolume) + + prometheusVolumes, prometheusVolumeMounts := getTLSPrometheusVolumeAndVolumeMount(appDb.Prometheus) + volumesToAdd = append(volumesToAdd, prometheusVolumes...) + volumeMounts = append(volumeMounts, prometheusVolumeMounts...) + + return volumesToAdd, volumeMounts +} + +func CAConfigMapName(appDb om.AppDBSpec, log *zap.SugaredLogger) string { + caName := fmt.Sprintf("%s-ca", appDb.Name()) + + tlsConfig := appDb.GetTLSConfig() + if tlsConfig.CA != "" { + caName = tlsConfig.CA + } else { + log.Debugf("No CA config map name has been supplied, defaulting to: %s", caName) + } + + return caName +} + +// tlsVolumes returns the podtemplatespec modification that adds all needed volumes +// and volumemounts for TLS. +func tlsVolumes(appDb om.AppDBSpec, podVars *env.PodEnvVars, log *zap.SugaredLogger) podtemplatespec.Modification { + volumesToAdd, volumeMounts := getTLSVolumesAndVolumeMounts(appDb, podVars, log) + volumesFunc := func(spec *corev1.PodTemplateSpec) { + for _, v := range volumesToAdd { + podtemplatespec.WithVolume(v)(spec) + } + } + + return podtemplatespec.Apply( + volumesFunc, + podtemplatespec.WithContainer( + construct.AgentName, + container.WithVolumeMounts(volumeMounts), + ), + podtemplatespec.WithContainer( + construct.MongodbName, + container.WithVolumeMounts(volumeMounts), + ), + ) +} + +func vaultModification(appDB om.AppDBSpec, podVars *env.PodEnvVars, opts AppDBStatefulSetOptions) podtemplatespec.Modification { + modification := podtemplatespec.NOOP() + if vault.IsVaultSecretBackend() { + appDBSecretsToInject := vault.AppDBSecretsToInject{Config: opts.VaultConfig} + if podVars != nil && podVars.ProjectID != "" { + appDBSecretsToInject.AgentApiKey = agents.ApiKeySecretName(podVars.ProjectID) + } + if appDB.GetSecurity().IsTLSEnabled() { + secretName := appDB.GetSecurity().MemberCertificateSecretName(appDB.Name()) + certs.OperatorGeneratedCertSuffix + appDBSecretsToInject.TLSSecretName = secretName + appDBSecretsToInject.TLSClusterHash = opts.CertHash + } + + appDBSecretsToInject.AutomationConfigSecretName = appDB.AutomationConfigSecretName() + appDBSecretsToInject.AutomationConfigPath = util.AppDBAutomationConfigKey + appDBSecretsToInject.AgentType = "automation-agent" + + if appDB.Prometheus != nil && appDB.Prometheus.TLSSecretRef.Name != "" && opts.PrometheusTLSCertHash != "" { + appDBSecretsToInject.PrometheusTLSCertHash = opts.PrometheusTLSCertHash + appDBSecretsToInject.PrometheusTLSPath = fmt.Sprintf("%s%s", appDB.Prometheus.TLSSecretRef.Name, certs.OperatorGeneratedCertSuffix) + } + + modification = podtemplatespec.Apply( + modification, + podtemplatespec.WithAnnotations(appDBSecretsToInject.AppDBAnnotations(appDB.Namespace)), + ) + + } else { + if ShouldEnableMonitoring(podVars) { + // AGENT-API-KEY volume + modification = podtemplatespec.Apply( + modification, + podtemplatespec.WithVolume(statefulset.CreateVolumeFromSecret(AgentAPIKeyVolumeName, agents.ApiKeySecretName(podVars.ProjectID))), + ) + } + } + return modification +} + +// customPersistenceConfig applies to the statefulset the modifications +// provided by the user through spec.persistence. +func customPersistenceConfig(om *om.MongoDBOpsManager) statefulset.Modification { + defaultPodSpecPersistence := newDefaultPodSpec().Persistence + // Two main branches - as the user can either define a single volume for data, logs and journal + // or three different volumes + if !om.Spec.AppDB.HasSeparateDataAndLogsVolumes() { + var config *mdbv1.PersistenceConfig + if om.Spec.AppDB.PodSpec.Persistence != nil && om.Spec.AppDB.PodSpec.Persistence.SingleConfig != nil { + config = om.Spec.AppDB.PodSpec.Persistence.SingleConfig + } + // Single persistence, needs to modify the only pvc we have + pvcModification := pvcFunc(om.Spec.AppDB.DataVolumeName(), config, *defaultPodSpecPersistence.SingleConfig, om.Labels) + + // We already have, by default, the data volume mount, + // here we also create the logs and journal one, as subpath from the same volume + logsVolumeMount := statefulset.CreateVolumeMount(om.Spec.AppDB.DataVolumeName(), util.PvcMountPathLogs, statefulset.WithSubPath(om.Spec.AppDB.LogsVolumeName())) + journalVolumeMount := statefulset.CreateVolumeMount(om.Spec.AppDB.DataVolumeName(), util.PvcMountPathJournal, statefulset.WithSubPath(util.PvcNameJournal)) + volumeMounts := []corev1.VolumeMount{journalVolumeMount, logsVolumeMount} + return statefulset.Apply( + statefulset.WithVolumeClaim(om.Spec.AppDB.DataVolumeName(), pvcModification), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + podtemplatespec.WithContainer(construct.AgentName, + container.WithVolumeMounts(volumeMounts), + ), + podtemplatespec.WithContainer(construct.MongodbName, + container.WithVolumeMounts(volumeMounts), + ), + ), + ), + ) + + } else { + // Here need to modify data and logs volumes, + // and create the journal one (which doesn't exist in Community, where this original STS is built) + dataModification := pvcFunc(om.Spec.AppDB.DataVolumeName(), om.Spec.AppDB.PodSpec.Persistence.MultipleConfig.Data, *defaultPodSpecPersistence.MultipleConfig.Data, om.Labels) + logsModification := pvcFunc(om.Spec.AppDB.LogsVolumeName(), om.Spec.AppDB.PodSpec.Persistence.MultipleConfig.Logs, *defaultPodSpecPersistence.MultipleConfig.Logs, om.Labels) + + journalVolumeMounts := statefulset.CreateVolumeMount(util.PvcNameJournal, util.PvcMountPathJournal) + journalVolumeClaim := pvcFunc(util.PvcNameJournal, om.Spec.AppDB.PodSpec.Persistence.MultipleConfig.Journal, *defaultPodSpecPersistence.MultipleConfig.Journal, om.Labels) + + return statefulset.Apply( + statefulset.WithVolumeClaim(util.PvcMountPathLogs, journalVolumeClaim), + statefulset.WithVolumeClaim(om.Spec.AppDB.DataVolumeName(), dataModification), + statefulset.WithVolumeClaim(om.Spec.AppDB.LogsVolumeName(), logsModification), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + podtemplatespec.WithContainer(construct.AgentName, + container.WithVolumeMounts([]corev1.VolumeMount{journalVolumeMounts}), + ), + podtemplatespec.WithContainer(construct.MongodbName, + container.WithVolumeMounts([]corev1.VolumeMount{journalVolumeMounts}), + ), + ), + ), + ) + } +} + +// ShouldEnableMonitoring returns true if we need to add monitoring container (along with volume mounts) in the current reconcile loop. +func ShouldEnableMonitoring(podVars *env.PodEnvVars) bool { + return GlobalMonitoringSettingEnabled() && podVars != nil && podVars.ProjectID != "" +} + +// GlobalMonitoringSettingEnabled returns global setting whether to enable or disable monitoring in appdb (OPS_MANAGER_MONITOR_APPDB env var) +func GlobalMonitoringSettingEnabled() bool { + return env.ReadBoolOrDefault(util.OpsManagerMonitorAppDB, util.OpsManagerMonitorAppDBDefault) // nolint:forbidigo +} + +// ShouldMountSSLMMSCAConfigMap returns true if we need to mount MMSCA to monitoring container in the current reconcile loop. +func ShouldMountSSLMMSCAConfigMap(podVars *env.PodEnvVars) bool { + return ShouldEnableMonitoring(podVars) && podVars.SSLMMSCAConfigMap != "" +} + +// AppDbStatefulSet fully constructs the AppDb StatefulSet that is ready to be sent to the Kubernetes API server. +func AppDbStatefulSet(opsManager om.MongoDBOpsManager, podVars *env.PodEnvVars, opts AppDBStatefulSetOptions, scaler interfaces.MultiClusterReplicaSetScaler, updateStrategyType appsv1.StatefulSetUpdateStrategyType, log *zap.SugaredLogger) (appsv1.StatefulSet, error) { + appDb := &opsManager.Spec.AppDB + + // If we can enable monitoring, let's fill in container modification function + monitoringModification := podtemplatespec.NOOP() + var podSpec *corev1.PodTemplateSpec + if appDb.PodSpec != nil && appDb.PodSpec.PodTemplateWrapper.PodTemplate != nil { + podSpec = appDb.PodSpec.PodTemplateWrapper.PodTemplate.DeepCopy() + } + + externalDomain := appDb.GetExternalDomainForMemberCluster(scaler.MemberClusterName()) + + if ShouldEnableMonitoring(podVars) { + monitoringModification = addMonitoringContainer(*appDb, *podVars, opts, externalDomain, log) + } else { + // Otherwise, let's remove for now every podTemplateSpec related to monitoring + // We will apply them when enabling monitoring + if podSpec != nil { + podSpec.Spec.Containers = removeContainerByName(podSpec.Spec.Containers, monitoringAgentContainerName) + } + } + + // We copy the Automation Agent command from community and add the agent startup parameters + automationAgentCommand := construct.AutomationAgentCommand(true, opsManager.Spec.AppDB.GetAgentLogLevel(), opsManager.Spec.AppDB.GetAgentLogFile(), opsManager.Spec.AppDB.GetAgentMaxLogFileDurationHours()) + idx := len(automationAgentCommand) - 1 + automationAgentCommand[idx] += appDb.AutomationAgent.StartupParameters.ToCommandLineArgs() + + automationAgentCommand[idx] += overrideLocalHostFlag(appDb, externalDomain) + + acVersionConfigMapVolume := statefulset.CreateVolumeFromConfigMap("automation-config-goal-version", opsManager.Spec.AppDB.AutomationConfigConfigMapName()) + acVersionMount := corev1.VolumeMount{ + Name: acVersionConfigMapVolume.Name, + ReadOnly: true, + MountPath: "/var/lib/automation/config/acVersion", + } + + // Here we ask to craete init containers which also creates required volumens. + // Note that we provide empty images for init containers. They are not important + // at this stage beucase later we will define our own init containers for non-static architecture. + mod := construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&opsManager.Spec.AppDB, scaler, opts.MongodbImage, opts.AgentImage, "", "", true) + if architectures.IsRunningStaticArchitecture(opsManager.Annotations) { + mod = construct.BuildMongoDBReplicaSetStatefulSetModificationFunction(&opsManager.Spec.AppDB, scaler, opts.MongodbImage, opts.AgentImage, "", "", false) + } + + sts := statefulset.New( + mod, + // create appdb statefulset from the community code + statefulset.WithName(opsManager.Spec.AppDB.NameForCluster(scaler.MemberClusterNum())), + statefulset.WithServiceName(opsManager.Spec.AppDB.HeadlessServiceNameForCluster(scaler.MemberClusterNum())), + + // If run in certified openshift bundle in disconnected environment with digest pinning we need to update + // mongod image as it is constructed from 2 env variables and version from spec, and it will not be replaced to sha256 digest properly. + // The official image provides both CMD and ENTRYPOINT. We're reusing the former and need to replace + // the latter with an empty string. + containerImageModification(construct.MongodbName, opts.MongodbImage, []string{""}), + // we don't need to update here the automation agent image for digest pinning, because it is defined in AGENT_IMAGE env var as full url with version + // if we run in certified bundle with digest pinning it will be properly updated to digest + customPersistenceConfig(&opsManager), + statefulset.WithUpdateStrategyType(updateStrategyType), + statefulset.WithOwnerReference(kube.BaseOwnerReference(&opsManager)), + statefulset.WithReplicas(scale.ReplicasThisReconciliation(scaler)), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + podtemplatespec.WithServiceAccount(appDBServiceAccount), + podtemplatespec.WithVolume(acVersionConfigMapVolume), + podtemplatespec.WithContainer(construct.AgentName, + container.Apply( + container.WithCommand(automationAgentCommand), + container.WithEnvs(appdbContainerEnv(*appDb)...), + container.WithVolumeMounts([]corev1.VolumeMount{acVersionMount}), + ), + ), + vaultModification(*appDb, podVars, opts), + appDbPodSpec(opts.InitAppDBImage, opsManager), + monitoringModification, + tlsVolumes(*appDb, podVars, log), + ), + ), + appDbLabels(&opsManager, scaler.MemberClusterNum()), + ) + + // We merge the podspec specified in the CR + if podSpec != nil { + sts.Spec = merge.StatefulSetSpecs(sts.Spec, appsv1.StatefulSetSpec{Template: *podSpec}) + } + return sts, nil +} + +// IsEnterprise returns whether the set image in activated with the enterprise module. +// By default, it should be true, but +// for safety mechanisms we implement a backdoor to deactivate the enterprise AC feature. +func IsEnterprise() bool { + overrideAssumption, err := strconv.ParseBool(os.Getenv(construct.MongoDBAssumeEnterpriseEnv)) // nolint:forbidigo + if err == nil { + return overrideAssumption + } + return true +} + +func containerImageModification(containerName string, image string, args []string) statefulset.Modification { + return func(sts *appsv1.StatefulSet) { + for i, c := range sts.Spec.Template.Spec.Containers { + if c.Name == containerName { + c.Image = image + c.Args = args + sts.Spec.Template.Spec.Containers[i] = c + break + } + } + } +} + +// getVolumeMountIndexByName returns the volume mount with the given name from the inut slice. +// It returns -1 if this doesn't exist +func getVolumeMountIndexByName(mounts []corev1.VolumeMount, name string) int { + for i, mount := range mounts { + if mount.Name == name { + return i + } + } + return -1 +} + +// addMonitoringContainer returns a podtemplatespec modification that adds the monitoring container to the AppDB Statefulset. +// Note that this replicates some code from the functions that do this for the base AppDB Statefulset. After many iterations +// this was deemed to be an acceptable compromise to make code clearer and more maintainable. +func addMonitoringContainer(appDB om.AppDBSpec, podVars env.PodEnvVars, opts AppDBStatefulSetOptions, externalDomain *string, log *zap.SugaredLogger) podtemplatespec.Modification { + var monitoringAcVolume corev1.Volume + var monitoringACFunc podtemplatespec.Modification + + monitoringConfigMapVolume := statefulset.CreateVolumeFromConfigMap("monitoring-automation-config-goal-version", appDB.MonitoringAutomationConfigConfigMapName()) + monitoringConfigMapVolumeFunc := podtemplatespec.WithVolume(monitoringConfigMapVolume) + + if vault.IsVaultSecretBackend() { + secretsToInject := vault.AppDBSecretsToInject{Config: opts.VaultConfig} + secretsToInject.AutomationConfigSecretName = appDB.MonitoringAutomationConfigSecretName() + secretsToInject.AutomationConfigPath = util.AppDBMonitoringAutomationConfigKey + secretsToInject.AgentType = "monitoring-agent" + monitoringACFunc = podtemplatespec.WithAnnotations(secretsToInject.AppDBAnnotations(appDB.Namespace)) + } else { + // Create a volume to store the monitoring automation config. + // This is different from the AC for the automation agent, since: + // - It contains entries for "MonitoringVersions" + // - It has empty entries for ReplicaSets and Processes + monitoringAcVolume = statefulset.CreateVolumeFromSecret("monitoring-automation-config", appDB.MonitoringAutomationConfigSecretName()) + monitoringACFunc = podtemplatespec.WithVolume(monitoringAcVolume) + } + // Construct the command by concatenating: + // 1. The base command - from community + command := construct.MongodbUserCommandWithAPIKeyExport + command += "agent/mongodb-agent" + command += " -healthCheckFilePath=" + monitoringAgentHealthStatusFilePathValue + command += " -serveStatusPort=5001" + command += getMonitoringAgentLogOptions(appDB) + + // 2. Add the cluster config file path + // If we are using k8s secrets, this is the same as community (and the same as the other agent container) + // But this is not possible in vault so we need two separate paths + if vault.IsVaultSecretBackend() { + command += " -cluster=/var/lib/automation/config/" + util.AppDBMonitoringAutomationConfigKey + } else { + command += " -cluster=/var/lib/automation/config/" + util.AppDBAutomationConfigKey + } + + // 2. Startup parameters for the agent to enable monitoring + startupParams := mdbv1.StartupParameters{ + "mmsApiKey": "${AGENT_API_KEY}", + "mmsGroupId": podVars.ProjectID, + } + + // 3. Startup parameters for the agent to enable TLS + if podVars.SSLMMSCAConfigMap != "" { + trustedCACertLocation := path.Join(caCertMountPath, util.CaCertMMS) + startupParams["sslTrustedMMSServerCertificate"] = trustedCACertLocation + } + + if podVars.SSLRequireValidMMSServerCertificates { + startupParams["sslRequireValidMMSServerCertificates"] = "true" + } + + // 4. Custom startup parameters provided in the CR + // By default appDB.AutomationAgent.StartupParameters apply to both agents + // if appDB.MonitoringAgent.StartupParameters is specified, it overrides the former + monitoringStartupParams := appDB.AutomationAgent.StartupParameters + if appDB.MonitoringAgent.StartupParameters != nil { + monitoringStartupParams = appDB.MonitoringAgent.StartupParameters + } + + for k, v := range monitoringStartupParams { + startupParams[k] = v + } + + command += startupParams.ToCommandLineArgs() + + command += overrideLocalHostFlag(&appDB, externalDomain) + + monitoringCommand := []string{"/bin/bash", "-c", command} + + // Add additional TLS volumes if needed + _, monitoringMounts := getTLSVolumesAndVolumeMounts(appDB, &podVars, log) + + return podtemplatespec.Apply( + monitoringACFunc, + monitoringConfigMapVolumeFunc, + // This is a function that reads the automation agent containers, copies it and modifies it. + // We do this since the two containers are very similar with just a few differences + func(podTemplateSpec *corev1.PodTemplateSpec) { + monitoringContainer := podtemplatespec.FindContainerByName(construct.AgentName, podTemplateSpec).DeepCopy() + monitoringContainer.Name = monitoringAgentContainerName + + // we ensure that the monitoring agent image is compatible with the input of Ops Manager we're using. + // once we make static containers the default, we can remove this code. + if opts.LegacyMonitoringAgentImage != "" { + monitoringContainer.Image = opts.LegacyMonitoringAgentImage + } + + // Replace the automation config volume + volumeMounts := monitoringContainer.VolumeMounts + if !vault.IsVaultSecretBackend() { + // Replace the automation config volume + acMountIndex := getVolumeMountIndexByName(volumeMounts, "automation-config") + if acMountIndex != -1 { + volumeMounts[acMountIndex].Name = monitoringAcVolume.Name + } + } + + configMapIndex := getVolumeMountIndexByName(volumeMounts, "automation-config-goal-input") + if configMapIndex != -1 { + volumeMounts[configMapIndex].Name = monitoringConfigMapVolume.Name + } + + tmpVolumeMountIndex := getVolumeMountIndexByName(volumeMounts, util.PvcNameTmp) + if tmpVolumeMountIndex != -1 { + volumeMounts[tmpVolumeMountIndex].SubPath = tmpSubpathName + } + + // Set up custom persistence options - see customPersistenceConfig() for an explanation + if appDB.HasSeparateDataAndLogsVolumes() { + journalVolumeMounts := statefulset.CreateVolumeMount(util.PvcNameJournal, util.PvcMountPathJournal) + volumeMounts = append(volumeMounts, journalVolumeMounts) + } else { + logsVolumeMount := statefulset.CreateVolumeMount(appDB.DataVolumeName(), util.PvcMountPathLogs, statefulset.WithSubPath(appDB.LogsVolumeName())) + journalVolumeMount := statefulset.CreateVolumeMount(appDB.DataVolumeName(), util.PvcMountPathJournal, statefulset.WithSubPath(util.PvcNameJournal)) + volumeMounts = append(volumeMounts, journalVolumeMount, logsVolumeMount) + } + + if !vault.IsVaultSecretBackend() { + // AGENT_API_KEY volume + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(AgentAPIKeyVolumeName, AgentAPIKeySecretPath)) + } + container.Apply( + container.WithVolumeMounts(volumeMounts), + container.WithCommand(monitoringCommand), + container.WithResourceRequirements(buildRequirementsFromPodSpec(*NewDefaultPodSpecWrapper(*appDB.PodSpec))), + container.WithVolumeMounts(monitoringMounts), + container.WithEnvs(appdbContainerEnv(appDB)...), + container.WithEnvs(readinessEnvironmentVariablesToEnvVars(appDB.AutomationAgent.ReadinessProbe.EnvironmentVariables)...), + )(monitoringContainer) + podTemplateSpec.Spec.Containers = append(podTemplateSpec.Spec.Containers, *monitoringContainer) + }, + ) +} + +// appdbContainerEnv returns the set of env var needed by the AppDB. +func appdbContainerEnv(appDbSpec om.AppDBSpec) []corev1.EnvVar { + envVars := []corev1.EnvVar{ + { + Name: podNamespaceEnv, + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}, + }, + { + Name: automationConfigMapEnv, + Value: appDbSpec.Name() + "-config", + }, + { + Name: headlessAgentEnv, + Value: "true", + }, + { + Name: clusterDomainEnv, + Value: appDbSpec.ClusterDomain, + }, + } + return envVars +} + +// Configure the -overrideLocalHost parameter which controls how the automation agent instances identify themselves. For multi-cluster deployments: +// - With externalDomain: Use {hostname}.{externalDomain} format for cross-cluster discovery via external DNS +// - Without externalDomain: Use Kubernetes internal DNS format with cluster domain suffix for in-cluster networking +func overrideLocalHostFlag(appDbSpec *om.AppDBSpec, externalDomain *string) string { + if externalDomain != nil { + return fmt.Sprintf(" -overrideLocalHost=$(hostname).%s", *externalDomain) + } else if appDbSpec.IsMultiCluster() { + return fmt.Sprintf(" -overrideLocalHost=$(hostname)-svc.${POD_NAMESPACE}.svc.%s", appDbSpec.GetClusterDomain()) + } + return "" +} diff --git a/controllers/operator/construct/appdb_construction_test.go b/controllers/operator/construct/appdb_construction_test.go new file mode 100644 index 000000000..d9c080852 --- /dev/null +++ b/controllers/operator/construct/appdb_construction_test.go @@ -0,0 +1,74 @@ +package construct + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +func TestAppDBAgentFlags(t *testing.T) { + agentStartupParameters := mdbv1.StartupParameters{ + "Key1": "Value1", + "Key2": "Value2", + } + om := omv1.NewOpsManagerBuilderDefault().Build() + om.Spec.AppDB.AutomationAgent.StartupParameters = agentStartupParameters + sts, err := AppDbStatefulSet(*om, &env.PodEnvVars{ProjectID: "abcd"}, + AppDBStatefulSetOptions{}, scalers.GetAppDBScaler(om, multicluster.LegacyCentralClusterName, 0, nil), v1.OnDeleteStatefulSetStrategyType, nil) + assert.NoError(t, err) + + command := sts.Spec.Template.Spec.Containers[0].Command + assert.Contains(t, command[len(command)-1], "-Key1=Value1", "-Key2=Value2") +} + +func TestResourceRequirements(t *testing.T) { + om := omv1.NewOpsManagerBuilderDefault().Build() + agentResourceRequirements := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: ParseQuantityOrZero("200"), + corev1.ResourceMemory: ParseQuantityOrZero("500M"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: ParseQuantityOrZero("100"), + corev1.ResourceMemory: ParseQuantityOrZero("200M"), + }, + } + + om.Spec.AppDB.PodSpec.PodTemplateWrapper = mdbv1.PodTemplateSpecWrapper{ + PodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "mongodb-agent", + Resources: agentResourceRequirements, + }, + }, + }, + }, + } + + sts, err := AppDbStatefulSet(*om, &env.PodEnvVars{ProjectID: "abcd"}, + AppDBStatefulSetOptions{}, scalers.GetAppDBScaler(om, "central", 0, nil), v1.OnDeleteStatefulSetStrategyType, nil) + assert.NoError(t, err) + + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name == "mongodb-agent" { + assert.Equal(t, agentResourceRequirements, c.Resources) + } + } +} diff --git a/controllers/operator/construct/backup_construction.go b/controllers/operator/construct/backup_construction.go new file mode 100644 index 000000000..d40eba5e6 --- /dev/null +++ b/controllers/operator/construct/backup_construction.go @@ -0,0 +1,217 @@ +package construct + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/lifecycle" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +const ( + BackupDaemonServicePort = 8443 + backupDaemonEnv = "BACKUP_DAEMON" + healthEndpointPortEnv = "HEALTH_ENDPOINT_PORT" + backupDaemonReadinessProbeCommand = "/opt/scripts/backup-daemon-readiness-probe" + backupDaemonLivenessProbeCommand = "/opt/scripts/backup-daemon-liveness-probe.sh" +) + +// BackupDaemonStatefulSet fully constructs the Backup StatefulSet. +func BackupDaemonStatefulSet(ctx context.Context, centralClusterSecretClient secrets.SecretClient, opsManager *omv1.MongoDBOpsManager, memberCluster multicluster.MemberCluster, log *zap.SugaredLogger, additionalOpts ...func(*OpsManagerStatefulSetOptions)) (appsv1.StatefulSet, error) { + opts := backupOptions(memberCluster, additionalOpts...)(opsManager) + if err := opts.updateHTTPSCertSecret(ctx, centralClusterSecretClient, memberCluster, opsManager.OwnerReferences, log); err != nil { + return appsv1.StatefulSet{}, err + } + + secretName := opsManager.Spec.Backup.QueryableBackupSecretRef.Name + opts.QueryableBackupPemSecretName = secretName + if secretName != "" { + // if the secret is specified, we must have a queryable.pem entry. + _, err := secret.ReadKey(ctx, memberCluster.SecretClient, "queryable.pem", kube.ObjectKey(opsManager.Namespace, secretName)) + if err != nil { + return appsv1.StatefulSet{}, xerrors.Errorf("error reading queryable.pem key from secret %s/%s: %w", opsManager.Namespace, secretName, err) + } + } + + backupSts := statefulset.New(backupDaemonStatefulSetFunc(opts)) + var err error + if opts.StatefulSetSpecOverride != nil { + backupSts.Spec = merge.StatefulSetSpecs(backupSts.Spec, *opts.StatefulSetSpecOverride) + } + + // the JVM env args must be determined after any potential stateful set override + // has taken place. + if err = setJvmArgsEnvVars(opsManager.Spec, util.BackupDaemonContainerName, &backupSts); err != nil { + return appsv1.StatefulSet{}, err + } + return backupSts, nil +} + +// backupOptions returns a function which returns the OpsManagerStatefulSetOptions to create the BackupDaemon StatefulSet. +func backupOptions(memberCluster multicluster.MemberCluster, additionalOpts ...func(opts *OpsManagerStatefulSetOptions)) func(opsManager *omv1.MongoDBOpsManager) OpsManagerStatefulSetOptions { + return func(opsManager *omv1.MongoDBOpsManager) OpsManagerStatefulSetOptions { + opts := getSharedOpsManagerOptions(opsManager) + + opts.Annotations = opsManager.Annotations + opts.ServicePort = BackupDaemonServicePort + if memberCluster.Legacy { + opts.ServiceName = opsManager.BackupDaemonServiceName() + opts.Name = opsManager.BackupDaemonStatefulSetName() + } else { + opts.ServiceName = opsManager.BackupDaemonHeadlessServiceNameForClusterIndex(memberCluster.Index) + opts.Name = opsManager.BackupDaemonStatefulSetNameForClusterIndex(memberCluster.Index) + } + opts.Replicas = opsManager.Spec.Backup.Members + opts.AppDBConnectionSecretName = opsManager.AppDBMongoConnectionStringSecretName() + + opts.LoggingConfiguration = opsManager.Spec.Backup.Logging + + if opsManager.Spec.Backup != nil { + if opsManager.Spec.Backup.StatefulSetConfiguration != nil { + opts.StatefulSetSpecOverride = &opsManager.Spec.Backup.StatefulSetConfiguration.SpecWrapper.Spec + } + if opsManager.Spec.Backup.HeadDB != nil { + opts.HeadDbPersistenceConfig = opsManager.Spec.Backup.HeadDB + } + } + + for _, additionalOpt := range additionalOpts { + additionalOpt(&opts) + } + + return opts + } +} + +// backupDaemonStatefulSetFunc constructs the Backup Daemon podTemplateSpec modification function. +func backupDaemonStatefulSetFunc(opts OpsManagerStatefulSetOptions) statefulset.Modification { + // PodSecurityContext is added in the backupAndOpsManagerSharedConfiguration + _, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + defaultConfig := mdbv1.PersistenceConfig{Storage: util.DefaultHeadDbStorageSize} + pvc := pvcFunc(util.PvcNameHeadDb, opts.HeadDbPersistenceConfig, defaultConfig, opts.Labels) + headDbMount := statefulset.CreateVolumeMount(util.PvcNameHeadDb, util.PvcMountPathHeadDb) + + caVolumeFunc := podtemplatespec.NOOP() + caVolumeMountFunc := container.NOOP() + + volumeMounts := []corev1.VolumeMount{headDbMount} + mmsMongoUriVolume := corev1.Volume{} + var mmsMongoUriMount corev1.VolumeMount + + if !vault.IsVaultSecretBackend() { + // configure the AppDB Connection String volume from a secret + mmsMongoUriVolume, mmsMongoUriMount = buildMmsMongoUriVolume(opts) + volumeMounts = append(volumeMounts, mmsMongoUriMount) + } + + return statefulset.Apply( + backupAndOpsManagerSharedConfiguration(opts), + statefulset.WithVolumeClaim(util.PvcNameHeadDb, pvc), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + // 70 minutes for Backup Damon (internal timeout is 65 minutes, see CLOUDP-61849) + podtemplatespec.WithTerminationGracePeriodSeconds(4200), + addUriVolume(mmsMongoUriVolume), + caVolumeFunc, + podtemplatespec.WithContainerByIndex(0, + container.Apply( + container.WithName(util.BackupDaemonContainerName), + container.WithEnvs(backupDaemonEnvVars()...), + container.WithLifecycle(buildBackupDaemonLifecycle()), + container.WithVolumeMounts(volumeMounts), + container.WithLivenessProbe(buildBackupDaemonLivenessProbe()), + container.WithReadinessProbe(buildBackupDaemonReadinessProbe()), + container.WithStartupProbe(buildBackupDaemonStartupProbe()), + caVolumeMountFunc, + configureContainerSecurityContext, + ), + )), + ), + ) +} + +func addUriVolume(volume corev1.Volume) podtemplatespec.Modification { + if !vault.IsVaultSecretBackend() { + return podtemplatespec.WithVolume(volume) + } + return podtemplatespec.NOOP() +} + +func backupDaemonEnvVars() []corev1.EnvVar { + return []corev1.EnvVar{ + { + // For the OM Docker image to run as Backup Daemon, the BACKUP_DAEMON env variable + // needs to be passed with any value.configureJvmParams + Name: backupDaemonEnv, + Value: "backup", + }, + { + // Specify the port of the backup daemon health endpoint for the liveness probe. + Name: healthEndpointPortEnv, + Value: fmt.Sprintf("%d", backupDaemonHealthPort), + }, + } +} + +func buildBackupDaemonLifecycle() lifecycle.Modification { + return lifecycle.WithPrestopCommand([]string{"/bin/sh", "-c", "/mongodb-ops-manager/bin/mongodb-mms stop_backup_daemon"}) +} + +// buildBackupDaemonReadinessProbe returns a probe modification which will add +// the readiness probe. +func buildBackupDaemonReadinessProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{backupDaemonReadinessProbeCommand}), + probes.WithFailureThreshold(3), + probes.WithInitialDelaySeconds(1), + probes.WithSuccessThreshold(1), + probes.WithPeriodSeconds(3), + probes.WithTimeoutSeconds(5), + ) +} + +// buildBackupDaemonLivenessProbe returns a probe modification which will add +// the liveness probe. +func buildBackupDaemonLivenessProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{backupDaemonLivenessProbeCommand}), + probes.WithFailureThreshold(10), + probes.WithInitialDelaySeconds(10), + probes.WithSuccessThreshold(1), + probes.WithPeriodSeconds(30), + probes.WithTimeoutSeconds(5), + ) +} + +// buildBackupDaemonStartupProbe returns a probe modification which will add +// the startup probe. +func buildBackupDaemonStartupProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{backupDaemonLivenessProbeCommand}), + probes.WithFailureThreshold(20), + probes.WithInitialDelaySeconds(1), + probes.WithSuccessThreshold(1), + probes.WithPeriodSeconds(30), + probes.WithTimeoutSeconds(5), + ) +} diff --git a/controllers/operator/construct/backup_construction_test.go b/controllers/operator/construct/backup_construction_test.go new file mode 100644 index 000000000..2b79002ee --- /dev/null +++ b/controllers/operator/construct/backup_construction_test.go @@ -0,0 +1,87 @@ +package construct + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" + + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func TestBuildBackupDaemonStatefulSet(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: client, + } + sts, err := BackupDaemonStatefulSet(ctx, secretsClient, omv1.NewOpsManagerBuilderDefault().SetName("test-om").Build(), multicluster.GetLegacyCentralMemberCluster(1, 0, client, secretsClient), zap.S()) + assert.NoError(t, err) + assert.Equal(t, "test-om-backup-daemon", sts.Name) + assert.Equal(t, util.BackupDaemonContainerName, sts.Spec.Template.Spec.Containers[0].Name) + assert.NotNil(t, sts.Spec.Template.Spec.Containers[0].ReadinessProbe) +} + +func TestBackupPodTemplate_TerminationTimeout(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: client, + } + set, err := BackupDaemonStatefulSet(ctx, secretsClient, omv1.NewOpsManagerBuilderDefault().SetName("test-om").Build(), multicluster.GetLegacyCentralMemberCluster(1, 0, client, secretsClient), zap.S()) + assert.NoError(t, err) + podSpecTemplate := set.Spec.Template + assert.Equal(t, int64(4200), *podSpecTemplate.Spec.TerminationGracePeriodSeconds) +} + +func TestBuildBackupDaemonContainer(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: client, + } + sts, err := BackupDaemonStatefulSet(ctx, secretsClient, omv1.NewOpsManagerBuilderDefault().SetVersion("4.2.0").Build(), multicluster.GetLegacyCentralMemberCluster(1, 0, client, secretsClient), zap.S(), + WithOpsManagerImage("quay.io/mongodb/mongodb-enterprise-ops-manager:4.2.0"), + ) + assert.NoError(t, err) + template := sts.Spec.Template + container := template.Spec.Containers[0] + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-ops-manager:4.2.0", container.Image) + + assert.Equal(t, util.BackupDaemonContainerName, container.Name) + + expectedProbe := probes.New(buildBackupDaemonReadinessProbe()) + assert.Equal(t, &expectedProbe, container.ReadinessProbe) + + expectedProbe = probes.New(buildBackupDaemonLivenessProbe()) + assert.Equal(t, &expectedProbe, container.LivenessProbe) + + expectedProbe = probes.New(buildBackupDaemonStartupProbe()) + assert.Equal(t, &expectedProbe, container.StartupProbe) + + assert.Equal(t, []string{"/bin/sh", "-c", "/mongodb-ops-manager/bin/mongodb-mms stop_backup_daemon"}, + container.Lifecycle.PreStop.Exec.Command) +} + +func TestMultipleBackupDaemons(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: client, + } + sts, err := BackupDaemonStatefulSet(ctx, secretsClient, omv1.NewOpsManagerBuilderDefault().SetVersion("4.2.0").SetBackupMembers(3).Build(), multicluster.GetLegacyCentralMemberCluster(1, 0, client, secretsClient), zap.S()) + assert.NoError(t, err) + assert.Equal(t, 3, int(*sts.Spec.Replicas)) +} diff --git a/controllers/operator/construct/construction_test.go b/controllers/operator/construct/construction_test.go new file mode 100644 index 000000000..40503b1fe --- /dev/null +++ b/controllers/operator/construct/construction_test.go @@ -0,0 +1,349 @@ +package construct + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/resource" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func TestBuildStatefulSet_PersistentFlagStatic(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + mdb := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).Build() + set := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Len(t, set.Spec.VolumeClaimTemplates, 1) + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 7) + assert.Len(t, set.Spec.Template.Spec.Containers[1].VolumeMounts, 7) + + mdb = mdbv1.NewReplicaSetBuilder().SetPersistent(util.BooleanRef(true)).Build() + set = DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Len(t, set.Spec.VolumeClaimTemplates, 1) + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 7) + assert.Len(t, set.Spec.Template.Spec.Containers[1].VolumeMounts, 7) + + // If no persistence is set then we still mount init scripts + mdb = mdbv1.NewReplicaSetBuilder().SetPersistent(util.BooleanRef(false)).Build() + set = DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Len(t, set.Spec.VolumeClaimTemplates, 0) + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 7) + assert.Len(t, set.Spec.Template.Spec.Containers[1].VolumeMounts, 7) +} + +func TestBuildStatefulSet_PersistentFlag(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.NonStatic)) + + mdb := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).Build() + set := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Len(t, set.Spec.VolumeClaimTemplates, 1) + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 8) + + mdb = mdbv1.NewReplicaSetBuilder().SetPersistent(util.BooleanRef(true)).Build() + set = DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Len(t, set.Spec.VolumeClaimTemplates, 1) + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 8) + + // If no persistence is set then we still mount init scripts + mdb = mdbv1.NewReplicaSetBuilder().SetPersistent(util.BooleanRef(false)).Build() + set = DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Len(t, set.Spec.VolumeClaimTemplates, 0) + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, 8) +} + +// TestBuildStatefulSet_PersistentVolumeClaimSingle checks that one persistent volume claim is created that is mounted by +// 3 points +func TestBuildStatefulSet_PersistentVolumeClaimSingle(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.NonStatic)) + + labels := map[string]string{"app": "foo"} + persistence := mdbv1.NewPersistenceBuilder("40G").SetStorageClass("fast").SetLabelSelector(labels) + podSpec := mdbv1.NewPodSpecWrapperBuilder().SetSinglePersistence(persistence).Build().MongoDbPodSpec + rs := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).SetPodSpec(&podSpec).Build() + set := DatabaseStatefulSet(*rs, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + checkPvClaims(t, set, []corev1.PersistentVolumeClaim{pvClaim(util.PvcNameData, "40G", stringutil.Ref("fast"), labels)}) + + checkMounts(t, set, []corev1.VolumeMount{ + {Name: util.PvMms, MountPath: util.PvcMmsHomeMountPath, SubPath: util.PvcMmsHome}, + {Name: util.PvMms, MountPath: util.PvcMountPathTmp, SubPath: util.PvcNameTmp}, + {Name: util.PvMms, MountPath: util.PvcMmsMountPath, SubPath: util.PvcMms}, + {Name: AgentAPIKeyVolumeName, MountPath: AgentAPIKeySecretPath}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathData, SubPath: util.PvcNameData}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathJournal, SubPath: util.PvcNameJournal}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathLogs, SubPath: util.PvcNameLogs}, + {Name: PvcNameDatabaseScripts, MountPath: PvcMountPathScripts, ReadOnly: true}, + }) +} + +// TestBuildStatefulSet_PersistentVolumeClaimSingle checks that one persistent volume claim is created that is mounted by +// 3 points +func TestBuildStatefulSet_PersistentVolumeClaimSingleStatic(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + labels := map[string]string{"app": "foo"} + persistence := mdbv1.NewPersistenceBuilder("40G").SetStorageClass("fast").SetLabelSelector(labels) + podSpec := mdbv1.NewPodSpecWrapperBuilder().SetSinglePersistence(persistence).Build().MongoDbPodSpec + rs := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).SetPodSpec(&podSpec).Build() + set := DatabaseStatefulSet(*rs, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + checkPvClaims(t, set, []corev1.PersistentVolumeClaim{pvClaim(util.PvcNameData, "40G", stringutil.Ref("fast"), labels)}) + + checkMounts(t, set, []corev1.VolumeMount{ + {Name: util.PvMms, MountPath: util.PvcMmsHomeMountPath, SubPath: util.PvcMmsHome}, + {Name: util.PvMms, MountPath: util.PvcMountPathTmp, SubPath: util.PvcNameTmp}, + {Name: util.PvMms, MountPath: util.PvcMmsMountPath, SubPath: util.PvcMms}, + {Name: AgentAPIKeyVolumeName, MountPath: AgentAPIKeySecretPath}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathData, SubPath: util.PvcNameData}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathJournal, SubPath: util.PvcNameJournal}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathLogs, SubPath: util.PvcNameLogs}, + }) +} + +// TestBuildStatefulSet_PersistentVolumeClaimMultiple checks multiple volumes for multiple mounts. Note, that subpaths +// for mount points are not created (unlike in single mode) +func TestBuildStatefulSet_PersistentVolumeClaimMultiple(t *testing.T) { + labels1 := map[string]string{"app": "bar"} + labels2 := map[string]string{"app": "foo"} + podSpec := mdbv1.NewPodSpecWrapperBuilder().SetMultiplePersistence( + mdbv1.NewPersistenceBuilder("40G").SetStorageClass("fast"), + mdbv1.NewPersistenceBuilder("3G").SetStorageClass("slow").SetLabelSelector(labels1), + mdbv1.NewPersistenceBuilder("500M").SetStorageClass("fast").SetLabelSelector(labels2), + ).Build() + + mdb := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).SetPodSpec(&podSpec.MongoDbPodSpec).Build() + set := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + checkPvClaims(t, set, []corev1.PersistentVolumeClaim{ + pvClaim(util.PvcNameData, "40G", stringutil.Ref("fast"), nil), + pvClaim(util.PvcNameJournal, "3G", stringutil.Ref("slow"), labels1), + pvClaim(util.PvcNameLogs, "500M", stringutil.Ref("fast"), labels2), + }) + + checkMounts(t, set, []corev1.VolumeMount{ + {Name: util.PvMms, MountPath: util.PvcMmsHomeMountPath, SubPath: util.PvcMmsHome}, + {Name: util.PvMms, MountPath: util.PvcMountPathTmp, SubPath: util.PvcNameTmp}, + {Name: util.PvMms, MountPath: util.PvcMmsMountPath, SubPath: util.PvcMms}, + {Name: AgentAPIKeyVolumeName, MountPath: AgentAPIKeySecretPath}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathData}, + {Name: PvcNameDatabaseScripts, MountPath: PvcMountPathScripts, ReadOnly: true}, + {Name: util.PvcNameJournal, MountPath: util.PvcMountPathJournal}, + {Name: util.PvcNameLogs, MountPath: util.PvcMountPathLogs}, + }) +} + +// TestBuildStatefulSet_PersistentVolumeClaimMultipleDefaults checks the scenario when storage is provided only for one +// mount point. Default values are expected to be used for two others +func TestBuildStatefulSet_PersistentVolumeClaimMultipleDefaults(t *testing.T) { + podSpec := mdbv1.NewPodSpecWrapperBuilder().SetMultiplePersistence( + mdbv1.NewPersistenceBuilder("40G").SetStorageClass("fast"), + nil, + nil). + Build() + mdb := mdbv1.NewReplicaSetBuilder().SetPersistent(nil).SetPodSpec(&podSpec.MongoDbPodSpec).Build() + set := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + checkPvClaims(t, set, []corev1.PersistentVolumeClaim{ + pvClaim(util.PvcNameData, "40G", stringutil.Ref("fast"), nil), + pvClaim(util.PvcNameJournal, util.DefaultJournalStorageSize, nil, nil), + pvClaim(util.PvcNameLogs, util.DefaultLogsStorageSize, nil, nil), + }) + + checkMounts(t, set, []corev1.VolumeMount{ + {Name: util.PvMms, MountPath: util.PvcMmsHomeMountPath, SubPath: util.PvcMmsHome}, + {Name: util.PvMms, MountPath: util.PvcMountPathTmp, SubPath: util.PvcNameTmp}, + {Name: util.PvMms, MountPath: util.PvcMmsMountPath, SubPath: util.PvcMms}, + {Name: AgentAPIKeyVolumeName, MountPath: AgentAPIKeySecretPath}, + {Name: util.PvcNameData, MountPath: util.PvcMountPathData}, + {Name: PvcNameDatabaseScripts, MountPath: PvcMountPathScripts, ReadOnly: true}, + {Name: util.PvcNameJournal, MountPath: util.PvcMountPathJournal}, + {Name: util.PvcNameLogs, MountPath: util.PvcMountPathLogs}, + }) +} + +func TestBuildAppDbStatefulSetDefault(t *testing.T) { + t.Setenv(util.OpsManagerMonitorAppDB, "false") + om := omv1.NewOpsManagerBuilderDefault().Build() + scaler := scalers.GetAppDBScaler(om, multicluster.LegacyCentralClusterName, 0, nil) + appDbSts, err := AppDbStatefulSet(*om, &env.PodEnvVars{ProjectID: "abcd"}, AppDBStatefulSetOptions{}, scaler, appsv1.OnDeleteStatefulSetStrategyType, nil) + assert.NoError(t, err) + podSpecTemplate := appDbSts.Spec.Template.Spec + assert.Len(t, podSpecTemplate.InitContainers, 1) + assert.Len(t, podSpecTemplate.Containers, 2, "Should contain mongodb and agent") + assert.Equal(t, "mongodb-agent", podSpecTemplate.Containers[0].Name) + assert.Equal(t, "mongod", podSpecTemplate.Containers[1].Name) +} + +func TestBasePodSpec_Affinity(t *testing.T) { + nodeAffinity := defaultNodeAffinity() + podAffinity := defaultPodAffinity() + + podSpec := mdbv1.NewPodSpecWrapperBuilder(). + SetNodeAffinity(nodeAffinity). + SetPodAffinity(podAffinity). + SetPodAntiAffinityTopologyKey("nodeId"). + Build() + mdb := mdbv1.NewReplicaSetBuilder(). + SetName("s"). + SetPodSpec(&podSpec.MongoDbPodSpec). + Build() + sts := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + spec := sts.Spec.Template.Spec + assert.Equal(t, nodeAffinity, *spec.Affinity.NodeAffinity) + assert.Equal(t, podAffinity, *spec.Affinity.PodAffinity) + assert.Len(t, spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, 1) + assert.Len(t, spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 0) + term := spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0] + assert.Equal(t, int32(100), term.Weight) + assert.Equal(t, map[string]string{PodAntiAffinityLabelKey: "s"}, term.PodAffinityTerm.LabelSelector.MatchLabels) + assert.Equal(t, "nodeId", term.PodAffinityTerm.TopologyKey) +} + +// TestBasePodSpec_AntiAffinityDefaultTopology checks that the default topology key is created if the topology key is +// not specified +func TestBasePodSpec_AntiAffinityDefaultTopology(t *testing.T) { + sts := DatabaseStatefulSet(*mdbv1.NewStandaloneBuilder().SetName("my-standalone").Build(), StandaloneOptions(GetPodEnvOptions()), zap.S()) + + spec := sts.Spec.Template.Spec + term := spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0] + assert.Equal(t, int32(100), term.Weight) + assert.Equal(t, map[string]string{PodAntiAffinityLabelKey: "my-standalone"}, term.PodAffinityTerm.LabelSelector.MatchLabels) + assert.Equal(t, util.DefaultAntiAffinityTopologyKey, term.PodAffinityTerm.TopologyKey) +} + +// TestBasePodSpec_ImagePullSecrets verifies that 'spec.ImagePullSecrets' is created only if env variable +// IMAGE_PULL_SECRETS is initialized +func TestBasePodSpec_ImagePullSecrets(t *testing.T) { + // Cleaning the state (there is no tear down in go test :( ) + defer mock.InitDefaultEnvVariables() + + sts := DatabaseStatefulSet(*mdbv1.NewStandaloneBuilder().Build(), StandaloneOptions(GetPodEnvOptions()), zap.S()) + + template := sts.Spec.Template + assert.Nil(t, template.Spec.ImagePullSecrets) + + t.Setenv(util.ImagePullSecrets, "foo") + + sts = DatabaseStatefulSet(*mdbv1.NewStandaloneBuilder().Build(), StandaloneOptions(GetPodEnvOptions()), zap.S()) + + template = sts.Spec.Template + assert.Equal(t, []corev1.LocalObjectReference{{Name: "foo"}}, template.Spec.ImagePullSecrets) +} + +// TestBasePodSpec_TerminationGracePeriodSeconds verifies that the TerminationGracePeriodSeconds is set to 600 seconds +func TestBasePodSpec_TerminationGracePeriodSeconds(t *testing.T) { + sts := DatabaseStatefulSet(*mdbv1.NewReplicaSetBuilder().Build(), ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.Equal(t, util.Int64Ref(600), sts.Spec.Template.Spec.TerminationGracePeriodSeconds) +} + +func checkPvClaims(t *testing.T, set appsv1.StatefulSet, expectedClaims []corev1.PersistentVolumeClaim) { + assert.Len(t, set.Spec.VolumeClaimTemplates, len(expectedClaims)) + + for i, c := range expectedClaims { + assert.Equal(t, c, set.Spec.VolumeClaimTemplates[i]) + } +} + +func checkMounts(t *testing.T, set appsv1.StatefulSet, expectedMounts []corev1.VolumeMount) { + assert.Len(t, set.Spec.Template.Spec.Containers[0].VolumeMounts, len(expectedMounts)) + + for i, c := range expectedMounts { + assert.Equal(t, c, set.Spec.Template.Spec.Containers[0].VolumeMounts[i]) + } +} + +func pvClaim(pvName, size string, storageClass *string, labels map[string]string) corev1.PersistentVolumeClaim { + quantity, _ := resource.ParseQuantity(size) + expectedClaim := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: quantity}, + }, + StorageClassName: storageClass, + }, + } + if len(labels) > 0 { + expectedClaim.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} + } + return expectedClaim +} + +func TestDefaultPodSpec_SecurityContext(t *testing.T) { + defer mock.InitDefaultEnvVariables() + + sts := DatabaseStatefulSet(*mdbv1.NewStandaloneBuilder().Build(), StandaloneOptions(GetPodEnvOptions()), zap.S()) + + spec := sts.Spec.Template.Spec + assert.Len(t, spec.InitContainers, 1) + assert.NotNil(t, spec.SecurityContext) + assert.NotNil(t, spec.InitContainers[0].SecurityContext) + assert.Equal(t, util.Int64Ref(util.FsGroup), spec.SecurityContext.FSGroup) + assert.Equal(t, util.Int64Ref(util.RunAsUser), spec.SecurityContext.RunAsUser) + assert.Equal(t, util.BooleanRef(true), spec.SecurityContext.RunAsNonRoot) + assert.Equal(t, util.BooleanRef(true), spec.InitContainers[0].SecurityContext.ReadOnlyRootFilesystem) + assert.Equal(t, util.BooleanRef(false), spec.InitContainers[0].SecurityContext.AllowPrivilegeEscalation) + + t.Setenv(util.ManagedSecurityContextEnv, "true") + + sts = DatabaseStatefulSet(*mdbv1.NewStandaloneBuilder().Build(), StandaloneOptions(GetPodEnvOptions()), zap.S()) + assert.Nil(t, sts.Spec.Template.Spec.SecurityContext) +} + +func TestPodSpec_Requirements(t *testing.T) { + podSpec := mdbv1.NewPodSpecWrapperBuilder(). + SetCpuRequests("0.1"). + SetMemoryRequest("512M"). + SetCpuLimit("0.3"). + SetMemoryLimit("1012M"). + Build() + + sts := DatabaseStatefulSet(*mdbv1.NewReplicaSetBuilder().SetPodSpec(&podSpec.MongoDbPodSpec).Build(), ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + podSpecTemplate := sts.Spec.Template + container := podSpecTemplate.Spec.Containers[0] + + expectedLimits := corev1.ResourceList{corev1.ResourceCPU: ParseQuantityOrZero("0.3"), corev1.ResourceMemory: ParseQuantityOrZero("1012M")} + expectedRequests := corev1.ResourceList{corev1.ResourceCPU: ParseQuantityOrZero("0.1"), corev1.ResourceMemory: ParseQuantityOrZero("512M")} + assert.Equal(t, expectedLimits, container.Resources.Limits) + assert.Equal(t, expectedRequests, container.Resources.Requests) +} + +func defaultPodVars() *env.PodEnvVars { + return &env.PodEnvVars{BaseURL: "http://localhost:8080", ProjectID: "myProject", User: "user@some.com"} +} + +func TestPodAntiAffinityOverride(t *testing.T) { + podAntiAffinity := defaultPodAntiAffinity() + + podSpec := mdbv1.NewPodSpecWrapperBuilder().Build() + + // override pod Anti Affinity + podSpec.PodTemplateWrapper.PodTemplate.Spec.Affinity.PodAntiAffinity = &podAntiAffinity + + mdb := mdbv1.NewReplicaSetBuilder(). + SetName("s"). + SetPodSpec(&podSpec.MongoDbPodSpec). + Build() + sts := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + spec := sts.Spec.Template.Spec + assert.Equal(t, podAntiAffinity, *spec.Affinity.PodAntiAffinity) +} diff --git a/controllers/operator/construct/database_construction.go b/controllers/operator/construct/database_construction.go new file mode 100644 index 000000000..1c732fec9 --- /dev/null +++ b/controllers/operator/construct/database_construction.go @@ -0,0 +1,1099 @@ +package construct + +import ( + "fmt" + "os" + "path" + "sort" + "strconv" + + "go.uber.org/zap" + "k8s.io/utils/ptr" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +const ( + // Volume constants + PvcNameDatabaseScripts = "database-scripts" + PvcMountPathScripts = "/opt/scripts" + + caCertMountPath = "/mongodb-automation/certs" + // CaCertName is the name of the volume with the CA Cert + CaCertName = "ca-cert-volume" + // AgentCertMountPath defines where in the Pod the ca cert will be mounted. + agentCertMountPath = "/mongodb-automation/" + util.AgentSecretName + + databaseLivenessProbeCommand = "/opt/scripts/probe.sh" + databaseReadinessProbeCommand = "/opt/scripts/readinessprobe" + + ControllerLabelName = "controller" + InitDatabaseContainerName = "mongodb-enterprise-init-database" + + // Database environment variable names + InitDatabaseVersionEnv = "INIT_DATABASE_VERSION" + DatabaseVersionEnv = "DATABASE_VERSION" + + // PodAntiAffinityLabelKey defines the anti affinity rule label. The main rule is to spread entities inside one statefulset + // (aka replicaset) to different locations, so pods having the same label shouldn't coexist on the node that has + // the same topology key + PodAntiAffinityLabelKey = "pod-anti-affinity" + + // AGENT_API_KEY secret path + AgentAPIKeySecretPath = "/mongodb-automation/agent-api-key" //nolint + AgentAPIKeyVolumeName = "agent-api-key" //nolint + + LogFileAutomationAgentEnv = "MDB_LOG_FILE_AUTOMATION_AGENT" + LogFileAutomationAgentVerboseEnv = "MDB_LOG_FILE_AUTOMATION_AGENT_VERBOSE" + LogFileAutomationAgentStderrEnv = "MDB_LOG_FILE_AUTOMATION_AGENT_STDERR" + LogFileMongoDBAuditEnv = "MDB_LOG_FILE_MONGODB_AUDIT" + LogFileMongoDBEnv = "MDB_LOG_FILE_MONGODB" + LogFileAgentMonitoringEnv = "MDB_LOG_FILE_MONITORING_AGENT" + LogFileAgentBackupEnv = "MDB_LOG_FILE_BACKUP_AGENT" +) + +type StsType int + +const ( + Undefined StsType = iota + ReplicaSet + Mongos + Config + Shard + Standalone + MultiReplicaSet +) + +// DatabaseStatefulSetOptions contains all the different values that are variable between +// StatefulSets. Depending on which StatefulSet is being built, a number of these will be pre-set, +// while the remainder will be configurable via configuration functions which modify this type. +type DatabaseStatefulSetOptions struct { + Replicas int + Name string + ServiceName string + PodSpec *mdbv1.PodSpecWrapper + PodVars *env.PodEnvVars + CurrentAgentAuthMode string + CertificateHash string + PrometheusTLSCertHash string + InternalClusterHash string + ServicePort int32 + Persistent *bool + OwnerReference []metav1.OwnerReference + AgentConfig *mdbv1.AgentConfig + StatefulSetSpecOverride *appsv1.StatefulSetSpec + StsType StsType + AdditionalMongodConfig *mdbv1.AdditionalMongodConfig + + InitDatabaseNonStaticImage string + DatabaseNonStaticImage string + MongodbImage string + AgentImage string + + Annotations map[string]string + VaultConfig vault.VaultConfiguration + ExtraEnvs []corev1.EnvVar + Labels map[string]string + StsLabels map[string]string + + // These fields are only relevant for multi-cluster + MultiClusterMode bool // should always be "false" in single-cluster + // This needs to be provided for the multi-cluster statefulsets as they contain a member index in the name. + // The name override is used for naming the statefulset and the pod affinity label. + // The certificate secrets and other dependencies named using the resource name will use the `Name` field. + StatefulSetNameOverride string // this needs to be overriden of the + HostNameOverrideConfigmapName string +} + +func (d DatabaseStatefulSetOptions) IsMongos() bool { + return d.StsType == Mongos +} + +func (d DatabaseStatefulSetOptions) GetStatefulSetName() string { + if d.StatefulSetNameOverride != "" { + return d.StatefulSetNameOverride + } + return d.Name +} + +// databaseStatefulSetSource is an interface which provides all the required fields to fully construct +// a database StatefulSet. +type databaseStatefulSetSource interface { + GetName() string + GetNamespace() string + + GetSecurity() *mdbv1.Security + + GetPrometheus() *mdbcv1.Prometheus + + GetAnnotations() map[string]string +} + +// StandaloneOptions returns a set of options which will configure a Standalone StatefulSet +func StandaloneOptions(additionalOpts ...func(options *DatabaseStatefulSetOptions)) func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + return func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + var stsSpec *appsv1.StatefulSetSpec = nil + if mdb.Spec.PodSpec.PodTemplateWrapper.PodTemplate != nil { + stsSpec = &appsv1.StatefulSetSpec{Template: *mdb.Spec.PodSpec.PodTemplateWrapper.PodTemplate} + } + + opts := DatabaseStatefulSetOptions{ + Replicas: 1, + Name: mdb.Name, + ServiceName: mdb.ServiceName(), + PodSpec: NewDefaultPodSpecWrapper(*mdb.Spec.PodSpec), + ServicePort: mdb.Spec.AdditionalMongodConfig.GetPortOrDefault(), + Persistent: mdb.Spec.Persistent, + OwnerReference: kube.BaseOwnerReference(&mdb), + AgentConfig: &mdb.Spec.Agent, + StatefulSetSpecOverride: stsSpec, + MultiClusterMode: mdb.Spec.IsMultiCluster(), + StsType: Standalone, + } + + for _, opt := range additionalOpts { + opt(&opts) + } + + return opts + } +} + +// ReplicaSetOptions returns a set of options which will configure a ReplicaSet StatefulSet +func ReplicaSetOptions(additionalOpts ...func(options *DatabaseStatefulSetOptions)) func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + return func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + var stsSpec *appsv1.StatefulSetSpec = nil + if mdb.Spec.PodSpec.PodTemplateWrapper.PodTemplate != nil { + stsSpec = &appsv1.StatefulSetSpec{Template: *mdb.Spec.PodSpec.PodTemplateWrapper.PodTemplate} + } + + opts := DatabaseStatefulSetOptions{ + Replicas: scale.ReplicasThisReconciliation(&mdb), + Name: mdb.Name, + ServiceName: mdb.ServiceName(), + Annotations: map[string]string{"type": "Replicaset"}, + PodSpec: NewDefaultPodSpecWrapper(*mdb.Spec.PodSpec), + ServicePort: mdb.Spec.AdditionalMongodConfig.GetPortOrDefault(), + Persistent: mdb.Spec.Persistent, + OwnerReference: kube.BaseOwnerReference(&mdb), + AgentConfig: &mdb.Spec.Agent, + StatefulSetSpecOverride: stsSpec, + Labels: mdb.Labels, + MultiClusterMode: mdb.Spec.IsMultiCluster(), + StsType: ReplicaSet, + } + + if mdb.Spec.DbCommonSpec.GetExternalDomain() != nil { + opts.HostNameOverrideConfigmapName = mdb.GetHostNameOverrideConfigmapName() + } + + for _, opt := range additionalOpts { + opt(&opts) + } + + return opts + } +} + +// shardedOptions group the shared logic for creating Shard, Config servers, and mongos options +func shardedOptions(cfg shardedOptionCfg, additionalOpts ...func(options *DatabaseStatefulSetOptions)) DatabaseStatefulSetOptions { + clusterComponentSpec := cfg.componentSpec.GetClusterSpecItem(cfg.memberClusterName) + statefulSetConfiguration := clusterComponentSpec.StatefulSetConfiguration + var statefulSetSpecOverride *appsv1.StatefulSetSpec + if statefulSetConfiguration != nil { + statefulSetSpecOverride = &statefulSetConfiguration.SpecWrapper.Spec + } + + podSpec := mdbv1.MongoDbPodSpec{} + if clusterComponentSpec.PodSpec != nil && clusterComponentSpec.PodSpec.Persistence != nil { + // Here, we explicitly ignore any other fields than Persistence + // Although we still support the PodTemplate field in the Sharded Cluster CRD, when preparing the + // ShardedClusterComponentSpec with functions prepareDesired[...]Configuration in the sharded controller, we + // store anything related to the pod template in the clusterSpecList.StatefulSetConfiguration fields + // The ShardOverrides.ClusterSpecList.PodSpec shouldn't contain anything relevant for the PodTemplate + podSpec = mdbv1.MongoDbPodSpec{Persistence: clusterComponentSpec.PodSpec.Persistence} + } + + opts := DatabaseStatefulSetOptions{ + Name: cfg.rsName, + ServiceName: cfg.serviceName, + PodSpec: NewDefaultPodSpecWrapper(podSpec), + ServicePort: cfg.componentSpec.GetAdditionalMongodConfig().GetPortOrDefault(), + OwnerReference: kube.BaseOwnerReference(&cfg.mdb), + AgentConfig: cfg.componentSpec.GetAgentConfig(), + StatefulSetSpecOverride: statefulSetSpecOverride, + Labels: cfg.mdb.Labels, + MultiClusterMode: cfg.mdb.Spec.IsMultiCluster(), + Persistent: cfg.persistent, + StsType: cfg.stsType, + } + + if cfg.mdb.Spec.IsMultiCluster() { + opts.HostNameOverrideConfigmapName = cfg.mdb.GetHostNameOverrideConfigmapName() + } + for _, opt := range additionalOpts { + opt(&opts) + } + + return opts +} + +type shardedOptionCfg struct { + mdb mdbv1.MongoDB + componentSpec *mdbv1.ShardedClusterComponentSpec + rsName string + serviceName string + memberClusterName string + stsType StsType + persistent *bool +} + +func (c shardedOptionCfg) hasExternalDomain() bool { + return c.mdb.Spec.DbCommonSpec.GetExternalDomain() != nil +} + +// ShardOptions returns a set of options which will configure single Shard StatefulSet +func ShardOptions(shardNum int, shardSpec *mdbv1.ShardedClusterComponentSpec, memberClusterName string, additionalOpts ...func(options *DatabaseStatefulSetOptions)) func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + return func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + cfg := shardedOptionCfg{ + mdb: mdb, + componentSpec: shardSpec, + rsName: mdb.ShardRsName(shardNum), + memberClusterName: memberClusterName, + serviceName: mdb.ShardServiceName(), + stsType: Shard, + persistent: mdb.Spec.Persistent, + } + + return shardedOptions(cfg, additionalOpts...) + } +} + +// ConfigServerOptions returns a set of options which will configure a Config Server StatefulSet +func ConfigServerOptions(configSrvSpec *mdbv1.ShardedClusterComponentSpec, memberClusterName string, additionalOpts ...func(options *DatabaseStatefulSetOptions)) func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + return func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + cfg := shardedOptionCfg{ + mdb: mdb, + componentSpec: configSrvSpec, + rsName: mdb.ConfigRsName(), + memberClusterName: memberClusterName, + serviceName: mdb.ConfigSrvServiceName(), + stsType: Config, + persistent: mdb.Spec.Persistent, + } + + return shardedOptions(cfg, additionalOpts...) + } +} + +// MongosOptions returns a set of options which will configure a Mongos StatefulSet +func MongosOptions(mongosSpec *mdbv1.ShardedClusterComponentSpec, memberClusterName string, additionalOpts ...func(options *DatabaseStatefulSetOptions)) func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + return func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions { + cfg := shardedOptionCfg{ + mdb: mdb, + componentSpec: mongosSpec, + rsName: mdb.MongosRsName(), + memberClusterName: memberClusterName, + serviceName: mdb.ServiceName(), + stsType: Mongos, + persistent: ptr.To(false), + } + + additionalOpts = append(additionalOpts, func(options *DatabaseStatefulSetOptions) { + if !cfg.mdb.Spec.IsMultiCluster() && cfg.hasExternalDomain() { + options.HostNameOverrideConfigmapName = cfg.mdb.GetHostNameOverrideConfigmapName() + } + }) + + return shardedOptions(cfg, additionalOpts...) + } +} + +func DatabaseStatefulSet(mdb mdbv1.MongoDB, stsOptFunc func(mdb mdbv1.MongoDB) DatabaseStatefulSetOptions, log *zap.SugaredLogger) appsv1.StatefulSet { + stsOptions := stsOptFunc(mdb) + dbSts := DatabaseStatefulSetHelper(&mdb, &stsOptions, log) + + if len(stsOptions.Annotations) > 0 { + dbSts.Annotations = merge.StringToStringMap(dbSts.Annotations, stsOptions.Annotations) + } + + if len(stsOptions.Labels) > 0 { + dbSts.Labels = merge.StringToStringMap(dbSts.Labels, stsOptions.Labels) + } + + if len(stsOptions.StsLabels) > 0 { + dbSts.Labels = merge.StringToStringMap(dbSts.Labels, stsOptions.StsLabels) + } + + if stsOptions.StatefulSetSpecOverride != nil { + dbSts.Spec = merge.StatefulSetSpecs(dbSts.Spec, *stsOptions.StatefulSetSpecOverride) + } + + return dbSts +} + +func DatabaseStatefulSetHelper(mdb databaseStatefulSetSource, stsOpts *DatabaseStatefulSetOptions, log *zap.SugaredLogger) appsv1.StatefulSet { + allSources := getAllMongoDBVolumeSources(mdb, *stsOpts, log) + + var extraEnvs []corev1.EnvVar + for _, source := range allSources { + if source.ShouldBeAdded() { + extraEnvs = append(extraEnvs, source.GetEnvs()...) + } + } + + extraEnvs = append(extraEnvs, ReadDatabaseProxyVarsFromEnv()...) + stsOpts.ExtraEnvs = extraEnvs + + templateFunc := buildMongoDBPodTemplateSpec(*stsOpts, mdb) + return statefulset.New(buildDatabaseStatefulSetConfigurationFunction(mdb, templateFunc, *stsOpts, log)) +} + +// buildVaultDatabaseSecretsToInject fully constructs the DatabaseSecretsToInject required to +// convert to annotations to configure vault. +func buildVaultDatabaseSecretsToInject(mdb databaseStatefulSetSource, opts DatabaseStatefulSetOptions) vault.DatabaseSecretsToInject { + secretsToInject := vault.DatabaseSecretsToInject{Config: opts.VaultConfig} + + if mdb.GetSecurity().ShouldUseX509(opts.CurrentAgentAuthMode) || mdb.GetSecurity().ShouldUseClientCertificates() { + secretName := mdb.GetSecurity().AgentClientCertificateSecretName(mdb.GetName()).Name + secretName = fmt.Sprintf("%s%s", secretName, certs.OperatorGeneratedCertSuffix) + secretsToInject.AgentCerts = secretName + } + + if mdb.GetSecurity().GetInternalClusterAuthenticationMode() == util.X509 { + secretName := mdb.GetSecurity().InternalClusterAuthSecretName(opts.Name) + secretName = fmt.Sprintf("%s%s", secretName, certs.OperatorGeneratedCertSuffix) + secretsToInject.InternalClusterAuth = secretName + secretsToInject.InternalClusterHash = opts.InternalClusterHash + } + + // Enable prometheus injection + prom := mdb.GetPrometheus() + if prom != nil && prom.TLSSecretRef.Name != "" { + // Only need to inject Prometheus TLS cert on Vault. Already done for Secret backend. + secretsToInject.Prometheus = fmt.Sprintf("%s%s", prom.TLSSecretRef.Name, certs.OperatorGeneratedCertSuffix) + secretsToInject.PrometheusTLSCertHash = opts.PrometheusTLSCertHash + } + + // add vault specific annotations + secretsToInject.AgentApiKey = agents.ApiKeySecretName(opts.PodVars.ProjectID) + if mdb.GetSecurity().IsTLSEnabled() { + secretName := mdb.GetSecurity().MemberCertificateSecretName(opts.Name) + secretsToInject.MemberClusterAuth = fmt.Sprintf("%s%s", secretName, certs.OperatorGeneratedCertSuffix) + secretsToInject.MemberClusterHash = opts.CertificateHash + } + return secretsToInject +} + +// buildDatabaseStatefulSetConfigurationFunction returns the function that will modify the StatefulSet +func buildDatabaseStatefulSetConfigurationFunction(mdb databaseStatefulSetSource, podTemplateSpecFunc podtemplatespec.Modification, opts DatabaseStatefulSetOptions, log *zap.SugaredLogger) statefulset.Modification { + podLabels := map[string]string{ + appLabelKey: opts.ServiceName, + ControllerLabelName: util.OperatorName, + PodAntiAffinityLabelKey: opts.Name, + } + + configurePodSpecSecurityContext, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + configureImagePullSecrets := podtemplatespec.NOOP() + name, found := env.Read(util.ImagePullSecrets) // nolint:forbidigo + if found { + configureImagePullSecrets = podtemplatespec.WithImagePullSecrets(name) + } + + secretsToInject := buildVaultDatabaseSecretsToInject(mdb, opts) + volumes, volumeMounts := getVolumesAndVolumeMounts(mdb, opts, secretsToInject.AgentCerts, secretsToInject.InternalClusterAuth) + + allSources := getAllMongoDBVolumeSources(mdb, opts, log) + for _, source := range allSources { + if source.ShouldBeAdded() { + volumes = append(volumes, source.GetVolumes()...) + volumeMounts = append(volumeMounts, source.GetVolumeMounts()...) + } + } + + var mounts []corev1.VolumeMount + var pvcFuncs map[string]persistentvolumeclaim.Modification + if opts.Persistent == nil || *opts.Persistent { + pvcFuncs, mounts = buildPersistentVolumeClaimsFuncs(opts) + volumeMounts = append(volumeMounts, mounts...) + } else { + volumes, volumeMounts = GetNonPersistentMongoDBVolumeMounts(volumes, volumeMounts) + } + + volumesFunc := func(spec *corev1.PodTemplateSpec) { + for _, v := range volumes { + podtemplatespec.WithVolume(v)(spec) + } + } + + keys := make([]string, 0, len(pvcFuncs)) + for k := range pvcFuncs { + keys = append(keys, k) + } + + // ensure consistent order of PVCs + sort.Strings(keys) + + volumeClaimFuncs := func(sts *appsv1.StatefulSet) { + for _, name := range keys { + statefulset.WithVolumeClaim(name, pvcFuncs[name])(sts) + } + } + + ssLabels := map[string]string{ + appLabelKey: opts.ServiceName, + } + + annotationFunc := statefulset.WithAnnotations(defaultPodAnnotations(opts.CertificateHash)) + podTemplateAnnotationFunc := podtemplatespec.NOOP() + + annotationFunc = statefulset.Apply( + annotationFunc, + statefulset.WithAnnotations(map[string]string{util.InternalCertAnnotationKey: opts.InternalClusterHash}), + ) + + if vault.IsVaultSecretBackend() { + podTemplateAnnotationFunc = podtemplatespec.Apply(podTemplateAnnotationFunc, podtemplatespec.WithAnnotations(secretsToInject.DatabaseAnnotations(mdb.GetNamespace()))) + } + + stsName := opts.GetStatefulSetName() + podAffinity := mdb.GetName() + if opts.StatefulSetNameOverride != "" { + stsName = opts.StatefulSetNameOverride + podAffinity = opts.StatefulSetNameOverride + } + + shareProcessNs := statefulset.NOOP() + secondContainerModification := podtemplatespec.NOOP() + + if architectures.IsRunningStaticArchitecture(mdb.GetAnnotations()) { + shareProcessNs = func(sts *appsv1.StatefulSet) { + a := true + sts.Spec.Template.Spec.ShareProcessNamespace = &a + } + secondContainerModification = podtemplatespec.WithContainerByIndex(1, container.WithVolumeMounts(volumeMounts)) + } + + var databaseImage string + if architectures.IsRunningStaticArchitecture(mdb.GetAnnotations()) { + databaseImage = opts.AgentImage + } else { + databaseImage = opts.DatabaseNonStaticImage + } + + return statefulset.Apply( + statefulset.WithLabels(ssLabels), + statefulset.WithName(stsName), + statefulset.WithNamespace(mdb.GetNamespace()), + statefulset.WithMatchLabels(podLabels), + statefulset.WithServiceName(opts.ServiceName), + statefulset.WithReplicas(opts.Replicas), + statefulset.WithOwnerReference(opts.OwnerReference), + annotationFunc, + volumeClaimFuncs, + shareProcessNs, + statefulset.WithPodSpecTemplate(podtemplatespec.Apply( + podTemplateAnnotationFunc, + podtemplatespec.WithAffinity(podAffinity, PodAntiAffinityLabelKey, 100), + podtemplatespec.WithTerminationGracePeriodSeconds(util.DefaultPodTerminationPeriodSeconds), + podtemplatespec.WithPodLabels(podLabels), + podtemplatespec.WithContainerByIndex(0, sharedDatabaseContainerFunc(databaseImage, *opts.PodSpec, volumeMounts, configureContainerSecurityContext, opts.ServicePort)), + secondContainerModification, + volumesFunc, + configurePodSpecSecurityContext, + configureImagePullSecrets, + podTemplateSpecFunc, + )), + ) +} + +func buildPersistentVolumeClaimsFuncs(opts DatabaseStatefulSetOptions) (map[string]persistentvolumeclaim.Modification, []corev1.VolumeMount) { + var claims map[string]persistentvolumeclaim.Modification + var mounts []corev1.VolumeMount + + podSpec := opts.PodSpec + // if persistence not set or if single one is set + if podSpec.Persistence == nil || + (podSpec.Persistence.SingleConfig == nil && podSpec.Persistence.MultipleConfig == nil) || + podSpec.Persistence.SingleConfig != nil { + var config *mdbv1.PersistenceConfig + if podSpec.Persistence != nil && podSpec.Persistence.SingleConfig != nil { + config = podSpec.Persistence.SingleConfig + } + // Single claim, multiple mounts using this claim. Note, that we use "subpaths" in the volume to mount to different + // physical folders + claims, mounts = createClaimsAndMountsSingleModeFunc(config, opts) + } else if podSpec.Persistence.MultipleConfig != nil { + defaultConfig := *podSpec.Default.Persistence.MultipleConfig + + // Multiple claims, multiple mounts. No subpaths are used and everything is mounted to the root of directory + claims, mounts = createClaimsAndMountsMultiModeFunc(opts.PodSpec.Persistence, defaultConfig, opts.Labels) + } + return claims, mounts +} + +func sharedDatabaseContainerFunc(databaseImage string, podSpecWrapper mdbv1.PodSpecWrapper, volumeMounts []corev1.VolumeMount, configureContainerSecurityContext container.Modification, port int32) container.Modification { + return container.Apply( + container.WithResourceRequirements(buildRequirementsFromPodSpec(podSpecWrapper)), + container.WithPorts([]corev1.ContainerPort{{ContainerPort: port}}), + container.WithImagePullPolicy(corev1.PullPolicy(env.ReadOrPanic(util.AutomationAgentImagePullPolicy))), // nolint:forbidigo + container.WithVolumeMounts(volumeMounts), + container.WithImage(databaseImage), + container.WithLivenessProbe(DatabaseLivenessProbe()), + container.WithReadinessProbe(DatabaseReadinessProbe()), + container.WithStartupProbe(DatabaseStartupProbe()), + configureContainerSecurityContext, + ) +} + +// getTLSPrometheusVolumeAndVolumeMount mounts Prometheus TLS Volumes from Secrets. +// +// These volumes will only be mounted when TLS has been enabled. They will +// contain the concatenated (PEM-format) certificates; which have been created +// by the Operator prior to this. +// +// Important: This function will not return Secret mounts in case of Vault backend! +// +// The Prometheus TLS Secret name is configured in: +// +// `spec.prometheus.tlsSecretRef.Name` +// +// The Secret will be mounted in: +// `/var/lib/mongodb-automation/secrets/prometheus`. +func getTLSPrometheusVolumeAndVolumeMount(prom *mdbcv1.Prometheus) ([]corev1.Volume, []corev1.VolumeMount) { + volumes := []corev1.Volume{} + volumeMounts := []corev1.VolumeMount{} + + if prom == nil || vault.IsVaultSecretBackend() { + return volumes, volumeMounts + } + + secretFunc := func(v *corev1.Volume) { v.Secret.Optional = util.BooleanRef(true) } + // Name of the Secret (PEM-format) with the concatenation of the certificate and key. + secretName := prom.TLSSecretRef.Name + certs.OperatorGeneratedCertSuffix + + secretVolume := statefulset.CreateVolumeFromSecret(util.PrometheusSecretVolumeName, secretName, secretFunc) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: util.SecretVolumeMountPathPrometheus, + Name: secretVolume.Name, + }) + volumes = append(volumes, secretVolume) + + return volumes, volumeMounts +} + +// getAllMongoDBVolumeSources returns a slice of MongoDBVolumeSource. These are used to determine which volumes +// and volume mounts should be added to the StatefulSet. +func getAllMongoDBVolumeSources(mdb databaseStatefulSetSource, databaseOpts DatabaseStatefulSetOptions, log *zap.SugaredLogger) []MongoDBVolumeSource { + caVolume := &caVolumeSource{ + opts: databaseOpts, + logger: log, + } + tlsVolume := &tlsVolumeSource{ + security: mdb.GetSecurity(), + databaseOpts: databaseOpts, + logger: log, + } + + var allVolumeSources []MongoDBVolumeSource + allVolumeSources = append(allVolumeSources, caVolume) + allVolumeSources = append(allVolumeSources, tlsVolume) + + return allVolumeSources +} + +// getVolumesAndVolumeMounts returns all volumes and mounts required for the StatefulSet. +func getVolumesAndVolumeMounts(mdb databaseStatefulSetSource, databaseOpts DatabaseStatefulSetOptions, agentCertsSecretName, internalClusterAuthSecretName string) ([]corev1.Volume, []corev1.VolumeMount) { + var volumesToAdd []corev1.Volume + var volumeMounts []corev1.VolumeMount + + prometheusVolumes, prometheusVolumeMounts := getTLSPrometheusVolumeAndVolumeMount(mdb.GetPrometheus()) + + volumesToAdd = append(volumesToAdd, prometheusVolumes...) + volumeMounts = append(volumeMounts, prometheusVolumeMounts...) + + if !vault.IsVaultSecretBackend() && mdb.GetSecurity().ShouldUseX509(databaseOpts.CurrentAgentAuthMode) || mdb.GetSecurity().ShouldUseClientCertificates() { + agentSecretVolume := statefulset.CreateVolumeFromSecret(util.AgentSecretName, agentCertsSecretName) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: agentCertMountPath, + Name: agentSecretVolume.Name, + ReadOnly: true, + }) + volumesToAdd = append(volumesToAdd, agentSecretVolume) + } + + // add volume for x509 cert used in internal cluster authentication + if !vault.IsVaultSecretBackend() && mdb.GetSecurity().GetInternalClusterAuthenticationMode() == util.X509 { + internalClusterAuthVolume := statefulset.CreateVolumeFromSecret(util.ClusterFileName, internalClusterAuthSecretName) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: util.InternalClusterAuthMountPath, + Name: internalClusterAuthVolume.Name, + ReadOnly: true, + }) + volumesToAdd = append(volumesToAdd, internalClusterAuthVolume) + } + + if !vault.IsVaultSecretBackend() { + volumesToAdd = append(volumesToAdd, statefulset.CreateVolumeFromSecret(AgentAPIKeyVolumeName, agents.ApiKeySecretName(databaseOpts.PodVars.ProjectID))) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(AgentAPIKeyVolumeName, AgentAPIKeySecretPath)) + } + + volumesToAdd, volumeMounts = GetNonPersistentAgentVolumeMounts(volumesToAdd, volumeMounts) + + return volumesToAdd, volumeMounts +} + +// buildMongoDBPodTemplateSpec constructs the podTemplateSpec for the MongoDB resource +func buildMongoDBPodTemplateSpec(opts DatabaseStatefulSetOptions, mdb databaseStatefulSetSource) podtemplatespec.Modification { + // scripts volume is shared by the init container and the AppDB, so the startup + // script can be copied over + scriptsVolume := statefulset.CreateVolumeFromEmptyDir("database-scripts") + databaseScriptsVolumeMount := databaseScriptsVolumeMount(true) + + volumes := []corev1.Volume{scriptsVolume} + volumeMounts := []corev1.VolumeMount{databaseScriptsVolumeMount} + + initContainerModifications := []func(*corev1.Container){buildDatabaseInitContainer(opts.InitDatabaseNonStaticImage)} + databaseContainerModifications := []func(*corev1.Container){container.Apply( + container.WithName(util.DatabaseContainerName), + container.WithImage(opts.DatabaseNonStaticImage), + container.WithEnvs(databaseEnvVars(opts)...), + container.WithCommand([]string{"/opt/scripts/agent-launcher.sh"}), + container.WithVolumeMounts(volumeMounts), + )} + + _, containerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + staticContainerMongodContainerModification := podtemplatespec.NOOP() + if architectures.IsRunningStaticArchitecture(mdb.GetAnnotations()) { + // we don't use initContainers therefore, we reset it here + initContainerModifications = []func(*corev1.Container){} + mongodModification := []func(*corev1.Container){container.Apply( + container.WithName(util.DatabaseContainerName), + container.WithArgs([]string{""}), + container.WithImage(opts.MongodbImage), + container.WithEnvs(databaseEnvVars(opts)...), + container.WithCommand([]string{"bash", "-c", "tail -F -n0 ${MDB_LOG_FILE_MONGODB} mongodb_marker"}), + containerSecurityContext, + )} + staticContainerMongodContainerModification = podtemplatespec.WithContainerByIndex(1, mongodModification...) + + // We are not setting the database-scripts volume on purpose, + // since we don't need to copy things from the init container over. + databaseContainerModifications = []func(*corev1.Container){container.Apply( + container.WithName(util.AgentContainerName), + container.WithImage(opts.AgentImage), + container.WithEnvs(databaseEnvVars(opts)...), + containerSecurityContext, + )} + } + + if opts.HostNameOverrideConfigmapName != "" { + volumes = append(volumes, statefulset.CreateVolumeFromConfigMap(opts.HostNameOverrideConfigmapName, opts.HostNameOverrideConfigmapName)) + modification := container.WithVolumeMounts([]corev1.VolumeMount{ + { + Name: opts.HostNameOverrideConfigmapName, + MountPath: "/opt/scripts/config", + }, + }) + + // we only need to add the volume modification if we actually use an init container + if len(initContainerModifications) > 0 { + initContainerModifications = append(initContainerModifications, modification) + } + + databaseContainerModifications = append(databaseContainerModifications, modification) + } + + serviceAccountName := getServiceAccountName(opts) + + mods := []podtemplatespec.Modification{ + sharedDatabaseConfiguration(opts, mdb), + podtemplatespec.WithServiceAccount(util.MongoDBServiceAccount), + podtemplatespec.WithServiceAccount(serviceAccountName), + podtemplatespec.WithVolumes(volumes), + podtemplatespec.WithContainerByIndex(0, databaseContainerModifications...), + staticContainerMongodContainerModification, + } + + if len(initContainerModifications) > 0 { + mods = append(mods, podtemplatespec.WithInitContainerByIndex(0, initContainerModifications...)) + } + + return podtemplatespec.Apply(mods...) +} + +// getServiceAccountName returns the serviceAccount to be used by the mongoDB pod, +// it uses the "serviceAccountName" specified in the podSpec of CR, if it's not specified returns +// the default serviceAccount name +func getServiceAccountName(opts DatabaseStatefulSetOptions) string { + podSpec := opts.PodSpec + + if podSpec != nil && podSpec.PodTemplateWrapper.PodTemplate != nil { + if podSpec.PodTemplateWrapper.PodTemplate.Spec.ServiceAccountName != "" { + return podSpec.PodTemplateWrapper.PodTemplate.Spec.ServiceAccountName + } + } + + return util.MongoDBServiceAccount +} + +// sharedDatabaseConfiguration is a function which applies all the shared configuration +// between the appDb and MongoDB resources +func sharedDatabaseConfiguration(opts DatabaseStatefulSetOptions, mdb databaseStatefulSetSource) podtemplatespec.Modification { + configurePodSpecSecurityContext, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + pullSecretsConfigurationFunc := podtemplatespec.NOOP() + if pullSecrets, ok := env.Read(util.ImagePullSecrets); ok { // nolint:forbidigo + pullSecretsConfigurationFunc = podtemplatespec.WithImagePullSecrets(pullSecrets) + } + + agentModification := podtemplatespec.WithContainerByIndex(0, + container.Apply( + container.WithResourceRequirements(buildRequirementsFromPodSpec(*opts.PodSpec)), + container.WithPorts([]corev1.ContainerPort{{ContainerPort: opts.ServicePort}}), + container.WithImagePullPolicy(corev1.PullPolicy(env.ReadOrPanic(util.AutomationAgentImagePullPolicy))), // nolint:forbidigo + container.WithLivenessProbe(DatabaseLivenessProbe()), + container.WithEnvs(startupParametersToAgentFlag(opts.AgentConfig.StartupParameters)), + container.WithEnvs(logConfigurationToEnvVars(opts.AgentConfig.StartupParameters, opts.AdditionalMongodConfig)...), + container.WithEnvs(readinessEnvironmentVariablesToEnvVars(opts.AgentConfig.ReadinessProbe.EnvironmentVariables)...), + configureContainerSecurityContext, + ), + ) + + staticMongodModification := podtemplatespec.NOOP() + if architectures.IsRunningStaticArchitecture(mdb.GetAnnotations()) { + // The mongod + staticMongodModification = podtemplatespec.WithContainerByIndex(1, + container.Apply( + container.WithArgs([]string{"tail -F -n0 \"${MDB_LOG_FILE_MONGODB}\""}), + container.WithResourceRequirements(buildRequirementsFromPodSpec(*opts.PodSpec)), + container.WithPorts([]corev1.ContainerPort{{ContainerPort: opts.ServicePort}}), + container.WithImagePullPolicy(corev1.PullPolicy(env.ReadOrPanic(util.AutomationAgentImagePullPolicy))), // nolint:forbidigo + container.WithEnvs(startupParametersToAgentFlag(opts.AgentConfig.StartupParameters)), + container.WithEnvs(logConfigurationToEnvVars(opts.AgentConfig.StartupParameters, opts.AdditionalMongodConfig)...), + configureContainerSecurityContext, + ), + ) + agentModification = podtemplatespec.WithContainerByIndex(0, + container.Apply( + container.WithImagePullPolicy(corev1.PullPolicy(env.ReadOrPanic(util.AutomationAgentImagePullPolicy))), // nolint:forbidigo + container.WithLivenessProbe(DatabaseLivenessProbe()), + container.WithEnvs(startupParametersToAgentFlag(opts.AgentConfig.StartupParameters)), + container.WithEnvs(logConfigurationToEnvVars(opts.AgentConfig.StartupParameters, opts.AdditionalMongodConfig)...), + container.WithEnvs(staticContainersEnvVars(mdb)...), + container.WithEnvs(readinessEnvironmentVariablesToEnvVars(opts.AgentConfig.ReadinessProbe.EnvironmentVariables)...), + container.WithArgs([]string{}), + container.WithCommand([]string{"/opt/scripts/agent-launcher.sh"}), + configureContainerSecurityContext, + ), + ) + } + + return podtemplatespec.Apply( + podtemplatespec.WithPodLabels(defaultPodLabels(opts.ServiceName, opts.Name)), + podtemplatespec.WithTerminationGracePeriodSeconds(util.DefaultPodTerminationPeriodSeconds), + pullSecretsConfigurationFunc, + configurePodSpecSecurityContext, + podtemplatespec.WithAffinity(opts.Name, PodAntiAffinityLabelKey, 100), + podtemplatespec.WithTopologyKey(opts.PodSpec.GetTopologyKeyOrDefault(), 0), + // The Agent + agentModification, + // AgentLoggingMongodConfig if static container + staticMongodModification, + ) +} + +// StartupParametersToAgentFlag takes a map representing key-value pairs +// of startup parameters +// and concatenates them into a single string that is then +// returned as env variable AGENT_FLAGS +func startupParametersToAgentFlag(parameters mdbv1.StartupParameters) corev1.EnvVar { + agentParams := "" + finalParameters := mdbv1.StartupParameters{} + // add default parameters if not already set + for key, value := range defaultAgentParameters() { + if _, ok := parameters[key]; !ok { + // add the default parameter + finalParameters[key] = value + } + } + for key, value := range parameters { + finalParameters[key] = value + } + // sort the parameters by key + keys := make([]string, 0, len(finalParameters)) + for k := range finalParameters { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + // Using comma as delimiter to split the string later + // in the agentlauncher script + agentParams += "-" + key + "=" + finalParameters[key] + "," + } + + return corev1.EnvVar{Name: "AGENT_FLAGS", Value: agentParams} +} + +// readinessEnvironmentVariablesToEnvVars returns the environment variables to bet set in the readinessProbe container +func readinessEnvironmentVariablesToEnvVars(parameters mdbv1.EnvironmentVariables) []corev1.EnvVar { + var finalParameters []corev1.EnvVar + for key, value := range parameters { + finalParameters = append(finalParameters, corev1.EnvVar{ + Name: key, + Value: value, + }) + } + sort.SliceStable(finalParameters, func(i, j int) bool { + return finalParameters[i].Name > finalParameters[j].Name + }) + + return finalParameters +} + +func defaultAgentParameters() mdbv1.StartupParameters { + return map[string]string{"logFile": path.Join(util.PvcMountPathLogs, "automation-agent.log")} +} + +func logConfigurationToEnvVars(parameters mdbv1.StartupParameters, additionalMongodConfig *mdbv1.AdditionalMongodConfig) []corev1.EnvVar { + var envVars []corev1.EnvVar + envVars = append(envVars, getAutomationLogEnvVars(parameters)...) + envVars = append(envVars, getAuditLogEnvVar(additionalMongodConfig)) + + // the following are hardcoded log files where we don't support changing the names + envVars = append(envVars, corev1.EnvVar{Name: LogFileMongoDBEnv, Value: path.Join(util.PvcMountPathLogs, "mongodb.log")}) + envVars = append(envVars, corev1.EnvVar{Name: LogFileAgentMonitoringEnv, Value: path.Join(util.PvcMountPathLogs, "monitoring-agent.log")}) + envVars = append(envVars, corev1.EnvVar{Name: LogFileAgentBackupEnv, Value: path.Join(util.PvcMountPathLogs, "backup-agent.log")}) + + return envVars +} + +func staticContainersEnvVars(mdb databaseStatefulSetSource) []corev1.EnvVar { + var envVars []corev1.EnvVar + if architectures.IsRunningStaticArchitecture(mdb.GetAnnotations()) { + envVars = append(envVars, corev1.EnvVar{Name: "MDB_STATIC_CONTAINERS_ARCHITECTURE", Value: "true"}) + } + return envVars +} + +func getAuditLogEnvVar(additionalMongodConfig *mdbv1.AdditionalMongodConfig) corev1.EnvVar { + auditLogFile := path.Join(util.PvcMountPathLogs, "mongodb-audit.log") + if additionalMongodConfig != nil { + if auditLogMap := maputil.ReadMapValueAsMap(additionalMongodConfig.ToMap(), "auditLog"); auditLogMap != nil { + auditLogDestination := maputil.ReadMapValueAsString(auditLogMap, "destination") + auditLogFilePath := maputil.ReadMapValueAsString(auditLogMap, "path") + if auditLogDestination == "file" && len(auditLogFile) > 0 { + auditLogFile = auditLogFilePath + } + } + } + + return corev1.EnvVar{Name: LogFileMongoDBAuditEnv, Value: auditLogFile} +} + +func getAutomationLogEnvVars(parameters mdbv1.StartupParameters) []corev1.EnvVar { + automationLogFile := path.Join(util.PvcMountPathLogs, "automation-agent.log") + if logFileValue, ok := parameters["logFile"]; ok && len(logFileValue) > 0 { + automationLogFile = logFileValue + } + + logFileDir, logFileName := path.Split(automationLogFile) + logFileExt := path.Ext(logFileName) + logFileWithoutExt := logFileName[0 : len(logFileName)-len(logFileExt)] + + verboseLogFile := fmt.Sprintf("%s%s-verbose%s", logFileDir, logFileWithoutExt, logFileExt) + stderrLogFile := fmt.Sprintf("%s%s-stderr%s", logFileDir, logFileWithoutExt, logFileExt) + return []corev1.EnvVar{ + {Name: LogFileAutomationAgentVerboseEnv, Value: verboseLogFile}, + {Name: LogFileAutomationAgentStderrEnv, Value: stderrLogFile}, + {Name: LogFileAutomationAgentEnv, Value: automationLogFile}, + } +} + +// databaseScriptsVolumeMount constructs the VolumeMount for the Database scripts +// this should be readonly for the Database, and not read only for the init container. +func databaseScriptsVolumeMount(readOnly bool) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: PvcNameDatabaseScripts, + MountPath: PvcMountPathScripts, + ReadOnly: readOnly, + } +} + +// buildDatabaseInitContainer builds the container specification for mongodb-enterprise-init-database image +func buildDatabaseInitContainer(initDatabaseImage string) container.Modification { + _, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + return container.Apply( + container.WithName(InitDatabaseContainerName), + container.WithImage(initDatabaseImage), + container.WithVolumeMounts([]corev1.VolumeMount{ + databaseScriptsVolumeMount(false), + }), + configureContainerSecurityContext, + ) +} + +func databaseEnvVars(opts DatabaseStatefulSetOptions) []corev1.EnvVar { + podVars := opts.PodVars + if podVars == nil { + return []corev1.EnvVar{} + } + vars := []corev1.EnvVar{ + { + Name: util.EnvVarLogLevel, + Value: podVars.LogLevel, + }, + { + Name: util.EnvVarBaseUrl, + Value: podVars.BaseURL, + }, + { + Name: util.EnvVarProjectId, + Value: podVars.ProjectID, + }, + { + Name: util.EnvVarUser, + Value: podVars.User, + }, + { + Name: util.EnvVarMultiClusterMode, + Value: fmt.Sprintf("%t", opts.MultiClusterMode), + }, + } + + if opts.PodVars.SSLRequireValidMMSServerCertificates { + vars = append(vars, + corev1.EnvVar{ + Name: util.EnvVarSSLRequireValidMMSCertificates, + Value: strconv.FormatBool(opts.PodVars.SSLRequireValidMMSServerCertificates), + }, + ) + } + + // This is only used for debugging + if useDebugAgent := os.Getenv(util.EnvVarDebug); useDebugAgent != "" { // nolint:forbidigo + zap.S().Debugf("running the agent in debug mode") + vars = append(vars, corev1.EnvVar{Name: util.EnvVarDebug, Value: useDebugAgent}) + } + + // This is only used for debugging + if agentVersion := os.Getenv(util.EnvVarAgentVersion); agentVersion != "" { // nolint:forbidigo + zap.S().Debugf("using a custom agent version: %s", agentVersion) + vars = append(vars, corev1.EnvVar{Name: util.EnvVarAgentVersion, Value: agentVersion}) + } + + // append any additional env vars specified. + vars = append(vars, opts.ExtraEnvs...) + + return vars +} + +func DatabaseLivenessProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{databaseLivenessProbeCommand}), + probes.WithInitialDelaySeconds(10), + probes.WithTimeoutSeconds(30), + probes.WithPeriodSeconds(30), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(6), + ) +} + +func DatabaseReadinessProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{databaseReadinessProbeCommand}), + probes.WithFailureThreshold(4), + probes.WithInitialDelaySeconds(5), + probes.WithPeriodSeconds(5), + ) +} + +func DatabaseStartupProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{databaseLivenessProbeCommand}), + probes.WithInitialDelaySeconds(1), + probes.WithTimeoutSeconds(30), + probes.WithPeriodSeconds(20), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(10), + ) +} + +func defaultPodAnnotations(certHash string) map[string]string { + return map[string]string{ + // This annotation is necessary to trigger a pod restart + // if the certificate secret is out of date. This happens if + // existing certificates have been replaced/rotated/renewed. + certs.CertHashAnnotationKey: certHash, + } +} + +// TODO: temprorary duplication to avoid circular imports +func NewDefaultPodSpecWrapper(podSpec mdbv1.MongoDbPodSpec) *mdbv1.PodSpecWrapper { + return &mdbv1.PodSpecWrapper{ + MongoDbPodSpec: podSpec, + Default: newDefaultPodSpec(), + } +} + +func newDefaultPodSpec() mdbv1.MongoDbPodSpec { + podSpecWrapper := mdbv1.NewEmptyPodSpecWrapperBuilder(). + SetPodAntiAffinityTopologyKey(util.DefaultAntiAffinityTopologyKey). + SetSinglePersistence(mdbv1.NewPersistenceBuilder(util.DefaultMongodStorageSize)). + SetMultiplePersistence(mdbv1.NewPersistenceBuilder(util.DefaultMongodStorageSize), + mdbv1.NewPersistenceBuilder(util.DefaultJournalStorageSize), + mdbv1.NewPersistenceBuilder(util.DefaultLogsStorageSize)). + Build() + + return podSpecWrapper.MongoDbPodSpec +} + +// GetNonPersistentMongoDBVolumeMounts returns two arrays of non-persistent, empty volumes and corresponding mounts for the database container. +func GetNonPersistentMongoDBVolumeMounts(volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) ([]corev1.Volume, []corev1.VolumeMount) { + volumes = append(volumes, statefulset.CreateVolumeFromEmptyDir(util.PvcNameData)) + + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathData, statefulset.WithSubPath(util.PvcNameData))) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathJournal, statefulset.WithSubPath(util.PvcNameJournal))) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathLogs, statefulset.WithSubPath(util.PvcNameLogs))) + + return volumes, volumeMounts +} + +// GetNonPersistentAgentVolumeMounts returns two arrays of non-persistent, empty volumes and corresponding mounts for the Agent container. +func GetNonPersistentAgentVolumeMounts(volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) ([]corev1.Volume, []corev1.VolumeMount) { + volumes = append(volumes, statefulset.CreateVolumeFromEmptyDir(util.PvMms)) + + // The agent reads and writes into its own directory. It also contains a subdirectory called downloads. + // This one is published by the Dockerfile + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.PvMms, util.PvcMmsMountPath, statefulset.WithSubPath(util.PvcMms))) + + // Runtime data for MMS + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.PvMms, util.PvcMmsHomeMountPath, statefulset.WithSubPath(util.PvcMmsHome))) + + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.PvMms, util.PvcMountPathTmp, statefulset.WithSubPath(util.PvcNameTmp))) + return volumes, volumeMounts +} diff --git a/controllers/operator/construct/database_construction_test.go b/controllers/operator/construct/database_construction_test.go new file mode 100644 index 000000000..e947e263e --- /dev/null +++ b/controllers/operator/construct/database_construction_test.go @@ -0,0 +1,390 @@ +package construct + +import ( + "path" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/utils/ptr" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) + mock.InitDefaultEnvVariables() +} + +func Test_buildDatabaseInitContainer(t *testing.T) { + modification := buildDatabaseInitContainer("quay.io/mongodb/mongodb-enterprise-init-database:latest") + container := &corev1.Container{} + modification(container) + expectedVolumeMounts := []corev1.VolumeMount{{ + Name: PvcNameDatabaseScripts, + MountPath: PvcMountPathScripts, + ReadOnly: false, + }} + expectedContainer := &corev1.Container{ + Name: InitDatabaseContainerName, + Image: "quay.io/mongodb/mongodb-enterprise-init-database:latest", + VolumeMounts: expectedVolumeMounts, + SecurityContext: &corev1.SecurityContext{ + ReadOnlyRootFilesystem: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + }, + } + assert.Equal(t, expectedContainer, container) +} + +func createShardSpecAndDefaultCluster(client kubernetesClient.Client, sc *mdbv1.MongoDB) (*mdbv1.ShardedClusterComponentSpec, multicluster.MemberCluster) { + shardSpec := sc.Spec.ShardSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec, multicluster.GetLegacyCentralMemberCluster(sc.Spec.MongodsPerShardCount, 0, client, secrets.SecretClient{KubeClient: client}) +} + +func createConfigSrvSpec(sc *mdbv1.MongoDB) *mdbv1.ShardedClusterComponentSpec { + shardSpec := sc.Spec.ConfigSrvSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec +} + +func createMongosSpec(sc *mdbv1.MongoDB) *mdbv1.ShardedClusterComponentSpec { + shardSpec := sc.Spec.ConfigSrvSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec +} + +func TestStatefulsetCreationPanicsIfEnvVariablesAreNotSet(t *testing.T) { + t.Run("Empty Image Pull Policy", func(t *testing.T) { + t.Setenv(util.AutomationAgentImagePullPolicy, "") + sc := mdbv1.NewClusterBuilder().Build() + + kubeClient, _ := mock.NewDefaultFakeClient(sc) + shardSpec, memberCluster := createShardSpecAndDefaultCluster(kubeClient, sc) + configServerSpec := createConfigSrvSpec(sc) + mongosSpec := createMongosSpec(sc) + + assert.Panics(t, func() { + DatabaseStatefulSet(*sc, ShardOptions(0, shardSpec, memberCluster.Name), zap.S()) + }) + assert.Panics(t, func() { + DatabaseStatefulSet(*sc, ConfigServerOptions(configServerSpec, memberCluster.Name), zap.S()) + }) + assert.Panics(t, func() { + DatabaseStatefulSet(*sc, MongosOptions(mongosSpec, memberCluster.Name), zap.S()) + }) + }) +} + +func TestStatefulsetCreationPanicsIfEnvVariablesAreNotSetStatic(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + t.Run("Empty Image Pull Policy", func(t *testing.T) { + t.Setenv(util.AutomationAgentImagePullPolicy, "") + sc := mdbv1.NewClusterBuilder().Build() + kubeClient, _ := mock.NewDefaultFakeClient(sc) + shardSpec, memberCluster := createShardSpecAndDefaultCluster(kubeClient, sc) + configServerSpec := createConfigSrvSpec(sc) + mongosSpec := createMongosSpec(sc) + assert.Panics(t, func() { + DatabaseStatefulSet(*sc, ShardOptions(0, shardSpec, memberCluster.Name), zap.S()) + }) + assert.Panics(t, func() { + DatabaseStatefulSet(*sc, ConfigServerOptions(configServerSpec, memberCluster.Name), zap.S()) + }) + assert.Panics(t, func() { + DatabaseStatefulSet(*sc, MongosOptions(mongosSpec, memberCluster.Name), zap.S()) + }) + }) +} + +func TestStatefulsetCreationSuccessful(t *testing.T) { + start := time.Now() + rs := mdbv1.NewReplicaSetBuilder().Build() + + _ = DatabaseStatefulSet(*rs, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + assert.True(t, time.Since(start) < time.Second*4) // we waited only a little (considering 2 seconds of wait as well) +} + +func TestDatabaseEnvVars(t *testing.T) { + envVars := defaultPodVars() + opts := DatabaseStatefulSetOptions{PodVars: envVars} + podEnv := databaseEnvVars(opts) + assert.Len(t, podEnv, 5) + + envVars = defaultPodVars() + envVars.SSLRequireValidMMSServerCertificates = true + opts = DatabaseStatefulSetOptions{PodVars: envVars} + + podEnv = databaseEnvVars(opts) + assert.Len(t, podEnv, 6) + assert.Equal(t, podEnv[5], corev1.EnvVar{ + Name: util.EnvVarSSLRequireValidMMSCertificates, + Value: "true", + }) + + envVars = defaultPodVars() + envVars.SSLMMSCAConfigMap = "custom-ca" + v := &caVolumeSource{} + extraEnvs := v.GetEnvs() + + opts = DatabaseStatefulSetOptions{PodVars: envVars, ExtraEnvs: extraEnvs} + trustedCACertLocation := path.Join(caCertMountPath, util.CaCertMMS) + podEnv = databaseEnvVars(opts) + assert.Len(t, podEnv, 6) + assert.Equal(t, podEnv[5], corev1.EnvVar{ + Name: util.EnvVarSSLTrustedMMSServerCertificate, + Value: trustedCACertLocation, + }) + + envVars = defaultPodVars() + envVars.SSLRequireValidMMSServerCertificates = true + envVars.SSLMMSCAConfigMap = "custom-ca" + opts = DatabaseStatefulSetOptions{PodVars: envVars, ExtraEnvs: extraEnvs} + podEnv = databaseEnvVars(opts) + assert.Len(t, podEnv, 7) + assert.Equal(t, podEnv[6], corev1.EnvVar{ + Name: util.EnvVarSSLTrustedMMSServerCertificate, + Value: trustedCACertLocation, + }) + assert.Equal(t, podEnv[5], corev1.EnvVar{ + Name: util.EnvVarSSLRequireValidMMSCertificates, + Value: "true", + }) +} + +func TestAgentFlags(t *testing.T) { + agentStartupParameters := mdbv1.StartupParameters{ + "key1": "Value1", + "key3": "Value3", + "message": "Hello", + "key2": "Value2", + // logFile is a default agent variable which we override for illustration in this test + "logFile": "/etc/agent.log", + } + + mdb := mdbv1.NewReplicaSetBuilder().SetAgentConfig(mdbv1.AgentConfig{StartupParameters: agentStartupParameters}).Build() + sts := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + variablesMap := env.ToMap(sts.Spec.Template.Spec.Containers[0].Env...) + val, ok := variablesMap["AGENT_FLAGS"] + assert.True(t, ok) + // AGENT_FLAGS environment variable is sorted + assert.Equal(t, val, "-key1=Value1,-key2=Value2,-key3=Value3,-logFile=/etc/agent.log,-message=Hello,") +} + +func TestLabelsAndAnotations(t *testing.T) { + labels := map[string]string{"l1": "val1", "l2": "val2"} + annotations := map[string]string{"a1": "val1", "a2": "val2"} + + mdb := mdbv1.NewReplicaSetBuilder().SetAnnotations(annotations).SetLabels(labels).Build() + sts := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + // add the default label to the map + labels["app"] = "test-mdb-svc" + assert.Equal(t, labels, sts.Labels) +} + +func TestLogConfigurationToEnvVars(t *testing.T) { + var parameters mdbv1.StartupParameters = map[string]string{ + "a": "1", + "logFile": "/var/log/mongodb-mms-automation/log.file", + } + additionalMongodConfig := mdbv1.NewEmptyAdditionalMongodConfig() + additionalMongodConfig.AddOption("auditLog", map[string]interface{}{ + "destination": "file", + "format": "JSON", + "path": "/var/log/mongodb-mms-automation/audit.log", + }) + + envVars := logConfigurationToEnvVars(parameters, additionalMongodConfig) + assert.Len(t, envVars, 7) + + logFileAutomationAgentEnvVar := corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: path.Join(util.PvcMountPathLogs, "log.file")} + logFileAutomationAgentVerboseEnvVar := corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: path.Join(util.PvcMountPathLogs, "log-verbose.file")} + logFileAutomationAgentStderrEnvVar := corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: path.Join(util.PvcMountPathLogs, "log-stderr.file")} + logFileAutomationAgentDefaultEnvVar := corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent.log")} + logFileAutomationAgentVerboseDefaultEnvVar := corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent-verbose.log")} + logFileAutomationAgentStderrDefaultEnvVar := corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent-stderr.log")} + logFileMongoDBAuditEnvVar := corev1.EnvVar{Name: LogFileMongoDBAuditEnv, Value: path.Join(util.PvcMountPathLogs, "audit.log")} + logFileMongoDBAuditDefaultEnvVar := corev1.EnvVar{Name: LogFileMongoDBAuditEnv, Value: path.Join(util.PvcMountPathLogs, "mongodb-audit.log")} + logFileMongoDBEnvVar := corev1.EnvVar{Name: LogFileMongoDBEnv, Value: path.Join(util.PvcMountPathLogs, "mongodb.log")} + logFileAgentMonitoringEnvVar := corev1.EnvVar{Name: LogFileAgentMonitoringEnv, Value: path.Join(util.PvcMountPathLogs, "monitoring-agent.log")} + logFileAgentBackupEnvVar := corev1.EnvVar{Name: LogFileAgentBackupEnv, Value: path.Join(util.PvcMountPathLogs, "backup-agent.log")} + + numberOfLogFilesInEnvVars := 7 + + t.Run("automation log is changed and audit log is changed", func(t *testing.T) { + envVars := logConfigurationToEnvVars(parameters, additionalMongodConfig) + assert.Len(t, envVars, numberOfLogFilesInEnvVars) + assert.Contains(t, envVars, logFileAutomationAgentEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentVerboseEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentStderrEnvVar) + assert.Contains(t, envVars, logFileMongoDBAuditEnvVar) + assert.Contains(t, envVars, logFileMongoDBEnvVar) + assert.Contains(t, envVars, logFileAgentMonitoringEnvVar) + assert.Contains(t, envVars, logFileAgentBackupEnvVar) + }) + + t.Run("automation log is changed and audit log is default", func(t *testing.T) { + envVars := logConfigurationToEnvVars(parameters, additionalMongodConfig) + assert.Len(t, envVars, numberOfLogFilesInEnvVars) + assert.Contains(t, envVars, logFileAutomationAgentEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentVerboseEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentStderrEnvVar) + assert.Contains(t, envVars, logFileMongoDBAuditEnvVar) + assert.Contains(t, envVars, logFileMongoDBEnvVar) + assert.Contains(t, envVars, logFileAgentMonitoringEnvVar) + assert.Contains(t, envVars, logFileAgentBackupEnvVar) + }) + + t.Run("automation log is default and audit log is changed", func(t *testing.T) { + envVars = logConfigurationToEnvVars(map[string]string{}, additionalMongodConfig) + assert.Len(t, envVars, numberOfLogFilesInEnvVars) + assert.Contains(t, envVars, logFileAutomationAgentDefaultEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentVerboseDefaultEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentStderrDefaultEnvVar) + assert.Contains(t, envVars, logFileMongoDBAuditEnvVar) + assert.Contains(t, envVars, logFileMongoDBEnvVar) + assert.Contains(t, envVars, logFileAgentMonitoringEnvVar) + assert.Contains(t, envVars, logFileAgentBackupEnvVar) + }) + + t.Run("all log files are default", func(t *testing.T) { + envVars = logConfigurationToEnvVars(map[string]string{"other": "value"}, mdbv1.NewEmptyAdditionalMongodConfig().AddOption("other", "value")) + assert.Len(t, envVars, numberOfLogFilesInEnvVars) + assert.Contains(t, envVars, logFileAutomationAgentDefaultEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentVerboseDefaultEnvVar) + assert.Contains(t, envVars, logFileAutomationAgentStderrDefaultEnvVar) + assert.Contains(t, envVars, logFileMongoDBAuditDefaultEnvVar) + assert.Contains(t, envVars, logFileMongoDBEnvVar) + assert.Contains(t, envVars, logFileAgentMonitoringEnvVar) + assert.Contains(t, envVars, logFileAgentBackupEnvVar) + }) +} + +func TestGetAutomationLogEnvVars(t *testing.T) { + t.Run("automation log file with extension", func(t *testing.T) { + envVars := getAutomationLogEnvVars(map[string]string{"logFile": "path/to/log.file"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: "path/to/log.file"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: "path/to/log-verbose.file"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: "path/to/log-stderr.file"}) + }) + + t.Run("automation log file without extension", func(t *testing.T) { + envVars := getAutomationLogEnvVars(map[string]string{"logFile": "path/to/logfile"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: "path/to/logfile"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: "path/to/logfile-verbose"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: "path/to/logfile-stderr"}) + }) + t.Run("invalid automation log file is not crashing", func(t *testing.T) { + envVars := getAutomationLogEnvVars(map[string]string{"logFile": "path/to/"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: "path/to/"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: "path/to/-verbose"}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: "path/to/-stderr"}) + }) + + t.Run("empty automation log file is falling back to default names", func(t *testing.T) { + envVars := getAutomationLogEnvVars(map[string]string{"logFile": ""}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent.log")}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent-verbose.log")}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent-stderr.log")}) + }) + + t.Run("not set logFile cause falling back to default names", func(t *testing.T) { + envVars := getAutomationLogEnvVars(map[string]string{}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent.log")}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentVerboseEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent-verbose.log")}) + assert.Contains(t, envVars, corev1.EnvVar{Name: LogFileAutomationAgentStderrEnv, Value: path.Join(util.PvcMountPathLogs, "automation-agent-stderr.log")}) + }) +} + +func TestDatabaseStatefulSet_StaticContainersEnvVars(t *testing.T) { + tests := []struct { + name string + defaultArchitecture string + annotations map[string]string + expectedEnvVar corev1.EnvVar + expectAgentContainer bool + }{ + { + name: "Default architecture - static, no annotations", + defaultArchitecture: string(architectures.Static), + annotations: nil, + expectedEnvVar: corev1.EnvVar{Name: "MDB_STATIC_CONTAINERS_ARCHITECTURE", Value: "true"}, + expectAgentContainer: true, + }, + { + name: "Default architecture - non-static, annotations - static", + defaultArchitecture: string(architectures.NonStatic), + annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.Static)}, + expectedEnvVar: corev1.EnvVar{Name: "MDB_STATIC_CONTAINERS_ARCHITECTURE", Value: "true"}, + expectAgentContainer: true, + }, + { + name: "Default architecture - non-static, no annotations", + defaultArchitecture: string(architectures.NonStatic), + annotations: nil, + expectedEnvVar: corev1.EnvVar{}, + expectAgentContainer: false, + }, + { + name: "Default architecture - static, annotations - non-static", + defaultArchitecture: string(architectures.Static), + annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.NonStatic)}, + expectedEnvVar: corev1.EnvVar{}, + expectAgentContainer: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, tt.defaultArchitecture) + + mdb := mdbv1.NewReplicaSetBuilder().SetAnnotations(tt.annotations).Build() + sts := DatabaseStatefulSet(*mdb, ReplicaSetOptions(GetPodEnvOptions()), zap.S()) + + agentContainerIdx := slices.IndexFunc(sts.Spec.Template.Spec.Containers, func(container corev1.Container) bool { + return container.Name == util.AgentContainerName + }) + if tt.expectAgentContainer { + require.NotEqual(t, -1, agentContainerIdx) + assert.Contains(t, sts.Spec.Template.Spec.Containers[agentContainerIdx].Env, tt.expectedEnvVar) + } else { + // In non-static architecture there is no agent container + // so the index should be -1. + require.Equal(t, -1, agentContainerIdx) + } + }) + } +} diff --git a/controllers/operator/construct/database_volumes.go b/controllers/operator/construct/database_volumes.go new file mode 100644 index 000000000..8e396f6e1 --- /dev/null +++ b/controllers/operator/construct/database_volumes.go @@ -0,0 +1,135 @@ +package construct + +import ( + "fmt" + "path" + + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +type MongoDBVolumeSource interface { + GetVolumes() []corev1.Volume + GetVolumeMounts() []corev1.VolumeMount + GetEnvs() []corev1.EnvVar + ShouldBeAdded() bool +} + +type caVolumeSource struct { + opts DatabaseStatefulSetOptions + logger *zap.SugaredLogger +} + +func (c *caVolumeSource) GetVolumes() []corev1.Volume { + return []corev1.Volume{statefulset.CreateVolumeFromConfigMap(CaCertName, c.opts.PodVars.SSLMMSCAConfigMap)} +} + +func (c *caVolumeSource) GetVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + MountPath: caCertMountPath, + Name: CaCertName, + ReadOnly: true, + }, + } +} + +func (c *caVolumeSource) GetEnvs() []corev1.EnvVar { + // A custom CA has been provided, point the trusted CA to the location of custom CAs + trustedCACertLocation := path.Join(caCertMountPath, util.CaCertMMS) + return []corev1.EnvVar{ + { + Name: util.EnvVarSSLTrustedMMSServerCertificate, + Value: trustedCACertLocation, + }, + } +} + +func (c *caVolumeSource) ShouldBeAdded() bool { + return c.opts.PodVars != nil && c.opts.PodVars.SSLMMSCAConfigMap != "" +} + +// tlsVolumeSource provides the volume and volumeMounts that need to be created for TLS. +type tlsVolumeSource struct { + security *mdbv1.Security + databaseOpts DatabaseStatefulSetOptions + logger *zap.SugaredLogger +} + +func (c *tlsVolumeSource) getVolumesAndMounts() ([]corev1.Volume, []corev1.VolumeMount) { + var volumes []corev1.Volume + var volumeMounts []corev1.VolumeMount + + security := c.security + databaseOpts := c.databaseOpts + + // We default each value to the "old-design" + tlsConfig := security.TLSConfig + if !security.IsTLSEnabled() { + return volumes, volumeMounts + } + + secretName := security.MemberCertificateSecretName(databaseOpts.Name) + + caName := fmt.Sprintf("%s-ca", databaseOpts.Name) + if tlsConfig != nil && tlsConfig.CA != "" { + caName = tlsConfig.CA + } else { + c.logger.Debugf("No CA name has been supplied, defaulting to: %s", caName) + } + + // This two functions modify the volumes to be optional (the absence of the referenced + // secret/configMap do not prevent the pods from starting) + optionalSecretFunc := func(v *corev1.Volume) { v.Secret.Optional = util.BooleanRef(true) } + optionalConfigMapFunc := func(v *corev1.Volume) { v.ConfigMap.Optional = util.BooleanRef(true) } + + secretMountPath := util.TLSCertMountPath + configmapMountPath := util.TLSCaMountPath + volumeSecretName := fmt.Sprintf("%s%s", secretName, certs.OperatorGeneratedCertSuffix) + + if !vault.IsVaultSecretBackend() { + secretVolume := statefulset.CreateVolumeFromSecret(util.SecretVolumeName, volumeSecretName, optionalSecretFunc) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: secretMountPath, + Name: secretVolume.Name, + ReadOnly: true, + }) + volumes = append(volumes, secretVolume) + } + + caVolume := statefulset.CreateVolumeFromConfigMap(tls.ConfigMapVolumeCAName, caName, optionalConfigMapFunc) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + MountPath: configmapMountPath, + Name: caVolume.Name, + ReadOnly: true, + }) + volumes = append(volumes, caVolume) + return volumes, volumeMounts +} + +func (c *tlsVolumeSource) GetVolumes() []corev1.Volume { + volumes, _ := c.getVolumesAndMounts() + return volumes +} + +func (c *tlsVolumeSource) GetVolumeMounts() []corev1.VolumeMount { + _, volumeMounts := c.getVolumesAndMounts() + return volumeMounts +} + +func (c *tlsVolumeSource) GetEnvs() []corev1.EnvVar { + return []corev1.EnvVar{} +} + +func (c *tlsVolumeSource) ShouldBeAdded() bool { + return c.security.IsTLSEnabled() +} diff --git a/controllers/operator/construct/jvm.go b/controllers/operator/construct/jvm.go new file mode 100644 index 000000000..985f8da2c --- /dev/null +++ b/controllers/operator/construct/jvm.go @@ -0,0 +1,138 @@ +package construct + +import ( + "fmt" + "strings" + + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +const ( + opsManagerPodMemPercentage = 90 + oneMB = 1048576 + backupDaemonHealthPort = 8090 +) + +// setJvmArgsEnvVars sets the correct environment variables for JVM size parameters. +// This method must be invoked on the final version of the StatefulSet (after user statefulSet spec +// was merged) +func setJvmArgsEnvVars(om omv1.MongoDBOpsManagerSpec, containerName string, sts *appsv1.StatefulSet) error { + jvmParamsEnvVars, err := buildJvmParamsEnvVars(om, containerName, sts.Spec.Template) + if err != nil { + return err + } + // pass Xmx java parameter to container (note, that we don't need to sort the env variables again + // as the jvm params order is consistent) + for _, envVar := range jvmParamsEnvVars { + omContainer := container.GetByName(containerName, sts.Spec.Template.Spec.Containers) + omContainer.Env = append(omContainer.Env, envVar) + } + return nil +} + +// buildJvmParamsEnvVars returns a slice of corev1.EnvVars that should be added to the Backup Daemon +// or Ops Manager containers. +func buildJvmParamsEnvVars(m omv1.MongoDBOpsManagerSpec, containerName string, template corev1.PodTemplateSpec) ([]corev1.EnvVar, error) { + mmsJvmEnvVar := corev1.EnvVar{Name: util.MmsJvmParamEnvVar} + backupJvmEnvVar := corev1.EnvVar{Name: util.BackupDaemonJvmParamEnvVar} + + omContainer := container.GetByName(containerName, template.Spec.Containers) + // calculate xmx from container's memory limit + memLimits := omContainer.Resources.Limits.Memory() + maxPodMem, err := getPercentOfQuantityAsInt(*memLimits, opsManagerPodMemPercentage) + if err != nil { + return []corev1.EnvVar{}, xerrors.Errorf("error calculating xmx from pod mem: %w", err) + } + + // calculate xms from container's memory request if it is set, otherwise xms=xmx + memRequests := omContainer.Resources.Requests.Memory() + minPodMem, err := getPercentOfQuantityAsInt(*memRequests, opsManagerPodMemPercentage) + if err != nil { + return []corev1.EnvVar{}, xerrors.Errorf("error calculating xms from pod mem: %w", err) + } + + // if only one of mem limits/requests is set, use that value for both xmx & xms + if minPodMem == 0 { + minPodMem = maxPodMem + } + if maxPodMem == 0 { + maxPodMem = minPodMem + } + + memParams := fmt.Sprintf("-Xmx%dm -Xms%dm", maxPodMem, minPodMem) + mmsJvmEnvVar.Value = buildJvmEnvVar(m.JVMParams, memParams) + backupJvmEnvVar.Value = buildJvmEnvVar(m.Backup.JVMParams, memParams) + + // If a debug port is set, add the JVM debug agent to the JVM parameters + if debugPort := getDebugPort(omContainer.Ports); debugPort > 0 { + mmsJvmEnvVar.Value += fmt.Sprintf(" -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=%d", debugPort) + } + + // Debug port reports the status of the AppDB, this can be used to determine if the Backup Daemon is in a good state. + // this exposes a /health endpoint which returns {"sync_db":"OK","backup_db":"OK","mms_db":"OK"} + // https://github.com/10gen/mms/blob/8c4047d67e157672051d37e340305d89ad20964a/server/src/main/com/xgen/svc/brs/grid/Daemon.java#L926 + backupJvmEnvVar.Value += fmt.Sprintf(" -DDAEMON.DEBUG.PORT=%d", backupDaemonHealthPort) + return []corev1.EnvVar{mmsJvmEnvVar, backupJvmEnvVar}, nil +} + +func getDebugPort(ports []corev1.ContainerPort) int32 { + if len(ports) == 0 { + return 0 + } + + for _, p := range ports { + if p.Name == "debug" { + return p.ContainerPort + } + } + + return 0 +} + +// getPercentOfQuantityAsInt returns the percentage of a given quantity as an int. +func getPercentOfQuantityAsInt(q resource.Quantity, percent int) (int, error) { + quantityAsInt, canConvert := q.AsInt64() + if !canConvert { + // the container's mem can't be converted to int64, use default of 5G + podMem, err := resource.ParseQuantity(util.DefaultMemoryOpsManager) + if err != nil { + return 0, err + } + quantityAsInt, canConvert = podMem.AsInt64() + if !canConvert { + return 0, xerrors.Errorf("cannot convert %s to int64", podMem.String()) + } + } + percentage := float64(percent) / 100.0 + + return int(float64(quantityAsInt)*percentage) / oneMB, nil +} + +// buildJvmEnvVar returns the string representation of the JVM environment variable +func buildJvmEnvVar(customParams []string, containerMemParams string) string { + jvmParams := strings.Join(customParams, " ") + + // if both mem limits and mem requests are unset/have value 0, we don't want to override om's default JVM xmx/xms params + if strings.Contains(containerMemParams, "-Xmx0m") { + return jvmParams + } + + if strings.Contains(jvmParams, "Xmx") { + return jvmParams + } + + if jvmParams != "" { + jvmParams += " " + } + + return jvmParams + containerMemParams +} diff --git a/controllers/operator/construct/jvm_test.go b/controllers/operator/construct/jvm_test.go new file mode 100644 index 000000000..e61709386 --- /dev/null +++ b/controllers/operator/construct/jvm_test.go @@ -0,0 +1,33 @@ +package construct + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildJvmEnvVar(t *testing.T) { + type args struct { + customParams []string + containerMemParams string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "custom params with ssl trust store and a container params", + args: args{ + customParams: []string{"-Djavax.net.ssl.trustStore=/etc/ssl"}, + containerMemParams: "-Xmx4291m -Xms4291m", + }, + want: "-Djavax.net.ssl.trustStore=/etc/ssl -Xmx4291m -Xms4291m", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, buildJvmEnvVar(tt.args.customParams, tt.args.containerMemParams), "buildJvmEnvVar(%v, %v)", tt.args.customParams, tt.args.containerMemParams) + }) + } +} diff --git a/controllers/operator/construct/multicluster/multicluster_replicaset.go b/controllers/operator/construct/multicluster/multicluster_replicaset.go new file mode 100644 index 000000000..af1421038 --- /dev/null +++ b/controllers/operator/construct/multicluster/multicluster_replicaset.go @@ -0,0 +1,116 @@ +package multicluster + +import ( + "fmt" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbmultiv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/pkg/handler" +) + +func MultiClusterReplicaSetOptions(additionalOpts ...func(options *construct.DatabaseStatefulSetOptions)) func(mdbm mdbmultiv1.MongoDBMultiCluster) construct.DatabaseStatefulSetOptions { + return func(mdbm mdbmultiv1.MongoDBMultiCluster) construct.DatabaseStatefulSetOptions { + stsSpec := appsv1.StatefulSetSpec{} + if mdbm.Spec.StatefulSetConfiguration != nil { + stsSpec = mdbm.Spec.StatefulSetConfiguration.SpecWrapper.Spec + } + opts := construct.DatabaseStatefulSetOptions{ + Name: mdbm.Name, + ServicePort: mdbm.Spec.GetAdditionalMongodConfig().GetPortOrDefault(), + Persistent: mdbm.Spec.Persistent, + AgentConfig: &mdbm.Spec.Agent, + PodSpec: construct.NewDefaultPodSpecWrapper(*mdbv1.NewMongoDbPodSpec()), + Labels: statefulSetLabels(mdbm.Name, mdbm.Namespace), + MultiClusterMode: true, + HostNameOverrideConfigmapName: mdbm.GetHostNameOverrideConfigmapName(), + StatefulSetSpecOverride: &stsSpec, + StsType: construct.MultiReplicaSet, + } + for _, opt := range additionalOpts { + opt(&opts) + } + + return opts + } +} + +func WithServiceName(serviceName string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.ServiceName = serviceName + } +} + +func WithClusterNum(clusterNum int) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.StatefulSetNameOverride = statefulSetName(options.Name, clusterNum) + } +} + +func WithMemberCount(memberCount int) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.Replicas = memberCount + } +} + +func WithStsOverride(stsOverride *appsv1.StatefulSetSpec) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + finalSpec := merge.StatefulSetSpecs(*options.StatefulSetSpecOverride, *stsOverride) + options.StatefulSetSpecOverride = &finalSpec + } +} + +func WithAnnotations(resourceName string, certHash string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.Annotations = statefulSetAnnotations(resourceName, certHash) + } +} + +func statefulSetName(mdbmName string, clusterNum int) string { + return fmt.Sprintf("%s-%d", mdbmName, clusterNum) +} + +func statefulSetLabels(mdbmName, mdbmNamespace string) map[string]string { + return map[string]string{ + "controller": "mongodb-enterprise-operator", + "mongodbmulticluster": fmt.Sprintf("%s-%s", mdbmName, mdbmNamespace), + } +} + +func statefulSetAnnotations(mdbmName string, certHash string) map[string]string { + return map[string]string{ + handler.MongoDBMultiResourceAnnotation: mdbmName, + certs.CertHashAnnotationKey: certHash, + } +} + +func PodLabel(mdbmName string) map[string]string { + return map[string]string{ + construct.ControllerLabelName: "mongodb-enterprise-operator", + construct.PodAntiAffinityLabelKey: mdbmName, + } +} + +func MultiClusterStatefulSet(mdbm mdbmultiv1.MongoDBMultiCluster, stsOptFunc func(mdbm mdbmultiv1.MongoDBMultiCluster) construct.DatabaseStatefulSetOptions) appsv1.StatefulSet { + stsOptions := stsOptFunc(mdbm) + dbSts := construct.DatabaseStatefulSetHelper(&mdbm, &stsOptions, nil) + + if len(stsOptions.Annotations) > 0 { + dbSts.Annotations = merge.StringToStringMap(dbSts.Annotations, stsOptions.Annotations) + } + + if len(stsOptions.Labels) > 0 { + dbSts.Labels = merge.StringToStringMap(dbSts.Labels, stsOptions.Labels) + } + + if stsOptions.StatefulSetSpecOverride != nil { + dbSts.Spec = merge.StatefulSetSpecs(dbSts.Spec, *stsOptions.StatefulSetSpecOverride) + } + + return dbSts +} diff --git a/controllers/operator/construct/multicluster/multicluster_replicaset_test.go b/controllers/operator/construct/multicluster/multicluster_replicaset_test.go new file mode 100644 index 000000000..cdb2b7dae --- /dev/null +++ b/controllers/operator/construct/multicluster/multicluster_replicaset_test.go @@ -0,0 +1,314 @@ +package multicluster + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + + mdbc "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" +) + +func init() { + mock.InitDefaultEnvVariables() +} + +func getMultiClusterMongoDB() mdbmulti.MongoDBMultiCluster { + spec := mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdb.DbCommonSpec{ + Version: "5.0.0", + ConnectionSpec: mdb.ConnectionSpec{ + SharedConnectionSpec: mdb.SharedConnectionSpec{ + OpsManagerConfig: &mdb.PrivateCloudConfig{ + ConfigMapRef: mdb.ConfigMapRef{ + Name: mock.TestProjectConfigMapName, + }, + }, + }, Credentials: mock.TestCredentialsSecretName, + }, + ResourceType: mdb.ReplicaSet, + Security: &mdb.Security{ + TLSConfig: &mdb.TLSConfig{}, + Authentication: &mdb.Authentication{ + Modes: []mdb.AuthMode{}, + }, + Roles: []mdb.MongoDbRole{}, + }, + }, + ClusterSpecList: mdb.ClusterSpecList{ + { + ClusterName: "foo", + Members: 3, + }, + }, + } + + return mdbmulti.MongoDBMultiCluster{Spec: spec, ObjectMeta: metav1.ObjectMeta{Name: "pod-aff", Namespace: mock.TestNamespace}} +} + +func TestMultiClusterStatefulSet(t *testing.T) { + t.Run("No override provided", func(t *testing.T) { + mdbm := getMultiClusterMongoDB() + opts := MultiClusterReplicaSetOptions( + WithClusterNum(0), + WithMemberCount(3), + construct.GetPodEnvOptions(), + ) + sts := MultiClusterStatefulSet(mdbm, opts) + + expectedReplicas := mdbm.Spec.ClusterSpecList[0].Members + assert.Equal(t, expectedReplicas, int(*sts.Spec.Replicas)) + }) + + t.Run("Override provided at clusterSpecList level only", func(t *testing.T) { + singleClusterOverride := &mdbc.StatefulSetConfiguration{SpecWrapper: mdbc.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(4)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + }, + }} + + mdbm := getMultiClusterMongoDB() + mdbm.Spec.ClusterSpecList[0].StatefulSetConfiguration = singleClusterOverride + + opts := MultiClusterReplicaSetOptions( + WithClusterNum(0), + WithMemberCount(3), + construct.GetPodEnvOptions(), + WithStsOverride(&singleClusterOverride.SpecWrapper.Spec), + ) + + sts := MultiClusterStatefulSet(mdbm, opts) + + expectedMatchLabels := singleClusterOverride.SpecWrapper.Spec.Selector.MatchLabels + expectedMatchLabels["app"] = "" + expectedMatchLabels["pod-anti-affinity"] = mdbm.Name + expectedMatchLabels["controller"] = "mongodb-enterprise-operator" + + assert.Equal(t, singleClusterOverride.SpecWrapper.Spec.Replicas, sts.Spec.Replicas) + assert.Equal(t, expectedMatchLabels, sts.Spec.Selector.MatchLabels) + }) + + t.Run("Override provided only at Spec level", func(t *testing.T) { + stsOverride := &mdbc.StatefulSetConfiguration{ + SpecWrapper: mdbc.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + ServiceName: "overrideservice", + }, + }, + } + + mdbm := getMultiClusterMongoDB() + mdbm.Spec.StatefulSetConfiguration = stsOverride + opts := MultiClusterReplicaSetOptions( + WithClusterNum(0), + WithMemberCount(3), + construct.GetPodEnvOptions(), + ) + + sts := MultiClusterStatefulSet(mdbm, opts) + + expectedReplicas := mdbm.Spec.ClusterSpecList[0].Members + assert.Equal(t, expectedReplicas, int(*sts.Spec.Replicas)) + + assert.Equal(t, stsOverride.SpecWrapper.Spec.ServiceName, sts.Spec.ServiceName) + }) + + t.Run("Override provided at both Spec and clusterSpecList level", func(t *testing.T) { + stsOverride := &mdbc.StatefulSetConfiguration{ + SpecWrapper: mdbc.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + ServiceName: "overrideservice", + }, + }, + } + + singleClusterOverride := &mdbc.StatefulSetConfiguration{ + SpecWrapper: mdbc.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + ServiceName: "clusteroverrideservice", + Replicas: ptr.To(int32(4)), + }, + }, + } + + mdbm := getMultiClusterMongoDB() + mdbm.Spec.StatefulSetConfiguration = stsOverride + + opts := MultiClusterReplicaSetOptions( + WithClusterNum(0), + WithMemberCount(3), + construct.GetPodEnvOptions(), + WithStsOverride(&singleClusterOverride.SpecWrapper.Spec), + ) + + sts := MultiClusterStatefulSet(mdbm, opts) + + assert.Equal(t, singleClusterOverride.SpecWrapper.Spec.ServiceName, sts.Spec.ServiceName) + assert.Equal(t, singleClusterOverride.SpecWrapper.Spec.Replicas, sts.Spec.Replicas) + }) +} + +func TestPVCOverride(t *testing.T) { + tests := []struct { + inp appsv1.StatefulSetSpec + out struct { + Storage int64 + AccessMode []corev1.PersistentVolumeAccessMode + } + }{ + { + inp: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: construct.ParseQuantityOrZero("20"), + }, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{}, + }, + }, + }, + }, + out: struct { + Storage int64 + AccessMode []corev1.PersistentVolumeAccessMode + }{ + Storage: 20, + AccessMode: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce"}, + }, + }, + { + inp: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"ReadWriteMany"}, + }, + }, + }, + }, + out: struct { + Storage int64 + AccessMode []corev1.PersistentVolumeAccessMode + }{ + Storage: 16000000000, + AccessMode: []corev1.PersistentVolumeAccessMode{"ReadWriteOnce", "ReadWriteMany"}, + }, + }, + } + + t.Setenv(util.NonStaticDatabaseEnterpriseImage, "some-registry") + t.Setenv(util.InitDatabaseImageUrlEnv, "some-registry") + + for _, tt := range tests { + mdbm := getMultiClusterMongoDB() + + stsOverrideConfiguration := &mdbc.StatefulSetConfiguration{SpecWrapper: mdbc.StatefulSetSpecWrapper{Spec: tt.inp}} + opts := MultiClusterReplicaSetOptions( + WithClusterNum(0), + WithMemberCount(3), + construct.GetPodEnvOptions(), + WithStsOverride(&stsOverrideConfiguration.SpecWrapper.Spec), + ) + sts := MultiClusterStatefulSet(mdbm, opts) + assert.Equal(t, tt.out.AccessMode, sts.Spec.VolumeClaimTemplates[0].Spec.AccessModes) + storage, _ := sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().AsInt64() + assert.Equal(t, tt.out.Storage, storage) + } +} + +func TestMultiClusterStatefulSet_StaticContainersEnvVars(t *testing.T) { + tests := []struct { + name string + defaultArchitecture string + annotations map[string]string + expectedEnvVar corev1.EnvVar + expectAgentContainer bool + }{ + { + name: "Default architecture - static, no annotations", + defaultArchitecture: string(architectures.Static), + annotations: nil, + expectedEnvVar: corev1.EnvVar{Name: "MDB_STATIC_CONTAINERS_ARCHITECTURE", Value: "true"}, + expectAgentContainer: true, + }, + { + name: "Default architecture - non-static, annotations - static", + defaultArchitecture: string(architectures.NonStatic), + annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.Static)}, + expectedEnvVar: corev1.EnvVar{Name: "MDB_STATIC_CONTAINERS_ARCHITECTURE", Value: "true"}, + expectAgentContainer: true, + }, + { + name: "Default architecture - non-static, no annotations", + defaultArchitecture: string(architectures.NonStatic), + annotations: nil, + expectedEnvVar: corev1.EnvVar{}, + expectAgentContainer: false, + }, + { + name: "Default architecture - static, annotations - non-static", + defaultArchitecture: string(architectures.Static), + annotations: map[string]string{architectures.ArchitectureAnnotation: string(architectures.NonStatic)}, + expectedEnvVar: corev1.EnvVar{}, + expectAgentContainer: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, tt.defaultArchitecture) + + mdbm := getMultiClusterMongoDB() + mdbm.Annotations = tt.annotations + opts := MultiClusterReplicaSetOptions( + WithClusterNum(0), + WithMemberCount(3), + construct.GetPodEnvOptions(), + ) + + sts := MultiClusterStatefulSet(mdbm, opts) + + agentContainerIdx := slices.IndexFunc(sts.Spec.Template.Spec.Containers, func(container corev1.Container) bool { + return container.Name == util.AgentContainerName + }) + if tt.expectAgentContainer { + require.NotEqual(t, -1, agentContainerIdx) + assert.Contains(t, sts.Spec.Template.Spec.Containers[agentContainerIdx].Env, tt.expectedEnvVar) + } else { + // In non-static architecture there is no agent container + // so the index should be -1. + require.Equal(t, -1, agentContainerIdx) + } + }) + } +} diff --git a/controllers/operator/construct/opsmanager_construction.go b/controllers/operator/construct/opsmanager_construction.go new file mode 100644 index 000000000..987da9e6a --- /dev/null +++ b/controllers/operator/construct/opsmanager_construction.go @@ -0,0 +1,702 @@ +package construct + +import ( + "context" + "fmt" + "net" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/lifecycle" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +const ( + appLabelKey = "app" + podAntiAffinityLabelKey = "pod-anti-affinity" +) + +// OpsManagerStatefulSetOptions contains all the different values that are variable between +// StatefulSets. Depending on which StatefulSet is being built, a number of these will be pre-set, +// while the remainder will be configurable via configuration functions which modify this type. +type OpsManagerStatefulSetOptions struct { + OwnerReference []metav1.OwnerReference + HTTPSCertSecretName string + CertHash string + AppDBTlsCAConfigMapName string + AppDBConnectionSecretName string + AppDBConnectionStringHash string + EnvVars []corev1.EnvVar + InitOpsManagerImage string + OpsManagerImage string + Name string + Replicas int + ServiceName string + Namespace string + OwnerName string + ServicePort int + QueryableBackupPemSecretName string + StatefulSetSpecOverride *appsv1.StatefulSetSpec + VaultConfig vault.VaultConfiguration + Labels map[string]string + kmip *KmipConfiguration + DebugPort int32 + // backup daemon only + HeadDbPersistenceConfig *mdbv1.PersistenceConfig + Annotations map[string]string + LoggingConfiguration *omv1.Logging +} + +type KmipClientConfiguration struct { + ClientCertificateSecretName string + ClientCertificatePasswordSecretName *string + ClientCertificatePasswordSecretKeyName *string +} + +type KmipConfiguration struct { + ServerConfiguration v1.KmipServerConfig + ClientConfigurations []KmipClientConfiguration +} + +func WithConnectionStringHash(hash string) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + opts.AppDBConnectionStringHash = hash + } +} + +func WithVaultConfig(config vault.VaultConfiguration) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + opts.VaultConfig = config + } +} + +func WithInitOpsManagerImage(initOpsManagerImage string) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + opts.InitOpsManagerImage = initOpsManagerImage + } +} + +func WithOpsManagerImage(opsManagerImage string) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + opts.OpsManagerImage = opsManagerImage + } +} + +func WithReplicas(replicas int) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + opts.Replicas = replicas + } +} + +func WithKmipConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, client kubernetesClient.Client, log *zap.SugaredLogger) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + if !opsManager.Spec.IsKmipEnabled() { + return + } + opts.kmip = &KmipConfiguration{ + ServerConfiguration: opsManager.Spec.Backup.Encryption.Kmip.Server, + ClientConfigurations: make([]KmipClientConfiguration, 0), + } + + mdbList := &mdbv1.MongoDBList{} + err := client.List(ctx, mdbList) + if err != nil { + log.Warnf("failed to fetch MongoDBList from Kubernetes: %v", err) + } + + for _, m := range mdbList.Items { + // Since KMIP integration requires the secret to be mounted into the backup daemon + // we might not be able to do any encrypted backups across namespaces. Such a backup + // would require syncing secrets across namespaces. + // I'm not adding any namespace validation, and we'll let the user handle such synchronization as + // the backup Daemon will hang in Pending until the secret is provided. + if m.Spec.Backup != nil && m.Spec.Backup.IsKmipEnabled() { + c := m.Spec.Backup.Encryption.Kmip.Client + config := KmipClientConfiguration{ + ClientCertificateSecretName: c.ClientCertificateSecretName(m.GetName()), + } + + clientCertificatePasswordSecret := &corev1.Secret{} + err := client.Get(ctx, kube.ObjectKey(m.GetNamespace(), c.ClientCertificatePasswordSecretName(m.GetName())), clientCertificatePasswordSecret) + if !apiErrors.IsNotFound(err) { + log.Warnf("failed to fetch the %s Secret from Kubernetes: %v", c.ClientCertificateSecretName(m.GetName()), err) + } else if err == nil { + clientCertificateSecretName := c.ClientCertificatePasswordSecretName(m.GetName()) + clientCertificatePasswordKey := c.ClientCertificatePasswordKeyName() + config.ClientCertificatePasswordSecretKeyName = &clientCertificateSecretName + config.ClientCertificatePasswordSecretName = &clientCertificatePasswordKey + } + + opts.kmip.ClientConfigurations = append(opts.kmip.ClientConfigurations, config) + } + } + } +} + +func WithStsOverride(stsOverride *appsv1.StatefulSetSpec) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + if stsOverride != nil { + finalSpec := merge.StatefulSetSpecs(*opts.StatefulSetSpecOverride, *stsOverride) + opts.StatefulSetSpecOverride = &finalSpec + } + } +} + +func WithDebugPort(port int32) func(opts *OpsManagerStatefulSetOptions) { + return func(opts *OpsManagerStatefulSetOptions) { + opts.DebugPort = port + } +} + +// updateHTTPSCertSecret updates the fields for the OpsManager HTTPS certificate in case the provided secret is of type kubernetes.io/tls. +func (opts *OpsManagerStatefulSetOptions) updateHTTPSCertSecret(ctx context.Context, centralClusterSecretClient secrets.SecretClient, memberCluster multicluster.MemberCluster, ownerReferences []metav1.OwnerReference, log *zap.SugaredLogger) error { + // Return immediately if no Certificate is provided + if opts.HTTPSCertSecretName == "" { + return nil + } + + var err error + var secretData map[string][]byte + var s corev1.Secret + var opsManagerSecretPath string + + if vault.IsVaultSecretBackend() { + opsManagerSecretPath = centralClusterSecretClient.VaultClient.OpsManagerSecretPath() + secretData, err = centralClusterSecretClient.VaultClient.ReadSecretBytes(fmt.Sprintf("%s/%s/%s", opsManagerSecretPath, opts.Namespace, opts.HTTPSCertSecretName)) + if err != nil { + return err + } + } else { + s, err = centralClusterSecretClient.KubeClient.GetSecret(ctx, kube.ObjectKey(opts.Namespace, opts.HTTPSCertSecretName)) + if err != nil { + return err + } + + // SecretTypeTLS is kubernetes.io/tls + // This is the standard way in K8S to have secrets that hold TLS certs + // And it is the one generated by cert manager + // These type of secrets contain tls.crt and tls.key entries + if s.Type != corev1.SecretTypeTLS { + return nil + } + secretData = s.Data + } + + data, err := certs.VerifyTLSSecretForStatefulSet(secretData, certs.Options{}) + if err != nil { + return err + } + + certHash := enterprisepem.ReadHashFromSecret(ctx, centralClusterSecretClient, opts.Namespace, opts.HTTPSCertSecretName, opsManagerSecretPath, log) + + // The operator concatenates the two fields of the secret into a PEM secret + err = certs.CreateOrUpdatePEMSecretWithPreviousCert(ctx, memberCluster.SecretClient, kube.ObjectKey(opts.Namespace, opts.HTTPSCertSecretName), certHash, data, ownerReferences, certs.OpsManager) + if err != nil { + return err + } + + opts.HTTPSCertSecretName = fmt.Sprintf("%s%s", opts.HTTPSCertSecretName, certs.OperatorGeneratedCertSuffix) + opts.CertHash = certHash + + return nil +} + +// OpsManagerStatefulSet is the base method for building StatefulSet shared by Ops Manager and Backup Daemon. +// Shouldn't be called by end users directly +func OpsManagerStatefulSet(ctx context.Context, centralClusterSecretClient secrets.SecretClient, opsManager *omv1.MongoDBOpsManager, memberCluster multicluster.MemberCluster, log *zap.SugaredLogger, additionalOpts ...func(*OpsManagerStatefulSetOptions)) (appsv1.StatefulSet, error) { + opts := opsManagerOptions(memberCluster, additionalOpts...)(opsManager) + + opts.Annotations = opsManager.Annotations + if err := opts.updateHTTPSCertSecret(ctx, centralClusterSecretClient, memberCluster, opsManager.OwnerReferences, log); err != nil { + return appsv1.StatefulSet{}, err + } + + secretName := opsManager.Spec.Backup.QueryableBackupSecretRef.Name + opts.QueryableBackupPemSecretName = secretName + if secretName != "" { + // if the secret is specified, we must have a queryable.pem entry. + _, err := secret.ReadKey(ctx, centralClusterSecretClient, "queryable.pem", kube.ObjectKey(opsManager.Namespace, secretName)) + if err != nil { + return appsv1.StatefulSet{}, xerrors.Errorf("error reading queryable.pem key from secret %s/%s: %w", opsManager.Namespace, secretName, err) + } + } + + opts.LoggingConfiguration = opsManager.Spec.Logging + + omSts := statefulset.New(opsManagerStatefulSetFunc(opts)) + var err error + if opts.StatefulSetSpecOverride != nil { + omSts.Spec = merge.StatefulSetSpecs(omSts.Spec, *opts.StatefulSetSpecOverride) + } + + // the JVM env args must be determined after any potential stateful set override + // has taken place. + if err = setJvmArgsEnvVars(opsManager.Spec, util.OpsManagerContainerName, &omSts); err != nil { + return appsv1.StatefulSet{}, err + } + return omSts, nil +} + +// getSharedOpsManagerOptions returns the options that are shared between both the OpsManager +// and BackupDaemon StatefulSets +func getSharedOpsManagerOptions(opsManager *omv1.MongoDBOpsManager) OpsManagerStatefulSetOptions { + return OpsManagerStatefulSetOptions{ + OwnerReference: kube.BaseOwnerReference(opsManager), + OwnerName: opsManager.Name, + HTTPSCertSecretName: opsManager.TLSCertificateSecretName(), + AppDBTlsCAConfigMapName: opsManager.Spec.AppDB.GetCAConfigMapName(), + EnvVars: opsManagerConfigurationToEnvVars(opsManager), + Namespace: opsManager.Namespace, + Labels: opsManager.Labels, + } +} + +// opsManagerOptions returns a function which returns the OpsManagerStatefulSetOptions to create the OpsManager StatefulSet +func opsManagerOptions(memberCluster multicluster.MemberCluster, additionalOpts ...func(opts *OpsManagerStatefulSetOptions)) func(opsManager *omv1.MongoDBOpsManager) OpsManagerStatefulSetOptions { + return func(opsManager *omv1.MongoDBOpsManager) OpsManagerStatefulSetOptions { + var stsSpec *appsv1.StatefulSetSpec = nil + if opsManager.Spec.StatefulSetConfiguration != nil { + stsSpec = &opsManager.Spec.StatefulSetConfiguration.SpecWrapper.Spec + } + + _, port := opsManager.GetSchemePort() + + opts := getSharedOpsManagerOptions(opsManager) + opts.ServicePort = int(port) + opts.ServiceName = opsManager.SvcName() + if memberCluster.Legacy { + opts.Name = opsManager.Name + } else { + opts.Name = fmt.Sprintf("%s-%d", opsManager.Name, memberCluster.Index) + } + opts.Replicas = memberCluster.Replicas + opts.StatefulSetSpecOverride = stsSpec + opts.AppDBConnectionSecretName = opsManager.AppDBMongoConnectionStringSecretName() + + for _, additionalOpt := range additionalOpts { + additionalOpt(&opts) + } + return opts + } +} + +// opsManagerStatefulSetFunc constructs the default Ops Manager StatefulSet modification function. +func opsManagerStatefulSetFunc(opts OpsManagerStatefulSetOptions) statefulset.Modification { + _, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + return statefulset.Apply( + backupAndOpsManagerSharedConfiguration(opts), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + // 5 minutes for Ops Manager just in case (its internal timeout is 20 seconds anyway) + podtemplatespec.WithTerminationGracePeriodSeconds(300), + podtemplatespec.WithContainerByIndex(0, + container.Apply( + configureContainerSecurityContext, + container.WithCommand([]string{"/opt/scripts/docker-entry-point.sh"}), + container.WithName(util.OpsManagerContainerName), + container.WithLivenessProbe(opsManagerLivenessProbe()), + container.WithStartupProbe(opsManagerStartupProbe()), + container.WithReadinessProbe(opsManagerReadinessProbe()), + container.WithLifecycle(buildOpsManagerLifecycle()), + container.WithEnvs(corev1.EnvVar{Name: "ENABLE_IRP", Value: "true"}), + ), + ), + )), + ) +} + +// backupAndOpsManagerSharedConfiguration returns a function which configures all of the shared +// options between the backup and Ops Manager StatefulSet +func backupAndOpsManagerSharedConfiguration(opts OpsManagerStatefulSetOptions) statefulset.Modification { + configurePodSpecSecurityContext, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + pullSecretsConfigurationFunc := podtemplatespec.NOOP() + if pullSecrets, ok := env.Read(util.ImagePullSecrets); ok { // nolint:forbidigo + pullSecretsConfigurationFunc = podtemplatespec.WithImagePullSecrets(pullSecrets) + } + var omVolumeMounts []corev1.VolumeMount + + var omVolumes []corev1.Volume + + if !architectures.IsRunningStaticArchitecture(opts.Annotations) { + omScriptsVolume := statefulset.CreateVolumeFromEmptyDir("ops-manager-scripts") + omVolumes = append(omVolumes, omScriptsVolume) + omScriptsVolumeMount := buildOmScriptsVolumeMount(true) + omVolumeMounts = append(omVolumeMounts, omScriptsVolumeMount) + } + + vaultSecrets := vault.OpsManagerSecretsToInject{Config: opts.VaultConfig} + if vault.IsVaultSecretBackend() { + vaultSecrets.GenKeyPath = fmt.Sprintf("%s-gen-key", opts.OwnerName) + } else { + genKeyVolume := statefulset.CreateVolumeFromSecret("gen-key", fmt.Sprintf("%s-gen-key", opts.OwnerName)) + genKeyVolumeMount := corev1.VolumeMount{ + Name: genKeyVolume.Name, + ReadOnly: true, + MountPath: util.GenKeyPath, + } + omVolumeMounts = append(omVolumeMounts, genKeyVolumeMount) + omVolumes = append(omVolumes, genKeyVolume) + } + + if opts.QueryableBackupPemSecretName != "" { + queryablePemVolume := statefulset.CreateVolumeFromSecret("queryable-pem", opts.QueryableBackupPemSecretName) + omVolumeMounts = append(omVolumeMounts, corev1.VolumeMount{ + Name: queryablePemVolume.Name, + ReadOnly: true, + MountPath: "/certs/", + }) + omVolumes = append(omVolumes, queryablePemVolume) + } + + omHTTPSVolumeFunc := podtemplatespec.NOOP() + + if vault.IsVaultSecretBackend() { + if opts.HTTPSCertSecretName != "" { + vaultSecrets.TLSSecretName = opts.HTTPSCertSecretName + vaultSecrets.TLSHash = opts.CertHash + } + vaultSecrets.AppDBConnection = opts.AppDBConnectionSecretName + vaultSecrets.AppDBConnectionVolume = AppDBConnectionStringPath + } else if opts.HTTPSCertSecretName != "" { + + omHTTPSCertificateVolume := statefulset.CreateVolumeFromSecret("om-https-certificate", opts.HTTPSCertSecretName) + omHTTPSVolumeFunc = podtemplatespec.WithVolume(omHTTPSCertificateVolume) + omVolumeMounts = append(omVolumeMounts, corev1.VolumeMount{ + Name: omHTTPSCertificateVolume.Name, + MountPath: util.MmsPemKeyFileDirInContainer, + }) + + } + + appDbTLSConfigMapVolumeFunc := podtemplatespec.NOOP() + if opts.AppDBTlsCAConfigMapName != "" { + appDbTLSVolume := statefulset.CreateVolumeFromConfigMap("appdb-ca-certificate", opts.AppDBTlsCAConfigMapName) + appDbTLSConfigMapVolumeFunc = podtemplatespec.WithVolume(appDbTLSVolume) + omVolumeMounts = append(omVolumeMounts, corev1.VolumeMount{ + Name: appDbTLSVolume.Name, + MountPath: util.AppDBMmsCaFileDirInContainer, + }) + } + + podtemplateAnnotation := podtemplatespec.WithAnnotations(map[string]string{ + "connectionStringHash": opts.AppDBConnectionStringHash, + }) + + if vault.IsVaultSecretBackend() { + podtemplateAnnotation = podtemplatespec.Apply( + podtemplateAnnotation, + podtemplatespec.WithAnnotations( + vaultSecrets.OpsManagerAnnotations(opts.Namespace), + ), + ) + } + + if !vault.IsVaultSecretBackend() { + // configure the AppDB Connection String volume from a secret + mmsMongoUriVolume, mmsMongoUriVolumeMount := buildMmsMongoUriVolume(opts) + omVolumeMounts = append(omVolumeMounts, mmsMongoUriVolumeMount) + omVolumes = append(omVolumes, mmsMongoUriVolume) + } + + labels := defaultPodLabels(opts.ServiceName, opts.Name) + + // get the labels from the opts and append it to final labels + stsLabels := defaultPodLabels(opts.ServiceName, opts.Name) + for k, v := range opts.Labels { + stsLabels[k] = v + } + + omVolumes, omVolumeMounts = getNonPersistentOpsManagerVolumeMounts(omVolumes, omVolumeMounts, opts) + + opts.EnvVars = append(opts.EnvVars, kmipEnvVars(opts)...) + omVolumes, omVolumeMounts = appendKmipVolumes(omVolumes, omVolumeMounts, opts) + + initContainerMod := podtemplatespec.NOOP() + + if !architectures.IsRunningStaticArchitecture(opts.Annotations) { + initContainerMod = podtemplatespec.WithInitContainerByIndex(0, + buildOpsManagerAndBackupInitContainer(opts.InitOpsManagerImage), + ) + } + + return statefulset.Apply( + statefulset.WithLabels(stsLabels), + statefulset.WithMatchLabels(labels), + statefulset.WithName(opts.Name), + statefulset.WithNamespace(opts.Namespace), + statefulset.WithOwnerReference(opts.OwnerReference), + statefulset.WithReplicas(opts.Replicas), + statefulset.WithServiceName(opts.ServiceName), + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + omHTTPSVolumeFunc, + appDbTLSConfigMapVolumeFunc, + podtemplateAnnotation, + podtemplatespec.WithVolumes(omVolumes), + configurePodSpecSecurityContext, + podtemplatespec.WithPodLabels(labels), + pullSecretsConfigurationFunc, + podtemplatespec.WithServiceAccount(util.OpsManagerServiceAccount), + podtemplatespec.WithAffinity(opts.Name, podAntiAffinityLabelKey, 100), + podtemplatespec.WithTopologyKey(util.DefaultAntiAffinityTopologyKey, 0), + initContainerMod, + podtemplatespec.WithContainerByIndex(0, + container.Apply( + container.WithResourceRequirements(defaultOpsManagerResourceRequirements()), + container.WithPorts(buildOpsManagerContainerPorts(opts.HTTPSCertSecretName, opts.DebugPort)), + container.WithImagePullPolicy(corev1.PullPolicy(env.ReadOrPanic(util.OpsManagerPullPolicy))), // nolint:forbidigo + container.WithImage(opts.OpsManagerImage), + container.WithEnvs(opts.EnvVars...), + container.WithEnvs(getOpsManagerHTTPSEnvVars(opts.HTTPSCertSecretName, opts.CertHash)...), + container.WithCommand([]string{"/opt/scripts/docker-entry-point.sh"}), + container.WithVolumeMounts(omVolumeMounts), + configureContainerSecurityContext, + ), + ), + ), + ), + ) +} + +func appendKmipVolumes(volumes []corev1.Volume, volumeMounts []corev1.VolumeMount, opts OpsManagerStatefulSetOptions) ([]corev1.Volume, []corev1.VolumeMount) { + if opts.kmip != nil { + volumes = append(volumes, statefulset.CreateVolumeFromConfigMap(util.KMIPServerCAName, opts.kmip.ServerConfiguration.CA)) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.KMIPServerCAName, util.KMIPServerCAHome, statefulset.WithReadOnly(true))) + + for _, cc := range opts.kmip.ClientConfigurations { + clientSecretName := util.KMIPClientSecretNamePrefix + cc.ClientCertificateSecretName + clientSecretPath := util.KMIPClientSecretsHome + "/" + cc.ClientCertificateSecretName + volumes = append(volumes, statefulset.CreateVolumeFromSecret(clientSecretName, cc.ClientCertificateSecretName)) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(clientSecretName, clientSecretPath, statefulset.WithReadOnly(true))) + } + } + return volumes, volumeMounts +} + +func kmipEnvVars(opts OpsManagerStatefulSetOptions) []corev1.EnvVar { + if opts.kmip != nil { + // At this point we are certain, this is correct. We checked it in kmipValidation + host, port, _ := net.SplitHostPort(opts.kmip.ServerConfiguration.URL) + return []corev1.EnvVar{ + { + Name: util.OmPropertyPrefix + "backup_kmip_server_host", + Value: host, + }, + { + Name: util.OmPropertyPrefix + "backup_kmip_server_port", + Value: port, + }, + { + Name: util.OmPropertyPrefix + "backup_kmip_server_ca_file", + Value: util.KMIPCAFileInContainer, + }, + } + } + return nil +} + +// opsManagerReadinessProbe creates the readiness probe. +// Note on 'PeriodSeconds': /monitor/health is a super lightweight method not doing any IO so we can make it more often. +func opsManagerReadinessProbe() probes.Modification { + return probes.Apply( + probes.WithInitialDelaySeconds(5), + probes.WithTimeoutSeconds(5), + probes.WithPeriodSeconds(5), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(12), + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Scheme: corev1.URISchemeHTTP, Port: intstr.FromInt(8080), Path: "/monitor/health"}, + }), + ) +} + +// opsManagerLivenessProbe creates the liveness probe. +func opsManagerLivenessProbe() probes.Modification { + return probes.Apply( + probes.WithInitialDelaySeconds(10), + probes.WithTimeoutSeconds(10), + probes.WithPeriodSeconds(30), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(24), + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Scheme: corev1.URISchemeHTTP, Port: intstr.FromInt(8080), Path: "/monitor/health"}, + }), + ) +} + +// opsManagerStartupProbe creates the startup probe. +func opsManagerStartupProbe() probes.Modification { + return probes.Apply( + probes.WithInitialDelaySeconds(1), + probes.WithTimeoutSeconds(10), + probes.WithPeriodSeconds(20), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(30), + probes.WithHandler(corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Scheme: corev1.URISchemeHTTP, Port: intstr.FromInt(8080), Path: "/monitor/health"}, + }), + ) +} + +// buildOpsManagerAndBackupInitContainer creates the init container which +// copies the entry point script in the OM/Backup container +func buildOpsManagerAndBackupInitContainer(initOpsManagerImage string) container.Modification { + _, configureContainerSecurityContext := podtemplatespec.WithDefaultSecurityContextsModifications() + + return container.Apply( + container.WithName(util.InitOpsManagerContainerName), + container.WithImage(initOpsManagerImage), + container.WithVolumeMounts([]corev1.VolumeMount{buildOmScriptsVolumeMount(false)}), + configureContainerSecurityContext, + ) +} + +func buildOmScriptsVolumeMount(readOnly bool) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: "ops-manager-scripts", + MountPath: "/opt/scripts", + ReadOnly: readOnly, + } +} + +func buildOpsManagerLifecycle() lifecycle.Modification { + return lifecycle.WithPrestopCommand([]string{"/bin/sh", "-c", "/mongodb-ops-manager/bin/mongodb-mms stop_mms"}) +} + +func getOpsManagerHTTPSEnvVars(httpsSecretName string, certHash string) []corev1.EnvVar { + if httpsSecretName != "" { + path := "server.pem" + if certHash != "" { + path = certHash + } + // Before creating the podTemplate, we need to add the new PemKeyFile + // configuration if required. + return []corev1.EnvVar{{ + Name: omv1.ConvertNameToEnvVarFormat(util.MmsPEMKeyFile), + Value: fmt.Sprintf("%s/%s", util.MmsPemKeyFileDirInContainer, path), + }} + } + return []corev1.EnvVar{} +} + +func defaultPodLabels(labelKey, antiAffinityKey string) map[string]string { + return map[string]string{ + appLabelKey: labelKey, + ControllerLabelName: util.OperatorName, + podAntiAffinityLabelKey: antiAffinityKey, + } +} + +// defaultOpsManagerResourceRequirements returns the default ResourceRequirements +// which are used by OpsManager and the BackupDaemon +func defaultOpsManagerResourceRequirements() corev1.ResourceRequirements { + defaultMemory, _ := resource.ParseQuantity(util.DefaultMemoryOpsManager) + return corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: defaultMemory, + }, + Requests: corev1.ResourceList{}, + } +} + +func buildOpsManagerContainerPorts(httpsCertSecretName string, debugPort int32) []corev1.ContainerPort { + if debugPort > 0 { + return []corev1.ContainerPort{ + {ContainerPort: getOpsManagerContainerPort(httpsCertSecretName), Name: "default"}, + {ContainerPort: debugPort, Name: "debug"}, // debug + } + } + + return []corev1.ContainerPort{{ContainerPort: getOpsManagerContainerPort(httpsCertSecretName)}} +} + +func getOpsManagerContainerPort(httpsSecretName string) int32 { + _, port := omv1.SchemePortFromAnnotation("http") + if httpsSecretName != "" { + _, port = omv1.SchemePortFromAnnotation("https") + } + return port +} + +// opsManagerConfigurationToEnvVars returns a list of corev1.EnvVar which should be passed +// to the container running Ops Manager +func opsManagerConfigurationToEnvVars(m *omv1.MongoDBOpsManager) []corev1.EnvVar { + var envVars []corev1.EnvVar + for name, value := range m.Spec.Configuration { + envVars = append(envVars, corev1.EnvVar{ + Name: omv1.ConvertNameToEnvVarFormat(name), Value: value, + }) + } + return envVars +} + +func hasReleasesVolumeMount(opts OpsManagerStatefulSetOptions) bool { + if opts.StatefulSetSpecOverride != nil { + for _, c := range opts.StatefulSetSpecOverride.Template.Spec.Containers { + for _, vm := range c.VolumeMounts { + if vm.MountPath == util.OpsManagerPvcMountDownloads { + return true + } + } + } + } + return false +} + +func getNonPersistentOpsManagerVolumeMounts(volumes []corev1.Volume, volumeMounts []corev1.VolumeMount, opts OpsManagerStatefulSetOptions) ([]corev1.Volume, []corev1.VolumeMount) { + volumes = append(volumes, statefulset.CreateVolumeFromEmptyDir(util.OpsManagerPvcNameData)) + + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcNameData, util.PvcMountPathTmp, statefulset.WithSubPath(util.PvcNameTmp))) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcNameData, util.OpsManagerPvcMountPathTmp, statefulset.WithSubPath(util.OpsManagerPvcNameTmp))) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcNameData, util.OpsManagerPvcMountPathLogs, statefulset.WithSubPath(util.OpsManagerPvcNameLogs))) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcNameData, util.OpsManagerPvcMountPathEtc, statefulset.WithSubPath(util.OpsManagerPvcNameEtc))) + + if opts.LoggingConfiguration != nil && opts.LoggingConfiguration.LogBackRef != nil { + volumes = append(volumes, statefulset.CreateVolumeFromConfigMap(util.OpsManagerPvcLogBackNameVolume, opts.LoggingConfiguration.LogBackRef.Name)) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcLogBackNameVolume, util.OpsManagerPvcLogbackMountPath, statefulset.WithSubPath(util.OpsManagerPvcLogbackSubPath))) + } + + if opts.LoggingConfiguration != nil && opts.LoggingConfiguration.LogBackAccessRef != nil { + volumes = append(volumes, statefulset.CreateVolumeFromConfigMap(util.OpsManagerPvcLogBackAccessNameVolume, opts.LoggingConfiguration.LogBackAccessRef.Name)) + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcLogBackAccessNameVolume, util.OpsManagerPvcLogbackAccessMountPath, statefulset.WithSubPath(util.OpsManagerPvcLogbackAccessSubPath))) + } + + // This content is used by the Ops Manager to download mongodbs. Mount it only if there's no downloads override (like in om_localmode-multiple-pv.yaml for example) + if !hasReleasesVolumeMount(opts) { + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcNameData, util.OpsManagerPvcMountDownloads, statefulset.WithSubPath(util.OpsManagerPvcNameDownloads))) + } + + // This content is populated by the docker-entry-point.sh. It's being copied from conf-template + volumeMounts = append(volumeMounts, statefulset.CreateVolumeMount(util.OpsManagerPvcNameData, util.OpsManagerPvcMountPathConf, statefulset.WithSubPath(util.OpsManagerPvcNameConf))) + + return volumes, volumeMounts +} diff --git a/controllers/operator/construct/opsmanager_construction_common.go b/controllers/operator/construct/opsmanager_construction_common.go new file mode 100644 index 000000000..e409ba4d6 --- /dev/null +++ b/controllers/operator/construct/opsmanager_construction_common.go @@ -0,0 +1,23 @@ +package construct + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + corev1 "k8s.io/api/core/v1" +) + +const ( + AppDBConnectionStringVolume = "mongodb-uri" + AppDBConnectionStringPath = "/mongodb-ops-manager/.mongodb-mms-connection-string" +) + +func buildMmsMongoUriVolume(opts OpsManagerStatefulSetOptions) (corev1.Volume, corev1.VolumeMount) { + mmsMongoUriVolume := statefulset.CreateVolumeFromSecret(AppDBConnectionStringVolume, opts.AppDBConnectionSecretName) + mmsMongoUriVolumeMount := corev1.VolumeMount{ + Name: mmsMongoUriVolume.Name, + ReadOnly: true, + MountPath: AppDBConnectionStringPath, + } + + return mmsMongoUriVolume, mmsMongoUriVolumeMount +} diff --git a/controllers/operator/construct/opsmanager_construction_test.go b/controllers/operator/construct/opsmanager_construction_test.go new file mode 100644 index 000000000..44c830e98 --- /dev/null +++ b/controllers/operator/construct/opsmanager_construction_test.go @@ -0,0 +1,465 @@ +package construct + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +func Test_buildOpsManagerAndBackupInitContainer(t *testing.T) { + modification := buildOpsManagerAndBackupInitContainer("test-registry:latest") + containerObj := &corev1.Container{} + modification(containerObj) + expectedVolumeMounts := []corev1.VolumeMount{{ + Name: "ops-manager-scripts", + MountPath: "/opt/scripts", + ReadOnly: false, + }} + expectedContainer := &corev1.Container{ + Name: util.InitOpsManagerContainerName, + Image: "test-registry:latest", + VolumeMounts: expectedVolumeMounts, + SecurityContext: &corev1.SecurityContext{ + ReadOnlyRootFilesystem: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + }, + } + assert.Equal(t, expectedContainer, containerObj) +} + +func TestBuildJvmParamsEnvVars_FromCustomContainerResource(t *testing.T) { + ctx := context.Background() + om := omv1.NewOpsManagerBuilderDefault(). + AddConfiguration(util.MmsCentralUrlPropKey, "http://om-svc"). + AddConfiguration("mms.adminEmailAddr", "cloud-manager-support@mongodb.com"). + Build() + om.Spec.JVMParams = []string{"-DFakeOptionEnabled"} + + omSts, err := createOpsManagerStatefulset(ctx, om) + assert.NoError(t, err) + template := omSts.Spec.Template + + unsetQuantity := *resource.NewQuantity(0, resource.BinarySI) + + template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = *resource.NewQuantity(268435456, resource.BinarySI) + template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = unsetQuantity + envVarLimitsOnly, err := buildJvmParamsEnvVars(om.Spec, util.OpsManagerContainerName, template) + assert.NoError(t, err) + + template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = *resource.NewQuantity(218435456, resource.BinarySI) + envVarLimitsAndReqs, err := buildJvmParamsEnvVars(om.Spec, util.OpsManagerContainerName, template) + assert.NoError(t, err) + + template.Spec.Containers[0].Resources.Limits[corev1.ResourceMemory] = unsetQuantity + envVarReqsOnly, err := buildJvmParamsEnvVars(om.Spec, util.OpsManagerContainerName, template) + assert.NoError(t, err) + + template.Spec.Containers[0].Resources.Requests[corev1.ResourceMemory] = unsetQuantity + envVarsNoLimitsOrReqs, err := buildJvmParamsEnvVars(om.Spec, util.OpsManagerContainerName, template) + assert.NoError(t, err) + + // if only memory requests are configured, xms and xmx should be 90% of mem request + assert.Equal(t, "-DFakeOptionEnabled -Xmx187m -Xms187m", envVarReqsOnly[0].Value) + // both are configured, xms should be 90% of mem request, and xmx 90% of mem limit + assert.Equal(t, "-DFakeOptionEnabled -Xmx230m -Xms187m", envVarLimitsAndReqs[0].Value) + // if only memory limits are configured, xms and xmx should be 90% of mem limits + assert.Equal(t, "-DFakeOptionEnabled -Xmx230m -Xms230m", envVarLimitsOnly[0].Value) + // if neither is configured, xms/xmx params should not be added to JVM params, keeping OM defaults + assert.Equal(t, "-DFakeOptionEnabled", envVarsNoLimitsOrReqs[0].Value) +} + +func createOpsManagerStatefulset(ctx context.Context, om *omv1.MongoDBOpsManager, additionalOpts ...func(*OpsManagerStatefulSetOptions)) (appsv1.StatefulSet, error) { + client, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: client, + } + + omSts, err := OpsManagerStatefulSet(ctx, secretsClient, om, multicluster.GetLegacyCentralMemberCluster(om.Spec.Replicas, 0, client, secretsClient), zap.S(), additionalOpts...) + return omSts, err +} + +func TestBuildJvmParamsEnvVars_FromDefaultPodSpec(t *testing.T) { + ctx := context.Background() + om := omv1.NewOpsManagerBuilderDefault(). + AddConfiguration(util.MmsCentralUrlPropKey, "http://om-svc"). + AddConfiguration("mms.adminEmailAddr", "cloud-manager-support@mongodb.com"). + Build() + + client, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: client, + } + + omSts, err := OpsManagerStatefulSet(ctx, secretsClient, om, multicluster.GetLegacyCentralMemberCluster(om.Spec.Replicas, 0, client, secretsClient), zap.S()) + assert.NoError(t, err) + template := omSts.Spec.Template + + envVar, err := buildJvmParamsEnvVars(om.Spec, util.OpsManagerContainerName, template) + assert.NoError(t, err) + // xmx and xms based calculated from default container memory, requests.mem=limits.mem=5GB + assert.Equal(t, "CUSTOM_JAVA_MMS_UI_OPTS", envVar[0].Name) + assert.Equal(t, "-Xmx4291m -Xms4291m", envVar[0].Value) + + assert.Equal(t, "CUSTOM_JAVA_DAEMON_OPTS", envVar[1].Name) + assert.Equal(t, "-Xmx4291m -Xms4291m -DDAEMON.DEBUG.PORT=8090", envVar[1].Value) +} + +func TestBuildOpsManagerStatefulSet(t *testing.T) { + ctx := context.Background() + t.Run("Env Vars Sorted", func(t *testing.T) { + om := omv1.NewOpsManagerBuilderDefault(). + AddConfiguration(util.MmsCentralUrlPropKey, "http://om-svc"). + AddConfiguration("mms.adminEmailAddr", "cloud-manager-support@mongodb.com"). + Build() + + sts, err := createOpsManagerStatefulset(ctx, om) + + assert.NoError(t, err) + + // env vars are in sorted order + expectedVars := []corev1.EnvVar{ + {Name: "ENABLE_IRP", Value: "true"}, + {Name: "OM_PROP_mms_adminEmailAddr", Value: "cloud-manager-support@mongodb.com"}, + {Name: "OM_PROP_mms_centralUrl", Value: "http://om-svc"}, + {Name: "CUSTOM_JAVA_MMS_UI_OPTS", Value: "-Xmx4291m -Xms4291m"}, + {Name: "CUSTOM_JAVA_DAEMON_OPTS", Value: "-Xmx4291m -Xms4291m -DDAEMON.DEBUG.PORT=8090"}, + } + env := sts.Spec.Template.Spec.Containers[0].Env + assert.Equal(t, expectedVars, env) + }) + t.Run("JVM params applied after StatefulSet merge", func(t *testing.T) { + requirements := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("6G")}, + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("400M")}, + } + + statefulSet := statefulset.New( + statefulset.WithPodSpecTemplate( + podtemplatespec.Apply( + podtemplatespec.WithContainer(util.OpsManagerContainerName, + container.Apply( + container.WithName(util.OpsManagerContainerName), + container.WithResourceRequirements(requirements), + ), + ), + ), + ), + ) + + om := omv1.NewOpsManagerBuilderDefault(). + SetStatefulSetSpec(statefulSet.Spec). + Build() + + sts, err := createOpsManagerStatefulset(ctx, om) + assert.NoError(t, err) + expectedVars := []corev1.EnvVar{ + {Name: "ENABLE_IRP", Value: "true"}, + {Name: "CUSTOM_JAVA_MMS_UI_OPTS", Value: "-Xmx5149m -Xms343m"}, + {Name: "CUSTOM_JAVA_DAEMON_OPTS", Value: "-Xmx5149m -Xms343m -DDAEMON.DEBUG.PORT=8090"}, + } + env := sts.Spec.Template.Spec.Containers[0].Env + assert.Equal(t, expectedVars, env) + }) +} + +func Test_buildOpsManagerStatefulSet(t *testing.T) { + ctx := context.Background() + sts, err := createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().SetName("test-om").Build()) + assert.NoError(t, err) + assert.Equal(t, "test-om", sts.Name) + assert.Equal(t, util.OpsManagerContainerName, sts.Spec.Template.Spec.Containers[0].Name) + assert.Equal(t, []string{"/opt/scripts/docker-entry-point.sh"}, + sts.Spec.Template.Spec.Containers[0].Command) +} + +func Test_buildOpsManagerStatefulSet_Secrets(t *testing.T) { + ctx := context.Background() + opsManager := omv1.NewOpsManagerBuilderDefault().SetName("test-om").Build() + sts, err := createOpsManagerStatefulset(ctx, opsManager) + assert.NoError(t, err) + + expectedSecretVolumeNames := []string{"test-om-gen-key", opsManager.AppDBMongoConnectionStringSecretName()} + var actualSecretVolumeNames []string + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Secret != nil { + actualSecretVolumeNames = append(actualSecretVolumeNames, v.Secret.SecretName) + } + } + + assert.Equal(t, expectedSecretVolumeNames, actualSecretVolumeNames) +} + +// TestOpsManagerPodTemplate_MergePodTemplate checks the custom pod template provided by the user. +// It's supposed to override the values produced by the Operator and leave everything else as is +func TestOpsManagerPodTemplate_MergePodTemplate(t *testing.T) { + ctx := context.Background() + expectedAnnotations := map[string]string{"customKey": "customVal", "connectionStringHash": ""} + expectedTolerations := []corev1.Toleration{{Key: "dedicated", Value: "database"}} + newContainer := corev1.Container{ + Name: "my-custom-container", + Image: "my-custom-image", + } + + podTemplateSpec := podtemplatespec.New( + podtemplatespec.WithAnnotations(expectedAnnotations), + podtemplatespec.WithServiceAccount("test-account"), + podtemplatespec.WithTolerations(expectedTolerations), + podtemplatespec.WithContainer("my-custom-container", + container.Apply( + container.WithName("my-custom-container"), + container.WithImage("my-custom-image"), + )), + ) + + om := omv1.NewOpsManagerBuilderDefault().Build() + + omSts, err := createOpsManagerStatefulset(ctx, om) + assert.NoError(t, err) + template := omSts.Spec.Template + + originalLabels := template.Labels + + operatorSts := appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Template: template, + }, + } + + mergedSpec := merge.StatefulSetSpecs(operatorSts.Spec, appsv1.StatefulSetSpec{ + Template: podTemplateSpec, + }) + + template = mergedSpec.Template + // Service account gets overriden by custom pod template + assert.Equal(t, "test-account", template.Spec.ServiceAccountName) + assert.Equal(t, expectedAnnotations, template.Annotations) + assert.Equal(t, expectedTolerations, template.Spec.Tolerations) + assert.Len(t, template.Spec.Containers, 2) + assert.Equal(t, newContainer, template.Spec.Containers[1]) + + // Some validation that the Operator-made config hasn't suffered + assert.Equal(t, originalLabels, template.Labels) + assert.NotNil(t, template.Spec.SecurityContext) + assert.Equal(t, util.Int64Ref(util.FsGroup), template.Spec.SecurityContext.FSGroup) + assert.Equal(t, util.OpsManagerContainerName, template.Spec.Containers[0].Name) +} + +// TestOpsManagerPodTemplate_PodSpec verifies that StatefulSetSpec is applied correctly to OpsManager/Backup pod template. +func TestOpsManagerPodTemplate_PodSpec(t *testing.T) { + ctx := context.Background() + omSts, err := createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().Build()) + assert.NoError(t, err) + + resourceLimits := buildSafeResourceList("1.0", "500M") + resourceRequests := buildSafeResourceList("0.5", "400M") + nodeAffinity := defaultNodeAffinity() + podAffinity := defaultPodAffinity() + + stsSpecOverride := appsv1.StatefulSetSpec{ + Template: podtemplatespec.New( + podtemplatespec.WithAffinity(omSts.Name, PodAntiAffinityLabelKey, 100), + podtemplatespec.WithPodAffinity(&podAffinity), + podtemplatespec.WithNodeAffinity(&nodeAffinity), + podtemplatespec.WithTopologyKey("rack", 0), + podtemplatespec.WithContainer(util.OpsManagerContainerName, + container.Apply( + container.WithName(util.OpsManagerContainerName), + container.WithResourceRequirements(corev1.ResourceRequirements{ + Limits: resourceLimits, + Requests: resourceRequests, + }), + ), + ), + ), + } + mergedSpec := merge.StatefulSetSpecs(omSts.Spec, stsSpecOverride) + assert.NoError(t, err) + + spec := mergedSpec.Template.Spec + assert.Equal(t, defaultNodeAffinity(), *spec.Affinity.NodeAffinity) + assert.Equal(t, defaultPodAffinity(), *spec.Affinity.PodAffinity) + assert.Len(t, spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, 1) + // the pod anti affinity term was overridden + term := spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0] + assert.Equal(t, "rack", term.PodAffinityTerm.TopologyKey) + + req := spec.Containers[0].Resources.Limits + assert.Len(t, req, 2) + cpu := req[corev1.ResourceCPU] + memory := req[corev1.ResourceMemory] + assert.Equal(t, "1", (&cpu).String()) + assert.Equal(t, int64(500000000), (&memory).Value()) + + req = spec.Containers[0].Resources.Requests + assert.Len(t, req, 2) + cpu = req[corev1.ResourceCPU] + memory = req[corev1.ResourceMemory] + assert.Equal(t, "500m", (&cpu).String()) + assert.Equal(t, int64(400000000), (&memory).Value()) +} + +// TestOpsManagerPodTemplate_SecurityContext verifies that security context is created correctly +// in OpsManager/BackupDaemon podTemplate. It's not built if 'MANAGED_SECURITY_CONTEXT' env var +// is set to 'true' +func TestOpsManagerPodTemplate_SecurityContext(t *testing.T) { + ctx := context.Background() + defer mock.InitDefaultEnvVariables() + + omSts, err := createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().Build()) + assert.NoError(t, err) + + podSpecTemplate := omSts.Spec.Template + spec := podSpecTemplate.Spec + assert.Len(t, spec.InitContainers, 1) + assert.Equal(t, spec.InitContainers[0].Name, "mongodb-enterprise-init-ops-manager") + assert.NotNil(t, spec.SecurityContext) + assert.Equal(t, util.Int64Ref(util.FsGroup), spec.SecurityContext.FSGroup) + + t.Setenv(util.ManagedSecurityContextEnv, "true") + + omSts, err = createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().Build()) + assert.NoError(t, err) + podSpecTemplate = omSts.Spec.Template + assert.Nil(t, podSpecTemplate.Spec.SecurityContext) +} + +func TestOpsManagerPodTemplate_TerminationTimeout(t *testing.T) { + ctx := context.Background() + omSts, err := createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().Build()) + assert.NoError(t, err) + podSpecTemplate := omSts.Spec.Template + assert.Equal(t, int64(300), *podSpecTemplate.Spec.TerminationGracePeriodSeconds) +} + +func TestOpsManagerPodTemplate_ImagePullPolicy(t *testing.T) { + ctx := context.Background() + defer mock.InitDefaultEnvVariables() + + omSts, err := createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().Build()) + assert.NoError(t, err) + + podSpecTemplate := omSts.Spec.Template + spec := podSpecTemplate.Spec + + assert.Nil(t, spec.ImagePullSecrets) + + t.Setenv(util.ImagePullSecrets, "my-cool-secret") + omSts, err = createOpsManagerStatefulset(ctx, omv1.NewOpsManagerBuilderDefault().Build()) + assert.NoError(t, err) + podSpecTemplate = omSts.Spec.Template + spec = podSpecTemplate.Spec + + assert.NotNil(t, spec.ImagePullSecrets) + assert.Equal(t, spec.ImagePullSecrets[0].Name, "my-cool-secret") +} + +// TestOpsManagerPodTemplate_Container verifies the default OM container built by 'opsManagerPodTemplate' method +func TestOpsManagerPodTemplate_Container(t *testing.T) { + const opsManagerImage = "quay.io/mongodb/mongodb-enterprise-ops-manager:4.2.0" + + ctx := context.Background() + om := omv1.NewOpsManagerBuilderDefault().SetVersion("4.2.0").Build() + sts, err := createOpsManagerStatefulset(ctx, om, WithOpsManagerImage(opsManagerImage)) + assert.NoError(t, err) + template := sts.Spec.Template + + assert.Len(t, template.Spec.Containers, 1) + containerObj := template.Spec.Containers[0] + assert.Equal(t, util.OpsManagerContainerName, containerObj.Name) + // TODO change when we stop using versioning + assert.Equal(t, opsManagerImage, containerObj.Image) + assert.Equal(t, corev1.PullNever, containerObj.ImagePullPolicy) + + assert.Equal(t, int32(util.OpsManagerDefaultPortHTTP), containerObj.Ports[0].ContainerPort) + assert.Equal(t, "/monitor/health", containerObj.ReadinessProbe.HTTPGet.Path) + assert.Equal(t, int32(8080), containerObj.ReadinessProbe.HTTPGet.Port.IntVal) + assert.Equal(t, "/monitor/health", containerObj.LivenessProbe.HTTPGet.Path) + assert.Equal(t, int32(8080), containerObj.LivenessProbe.HTTPGet.Port.IntVal) + assert.Equal(t, "/monitor/health", containerObj.StartupProbe.HTTPGet.Path) + assert.Equal(t, int32(8080), containerObj.StartupProbe.HTTPGet.Port.IntVal) + + assert.Equal(t, []string{"/opt/scripts/docker-entry-point.sh"}, containerObj.Command) + assert.Equal(t, []string{"/bin/sh", "-c", "/mongodb-ops-manager/bin/mongodb-mms stop_mms"}, + containerObj.Lifecycle.PreStop.Exec.Command) +} + +func defaultNodeAffinity() corev1.NodeAffinity { + return corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "dc", + Values: []string{"US-EAST"}, + }}, + }, + }, + }, + } +} + +func defaultPodAffinity() corev1.PodAffinity { + return corev1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ + Weight: 50, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "web-server"}}, + TopologyKey: "rack", + }, + }}, + } +} + +func defaultPodAntiAffinity() corev1.PodAntiAffinity { + return corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ + Weight: 77, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "web-server"}}, + TopologyKey: "rack", + }, + }}, + } +} + +// buildSafeResourceList returns a ResourceList but should not be called +// with dynamic values. This function ignores errors in the parsing of +// resource.Quantities and as a result, should only be used in tests with +// pre-set valid values. +func buildSafeResourceList(cpu, memory string) corev1.ResourceList { + res := corev1.ResourceList{} + if q := ParseQuantityOrZero(cpu); !q.IsZero() { + res[corev1.ResourceCPU] = q + } + if q := ParseQuantityOrZero(memory); !q.IsZero() { + res[corev1.ResourceMemory] = q + } + return res +} diff --git a/controllers/operator/construct/proxy.go b/controllers/operator/construct/proxy.go new file mode 100644 index 000000000..395648168 --- /dev/null +++ b/controllers/operator/construct/proxy.go @@ -0,0 +1,47 @@ +package construct + +import ( + "os" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" +) + +const ( + // MDB_PROPAGATE_PROXY_ENV needs to be configurable in the operator environment to support configuring whether the proxy environment + // variables should be propagated to the database containers. A valid case for this is a multi-cluster environment where the operator + // might have to use a proxy to connect to OM/CM, but the mongodb agents in different clusters don't have to. + PropagateProxyEnv = "MDB_PROPAGATE_PROXY_ENV" +) + +// below proxy handling has been inspired by the proxy handling in operator-lib +// here: https://github.com/operator-framework/operator-lib/blob/e40c80627593fa6eaad3e2cb1380e3e838afe56c/proxy/proxy.go#L30 + +// ProxyEnvNames are standard environment variables for proxies +var ProxyEnvNames = []string{"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY"} + +// ReadDatabaseProxyVarsFromEnv retrieves the standard proxy-related environment +// variables from the running environment and returns a slice of corev1 EnvVar +// containing upper and lower case versions of those variables. +func ReadDatabaseProxyVarsFromEnv() []corev1.EnvVar { + propagateProxyVar, _ := os.LookupEnv(PropagateProxyEnv) // nolint:forbidigo + propagateProxy, _ := strconv.ParseBool(propagateProxyVar) + if !propagateProxy { + return nil + } + var envVars []corev1.EnvVar + for _, s := range ProxyEnvNames { + value, isSet := os.LookupEnv(s) // nolint:forbidigo + if isSet { + envVars = append(envVars, corev1.EnvVar{ + Name: s, + Value: value, + }, corev1.EnvVar{ + Name: strings.ToLower(s), + Value: value, + }) + } + } + return envVars +} diff --git a/controllers/operator/construct/proxy_test.go b/controllers/operator/construct/proxy_test.go new file mode 100644 index 000000000..4deb366cc --- /dev/null +++ b/controllers/operator/construct/proxy_test.go @@ -0,0 +1,88 @@ +package construct + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + corev1 "k8s.io/api/core/v1" +) + +func TestReadProxyVarsFromEnv(t *testing.T) { + tests := []struct { + name string + operatorEnv map[string]string + expectedVars []corev1.EnvVar + }{ + { + name: "Do not propagate proxy env explicitly", + operatorEnv: map[string]string{ + "MDB_PROPAGATE_PROXY_ENV": "false", + "NO_PROXY": "google.com", + }, + expectedVars: nil, + }, + { + name: "Do not propagate proxy env by default", + operatorEnv: map[string]string{ + "HTTP_PROXY": "http://example-http-proxy:7312", + "HTTPS_PROXY": "https://secure-proxy:3242", + }, + expectedVars: nil, + }, + { + name: "Propagate proxy environment variables", + operatorEnv: map[string]string{ + "MDB_PROPAGATE_PROXY_ENV": "true", + "HTTP_PROXY": "http://example-http-proxy:7312", + "HTTPS_PROXY": "https://secure-proxy:3242", + }, + expectedVars: []corev1.EnvVar{ + { + Name: "HTTP_PROXY", + Value: "http://example-http-proxy:7312", + }, + { + Name: "http_proxy", + Value: "http://example-http-proxy:7312", + }, + { + Name: "HTTPS_PROXY", + Value: "https://secure-proxy:3242", + }, + { + Name: "https_proxy", + Value: "https://secure-proxy:3242", + }, + }, + }, + { + name: "Propagate only proxy environment variables", + operatorEnv: map[string]string{ + "MDB_PROPAGATE_PROXY_ENV": "true", + "HTTPS_PROXY": "https://secure-proxy:3242", + "DEFAULT_AGENT_VERSION": "13.0.2341", + "MAX_SURGE": "23415", + }, + expectedVars: []corev1.EnvVar{ + { + Name: "HTTPS_PROXY", + Value: "https://secure-proxy:3242", + }, + { + Name: "https_proxy", + Value: "https://secure-proxy:3242", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for key, val := range tt.operatorEnv { + t.Setenv(key, val) + } + assert.Equal(t, ReadDatabaseProxyVarsFromEnv(), tt.expectedVars) + }) + } +} diff --git a/controllers/operator/construct/pvc.go b/controllers/operator/construct/pvc.go new file mode 100644 index 000000000..904ce9c70 --- /dev/null +++ b/controllers/operator/construct/pvc.go @@ -0,0 +1,59 @@ +package construct + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// pvcFunc convenience function to build a PersistentVolumeClaim. It accepts two config parameters - the one specified by +// the customers and the default one configured by the Operator. Putting the default one to the signature ensures the +// calling code doesn't forget to think about default values in case the user hasn't provided values. +func pvcFunc(name string, config *mdbv1.PersistenceConfig, defaultConfig mdbv1.PersistenceConfig, labels map[string]string) persistentvolumeclaim.Modification { + selectorFunc := persistentvolumeclaim.NOOP() + storageClassNameFunc := persistentvolumeclaim.NOOP() + if config != nil { + if config.LabelSelector != nil { + selectorFunc = persistentvolumeclaim.WithLabelSelector(&config.LabelSelector.LabelSelector) + } + if config.StorageClass != nil { + storageClassNameFunc = persistentvolumeclaim.WithStorageClassName(*config.StorageClass) + } + } + return persistentvolumeclaim.Apply( + persistentvolumeclaim.WithName(name), + persistentvolumeclaim.WithAccessModes(corev1.ReadWriteOnce), + persistentvolumeclaim.WithResourceRequests(buildStorageRequirements(config, defaultConfig)), + persistentvolumeclaim.WithLabels(labels), + selectorFunc, + storageClassNameFunc, + ) +} + +func createClaimsAndMountsMultiModeFunc(persistence *mdbv1.Persistence, defaultConfig mdbv1.MultiplePersistenceConfig, labels map[string]string) (map[string]persistentvolumeclaim.Modification, []corev1.VolumeMount) { + mounts := []corev1.VolumeMount{ + statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathData), + statefulset.CreateVolumeMount(util.PvcNameJournal, util.PvcMountPathJournal), + statefulset.CreateVolumeMount(util.PvcNameLogs, util.PvcMountPathLogs), + } + return map[string]persistentvolumeclaim.Modification{ + util.PvcNameData: pvcFunc(util.PvcNameData, persistence.MultipleConfig.Data, *defaultConfig.Data, labels), + util.PvcNameJournal: pvcFunc(util.PvcNameJournal, persistence.MultipleConfig.Journal, *defaultConfig.Journal, labels), + util.PvcNameLogs: pvcFunc(util.PvcNameLogs, persistence.MultipleConfig.Logs, *defaultConfig.Logs, labels), + }, mounts +} + +func createClaimsAndMountsSingleModeFunc(config *mdbv1.PersistenceConfig, opts DatabaseStatefulSetOptions) (map[string]persistentvolumeclaim.Modification, []corev1.VolumeMount) { + mounts := []corev1.VolumeMount{ + statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathData, statefulset.WithSubPath(util.PvcNameData)), + statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathJournal, statefulset.WithSubPath(util.PvcNameJournal)), + statefulset.CreateVolumeMount(util.PvcNameData, util.PvcMountPathLogs, statefulset.WithSubPath(util.PvcNameLogs)), + } + return map[string]persistentvolumeclaim.Modification{ + util.PvcNameData: pvcFunc(util.PvcNameData, config, *opts.PodSpec.Default.Persistence.SingleConfig, opts.Labels), + }, mounts +} diff --git a/controllers/operator/construct/resourcerequirements.go b/controllers/operator/construct/resourcerequirements.go new file mode 100644 index 000000000..166596f2f --- /dev/null +++ b/controllers/operator/construct/resourcerequirements.go @@ -0,0 +1,71 @@ +package construct + +import ( + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/resource" + + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +// buildStorageRequirements returns a corev1.ResourceList definition for storage requirements. +// This is used by the StatefulSet PersistentVolumeClaimTemplate. +func buildStorageRequirements(persistenceConfig *mdbv1.PersistenceConfig, defaultConfig mdbv1.PersistenceConfig) corev1.ResourceList { + res := corev1.ResourceList{} + + if q := ParseQuantityOrZero(mdbv1.GetStorageOrDefault(persistenceConfig, defaultConfig)); !q.IsZero() { + res[corev1.ResourceStorage] = q + } + + return res +} + +// buildRequirementsFromPodSpec takes a podSpec, and builds a ResourceRequirements +// taking into consideration the default values of the given podSpec +func buildRequirementsFromPodSpec(podSpec mdbv1.PodSpecWrapper) corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Limits: buildLimitsRequirements(&podSpec), + Requests: buildRequestsRequirements(&podSpec), + } +} + +// buildLimitsRequirements returns a corev1.ResourceList definition for limits for CPU and Memory Requirements +// This is used by the StatefulSet containers to allocate resources per Pod. +func buildLimitsRequirements(reqs *mdbv1.PodSpecWrapper) corev1.ResourceList { + res := corev1.ResourceList{} + + if q := ParseQuantityOrZero(reqs.GetCpuOrDefault()); !q.IsZero() { + res[corev1.ResourceCPU] = q + } + if q := ParseQuantityOrZero(reqs.GetMemoryOrDefault()); !q.IsZero() { + res[corev1.ResourceMemory] = q + } + + return res +} + +// buildRequestsRequirements returns a corev1.ResourceList definition for requests for CPU and Memory Requirements +// This is used by the StatefulSet containers to allocate resources per Pod. +func buildRequestsRequirements(reqs *mdbv1.PodSpecWrapper) corev1.ResourceList { + res := corev1.ResourceList{} + + if q := ParseQuantityOrZero(reqs.GetCpuRequestsOrDefault()); !q.IsZero() { + res[corev1.ResourceCPU] = q + } + if q := ParseQuantityOrZero(reqs.GetMemoryRequestsOrDefault()); !q.IsZero() { + res[corev1.ResourceMemory] = q + } + + return res +} + +// TODO: this function needs to be unexported - refactor tests and make this private +func ParseQuantityOrZero(qty string) resource.Quantity { + q, err := resource.ParseQuantity(qty) + if err != nil && qty != "" { + zap.S().Infof("Error converting %s to `resource.Quantity`", qty) + } + + return q +} diff --git a/controllers/operator/construct/resourcerequirements_test.go b/controllers/operator/construct/resourcerequirements_test.go new file mode 100644 index 000000000..5ecd47ca7 --- /dev/null +++ b/controllers/operator/construct/resourcerequirements_test.go @@ -0,0 +1,115 @@ +package construct + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +func TestStorageRequirements(t *testing.T) { + // value is provided - the default is ignored + podSpec := mdbv1.NewEmptyPodSpecWrapperBuilder(). + SetSinglePersistence(mdbv1.NewPersistenceBuilder("40G")). + SetDefault(mdbv1.NewPodSpecWrapperBuilder().SetSinglePersistence(mdbv1.NewPersistenceBuilder("12G"))). + Build() + + req := buildStorageRequirements(podSpec.Persistence.SingleConfig, *podSpec.Default.Persistence.SingleConfig) + + assert.Len(t, req, 1) + quantity := req[corev1.ResourceStorage] + assert.Equal(t, int64(40000000000), (&quantity).Value()) + + // value is not provided - the default is used + podSpec = mdbv1.NewEmptyPodSpecWrapperBuilder(). + SetDefault(mdbv1.NewPodSpecWrapperBuilder().SetSinglePersistence(mdbv1.NewPersistenceBuilder("5G"))). + Build() + + req = buildStorageRequirements(podSpec.Persistence.SingleConfig, *podSpec.Default.Persistence.SingleConfig) + + assert.Len(t, req, 1) + quantity = req[corev1.ResourceStorage] + assert.Equal(t, int64(5000000000), (&quantity).Value()) + + // value is not provided and default is empty - the parameter must not be set at all + podSpec = mdbv1.NewEmptyPodSpecWrapperBuilder().Build() + req = buildStorageRequirements(podSpec.Persistence.SingleConfig, *podSpec.Default.Persistence.SingleConfig) + + assert.Len(t, req, 0) +} + +func TestBuildLimitsRequirements(t *testing.T) { + // values are provided - the defaults are ignored + podSpec := mdbv1.NewEmptyPodSpecWrapperBuilder().SetCpuLimit("0.1").SetMemoryLimit("512M"). + SetDefault(mdbv1.NewPodSpecWrapperBuilder().SetCpuLimit("0.5").SetMemoryLimit("1G")). + Build() + + req := buildLimitsRequirements(podSpec) + + assert.Len(t, req, 2) + cpu := req[corev1.ResourceCPU] + memory := req[corev1.ResourceMemory] + assert.Equal(t, "100m", (&cpu).String()) + assert.Equal(t, int64(512000000), (&memory).Value()) + + // values are not provided - the defaults are used + podSpec = mdbv1.NewEmptyPodSpecWrapperBuilder(). + SetDefault(mdbv1.NewPodSpecWrapperBuilder().SetCpuLimit("0.8").SetMemoryLimit("10G")). + Build() + + req = buildLimitsRequirements(podSpec) + + assert.Len(t, req, 2) + cpu = req[corev1.ResourceCPU] + memory = req[corev1.ResourceMemory] + assert.Equal(t, "800m", (&cpu).String()) + assert.Equal(t, int64(10000000000), (&memory).Value()) + + // value are not provided and default are empty - the parameters must not be set at all + podSpec = mdbv1.NewEmptyPodSpecWrapperBuilder().Build() + req = buildLimitsRequirements(podSpec) + + assert.Len(t, req, 0) +} + +func TestBuildRequestsRequirements(t *testing.T) { + // values are provided - the defaults are ignored + podSpec := mdbv1.NewEmptyPodSpecWrapperBuilder().SetCpuRequests("0.1").SetMemoryRequest("512M"). + SetDefault(mdbv1.NewPodSpecWrapperBuilder().SetCpuRequests("0.5").SetMemoryRequest("1G")). + Build() + + req := buildRequestsRequirements(podSpec) + + assert.Len(t, req, 2) + cpu := req[corev1.ResourceCPU] + memory := req[corev1.ResourceMemory] + assert.Equal(t, "100m", (&cpu).String()) + assert.Equal(t, int64(512000000), (&memory).Value()) + + // values are not provided - the defaults are used + podSpec = mdbv1.NewEmptyPodSpecWrapperBuilder(). + SetDefault(mdbv1.NewPodSpecWrapperBuilder().SetCpuRequests("0.8").SetMemoryRequest("10G")). + Build() + req = buildRequestsRequirements(podSpec) + + assert.Len(t, req, 2) + cpu = req[corev1.ResourceCPU] + memory = req[corev1.ResourceMemory] + assert.Equal(t, "800m", (&cpu).String()) + assert.Equal(t, int64(10000000000), (&memory).Value()) + + // value are not provided and default are empty - the parameters must not be set at all + podSpec = mdbv1.NewEmptyPodSpecWrapperBuilder().Build() + req = buildRequestsRequirements(podSpec) + + assert.Len(t, req, 0) +} diff --git a/controllers/operator/construct/scalers/appdb_scaler.go b/controllers/operator/construct/scalers/appdb_scaler.go new file mode 100644 index 000000000..2dbd4057e --- /dev/null +++ b/controllers/operator/construct/scalers/appdb_scaler.go @@ -0,0 +1,56 @@ +package scalers + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers/interfaces" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +func GetAppDBScaler(opsManager *om.MongoDBOpsManager, memberClusterName string, memberClusterNum int, prevMembers []multicluster.MemberCluster) interfaces.MultiClusterReplicaSetScaler { + if opsManager.Spec.AppDB.IsMultiCluster() { + return NewMultiClusterReplicaSetScaler("AppDB", opsManager.Spec.AppDB.ClusterSpecList, memberClusterName, memberClusterNum, prevMembers) + } else { + return NewAppDBSingleClusterScaler(opsManager) + } +} + +// this is the implementation that originally was in om.MongoDBOpsManager +type appDBSingleClusterScaler struct { + opsManager *om.MongoDBOpsManager +} + +func NewAppDBSingleClusterScaler(opsManager *om.MongoDBOpsManager) interfaces.MultiClusterReplicaSetScaler { + return &appDBSingleClusterScaler{ + opsManager: opsManager, + } +} + +func (s *appDBSingleClusterScaler) ForcedIndividualScaling() bool { + return false +} + +func (s *appDBSingleClusterScaler) DesiredReplicas() int { + return s.opsManager.Spec.AppDB.Members +} + +func (s *appDBSingleClusterScaler) TargetReplicas() int { + return s.DesiredReplicas() +} + +func (s *appDBSingleClusterScaler) CurrentReplicas() int { + return s.opsManager.Status.AppDbStatus.Members +} + +func (s *appDBSingleClusterScaler) ScalingFirstTime() bool { + return true +} + +func (s *appDBSingleClusterScaler) MemberClusterName() string { + return multicluster.LegacyCentralClusterName +} + +func (s *appDBSingleClusterScaler) MemberClusterNum() int { + return 0 +} + +func (s *appDBSingleClusterScaler) ScalerDescription() string { return "AppDB" } diff --git a/controllers/operator/construct/scalers/appdb_scaler_test.go b/controllers/operator/construct/scalers/appdb_scaler_test.go new file mode 100644 index 000000000..bc3db631a --- /dev/null +++ b/controllers/operator/construct/scalers/appdb_scaler_test.go @@ -0,0 +1,302 @@ +package scalers + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +func TestAppDBMultiClusterScaler(t *testing.T) { + testCases := []struct { + name string + memberClusterName string + memberClusterNum int + clusterSpecList mdbv1.ClusterSpecList + prevMembers []multicluster.MemberCluster + expectedDesiredReplicas int + expectedCurrentReplicas int + expectedReplicasThisReconciliation int + }{ + { + name: "no previous members", + memberClusterName: "cluster-1", + memberClusterNum: 0, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 2, + }, + { + ClusterName: "cluster-3", + Members: 2, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 0}, + {Name: "cluster-2", Index: 1, Replicas: 0}, + {Name: "cluster-3", Index: 2, Replicas: 0}, + }, + expectedDesiredReplicas: 3, + expectedCurrentReplicas: 0, + expectedReplicasThisReconciliation: 3, + }, + { + name: "scaling up one member", + memberClusterName: "cluster-1", + memberClusterNum: 0, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 2, + }, + { + ClusterName: "cluster-3", + Members: 2, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 1}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 3, + expectedCurrentReplicas: 1, + expectedReplicasThisReconciliation: 2, + }, + { + name: "scaling down one member", + memberClusterName: "cluster-2", + memberClusterNum: 1, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 0, + }, + { + ClusterName: "cluster-3", + Members: 2, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 0, + expectedCurrentReplicas: 2, + expectedReplicasThisReconciliation: 1, + }, + { + name: "scaling up multiple members cluster currently scaling", + memberClusterName: "cluster-2", + memberClusterNum: 1, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 4, + }, + { + ClusterName: "cluster-3", + Members: 3, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 4, + expectedCurrentReplicas: 2, + expectedReplicasThisReconciliation: 3, + }, + { + name: "scaling up multiple members cluster currently not scaling", + memberClusterName: "cluster-3", + memberClusterNum: 2, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 4, + }, + { + ClusterName: "cluster-3", + Members: 3, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 2, + expectedCurrentReplicas: 2, + expectedReplicasThisReconciliation: 2, + }, + { + name: "scaling down multiple members cluster currently scaling", + memberClusterName: "cluster-2", + memberClusterNum: 1, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 0, + }, + { + ClusterName: "cluster-3", + Members: 0, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 0, + expectedCurrentReplicas: 2, + expectedReplicasThisReconciliation: 1, + }, + { + name: "scaling down multiple members cluster currently not scaling", + memberClusterName: "cluster-3", + memberClusterNum: 2, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 0, + }, + { + ClusterName: "cluster-3", + Members: 0, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 2, + expectedCurrentReplicas: 2, + expectedReplicasThisReconciliation: 2, + }, + { + name: "no scaling required", + memberClusterName: "cluster-3", + memberClusterNum: 2, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 2, + }, + { + ClusterName: "cluster-3", + Members: 2, + }, + }, + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + }, + expectedDesiredReplicas: 2, + expectedCurrentReplicas: 2, + expectedReplicasThisReconciliation: 2, + }, + { + name: "adding a new cluster to an already populated config", + memberClusterName: "cluster-4", + memberClusterNum: 3, + clusterSpecList: mdbv1.ClusterSpecList{ + { + ClusterName: "cluster-1", + Members: 3, + }, + { + ClusterName: "cluster-2", + Members: 2, + }, + { + ClusterName: "cluster-3", + Members: 2, + }, + { + ClusterName: "cluster-4", + Members: 3, + }, + }, + + prevMembers: []multicluster.MemberCluster{ + {Name: "cluster-1", Index: 0, Replicas: 3}, + {Name: "cluster-2", Index: 1, Replicas: 2}, + {Name: "cluster-3", Index: 2, Replicas: 2}, + {Name: "cluster-4", Index: 3, Replicas: 0}, + }, + + expectedDesiredReplicas: 3, + expectedCurrentReplicas: 0, + expectedReplicasThisReconciliation: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + builder := opsManagerBuilder().SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster) + opsManager := builder.SetAppDBClusterSpecList(tc.clusterSpecList).Build() + scaler := GetAppDBScaler(opsManager, tc.memberClusterName, tc.memberClusterNum, tc.prevMembers) + + assert.Equal(t, tc.expectedDesiredReplicas, scaler.DesiredReplicas(), "Desired replicas") + assert.Equal(t, tc.expectedCurrentReplicas, scaler.CurrentReplicas(), "Current replicas") + assert.Equal(t, tc.expectedReplicasThisReconciliation, scale.ReplicasThisReconciliation(scaler), "Replicas this reconciliation") + }) + } +} + +func opsManagerBuilder() *omv1.OpsManagerBuilder { + spec := omv1.MongoDBOpsManagerSpec{ + Version: "5.0.0", + AppDB: *omv1.DefaultAppDbBuilder().Build(), + AdminSecret: "om-admin", + } + resource := omv1.MongoDBOpsManager{Spec: spec, ObjectMeta: metav1.ObjectMeta{Name: "test-om", Namespace: mock.TestNamespace}} + return omv1.NewOpsManagerBuilderFromResource(resource) +} diff --git a/controllers/operator/construct/scalers/interfaces/interfaces.go b/controllers/operator/construct/scalers/interfaces/interfaces.go new file mode 100644 index 000000000..bb9371a6c --- /dev/null +++ b/controllers/operator/construct/scalers/interfaces/interfaces.go @@ -0,0 +1,13 @@ +package interfaces + +import "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + +type MultiClusterReplicaSetScaler interface { + scale.ReplicaSetScaler + ScalingFirstTime() bool + TargetReplicas() int + MemberClusterName() string + MemberClusterNum() int + // ScalerDescription contains the name of the component associated to that scaler (shard, config server, AppDB...) + ScalerDescription() string +} diff --git a/controllers/operator/construct/scalers/replicaset_scaler.go b/controllers/operator/construct/scalers/replicaset_scaler.go new file mode 100644 index 000000000..5e6e267d6 --- /dev/null +++ b/controllers/operator/construct/scalers/replicaset_scaler.go @@ -0,0 +1,153 @@ +package scalers + +import ( + "fmt" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +// MultiClusterReplicaSetScaler is a generic scaler that can be user in any multi-cluster replica set. +type MultiClusterReplicaSetScaler struct { + clusterSpecList mdbv1.ClusterSpecList + memberClusterName string + memberClusterNum int + prevMembers []multicluster.MemberCluster + scalerDescription string +} + +func NewMultiClusterReplicaSetScaler(scalerDescription string, clusterSpecList mdbv1.ClusterSpecList, memberClusterName string, memberClusterNum int, prevMembers []multicluster.MemberCluster) *MultiClusterReplicaSetScaler { + return &MultiClusterReplicaSetScaler{ + scalerDescription: scalerDescription, + clusterSpecList: clusterSpecList, + memberClusterName: memberClusterName, + memberClusterNum: memberClusterNum, + prevMembers: prevMembers, + } +} + +func getMemberClusterItemByClusterName(clusterSpecList mdbv1.ClusterSpecList, memberClusterName string) mdbv1.ClusterSpecItem { + for _, clusterSpec := range clusterSpecList { + if clusterSpec.ClusterName == memberClusterName { + return clusterSpec + } + } + + // In case the member cluster is not found in the cluster spec list, we return an empty ClusterSpecItem + // with 0 members to handle the case of removing a cluster from the spec list without a panic. + return mdbv1.ClusterSpecItem{ + ClusterName: memberClusterName, + Members: 0, + } +} + +func (s *MultiClusterReplicaSetScaler) ForcedIndividualScaling() bool { + // When scaling ReplicaSet for the first time, it's safe to add all the members. + // When adding a new cluster, we want to force individual scaling because ReplicasThisReconciliation + // short circuits the one-by-one scaling when individual scaling is disabled and starting replicas is zero. + if s.ScalingFirstTime() { + return false + } else { + return true + } +} + +// DesiredReplicas returns desired replicas for the statefulset in one member cluster. +// Important: if other scalers (for other statefulsets) are still scaling, then this scaler will return +// the previous member count instead of the true desired member count to guarantee that we change only +// one member of the replica set across all scalers (for statefulsets in different member clusters). +func (s *MultiClusterReplicaSetScaler) DesiredReplicas() int { + if s.ScalingFirstTime() { + return getMemberClusterItemByClusterName(s.clusterSpecList, s.memberClusterName).Members + } + previousMembers := 0 + for _, memberCluster := range s.prevMembers { + if memberCluster.Name == s.memberClusterName { + previousMembers = memberCluster.Replicas + break + } + } + + // Example: + // spec: + // cluster-1: 3 + // cluster-2: 5 + // cluster-3: 1 + // + // previous: + // cluster-1: 3 + // cluster-2: 2 + // cluster-3: 0 + // + // scaler cluster-1: + // current: 3 + // desired: 3 -> return previousMembers + // replicasThisReconcile: 3 + // + // scaler cluster-2: + // current: 2 + // desired: 5 -> return replicasInSpec + // replicasThisReconcile: 3 + // + // scaler cluster-3: + // current: 0 + // desired: 0 -> return previousMembers + // replicasThisReconcile: 0 + for _, memberCluster := range s.prevMembers { + replicasInSpec := getMemberClusterItemByClusterName(s.clusterSpecList, memberCluster.Name).Members + // find the first cluster with a different desired spec + if replicasInSpec != memberCluster.Replicas { + // if it's a different cluster, we don't scale this cluster up or down + if memberCluster.Name != s.memberClusterName { + return previousMembers + } else { + return replicasInSpec + } + } + } + return previousMembers +} + +// TargetReplicas always returns the true replicas that the statefulset should have in this cluster regardless +// whether other scalers are still scaling or not. +func (s *MultiClusterReplicaSetScaler) TargetReplicas() int { + return getMemberClusterItemByClusterName(s.clusterSpecList, s.memberClusterName).Members +} + +func (s *MultiClusterReplicaSetScaler) CurrentReplicas() int { + for _, memberCluster := range s.prevMembers { + if memberCluster.Name == s.memberClusterName { + return memberCluster.Replicas + } + } + return 0 +} + +func (s *MultiClusterReplicaSetScaler) ScalingFirstTime() bool { + for _, memberCluster := range s.prevMembers { + if memberCluster.Replicas != 0 { + return false + } + } + return true +} + +func (s *MultiClusterReplicaSetScaler) MemberClusterName() string { + return s.memberClusterName +} + +func (s *MultiClusterReplicaSetScaler) MemberClusterNum() int { + return s.memberClusterNum +} + +func (s *MultiClusterReplicaSetScaler) ScalerDescription() string { + return s.scalerDescription +} + +func (s *MultiClusterReplicaSetScaler) String() string { + return fmt.Sprintf("{MultiClusterReplicaSetScaler (%s): still scaling: %t (finishing this reconcile: %t), clusterName=%s, clusterIdx=%d, current/target replicas:%d/%d, "+ + "replicas this reconciliation: %d, scaling first time: %t}", s.scalerDescription, s.CurrentReplicas() != s.TargetReplicas(), scale.ReplicasThisReconciliation(s) == s.TargetReplicas(), s.memberClusterName, s.memberClusterNum, + s.CurrentReplicas(), s.TargetReplicas(), scale.ReplicasThisReconciliation(s), s.ScalingFirstTime()) +} diff --git a/controllers/operator/construct/testing_utils.go b/controllers/operator/construct/testing_utils.go new file mode 100644 index 000000000..5ec622d92 --- /dev/null +++ b/controllers/operator/construct/testing_utils.go @@ -0,0 +1,11 @@ +package construct + +import ( + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func GetPodEnvOptions() func(options *DatabaseStatefulSetOptions) { + return func(options *DatabaseStatefulSetOptions) { + options.PodVars = &env.PodEnvVars{ProjectID: "abcd"} + } +} diff --git a/controllers/operator/controlledfeature/controlled_feature.go b/controllers/operator/controlledfeature/controlled_feature.go new file mode 100644 index 000000000..3bcac8b27 --- /dev/null +++ b/controllers/operator/controlledfeature/controlled_feature.go @@ -0,0 +1,127 @@ +package controlledfeature + +import ( + "go.uber.org/zap" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +type ControlledFeature struct { + ManagementSystem ManagementSystem `json:"externalManagementSystem"` + Policies []Policy `json:"policies"` +} + +type ManagementSystem struct { + Name string `json:"name"` + Version string `json:"version"` +} + +type PolicyType string + +const ( + ExternallyManaged PolicyType = "EXTERNALLY_MANAGED_LOCK" + DisableAuthenticationMechanisms PolicyType = "DISABLE_AUTHENTICATION_MECHANISMS" + DisableMongodConfig PolicyType = "DISABLE_SET_MONGOD_CONFIG" + DisableMongodVersion PolicyType = "DISABLE_SET_MONGOD_VERSION" +) + +type Policy struct { + PolicyType PolicyType `json:"policy"` + DisabledParams []string `json:"disabledParams"` +} + +func newControlledFeature(options ...func(*ControlledFeature)) *ControlledFeature { + cf := &ControlledFeature{ + ManagementSystem: ManagementSystem{ + Name: util.OperatorName, + Version: util.OperatorVersion, + }, + } + + for _, op := range options { + op(cf) + } + + return cf +} + +func OptionExternallyManaged(cf *ControlledFeature) { + cf.Policies = append(cf.Policies, Policy{PolicyType: ExternallyManaged, DisabledParams: make([]string, 0)}) +} + +func OptionDisableAuthenticationMechanism(cf *ControlledFeature) { + cf.Policies = append(cf.Policies, Policy{PolicyType: DisableAuthenticationMechanisms, DisabledParams: make([]string, 0)}) +} + +func OptionDisableMongodbConfig(disabledParams []string) func(*ControlledFeature) { + return func(cf *ControlledFeature) { + cf.Policies = append(cf.Policies, Policy{PolicyType: DisableMongodConfig, DisabledParams: disabledParams}) + } +} + +func OptionDisableMongodbVersion(cf *ControlledFeature) { + cf.Policies = append(cf.Policies, Policy{PolicyType: DisableMongodVersion}) +} + +type Updater interface { + UpdateControlledFeature(cf *ControlledFeature) error +} + +type Getter interface { + GetControlledFeature() (*ControlledFeature, error) +} + +// EnsureFeatureControls updates the controlled feature based on the provided MongoDB +// resource if the version of Ops Manager supports it +func EnsureFeatureControls(mdb mdbv1.MongoDB, updater Updater, omVersion versionutil.OpsManagerVersion, log *zap.SugaredLogger) workflow.Status { + if !ShouldUseFeatureControls(omVersion) { + log.Debugf("Ops Manager version is %s, which does not support Feature Controls API", omVersion) + return workflow.OK() + } + + cf := buildFeatureControlsByMdb(mdb) + log.Debug("Configuring feature controls") + if err := updater.UpdateControlledFeature(cf); err != nil { + return workflow.Failed(err) + } + return workflow.OK() +} + +// ClearFeatureControls cleares the controlled feature if the version of OpsManager supports it +func ClearFeatureControls(updater Updater, omVersion versionutil.OpsManagerVersion, log *zap.SugaredLogger) workflow.Status { + if !ShouldUseFeatureControls(omVersion) { + log.Debugf("Ops Manager version is %s, which does not support Feature Controls API", omVersion) + return workflow.OK() + } + cf := newControlledFeature() + // cf.Policies needs to be an empty list, instead of a nil pointer, for a valid API call. + cf.Policies = make([]Policy, 0) + if err := updater.UpdateControlledFeature(cf); err != nil { + return workflow.Failed(err) + } + return workflow.OK() +} + +// ShouldUseFeatureControls returns a boolean indicating if the feature controls +// should be enabled for the given version of Ops Manager +func ShouldUseFeatureControls(version versionutil.OpsManagerVersion) bool { + // if we were not successfully able to determine a version + // from Ops Manager, we can assume it is a legacy version + if version.IsUnknown() { + return false + } + + // feature controls are enabled on Cloud Manager, e.g. v20191112 + if version.IsCloudManager() { + return true + } + + if _, err := version.Semver(); err != nil { + return false + } + + return true +} diff --git a/controllers/operator/controlledfeature/controlled_feature_test.go b/controllers/operator/controlledfeature/controlled_feature_test.go new file mode 100644 index 000000000..02e61dbe0 --- /dev/null +++ b/controllers/operator/controlledfeature/controlled_feature_test.go @@ -0,0 +1,30 @@ +package controlledfeature + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +func TestShouldUseFeatureControls(t *testing.T) { + // All OM versions that we support now support feature controls + assert.True(t, ShouldUseFeatureControls(toOMVersion("4.4.0"))) + assert.True(t, ShouldUseFeatureControls(toOMVersion("4.4.1"))) + assert.True(t, ShouldUseFeatureControls(toOMVersion("5.0.1"))) + + // Cloud Manager also supports it + assert.True(t, ShouldUseFeatureControls(toOMVersion("v20020201"))) +} + +func toOMVersion(versionString string) versionutil.OpsManagerVersion { + if versionString == "" { + return versionutil.OpsManagerVersion{} + } + + return versionutil.OpsManagerVersion{ + VersionString: fmt.Sprintf("%s.56729.20191105T2247Z", versionString), + } +} diff --git a/controllers/operator/controlledfeature/feature_by_mdb.go b/controllers/operator/controlledfeature/feature_by_mdb.go new file mode 100644 index 000000000..02d8bab1a --- /dev/null +++ b/controllers/operator/controlledfeature/feature_by_mdb.go @@ -0,0 +1,60 @@ +package controlledfeature + +import ( + "sort" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// buildFeatureControlsByMdb builds the controlled feature by MongoDB resource +func buildFeatureControlsByMdb(mdb mdbv1.MongoDB) *ControlledFeature { + var cf *ControlledFeature + controlledFeatures := []func(*ControlledFeature){OptionExternallyManaged} + controlledFeatures = append(controlledFeatures, authentication(mdb)...) + controlledFeatures = append(controlledFeatures, mongodbParams(mdb)...) + controlledFeatures = append(controlledFeatures, mdbVersion()...) + + cf = newControlledFeature(controlledFeatures...) + + return cf +} + +// mongodbParams enables mongodb options policy if any of additional mongod configurations are specified for MongoDB spec +func mongodbParams(mdb mdbv1.MongoDB) []func(*ControlledFeature) { + var disabledMongodbParams []string + disabledMongodbParams = append(disabledMongodbParams, mdb.Spec.AdditionalMongodConfig.ToFlatList()...) + if mdb.Spec.MongosSpec != nil { + disabledMongodbParams = append(disabledMongodbParams, mdb.Spec.MongosSpec.AdditionalMongodConfig.ToFlatList()...) + } + if mdb.Spec.ConfigSrvSpec != nil { + disabledMongodbParams = append(disabledMongodbParams, mdb.Spec.ConfigSrvSpec.AdditionalMongodConfig.ToFlatList()...) + } + if mdb.Spec.ShardSpec != nil { + disabledMongodbParams = append(disabledMongodbParams, mdb.Spec.ShardSpec.AdditionalMongodConfig.ToFlatList()...) + } + if len(disabledMongodbParams) > 0 { + // We need to ensure no duplicates + var deduplicatedParams []string + for _, v := range disabledMongodbParams { + if !stringutil.Contains(deduplicatedParams, v) { + deduplicatedParams = append(deduplicatedParams, v) + } + } + sort.Strings(deduplicatedParams) + return []func(*ControlledFeature){OptionDisableMongodbConfig(deduplicatedParams)} + } + return []func(*ControlledFeature){} +} + +// authentication enables authentication feature only if Authentication is enabled in mdb +func authentication(mdb mdbv1.MongoDB) []func(*ControlledFeature) { + if mdb.Spec.Security.Authentication != nil { + return []func(*ControlledFeature){OptionDisableAuthenticationMechanism} + } + return []func(*ControlledFeature){} +} + +func mdbVersion() []func(*ControlledFeature) { + return []func(*ControlledFeature){OptionDisableMongodbVersion} +} diff --git a/controllers/operator/controlledfeature/feature_by_mdb_test.go b/controllers/operator/controlledfeature/feature_by_mdb_test.go new file mode 100644 index 000000000..b63395177 --- /dev/null +++ b/controllers/operator/controlledfeature/feature_by_mdb_test.go @@ -0,0 +1,63 @@ +package controlledfeature + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestBuildFeatureControlsByMdb_MongodbParams(t *testing.T) { + t.Run("Feature controls for replica set additional params", func(t *testing.T) { + config := mdbv1.NewAdditionalMongodConfig("storage.journal.enabled", true).AddOption("storage.indexBuildRetry", true) + rs := mdbv1.NewReplicaSetBuilder().SetAdditionalConfig(config).Build() + controlledFeature := buildFeatureControlsByMdb(*rs) + + expectedControlledFeature := &ControlledFeature{ + ManagementSystem: ManagementSystem{ + Name: util.OperatorName, + Version: util.OperatorVersion, + }, + Policies: []Policy{ + {PolicyType: ExternallyManaged, DisabledParams: make([]string, 0)}, + { + PolicyType: DisableMongodConfig, + DisabledParams: []string{"storage.indexBuildRetry", "storage.journal.enabled"}, + }, + {PolicyType: DisableMongodVersion}, + }, + } + assert.Equal(t, expectedControlledFeature, controlledFeature) + }) + t.Run("Feature controls for sharded cluster additional params", func(t *testing.T) { + shardConfig := mdbv1.NewAdditionalMongodConfig("storage.journal.enabled", true).AddOption("storage.indexBuildRetry", true) + mongosConfig := mdbv1.NewAdditionalMongodConfig("systemLog.verbosity", 2) + configSrvConfig := mdbv1.NewAdditionalMongodConfig("systemLog.verbosity", 5).AddOption("systemLog.traceAllExceptions", true) + + rs := mdbv1.NewClusterBuilder(). + SetShardAdditionalConfig(shardConfig). + SetMongosAdditionalConfig(mongosConfig). + SetConfigSrvAdditionalConfig(configSrvConfig). + Build() + controlledFeature := buildFeatureControlsByMdb(*rs) + + expectedControlledFeature := &ControlledFeature{ + ManagementSystem: ManagementSystem{ + Name: util.OperatorName, + Version: util.OperatorVersion, + }, + Policies: []Policy{ + {PolicyType: ExternallyManaged, DisabledParams: make([]string, 0)}, + { + PolicyType: DisableMongodConfig, + // The options have been deduplicated and contain the list of all options for each sharded cluster member + DisabledParams: []string{"storage.indexBuildRetry", "storage.journal.enabled", "systemLog.traceAllExceptions", "systemLog.verbosity"}, + }, + {PolicyType: DisableMongodVersion}, + }, + } + assert.Equal(t, expectedControlledFeature, controlledFeature) + }) +} diff --git a/controllers/operator/create/create.go b/controllers/operator/create/create.go new file mode 100644 index 000000000..3fb49524e --- /dev/null +++ b/controllers/operator/create/create.go @@ -0,0 +1,607 @@ +package create + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "time" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/status/pvc" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + mekoService "github.com/10gen/ops-manager-kubernetes/pkg/kube/service" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/placeholders" + enterprisests "github.com/10gen/ops-manager-kubernetes/pkg/statefulset" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +var ( + externalConnectivityPortName = "external-connectivity-port" + internalConnectivityPortName = "internal-connectivity-port" + backupPortName = "backup-port" + appLabelKey = "app" + podNameLabelKey = "statefulset.kubernetes.io/pod-name" +) + +// DatabaseInKubernetes creates (updates if it exists) the StatefulSet with its Service. +// It returns any errors coming from Kubernetes API. +func DatabaseInKubernetes(ctx context.Context, client kubernetesClient.Client, mdb mdbv1.MongoDB, sts appsv1.StatefulSet, config func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions, log *zap.SugaredLogger) error { + opts := config(mdb) + set, err := enterprisests.CreateOrUpdateStatefulset(ctx, client, mdb.Namespace, log, &sts) + if err != nil { + return err + } + + namespacedName := kube.ObjectKey(mdb.Namespace, set.Spec.ServiceName) + internalService := BuildService(namespacedName, &mdb, &set.Spec.ServiceName, nil, opts.ServicePort, omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeClusterIP}) + + // Adds Prometheus Port if Prometheus has been enabled. + prom := mdb.GetPrometheus() + if prom != nil { + //nolint:gosec // suppressing integer overflow warning for int32(prom.GetPort()) + internalService.Spec.Ports = append(internalService.Spec.Ports, corev1.ServicePort{Port: int32(prom.GetPort()), Name: "prometheus"}) + } + err = mekoService.CreateOrUpdateService(ctx, client, internalService) + if err != nil { + return err + } + + for podNum := 0; podNum < mdb.GetSpec().Replicas(); podNum++ { + namespacedName = kube.ObjectKey(mdb.Namespace, dns.GetExternalServiceName(set.Name, podNum)) + if mdb.Spec.ExternalAccessConfiguration == nil { + if err := mekoService.DeleteServiceIfItExists(ctx, client, namespacedName); err != nil { + return err + } + continue + } + + if mdb.Spec.ExternalAccessConfiguration != nil { + if err = createExternalServices(ctx, client, mdb, opts, namespacedName, set, podNum, log); err != nil { + return err + } + } + } + + return nil +} + +// HandlePVCResize handles the state machine of a PVC resize. +// Note: it modifies the desiredSTS.annotation to trigger a rolling restart later +// We leverage workflowStatus.WithAdditionalOptions(...) to merge/update/add to existing mdb.status.pvc +func HandlePVCResize(ctx context.Context, memberClient kubernetesClient.Client, desiredSts *appsv1.StatefulSet, log *zap.SugaredLogger) workflow.Status { + existingStatefulSet, stsErr := memberClient.GetStatefulSet(ctx, kube.ObjectKey(desiredSts.Namespace, desiredSts.Name)) + if stsErr != nil { + // if we are here it means its first reconciling, we can skip the whole pvc state machine + if apiErrors.IsNotFound(stsErr) { + return workflow.OK() + } else { + return workflow.Failed(stsErr) + } + } + + pvcResizes := resourceStorageHasChanged(existingStatefulSet.Spec.VolumeClaimTemplates, desiredSts.Spec.VolumeClaimTemplates) + + increaseStorageOfAtLeastOnePVC := false + // we have decreased the storage for at least one pvc, we do not support that + for _, pvcResize := range pvcResizes { + if pvcResize.resizeIndicator == 1 { + log.Debug("Can't update the stateful set, as we cannot decrease the pvc size") + return workflow.Failed(xerrors.Errorf("can't update pvc and statefulset to a smaller storage, from: %s - to:%s", pvcResize.from, pvcResize.to)) + } + if pvcResize.resizeIndicator == -1 { + log.Infof("Detected PVC size expansion; for pvc %s, from: %s to: %s", pvcResize.pvcName, pvcResize.from, pvcResize.to) + increaseStorageOfAtLeastOnePVC = true + } + } + + // The sts claim has been increased (based on resourceChangeIndicator) for at least one PVC, + // and we are not in the middle of a resize (that means pvcPhase is pvc.PhaseNoAction) for this statefulset. + // This means we want to start one + if increaseStorageOfAtLeastOnePVC { + err := enterprisests.AddPVCAnnotation(desiredSts) + if err != nil { + return workflow.Failed(xerrors.Errorf("can't add pvc annotation, err: %s", err)) + } + log.Infof("Detected PVC size expansion; patching all pvcs and increasing the size for sts: %s", desiredSts.Name) + if err := resizePVCsStorage(memberClient, desiredSts); err != nil { + return workflow.Failed(xerrors.Errorf("can't resize pvc, err: %s", err)) + } + + finishedResizing, err := hasFinishedResizing(ctx, memberClient, desiredSts) + if err != nil { + return workflow.Failed(err) + } + if finishedResizing { + log.Info("PVCs finished resizing") + log.Info("Deleting StatefulSet and orphan pods") + // Cascade delete the StatefulSet + deletePolicy := metav1.DeletePropagationOrphan + if err := memberClient.Delete(context.TODO(), desiredSts, client.PropagationPolicy(deletePolicy)); err != nil && !apiErrors.IsNotFound(err) { + return workflow.Failed(xerrors.Errorf("error deleting sts, err: %s", err)) + } + + deletedIsStatefulset := checkStatefulsetIsDeleted(ctx, memberClient, desiredSts, 1*time.Second, log) + + if !deletedIsStatefulset { + log.Info("deletion has not been reflected in kube yet, restarting the reconcile") + return workflow.Pending("STS has been orphaned but not yet reflected in kubernetes. " + + "Restarting the reconcile").WithAdditionalOptions(status.NewPVCsStatusOption(&status.PVC{Phase: pvc.PhasePVCResize, StatefulsetName: desiredSts.Name})) + } + log.Info("Statefulset have been orphaned") + return workflow.OK().WithAdditionalOptions(status.NewPVCsStatusOption(&status.PVC{Phase: pvc.PhaseSTSOrphaned, StatefulsetName: desiredSts.Name})) + } else { + log.Info("PVCs are still resizing, waiting until it has finished") + return workflow.Pending("PVC resizes has not finished; current state of sts: %s: %s", desiredSts.Name, pvc.PhasePVCResize).WithAdditionalOptions(status.NewPVCsStatusOption(&status.PVC{Phase: pvc.PhasePVCResize, StatefulsetName: desiredSts.Name})) + } + } + + return workflow.OK() +} + +func checkStatefulsetIsDeleted(ctx context.Context, memberClient kubernetesClient.Client, desiredSts *appsv1.StatefulSet, sleepDuration time.Duration, log *zap.SugaredLogger) bool { + // After deleting the statefulset it can take seconds to be reflected in kubernetes. + // In case it is still not reflected + deletedIsStatefulset := false + for i := 0; i < 3; i++ { + time.Sleep(sleepDuration) + _, stsErr := memberClient.GetStatefulSet(ctx, kube.ObjectKey(desiredSts.Namespace, desiredSts.Name)) + if apiErrors.IsNotFound(stsErr) { + deletedIsStatefulset = true + break + } else { + log.Info("Statefulset still exists, attempting again") + } + } + return deletedIsStatefulset +} + +func hasFinishedResizing(ctx context.Context, memberClient kubernetesClient.Client, desiredSts *appsv1.StatefulSet) (bool, error) { + pvcList := corev1.PersistentVolumeClaimList{} + if err := memberClient.List(ctx, &pvcList); err != nil { + return false, err + } + + finishedResizing := true + for _, currentPVC := range pvcList.Items { + if template, index := getMatchingPVCTemplateFromSTS(desiredSts, ¤tPVC); template != nil { + if currentPVC.Status.Capacity.Storage().Cmp(*desiredSts.Spec.VolumeClaimTemplates[index].Spec.Resources.Requests.Storage()) != 0 { + finishedResizing = false + } + } + } + return finishedResizing, nil +} + +// resizePVCsStorage takes the sts we want to create and update all matching pvc with the new storage +func resizePVCsStorage(client kubernetesClient.Client, statefulSetToCreate *appsv1.StatefulSet) error { + pvcList := corev1.PersistentVolumeClaimList{} + + // this is to ensure that requests to a potentially not allowed resource is not blocking the operator until the end + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + if err := client.List(ctx, &pvcList); err != nil { + return err + } + for _, existingPVC := range pvcList.Items { + if template, _ := getMatchingPVCTemplateFromSTS(statefulSetToCreate, &existingPVC); template != nil { + existingPVC.Spec.Resources.Requests[corev1.ResourceStorage] = *template.Spec.Resources.Requests.Storage() + if err := client.Update(ctx, &existingPVC); err != nil { + return err + } + } + } + return nil +} + +func getMatchingPVCTemplateFromSTS(statefulSet *appsv1.StatefulSet, pvc *corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, int) { + for i, claimTemplate := range statefulSet.Spec.VolumeClaimTemplates { + expectedPrefix := fmt.Sprintf("%s-%s", claimTemplate.Name, statefulSet.Name) + + // Regex to match expectedPrefix followed by a dash and a number (ordinal) + regexPattern := fmt.Sprintf("^%s-[0-9]+$", regexp.QuoteMeta(expectedPrefix)) + if matched, _ := regexp.MatchString(regexPattern, pvc.Name); matched { + return &claimTemplate, i + } + } + return nil, -1 +} + +// createExternalServices creates the external services. +// For sharded clusters: services are only created for mongos. +func createExternalServices(ctx context.Context, client kubernetesClient.Client, mdb mdbv1.MongoDB, opts construct.DatabaseStatefulSetOptions, namespacedName client.ObjectKey, set *appsv1.StatefulSet, podNum int, log *zap.SugaredLogger) error { + if mdb.IsShardedCluster() && !opts.IsMongos() { + return nil + } + externalService := BuildService(namespacedName, &mdb, &set.Spec.ServiceName, ptr.To(dns.GetPodName(set.Name, podNum)), opts.ServicePort, omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeLoadBalancer}) + + if mdb.Spec.DbCommonSpec.GetExternalDomain() != nil { + // When an external domain is defined, we put it into process.hostname in automation config. Because of that we need to define additional well-defined port for backups. + // This backup port is not needed when we use headless service, because then agent is resolving DNS directly to pod's IP and that allows to connect + // to any port in a pod, even ephemeral one. + // When we put any other address than headless service into process.hostname: non-headless service fqdn (e.g. in multi cluster using service mesh) or + // external domain (e.g. for multi-cluster no-mesh), then we need to define backup port. + // In the agent process, we pass -ephemeralPortOffset 1 argument to define, that backup port should be a standard port+1. + backupPort := GetNonEphemeralBackupPort(opts.ServicePort) + externalService.Spec.Ports = append(externalService.Spec.Ports, corev1.ServicePort{Port: backupPort, TargetPort: intstr.FromInt32(backupPort), Name: "backup"}) + } + + if mdb.Spec.ExternalAccessConfiguration.ExternalService.SpecWrapper != nil { + externalService.Spec = merge.ServiceSpec(externalService.Spec, mdb.Spec.ExternalAccessConfiguration.ExternalService.SpecWrapper.Spec) + } + externalService.Annotations = merge.StringToStringMap(externalService.Annotations, mdb.Spec.ExternalAccessConfiguration.ExternalService.Annotations) + + placeholderReplacer := GetSingleClusterMongoDBPlaceholderReplacer(mdb.Name, set.Name, mdb.Namespace, mdb.ServiceName(), mdb.Spec.GetExternalDomain(), mdb.Spec.GetClusterDomain(), podNum, mdb.GetResourceType()) + if processedAnnotations, replacedFlag, err := placeholderReplacer.ProcessMap(externalService.Annotations); err != nil { + return xerrors.Errorf("failed to process annotations in service %s: %w", externalService.Name, err) + } else if replacedFlag { + log.Debugf("Replaced placeholders in annotations in external service: %s. Annotations before: %+v, annotations after: %+v", externalService.Name, externalService.Annotations, processedAnnotations) + externalService.Annotations = processedAnnotations + } + + err := mekoService.CreateOrUpdateService(ctx, client, externalService) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to created external service: %s, err: %w", externalService.Name, err) + } + return nil +} + +const ( + PlaceholderPodIndex = "podIndex" + PlaceholderNamespace = "namespace" + PlaceholderResourceName = "resourceName" + PlaceholderPodName = "podName" + PlaceholderStatefulSetName = "statefulSetName" + PlaceholderExternalServiceName = "externalServiceName" + PlaceholderMongosProcessDomain = "mongosProcessDomain" + PlaceholderMongodProcessDomain = "mongodProcessDomain" + PlaceholderMongosProcessFQDN = "mongosProcessFQDN" + PlaceholderMongodProcessFQDN = "mongodProcessFQDN" + PlaceholderClusterName = "clusterName" + PlaceholderClusterIndex = "clusterIndex" +) + +func GetSingleClusterMongoDBPlaceholderReplacer(resourceName string, statefulSetName string, namespace string, serviceName string, externalDomain *string, clusterDomain string, podIdx int, resourceType mdbv1.ResourceType) *placeholders.Replacer { + podName := dns.GetPodName(statefulSetName, podIdx) + placeholderValues := map[string]string{ + PlaceholderPodIndex: fmt.Sprintf("%d", podIdx), + PlaceholderNamespace: namespace, + PlaceholderResourceName: resourceName, + PlaceholderPodName: podName, + PlaceholderStatefulSetName: statefulSetName, + PlaceholderExternalServiceName: dns.GetExternalServiceName(statefulSetName, podIdx), + } + + if resourceType == mdbv1.ShardedCluster { + placeholderValues[PlaceholderMongosProcessDomain] = dns.GetServiceFQDN(serviceName, namespace, clusterDomain) + placeholderValues[PlaceholderMongosProcessFQDN] = dns.GetPodFQDN(podName, serviceName, namespace, clusterDomain, externalDomain) + if externalDomain != nil { + placeholderValues[PlaceholderMongosProcessDomain] = *externalDomain + } + } else { + placeholderValues[PlaceholderMongodProcessDomain] = dns.GetServiceFQDN(serviceName, namespace, clusterDomain) + placeholderValues[PlaceholderMongodProcessFQDN] = dns.GetPodFQDN(podName, serviceName, namespace, clusterDomain, externalDomain) + if externalDomain != nil { + placeholderValues[PlaceholderMongodProcessDomain] = *externalDomain + } + } + + return placeholders.New(placeholderValues) +} + +func GetMultiClusterMongoDBPlaceholderReplacer(name string, stsName string, namespace string, clusterName string, clusterNum int, externalDomain *string, clusterDomain string, podIdx int) *placeholders.Replacer { + podName := dns.GetMultiPodName(stsName, clusterNum, podIdx) + serviceDomain := dns.GetServiceDomain(namespace, clusterDomain, externalDomain) + placeholderValues := map[string]string{ + PlaceholderPodIndex: fmt.Sprintf("%d", podIdx), + PlaceholderNamespace: namespace, + PlaceholderResourceName: name, + PlaceholderPodName: podName, + PlaceholderStatefulSetName: dns.GetMultiStatefulSetName(stsName, clusterNum), + PlaceholderExternalServiceName: dns.GetMultiExternalServiceName(stsName, clusterNum, podIdx), + PlaceholderMongodProcessDomain: serviceDomain, + PlaceholderMongodProcessFQDN: dns.GetMultiClusterPodServiceFQDN(stsName, namespace, clusterNum, externalDomain, podIdx, clusterDomain), + PlaceholderClusterName: clusterName, + PlaceholderClusterIndex: fmt.Sprintf("%d", clusterNum), + } + + if strings.HasSuffix(stsName, "mongos") { + placeholderValues[PlaceholderMongosProcessDomain] = serviceDomain + placeholderValues[PlaceholderMongosProcessFQDN] = dns.GetMultiClusterPodServiceFQDN(stsName, namespace, clusterNum, externalDomain, podIdx, clusterDomain) + if externalDomain != nil { + placeholderValues[PlaceholderMongosProcessDomain] = *externalDomain + } + } else { + placeholderValues[PlaceholderMongodProcessDomain] = serviceDomain + placeholderValues[PlaceholderMongodProcessFQDN] = dns.GetMultiClusterPodServiceFQDN(stsName, namespace, clusterNum, externalDomain, podIdx, clusterDomain) + if externalDomain != nil { + placeholderValues[PlaceholderMongodProcessDomain] = *externalDomain + } + } + + return placeholders.New(placeholderValues) +} + +// AppDBInKubernetes creates or updates the StatefulSet and Service required for the AppDB. +func AppDBInKubernetes(ctx context.Context, client kubernetesClient.Client, opsManager *omv1.MongoDBOpsManager, sts appsv1.StatefulSet, serviceSelectorLabel string, log *zap.SugaredLogger) error { + set, err := enterprisests.CreateOrUpdateStatefulset(ctx, client, opsManager.Namespace, log, &sts) + if err != nil { + return err + } + + namespacedName := kube.ObjectKey(opsManager.Namespace, set.Spec.ServiceName) + internalService := BuildService(namespacedName, opsManager, ptr.To(serviceSelectorLabel), nil, opsManager.Spec.AppDB.AdditionalMongodConfig.GetPortOrDefault(), omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeClusterIP}) + + // Adds Prometheus Port if Prometheus has been enabled. + prom := opsManager.Spec.AppDB.Prometheus + if prom != nil { + //nolint:gosec // suppressing integer overflow warning for int32(prom.GetPort()) + internalService.Spec.Ports = append(internalService.Spec.Ports, corev1.ServicePort{Port: int32(prom.GetPort()), Name: "prometheus"}) + } + + return mekoService.CreateOrUpdateService(ctx, client, internalService) +} + +// BackupDaemonInKubernetes creates or updates the StatefulSet and Services required. +func BackupDaemonInKubernetes(ctx context.Context, client kubernetesClient.Client, opsManager *omv1.MongoDBOpsManager, sts appsv1.StatefulSet, log *zap.SugaredLogger) (bool, error) { + set, err := enterprisests.CreateOrUpdateStatefulset(ctx, client, opsManager.Namespace, log, &sts) + if err != nil { + // Check if it is a k8s error or a custom one + var statefulSetCantBeUpdatedError enterprisests.StatefulSetCantBeUpdatedError + if !errors.As(err, &statefulSetCantBeUpdatedError) { + return false, err + } + + // In this case, we delete the old Statefulset + log.Debug("Deleting the old backup stateful set and creating a new one") + stsNamespacedName := kube.ObjectKey(sts.Namespace, sts.Name) + err = client.DeleteStatefulSet(ctx, stsNamespacedName) + if err != nil { + return false, xerrors.Errorf("failed while trying to delete previous backup daemon statefulset: %w", err) + } + return true, nil + } + namespacedName := kube.ObjectKey(opsManager.Namespace, set.Spec.ServiceName) + internalService := BuildService(namespacedName, opsManager, &set.Spec.ServiceName, nil, construct.BackupDaemonServicePort, omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeClusterIP}) + err = mekoService.CreateOrUpdateService(ctx, client, internalService) + return false, err +} + +// OpsManagerInKubernetes creates all of the required Kubernetes resources for Ops Manager. +// It creates the StatefulSet and all required services. +func OpsManagerInKubernetes(ctx context.Context, memberCluster multicluster.MemberCluster, opsManager *omv1.MongoDBOpsManager, sts appsv1.StatefulSet, log *zap.SugaredLogger) error { + set, err := enterprisests.CreateOrUpdateStatefulset(ctx, memberCluster.Client, opsManager.Namespace, log, &sts) + if err != nil { + return err + } + + _, port := opsManager.GetSchemePort() + + namespacedName := kube.ObjectKey(opsManager.Namespace, set.Spec.ServiceName) + internalService := BuildService(namespacedName, opsManager, &set.Spec.ServiceName, nil, port, getInternalServiceDefinition(opsManager)) + + // add queryable backup port to service + if opsManager.Spec.Backup.Enabled { + if err := addQueryableBackupPortToService(opsManager, &internalService, internalConnectivityPortName); err != nil { + return err + } + } + + if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, internalService); err != nil { + return err + } + + namespacedName = kube.ObjectKey(opsManager.Namespace, opsManager.ExternalSvcName()) + if externalConnectivity := opsManager.GetExternalConnectivityConfigurationForMemberCluster(memberCluster.Name); externalConnectivity != nil { + svc := BuildService(namespacedName, opsManager, &set.Spec.ServiceName, nil, port, *externalConnectivity) + + // Need to create queryable backup service + if opsManager.Spec.Backup.Enabled { + if err := addQueryableBackupPortToService(opsManager, &svc, externalConnectivityPortName); err != nil { + return err + } + } + + if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil { + return err + } + } else { + if err := mekoService.DeleteServiceIfItExists(ctx, memberCluster.Client, namespacedName); err != nil { + return err + } + } + + return nil +} + +func getInternalServiceDefinition(opsManager *omv1.MongoDBOpsManager) omv1.MongoDBOpsManagerServiceDefinition { + if opsManager.Spec.InternalConnectivity != nil { + return *opsManager.Spec.InternalConnectivity + } + serviceDefinition := omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeClusterIP} + // For multicluster OpsManager we create ClusterIP services by default, based on the assumption + // that the multi-cluster architecture is a multi-network configuration in most cases, + // which makes headless services not resolve across different clusters. + // https://github.com/istio/istio/issues/36733 + // The Spec.InternalConnectivity field allows for explicitly configuring headless services + // and adding additional annotations to the service used for internal connectivity. + if opsManager.Spec.IsMultiCluster() { + serviceDefinition.ClusterIP = ptr.To("") + } + return serviceDefinition +} + +// addQueryableBackupPortToService adds the backup port to the existing external Ops Manager service. +// this function assumes externalService is not nil. +func addQueryableBackupPortToService(opsManager *omv1.MongoDBOpsManager, service *corev1.Service, portName string) error { + backupSvcPort, err := opsManager.Spec.BackupDaemonSvcPort() + if err != nil { + return xerrors.Errorf("can't parse queryable backup port: %w", err) + } + service.Spec.Ports[0].Name = portName + service.Spec.Ports = append(service.Spec.Ports, corev1.ServicePort{ + Name: backupPortName, + Port: backupSvcPort, + }) + return nil +} + +// BuildService creates the Kube Service. If it should be seen externally it makes it of type NodePort that will assign +// some random port in the range 30000-32767 +// Note that itself service has no dedicated IP by default ("clusterIP: None") as all mongo entities should be directly +// addressable. +// This function will update a Service object if passed, or return a new one if passed nil, this is to be able to update +// Services and to not change any attribute they might already have that needs to be maintained. +// +// When appLabel is specified, then the selector is targeting all pods (round-robin service). Usable for e.g. OpsManager service. +// When podLabel is specified, then the selector is targeting only a single pod. Used for external services or multi-cluster services. +func BuildService(namespacedName types.NamespacedName, owner v1.CustomResourceReadWriter, appLabel *string, podLabel *string, port int32, mongoServiceDefinition omv1.MongoDBOpsManagerServiceDefinition) corev1.Service { + labels := map[string]string{ + construct.ControllerLabelName: util.OperatorName, + } + + selectorLabels := map[string]string{ + construct.ControllerLabelName: util.OperatorName, + } + + if appLabel != nil { + labels[appLabelKey] = *appLabel + selectorLabels[appLabelKey] = *appLabel + } + + if podLabel != nil { + labels[podNameLabelKey] = *podLabel + selectorLabels[podNameLabelKey] = *podLabel + } + + if _, ok := owner.(*mdbv1.MongoDB); ok { + labels[mdbv1.LabelMongoDBResourceOwner] = owner.GetName() + } + + svcBuilder := service.Builder(). + SetNamespace(namespacedName.Namespace). + SetName(namespacedName.Name). + SetOwnerReferences(kube.BaseOwnerReference(owner)). + SetLabels(labels). + SetSelector(selectorLabels). + SetServiceType(mongoServiceDefinition.Type). + SetPublishNotReadyAddresses(true) + + serviceType := mongoServiceDefinition.Type + switch serviceType { + case corev1.ServiceTypeNodePort: + // Service will have a NodePort + svcPort := corev1.ServicePort{TargetPort: intstr.FromInt(int(port)), Name: "mongodb"} + svcPort.NodePort = mongoServiceDefinition.Port + if mongoServiceDefinition.Port != 0 { + svcPort.Port = mongoServiceDefinition.Port + } else { + svcPort.Port = port + } + svcBuilder.AddPort(&svcPort).SetClusterIP("") + case corev1.ServiceTypeLoadBalancer: + svcPort := corev1.ServicePort{TargetPort: intstr.FromInt(int(port)), Name: "mongodb"} + if mongoServiceDefinition.Port != 0 { + svcPort.Port = mongoServiceDefinition.Port + } else { + svcPort.Port = port + } + svcBuilder.AddPort(&svcPort).SetClusterIP("") + case corev1.ServiceTypeClusterIP: + if mongoServiceDefinition.ClusterIP == nil { + svcBuilder.SetClusterIP("None") + } else { + svcBuilder.SetClusterIP(*mongoServiceDefinition.ClusterIP) + } + // Service will have a named Port + svcBuilder.AddPort(&corev1.ServicePort{Port: port, Name: "mongodb"}) + default: + // Service will have a regular Port (unnamed) + svcBuilder.AddPort(&corev1.ServicePort{Port: port}) + } + + if mongoServiceDefinition.Annotations != nil { + svcBuilder.SetAnnotations(mongoServiceDefinition.Annotations) + } + + if mongoServiceDefinition.LoadBalancerIP != "" { + svcBuilder.SetLoadBalancerIP(mongoServiceDefinition.LoadBalancerIP) + } + + if mongoServiceDefinition.ExternalTrafficPolicy != "" { + svcBuilder.SetExternalTrafficPolicy(mongoServiceDefinition.ExternalTrafficPolicy) + } + + return svcBuilder.Build() +} + +// GetNonEphemeralBackupPort returns port number that will be used when we instruct the agent to use non-ephemeral port for backup monogod. +// Non-ephemeral ports are used when we set process.hostname for anything other than headless service FQDN. +func GetNonEphemeralBackupPort(mongodPort int32) int32 { + return mongodPort + 1 +} + +type pvcResize struct { + pvcName string + resizeIndicator int + from string + to string +} + +// resourceStorageHasChanged returns 0 if both storage sizes are equal or not exist at all, +// +// 1: toCreateVolumeClaims < desiredVolumeClaims → decrease storage +// -1: toCreateVolumeClaims > desiredVolumeClaims → increase storage +// 0: toCreateVolumeClaims = desiredVolumeClaims → storage stays same +func resourceStorageHasChanged(existingVolumeClaims []corev1.PersistentVolumeClaim, desiredVolumeClaims []corev1.PersistentVolumeClaim) []pvcResize { + existingClaimByName := map[string]*corev1.PersistentVolumeClaim{} + var pvcResizes []pvcResize + + for _, existingClaim := range existingVolumeClaims { + existingClaimByName[existingClaim.Name] = &existingClaim + } + + for _, desiredClaim := range desiredVolumeClaims { + // if the desiredClaim does not exist in the list of claims, then we don't need to consider resizing, since + // its most likely a new one + if existingPVCClaim, ok := existingClaimByName[desiredClaim.Name]; ok { + desiredPVCClaimStorage := desiredClaim.Spec.Resources.Requests.Storage() + existingPVCClaimStorage := existingPVCClaim.Spec.Resources.Requests.Storage() + if desiredPVCClaimStorage != nil && existingPVCClaimStorage != nil { + pvcResizes = append(pvcResizes, pvcResize{ + pvcName: desiredClaim.Name, + resizeIndicator: existingPVCClaimStorage.Cmp(*desiredPVCClaimStorage), + from: existingPVCClaimStorage.String(), + to: desiredPVCClaimStorage.String(), + }) + } + } + } + + return pvcResizes +} diff --git a/controllers/operator/create/create_test.go b/controllers/operator/create/create_test.go new file mode 100644 index 000000000..37498a424 --- /dev/null +++ b/controllers/operator/create/create_test.go @@ -0,0 +1,1403 @@ +package create + +import ( + "context" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func init() { + mock.InitDefaultEnvVariables() +} + +func TestBuildService(t *testing.T) { + mdb := mdbv1.NewReplicaSetBuilder().Build() + svc := BuildService(kube.ObjectKey(mock.TestNamespace, "my-svc"), mdb, ptr.To("label"), nil, 2000, omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeClusterIP, + Port: 2000, + LoadBalancerIP: "loadbalancerip", + }) + + assert.Len(t, svc.OwnerReferences, 1) + assert.Equal(t, mdb.Name, svc.OwnerReferences[0].Name) + assert.Equal(t, mdb.GetObjectKind().GroupVersionKind().Kind, svc.OwnerReferences[0].Kind) + assert.Equal(t, mock.TestNamespace, svc.Namespace) + assert.Equal(t, "my-svc", svc.Name) + assert.Equal(t, "loadbalancerip", svc.Spec.LoadBalancerIP) + assert.Equal(t, "None", svc.Spec.ClusterIP) + assert.Equal(t, int32(2000), svc.Spec.Ports[0].Port) + assert.Equal(t, "label", svc.Labels[appLabelKey]) + assert.NotContains(t, svc.Labels, podNameLabelKey) + assert.True(t, svc.Spec.PublishNotReadyAddresses) + + // test podName label not nil + svc = BuildService(kube.ObjectKey(mock.TestNamespace, "my-svc"), mdb, nil, ptr.To("podName"), 2000, omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeClusterIP, + Port: 2000, + LoadBalancerIP: "loadbalancerip", + }) + + assert.Len(t, svc.OwnerReferences, 1) + assert.Equal(t, mdb.Name, svc.OwnerReferences[0].Name) + assert.Equal(t, mdb.GetObjectKind().GroupVersionKind().Kind, svc.OwnerReferences[0].Kind) + assert.Equal(t, mock.TestNamespace, svc.Namespace) + assert.Equal(t, "my-svc", svc.Name) + assert.Equal(t, "loadbalancerip", svc.Spec.LoadBalancerIP) + assert.Equal(t, "None", svc.Spec.ClusterIP) + assert.Equal(t, int32(2000), svc.Spec.Ports[0].Port) + assert.NotContains(t, svc.Labels, appLabelKey) + assert.Equal(t, "podName", svc.Labels[podNameLabelKey]) + assert.True(t, svc.Spec.PublishNotReadyAddresses) +} + +func TestOpsManagerInKubernetes_InternalConnectivityOverride(t *testing.T) { + ctx := context.Background() + testOm := omv1.NewOpsManagerBuilderDefault(). + SetName("test-om"). + SetInternalConnectivity(omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: ptr.To("0.0.12.0"), + Port: 5000, + }). + SetAppDBPassword("my-secret", "password").SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: true, + }).AddConfiguration("brs.queryable.proxyPort", "1234"). + Build() + + fakeClient, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: fakeClient, + } + + memberCluster := multicluster.GetLegacyCentralMemberCluster(testOm.Spec.Replicas, 0, fakeClient, secretsClient) + sts, err := construct.OpsManagerStatefulSet(ctx, secretsClient, testOm, memberCluster, zap.S()) + assert.NoError(t, err) + + err = OpsManagerInKubernetes(ctx, memberCluster, testOm, sts, zap.S()) + assert.NoError(t, err) + + svc, err := fakeClient.GetService(ctx, kube.ObjectKey(testOm.Namespace, testOm.SvcName())) + assert.NoError(t, err, "Internal service exists") + + assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP, "The operator creates a ClusterIP service if explicitly requested to do so.") + assert.Equal(t, svc.Spec.ClusterIP, "0.0.12.0", "The operator configures the requested ClusterIP for the service") + + assert.Len(t, svc.Spec.Ports, 2, "Backup port should have been added to existing internal service") + + port0 := svc.Spec.Ports[0] + assert.Equal(t, internalConnectivityPortName, port0.Name) + + port1 := svc.Spec.Ports[1] + assert.Equal(t, backupPortName, port1.Name) + assert.Equal(t, int32(1234), port1.Port) +} + +func TestOpsManagerInKubernetes_DefaultInternalServiceForMultiCluster(t *testing.T) { + ctx := context.Background() + testOm := omv1.NewOpsManagerBuilderDefault(). + SetName("test-om"). + SetOpsManagerTopology(mdbv1.ClusterTopologyMultiCluster). + SetAppDBPassword("my-secret", "password").SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: true, + }).AddConfiguration("brs.queryable.proxyPort", "1234"). + Build() + + fakeClient, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: fakeClient, + } + + memberCluster := multicluster.GetLegacyCentralMemberCluster(testOm.Spec.Replicas, 0, fakeClient, secretsClient) + sts, err := construct.OpsManagerStatefulSet(ctx, secretsClient, testOm, memberCluster, zap.S()) + assert.NoError(t, err) + + err = OpsManagerInKubernetes(ctx, memberCluster, testOm, sts, zap.S()) + assert.NoError(t, err) + + svc, err := fakeClient.GetService(ctx, kube.ObjectKey(testOm.Namespace, testOm.SvcName())) + assert.NoError(t, err, "Internal service exists") + + assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP, "Default internal service for OM multicluster is of type ClusterIP") + assert.Equal(t, svc.Spec.ClusterIP, "", "Default internal service for OM multicluster is not a headless service") +} + +func TestOpsManagerInKubernetes_ClusterSpecificExternalConnectivity(t *testing.T) { + memberClusterName1 := "member-cluster-1" + memberClusterName2 := "member-cluster-2" + memberClusterName3 := "member-cluster-3" + + type testCase struct { + clusterSpecList []omv1.ClusterSpecOMItem + commonExternalConnectivity *omv1.MongoDBOpsManagerServiceDefinition + expectedServices map[string]corev1.Service + } + + testCases := map[string]testCase{ + "no common external connectivity + cluster specific": { + clusterSpecList: []omv1.ClusterSpecOMItem{ + { + ClusterName: memberClusterName1, + Members: 1, + MongoDBOpsManagerExternalConnectivity: &omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeNodePort, + Port: 30006, + }, + }, + { + ClusterName: memberClusterName2, + Members: 1, + }, + { + ClusterName: memberClusterName3, + Members: 1, + MongoDBOpsManagerExternalConnectivity: &omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeLoadBalancer, + Port: 8080, + LoadBalancerIP: "10.10.10.1", + }, + }, + }, + commonExternalConnectivity: nil, + expectedServices: map[string]corev1.Service{ + memberClusterName1: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-svc-ext", + ResourceVersion: "1", + Labels: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "mongodb.com/v1", + Name: "test-om", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "external-connectivity-port", + Port: 30006, + TargetPort: intstr.FromInt32(8080), + NodePort: 30006, + }, + { + Name: "backup-port", + Port: 1234, + TargetPort: intstr.FromInt32(0), + NodePort: 0, + }, + }, + Selector: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + Type: corev1.ServiceTypeNodePort, + PublishNotReadyAddresses: true, + }, + }, + memberClusterName3: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-svc-ext", + ResourceVersion: "1", + Labels: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "mongodb.com/v1", + Name: "test-om", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "external-connectivity-port", + Port: 8080, + TargetPort: intstr.FromInt32(8080), + }, + { + Name: "backup-port", + Port: 1234, + TargetPort: intstr.FromInt32(0), + NodePort: 0, + }, + }, + Selector: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerIP: "10.10.10.1", + PublishNotReadyAddresses: true, + }, + }, + }, + }, + "common external connectivity + cluster specific": { + clusterSpecList: []omv1.ClusterSpecOMItem{ + { + ClusterName: memberClusterName1, + Members: 1, + MongoDBOpsManagerExternalConnectivity: &omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeNodePort, + Port: 30006, + }, + }, + { + ClusterName: memberClusterName2, + Members: 1, + }, + { + ClusterName: memberClusterName3, + Members: 1, + MongoDBOpsManagerExternalConnectivity: &omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeLoadBalancer, + Port: 8080, + LoadBalancerIP: "10.10.10.1", + }, + }, + }, + commonExternalConnectivity: &omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeLoadBalancer, + Port: 5005, + LoadBalancerIP: "20.20.20.2", + Annotations: map[string]string{ + "test-annotation": "test-value", + }, + }, + expectedServices: map[string]corev1.Service{ + memberClusterName1: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-svc-ext", + ResourceVersion: "1", + Labels: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "mongodb.com/v1", + Name: "test-om", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "external-connectivity-port", + Port: 30006, + TargetPort: intstr.FromInt32(8080), + NodePort: 30006, + }, + { + Name: "backup-port", + Port: 1234, + TargetPort: intstr.FromInt32(0), + NodePort: 0, + }, + }, + Selector: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + Type: corev1.ServiceTypeNodePort, + PublishNotReadyAddresses: true, + }, + }, + memberClusterName2: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-svc-ext", + ResourceVersion: "1", + Labels: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "mongodb.com/v1", + Name: "test-om", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + Annotations: map[string]string{ + "test-annotation": "test-value", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "external-connectivity-port", + Port: 5005, + TargetPort: intstr.FromInt32(8080), + }, + { + Name: "backup-port", + Port: 1234, + TargetPort: intstr.FromInt32(0), + NodePort: 0, + }, + }, + Selector: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerIP: "20.20.20.2", + PublishNotReadyAddresses: true, + }, + }, + memberClusterName3: { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-om-svc-ext", + ResourceVersion: "1", + Labels: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "mongodb.com/v1", + Name: "test-om", + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "external-connectivity-port", + Port: 8080, + TargetPort: intstr.FromInt32(8080), + }, + { + Name: "backup-port", + Port: 1234, + TargetPort: intstr.FromInt32(0), + NodePort: 0, + }, + }, + Selector: map[string]string{ + "app": "test-om-svc", + construct.ControllerLabelName: "mongodb-enterprise-operator", + }, + Type: corev1.ServiceTypeLoadBalancer, + LoadBalancerIP: "10.10.10.1", + PublishNotReadyAddresses: true, + }, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + testOmBuilder := omv1.NewOpsManagerBuilderDefault(). + SetName("test-om"). + SetOpsManagerTopology(mdbv1.ClusterTopologyMultiCluster). + SetAppDBPassword("my-secret", "password"). + SetBackup(omv1.MongoDBOpsManagerBackup{Enabled: true}). + AddConfiguration("brs.queryable.proxyPort", "1234"). + SetOpsManagerClusterSpecList(tc.clusterSpecList) + + if tc.commonExternalConnectivity != nil { + testOmBuilder.SetExternalConnectivity(*tc.commonExternalConnectivity) + } + testOm := testOmBuilder.Build() + + memberClusters := make([]multicluster.MemberCluster, len(tc.clusterSpecList)) + for clusterIndex, clusterSpecItem := range tc.clusterSpecList { + fakeClient, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: fakeClient, + } + + memberClusters[clusterIndex] = multicluster.MemberCluster{ + Name: clusterSpecItem.ClusterName, + Index: clusterIndex, + Replicas: clusterSpecItem.Members, + Client: fakeClient, + SecretClient: secretsClient, + Active: true, + Healthy: true, + Legacy: false, + } + } + + for _, memberCluster := range memberClusters { + ctx := context.Background() + sts, err := construct.OpsManagerStatefulSet(ctx, memberCluster.SecretClient, testOm, memberCluster, zap.S()) + assert.NoError(t, err) + + err = OpsManagerInKubernetes(ctx, memberCluster, testOm, sts, zap.S()) + assert.NoError(t, err) + + expectedService, ok := tc.expectedServices[memberCluster.Name] + svc, err := memberCluster.Client.GetService(ctx, kube.ObjectKey(testOm.Namespace, testOm.ExternalSvcName())) + if ok { + assert.NoError(t, err) + assert.Equal(t, expectedService, svc, "service for cluster %s does not match", memberCluster.Name) + } else { + assert.Error(t, err) + } + } + }) + } +} + +func TestBackupServiceCreated_NoExternalConnectivity(t *testing.T) { + ctx := context.Background() + testOm := omv1.NewOpsManagerBuilderDefault(). + SetName("test-om"). + SetAppDBPassword("my-secret", "password").SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: true, + }).AddConfiguration("brs.queryable.proxyPort", "1234"). + Build() + + fakeClient, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: fakeClient, + } + + memberCluster := multicluster.GetLegacyCentralMemberCluster(testOm.Spec.Replicas, 0, fakeClient, secretsClient) + sts, err := construct.OpsManagerStatefulSet(ctx, secretsClient, testOm, memberCluster, zap.S()) + assert.NoError(t, err) + + err = OpsManagerInKubernetes(ctx, memberCluster, testOm, sts, zap.S()) + assert.NoError(t, err) + + _, err = fakeClient.GetService(ctx, kube.ObjectKey(testOm.Namespace, testOm.SvcName()+"-ext")) + assert.Error(t, err, "No external service should have been created") + + svc, err := fakeClient.GetService(ctx, kube.ObjectKey(testOm.Namespace, testOm.SvcName())) + assert.NoError(t, err, "Internal service exists") + + assert.Equal(t, svc.Spec.Type, corev1.ServiceTypeClusterIP, "Default internal service is of type ClusterIP") + assert.Equal(t, svc.Spec.ClusterIP, corev1.ClusterIPNone, "Default internal service is a headless service") + + assert.Len(t, svc.Spec.Ports, 2, "Backup port should have been added to existing internal service") + + port0 := svc.Spec.Ports[0] + assert.Equal(t, internalConnectivityPortName, port0.Name) + + port1 := svc.Spec.Ports[1] + assert.Equal(t, backupPortName, port1.Name) + assert.Equal(t, int32(1234), port1.Port) +} + +func TestBackupServiceCreated_ExternalConnectivity(t *testing.T) { + ctx := context.Background() + testOm := omv1.NewOpsManagerBuilderDefault(). + SetName("test-om"). + SetAppDBPassword("my-secret", "password"). + SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: true, + }).AddConfiguration("brs.queryable.proxyPort", "1234"). + SetExternalConnectivity(omv1.MongoDBOpsManagerServiceDefinition{ + Type: corev1.ServiceTypeNodePort, + Port: 5000, + }). + Build() + fakeClient, _ := mock.NewDefaultFakeClient() + secretsClient := secrets.SecretClient{ + VaultClient: &vault.VaultClient{}, + KubeClient: fakeClient, + } + memberCluster := multicluster.GetLegacyCentralMemberCluster(testOm.Spec.Replicas, 0, fakeClient, secretsClient) + sts, err := construct.OpsManagerStatefulSet(ctx, secretsClient, testOm, memberCluster, zap.S()) + assert.NoError(t, err) + + err = OpsManagerInKubernetes(ctx, memberCluster, testOm, sts, zap.S()) + assert.NoError(t, err) + + externalService, err := fakeClient.GetService(ctx, kube.ObjectKey(testOm.Namespace, testOm.SvcName()+"-ext")) + assert.NoError(t, err, "An External service should have been created") + + assert.Len(t, externalService.Spec.Ports, 2, "Backup port should have been added to existing external service") + + port0 := externalService.Spec.Ports[0] + assert.Equal(t, externalConnectivityPortName, port0.Name) + assert.Equal(t, int32(5000), port0.Port) + assert.Equal(t, intstr.FromInt(8080), port0.TargetPort) + assert.Equal(t, int32(5000), port0.NodePort) + + port1 := externalService.Spec.Ports[1] + assert.Equal(t, backupPortName, port1.Name) + assert.Equal(t, int32(1234), port1.Port) +} + +func TestDatabaseInKubernetes_ExternalServicesWithoutExternalDomain(t *testing.T) { + ctx := context.Background() + svc := corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "mdb-0-svc-external"}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + TargetPort: intstr.IntOrString{IntVal: 27017}, + }, + }, + Type: corev1.ServiceTypeLoadBalancer, + PublishNotReadyAddresses: true, + }, + } + + service1 := svc + service1.Name = "mdb-0-svc-external" + service2 := svc + service2.Name = "mdb-1-svc-external" + expectedServices := []corev1.Service{service1, service2} + + testDatabaseInKubernetesExternalServices(ctx, t, mdbv1.ExternalAccessConfiguration{}, expectedServices) +} + +func TestDatabaseInKubernetes_ExternalServicesWithExternalDomainHaveAdditionalBackupPort(t *testing.T) { + ctx := context.Background() + svc := corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "mdb-0-svc-external"}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + TargetPort: intstr.IntOrString{IntVal: 27017}, + }, + { + Name: "backup", + TargetPort: intstr.IntOrString{IntVal: 27018}, + }, + }, + Type: corev1.ServiceTypeLoadBalancer, + PublishNotReadyAddresses: true, + }, + } + + service1 := svc + service1.Name = "mdb-0-svc-external" + service2 := svc + service2.Name = "mdb-1-svc-external" + expectedServices := []corev1.Service{service1, service2} + + testDatabaseInKubernetesExternalServices(ctx, t, mdbv1.ExternalAccessConfiguration{ExternalDomain: ptr.To("example.com")}, expectedServices) +} + +func TestDatabaseInKubernetes_ExternalServicesWithServiceSpecOverrides(t *testing.T) { + ctx := context.Background() + svc := corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "mdb-0-svc-external", Annotations: map[string]string{ + "key": "value", + }}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + TargetPort: intstr.IntOrString{IntVal: 27017}, + }, + { + Name: "backup", + TargetPort: intstr.IntOrString{IntVal: 27018}, + }, + }, + Type: corev1.ServiceTypeNodePort, + PublishNotReadyAddresses: true, + }, + } + + service1 := svc + service1.Name = "mdb-0-svc-external" + service2 := svc + service2.Name = "mdb-1-svc-external" + expectedServices := []corev1.Service{service1, service2} + + externalAccessConfiguration := mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("example.com"), + ExternalService: mdbv1.ExternalServiceConfiguration{ + SpecWrapper: &mdbv1.ServiceSpecWrapper{Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + }}, + Annotations: map[string]string{ + "key": "value", + }, + }, + } + testDatabaseInKubernetesExternalServices(ctx, t, externalAccessConfiguration, expectedServices) +} + +const ( + defaultResourceName = "mdb" + defaultNamespace = "my-namespace" +) + +func TestDatabaseInKubernetes_ExternalServicesWithPlaceholders(t *testing.T) { + ctx := context.Background() + svc := corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "mdb-0-svc-external", Annotations: map[string]string{ + "key": "value", + }}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + TargetPort: intstr.IntOrString{IntVal: 27017}, + }, + }, + Type: corev1.ServiceTypeNodePort, + PublishNotReadyAddresses: true, + }, + } + + service1 := svc + service1.Name = "mdb-0-svc-external" + service2 := svc + service2.Name = "mdb-1-svc-external" + externalAccessConfiguration := mdbv1.ExternalAccessConfiguration{ + ExternalService: mdbv1.ExternalServiceConfiguration{ + SpecWrapper: &mdbv1.ServiceSpecWrapper{Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + }}, + Annotations: map[string]string{ + PlaceholderPodIndex: "{podIndex}", + PlaceholderNamespace: "{namespace}", + PlaceholderResourceName: "{resourceName}", + PlaceholderPodName: "{podName}", + PlaceholderStatefulSetName: "{statefulSetName}", + PlaceholderExternalServiceName: "{externalServiceName}", + PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + }, + }, + } + + podIndex := 0 + podName := fmt.Sprintf("%s-%d", defaultResourceName, podIndex) + mongodProcessDomain := fmt.Sprintf("%s-svc.%s.svc.cluster.local", defaultResourceName, defaultNamespace) + service1.Annotations = map[string]string{ + PlaceholderPodIndex: fmt.Sprintf("%d", podIndex), + PlaceholderNamespace: defaultNamespace, + PlaceholderResourceName: defaultResourceName, + PlaceholderPodName: podName, + PlaceholderStatefulSetName: defaultResourceName, + PlaceholderExternalServiceName: fmt.Sprintf("%s-svc-external", podName), + PlaceholderMongodProcessDomain: mongodProcessDomain, + PlaceholderMongodProcessFQDN: fmt.Sprintf("%s.%s", podName, mongodProcessDomain), + } + + podIndex = 1 + podName = fmt.Sprintf("%s-%d", defaultResourceName, podIndex) + service2.Annotations = map[string]string{ + PlaceholderPodIndex: fmt.Sprintf("%d", podIndex), + PlaceholderNamespace: defaultNamespace, + PlaceholderResourceName: defaultResourceName, + PlaceholderPodName: podName, + PlaceholderStatefulSetName: defaultResourceName, + PlaceholderExternalServiceName: fmt.Sprintf("%s-svc-external", podName), + PlaceholderMongodProcessDomain: mongodProcessDomain, + PlaceholderMongodProcessFQDN: fmt.Sprintf("%s.%s", podName, mongodProcessDomain), + } + + testDatabaseInKubernetesExternalServices(ctx, t, externalAccessConfiguration, []corev1.Service{service1, service2}) +} + +func TestDatabaseInKubernetes_ExternalServicesWithPlaceholders_WithExternalDomain(t *testing.T) { + ctx := context.Background() + svc := corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "mdb-0-svc-external", Annotations: map[string]string{ + "key": "value", + }}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "mongodb", + TargetPort: intstr.IntOrString{IntVal: 27017}, + }, + { + Name: "backup", + TargetPort: intstr.IntOrString{IntVal: 27018}, + }, + }, + Type: corev1.ServiceTypeNodePort, + PublishNotReadyAddresses: true, + }, + } + + service1 := svc + service1.Name = "mdb-0-svc-external" + service2 := svc + service2.Name = "mdb-1-svc-external" + externalDomain := "external.domain.example.com" + externalAccessConfiguration := mdbv1.ExternalAccessConfiguration{ + ExternalDomain: &externalDomain, + ExternalService: mdbv1.ExternalServiceConfiguration{ + SpecWrapper: &mdbv1.ServiceSpecWrapper{Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + }}, + Annotations: map[string]string{ + PlaceholderPodIndex: "{podIndex}", + PlaceholderNamespace: "{namespace}", + PlaceholderResourceName: "{resourceName}", + PlaceholderPodName: "{podName}", + PlaceholderStatefulSetName: "{statefulSetName}", + PlaceholderExternalServiceName: "{externalServiceName}", + PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + }, + }, + } + + podIndex := 0 + podName := fmt.Sprintf("%s-%d", defaultResourceName, podIndex) + mongodProcessDomain := externalDomain + service1.Annotations = map[string]string{ + PlaceholderPodIndex: fmt.Sprintf("%d", podIndex), + PlaceholderNamespace: defaultNamespace, + PlaceholderResourceName: defaultResourceName, + PlaceholderPodName: podName, + PlaceholderStatefulSetName: defaultResourceName, + PlaceholderExternalServiceName: fmt.Sprintf("%s-svc-external", podName), + PlaceholderMongodProcessDomain: mongodProcessDomain, + PlaceholderMongodProcessFQDN: fmt.Sprintf("%s.%s", podName, mongodProcessDomain), + } + + podIndex = 1 + podName = fmt.Sprintf("%s-%d", defaultResourceName, podIndex) + service2.Annotations = map[string]string{ + PlaceholderPodIndex: fmt.Sprintf("%d", podIndex), + PlaceholderNamespace: defaultNamespace, + PlaceholderResourceName: defaultResourceName, + PlaceholderPodName: podName, + PlaceholderStatefulSetName: defaultResourceName, + PlaceholderExternalServiceName: fmt.Sprintf("%s-svc-external", podName), + PlaceholderMongodProcessDomain: mongodProcessDomain, + PlaceholderMongodProcessFQDN: fmt.Sprintf("%s.%s", podName, mongodProcessDomain), + } + + testDatabaseInKubernetesExternalServices(ctx, t, externalAccessConfiguration, []corev1.Service{service1, service2}) +} + +func testDatabaseInKubernetesExternalServices(ctx context.Context, t *testing.T, externalAccessConfiguration mdbv1.ExternalAccessConfiguration, expectedServices []corev1.Service) { + log := zap.S() + fakeClient, _ := mock.NewDefaultFakeClient() + mdb := mdbv1.NewReplicaSetBuilder(). + SetName(defaultResourceName). + SetNamespace(defaultNamespace). + SetMembers(2). + Build() + mdb.Spec.ExternalAccessConfiguration = &externalAccessConfiguration + + sts := construct.DatabaseStatefulSet(*mdb, construct.ReplicaSetOptions(construct.GetPodEnvOptions()), log) + err := DatabaseInKubernetes(ctx, fakeClient, *mdb, sts, construct.ReplicaSetOptions(), log) + assert.NoError(t, err) + + // we only test a subset of fields from service spec, which are the most relevant for external services + for _, expectedService := range expectedServices { + actualService, err := fakeClient.GetService(ctx, types.NamespacedName{Name: expectedService.GetName(), Namespace: defaultNamespace}) + require.NoError(t, err, "serviceName: %s", expectedService.GetName()) + require.NotNil(t, actualService) + require.Len(t, actualService.Spec.Ports, len(expectedService.Spec.Ports)) + for i, expectedPort := range expectedService.Spec.Ports { + actualPort := actualService.Spec.Ports[i] + assert.Equal(t, expectedPort.Name, actualPort.Name) + assert.Equal(t, expectedPort.TargetPort.IntVal, actualPort.TargetPort.IntVal) + } + assert.Equal(t, expectedService.Spec.Type, actualService.Spec.Type) + assert.True(t, expectedService.Spec.PublishNotReadyAddresses, actualService.Spec.PublishNotReadyAddresses) + if expectedService.Annotations != nil { + assert.Equal(t, expectedService.Annotations, actualService.Annotations) + } + } + + // disable external access -> remove external services + mdb.Spec.ExternalAccessConfiguration = nil + err = DatabaseInKubernetes(ctx, fakeClient, *mdb, sts, construct.ReplicaSetOptions(), log) + assert.NoError(t, err) + + for _, expectedService := range expectedServices { + _, err := fakeClient.GetService(ctx, types.NamespacedName{Name: expectedService.GetName(), Namespace: defaultNamespace}) + assert.True(t, errors.IsNotFound(err)) + } +} + +func TestDatabaseInKubernetesExternalServicesSharded(t *testing.T) { + ctx := context.Background() + log := zap.S() + fakeClient, _ := mock.NewDefaultFakeClient() + mdb := mdbv1.NewDefaultShardedClusterBuilder(). + SetName("mdb"). + SetNamespace("my-namespace"). + SetMongosCountSpec(2). + SetShardCountSpec(1). + SetConfigServerCountSpec(1). + Build() + + mdb.Spec.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{} + + createShardSts(ctx, t, mdb, log, fakeClient) + + createMongosSts(ctx, t, mdb, log, fakeClient) + + actualService, err := fakeClient.GetService(ctx, types.NamespacedName{Name: "mdb-mongos-0-svc-external", Namespace: "my-namespace"}) + require.NoError(t, err) + require.NotNil(t, actualService) + + actualService, err = fakeClient.GetService(ctx, types.NamespacedName{Name: "mdb-mongos-1-svc-external", Namespace: "my-namespace"}) + require.NoError(t, err) + require.NotNil(t, actualService) + + _, err = fakeClient.GetService(ctx, types.NamespacedName{Name: "mdb-config-0-svc-external", Namespace: "my-namespace"}) + require.Errorf(t, err, "expected no config service") + + _, err = fakeClient.GetService(ctx, types.NamespacedName{Name: "mdb-0-svc-external", Namespace: "my-namespace"}) + require.Errorf(t, err, "expected no shard service") +} + +func createShardSpecAndDefaultCluster(client kubernetesClient.Client, sc *mdbv1.MongoDB) (*mdbv1.ShardedClusterComponentSpec, multicluster.MemberCluster) { + shardSpec := sc.Spec.ShardSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + PodSpec: sc.Spec.PodSpec, + }, + } + + return shardSpec, multicluster.GetLegacyCentralMemberCluster(sc.Spec.MongodsPerShardCount, 0, client, secrets.SecretClient{KubeClient: client}) +} + +func createMongosSpec(sc *mdbv1.MongoDB) *mdbv1.ShardedClusterComponentSpec { + shardSpec := sc.Spec.ConfigSrvSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec +} + +func createShardSts(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDB, log *zap.SugaredLogger, kubeClient kubernetesClient.Client) { + shardSpec, memberCluster := createShardSpecAndDefaultCluster(kubeClient, mdb) + sts := construct.DatabaseStatefulSet(*mdb, construct.ShardOptions(1, shardSpec, memberCluster.Name, construct.GetPodEnvOptions()), log) + err := DatabaseInKubernetes(ctx, kubeClient, *mdb, sts, construct.ShardOptions(1, shardSpec, memberCluster.Name), log) + assert.NoError(t, err) +} + +func createMongosSts(ctx context.Context, t *testing.T, mdb *mdbv1.MongoDB, log *zap.SugaredLogger, kubeClient kubernetesClient.Client) { + mongosSpec := createMongosSpec(mdb) + sts := construct.DatabaseStatefulSet(*mdb, construct.MongosOptions(mongosSpec, multicluster.LegacyCentralClusterName, construct.GetPodEnvOptions()), log) + err := DatabaseInKubernetes(ctx, kubeClient, *mdb, sts, construct.MongosOptions(mongosSpec, multicluster.LegacyCentralClusterName), log) + assert.NoError(t, err) +} + +func TestResizePVCsStorage(t *testing.T) { + fakeClient, _ := mock.NewDefaultFakeClient() + + initialSts := createStatefulSet("20Gi", "20Gi", "20Gi") + + // Create the StatefulSet that we want to resize the PVC to + err := fakeClient.CreateStatefulSet(context.TODO(), *initialSts) + assert.NoError(t, err) + + for _, template := range initialSts.Spec.VolumeClaimTemplates { + for i := range *initialSts.Spec.Replicas { + pvc := createPVCFromTemplate(template, initialSts.Name, i) + err = fakeClient.Create(context.TODO(), pvc) + assert.NoError(t, err) + } + } + + err = resizePVCsStorage(fakeClient, createStatefulSet("30Gi", "30Gi", "20Gi")) + assert.NoError(t, err) + + pvcList := corev1.PersistentVolumeClaimList{} + err = fakeClient.List(context.TODO(), &pvcList) + assert.NoError(t, err) + + for _, pvc := range pvcList.Items { + if strings.HasPrefix(pvc.Name, "data") { + assert.Equal(t, pvc.Spec.Resources.Requests.Storage().String(), "30Gi") + } else if strings.HasPrefix(pvc.Name, "journal") { + assert.Equal(t, pvc.Spec.Resources.Requests.Storage().String(), "30Gi") + } else if strings.HasPrefix(pvc.Name, "logs") { + assert.Equal(t, pvc.Spec.Resources.Requests.Storage().String(), "20Gi") + } else { + t.Fatal("no pvc was compared while we should have at least detected and compared one") + } + } +} + +// Helper function to create a StatefulSet +func createStatefulSet(size1, size2, size3 string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(3)), + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(size1), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "journal", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(size2), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "logs", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(size3), + }, + }, + }, + }, + }, + }, + } +} + +func createPVCFromTemplate(pvcTemplate corev1.PersistentVolumeClaim, stsName string, ordinal int32) *corev1.PersistentVolumeClaim { + pvcName := fmt.Sprintf("%s-%s-%d", pvcTemplate.Name, stsName, ordinal) + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: "default", + }, + Spec: pvcTemplate.Spec, + } +} + +func TestResourceStorageHasChanged(t *testing.T) { + type args struct { + existingPVC []corev1.PersistentVolumeClaim + toCreatePVC []corev1.PersistentVolumeClaim + } + tests := []struct { + name string + args args + want []pvcResize + }{ + { + name: "empty", + want: nil, + }, + { + name: "existing is larger", + args: args{ + existingPVC: []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("2Gi")}, + }, + }, + }, + }, + toCreatePVC: []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, + }, + }, + want: []pvcResize{{resizeIndicator: 1, from: "2Gi", to: "1Gi"}}, + }, + { + name: "toCreate is larger", + args: args{ + existingPVC: []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, + }, + toCreatePVC: []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("2Gi")}, + }, + }, + }, + }, + }, + want: []pvcResize{{resizeIndicator: -1, from: "1Gi", to: "2Gi"}}, + }, + { + name: "both are equal", + args: args{ + existingPVC: []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, + }, + toCreatePVC: []corev1.PersistentVolumeClaim{ + { + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, + }, + }, + want: []pvcResize{{resizeIndicator: 0, from: "1Gi", to: "1Gi"}}, + }, + { + name: "none exist", + args: args{ + existingPVC: []corev1.PersistentVolumeClaim{}, + toCreatePVC: []corev1.PersistentVolumeClaim{}, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, resourceStorageHasChanged(tt.args.existingPVC, tt.args.toCreatePVC), "resourceStorageHasChanged(%v, %v)", tt.args.existingPVC, tt.args.toCreatePVC) + }) + } +} + +func TestHasFinishedResizing(t *testing.T) { + stsName := "test" + desiredSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: stsName}, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "logs", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("30Gi"), + }, + }, + }, + }, + }, + }, + } + + ctx := context.TODO() + { + fakeClient, _ := mock.NewDefaultFakeClient() + // Scenario 1: All PVCs have finished resizing + pvc1 := createPVCWithCapacity("data-"+stsName+"-0", "20Gi") + pvc2 := createPVCWithCapacity("logs-"+stsName+"-0", "30Gi") + notPartOfSts := createPVCWithCapacity("random-sts-0", "30Gi") + err := fakeClient.Create(ctx, pvc1) + assert.NoError(t, err) + err = fakeClient.Create(ctx, pvc2) + assert.NoError(t, err) + err = fakeClient.Create(ctx, notPartOfSts) + assert.NoError(t, err) + + finished, err := hasFinishedResizing(ctx, fakeClient, desiredSts) + assert.NoError(t, err) + assert.True(t, finished, "PVCs should be finished resizing") + } + + { + // Scenario 2: Some PVCs are still resizing + fakeClient, _ := mock.NewDefaultFakeClient() + pvc2Incomplete := createPVCWithCapacity("logs-"+stsName+"-0", "10Gi") + err := fakeClient.Create(ctx, pvc2Incomplete) + assert.NoError(t, err) + + finished, err := hasFinishedResizing(ctx, fakeClient, desiredSts) + assert.NoError(t, err) + assert.False(t, finished, "PVCs should not be finished resizing") + } +} + +// Helper function to create a PVC with a specific capacity and status +func createPVCWithCapacity(name string, capacity string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(capacity), + }, + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(capacity), + }, + }, + } +} + +func TestGetMatchingPVCTemplateFromSTS(t *testing.T) { + statefulSet := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-sts", + }, + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data-pvc", + }, + Spec: corev1.PersistentVolumeClaimSpec{}, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "logs-pvc", + }, + Spec: corev1.PersistentVolumeClaimSpec{}, + }, + }, + }, + } + + tests := []struct { + name string + pvcName string + expectedTemplate *corev1.PersistentVolumeClaim + expectedIndex int + }{ + { + name: "Matching data-pvc with ordinal 0", + pvcName: "data-pvc-example-sts-0", + expectedTemplate: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "data-pvc", + }, + }, + expectedIndex: 0, + }, + { + name: "Matching logs-pvc with ordinal 1", + pvcName: "logs-pvc-example-sts-1", + expectedTemplate: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "logs-pvc", + }, + }, + expectedIndex: 1, + }, + { + name: "Non-matching PVC name", + pvcName: "cache-pvc-example-sts-0", + expectedTemplate: nil, + expectedIndex: -1, + }, + { + name: "Matching data-pvc with high ordinal", + pvcName: "data-pvc-example-sts-1000", + expectedTemplate: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "data-pvc", + }, + }, + expectedIndex: 0, + }, + { + name: "PVC name with similar prefix but different StatefulSet name", + pvcName: "data-pvc-other-sts-0", + expectedTemplate: nil, + expectedIndex: -1, + }, + { + name: "Not matching logs-pvc without ordinal", + pvcName: "logs-pvc-example-sts", + expectedTemplate: nil, + expectedIndex: -1, + }, + { + name: "Empty PVC name", + pvcName: "", + expectedTemplate: nil, + expectedIndex: -1, + }, + { + name: "PVC name with extra suffix", + pvcName: "data-pvc-example-sts-extra-0", + expectedTemplate: nil, + expectedIndex: -1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.pvcName, + }, + } + + template, index := getMatchingPVCTemplateFromSTS(statefulSet, p) + + if tt.expectedTemplate == nil { + assert.Nil(t, template, "Expected no matching PVC template") + } else { + if assert.NotNil(t, template, "Expected a matching PVC template") { + assert.Equal(t, tt.expectedTemplate.Name, template.Name, "PVC template name should match") + } + } + + assert.Equal(t, tt.expectedIndex, index, "PVC template index should match") + }) + } +} + +func TestCheckStatefulsetIsDeleted(t *testing.T) { + ctx := context.TODO() + sleepDuration := 10 * time.Millisecond + log := zap.NewNop().Sugar() + + namespace := "default" + stsName := "test-sts" + desiredSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: stsName, + Namespace: namespace, + }, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(3))}, + } + + t.Run("StatefulSet is deleted", func(t *testing.T) { + fakeClient, _ := mock.NewDefaultFakeClient() + err := fakeClient.CreateStatefulSet(ctx, *desiredSts) + assert.NoError(t, err) + + // Simulate the deletion by deleting the StatefulSet + err = fakeClient.DeleteStatefulSet(ctx, kube.ObjectKey(desiredSts.Namespace, desiredSts.Name)) + assert.NoError(t, err) + + // Check if the StatefulSet is detected as deleted + result := checkStatefulsetIsDeleted(ctx, fakeClient, desiredSts, sleepDuration, log) + + assert.True(t, result, "StatefulSet should be detected as deleted") + }) + + t.Run("StatefulSet is not deleted", func(t *testing.T) { + fakeClient, _ := mock.NewDefaultFakeClient() + err := fakeClient.CreateStatefulSet(ctx, *desiredSts) + assert.NoError(t, err) + + // Do not delete the StatefulSet, to simulate it still existing + // Check if the StatefulSet is detected as not deleted + result := checkStatefulsetIsDeleted(ctx, fakeClient, desiredSts, sleepDuration, log) + + assert.False(t, result, "StatefulSet should not be detected as deleted") + }) + + t.Run("StatefulSet is deleted after some retries", func(t *testing.T) { + fakeClient, _ := mock.NewDefaultFakeClient() + err := fakeClient.CreateStatefulSet(ctx, *desiredSts) + assert.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + // Use a goroutine to delete the StatefulSet after a delay, making it race-safe + go func() { + defer wg.Done() + time.Sleep(20 * time.Millisecond) // Wait for a bit longer than the first sleep + err = fakeClient.DeleteStatefulSet(ctx, kube.ObjectKey(desiredSts.Namespace, desiredSts.Name)) + assert.NoError(t, err) + }() + + // Check if the StatefulSet is detected as deleted after retries + result := checkStatefulsetIsDeleted(ctx, fakeClient, desiredSts, sleepDuration, log) + + wg.Wait() + + assert.True(t, result, "StatefulSet should be detected as deleted after retries") + }) +} diff --git a/controllers/operator/database_statefulset_options.go b/controllers/operator/database_statefulset_options.go new file mode 100644 index 000000000..70eae9822 --- /dev/null +++ b/controllers/operator/database_statefulset_options.go @@ -0,0 +1,129 @@ +package operator + +import ( + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +// CurrentAgentAuthMechanism will assign the given value as the current authentication mechanism. +func CurrentAgentAuthMechanism(mode string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.CurrentAgentAuthMode = mode + } +} + +// PodEnvVars will assign the given env vars which will used during StatefulSet construction. +func PodEnvVars(vars *env.PodEnvVars) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.PodVars = vars + } +} + +// Replicas will set the given number of replicas when building a StatefulSet. +func Replicas(replicas int) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.Replicas = replicas + } +} + +func Name(name string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.Name = name + } +} + +func StatefulSetNameOverride(statefulSetNameOverride string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.StatefulSetNameOverride = statefulSetNameOverride + } +} + +func ServiceName(serviceName string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.ServiceName = serviceName + } +} + +// CertificateHash will assign the given CertificateHash during StatefulSet construction. +func CertificateHash(hash string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.CertificateHash = hash + } +} + +// InternalClusterHash will assign the given InternalClusterHash during StatefulSet construction. +func InternalClusterHash(hash string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.InternalClusterHash = hash + } +} + +func PrometheusTLSCertHash(hash string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.PrometheusTLSCertHash = hash + } +} + +// WithLabels will assign the provided labels during the statefulset construction +func WithLabels(labels map[string]string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.Labels = labels + } +} + +// WithStsLabels will assign the provided labels during the statefulset construction +func WithStsLabels(labels map[string]string) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.StsLabels = labels + } +} + +// WithVaultConfig sets the vault configuration to extract annotations for the statefulset. +func WithVaultConfig(config vault.VaultConfiguration) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.VaultConfig = config + } +} + +func WithAdditionalMongodConfig(additionalMongodConfig *mdbv1.AdditionalMongodConfig) func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.AdditionalMongodConfig = additionalMongodConfig + } +} + +func WithDefaultConfigSrvStorageSize() func(options *construct.DatabaseStatefulSetOptions) { + return func(options *construct.DatabaseStatefulSetOptions) { + options.PodSpec.Default.Persistence.SingleConfig.Storage = util.DefaultConfigSrvStorageSize + } +} + +// WithInitDatabaseNonStaticImage sets the InitDatabaseNonStaticImage field. +func WithInitDatabaseNonStaticImage(image string) func(*construct.DatabaseStatefulSetOptions) { + return func(opts *construct.DatabaseStatefulSetOptions) { + opts.InitDatabaseNonStaticImage = image + } +} + +// WithDatabaseNonStaticImage sets the DatabaseNonStaticImage field. +func WithDatabaseNonStaticImage(image string) func(*construct.DatabaseStatefulSetOptions) { + return func(opts *construct.DatabaseStatefulSetOptions) { + opts.DatabaseNonStaticImage = image + } +} + +// WithMongodbImage sets the MongodbImage field. +func WithMongodbImage(image string) func(*construct.DatabaseStatefulSetOptions) { + return func(opts *construct.DatabaseStatefulSetOptions) { + opts.MongodbImage = image + } +} + +// WithAgentImage sets the AgentImage field. +func WithAgentImage(image string) func(*construct.DatabaseStatefulSetOptions) { + return func(opts *construct.DatabaseStatefulSetOptions) { + opts.AgentImage = image + } +} diff --git a/controllers/operator/inspect/statefulset_inspector.go b/controllers/operator/inspect/statefulset_inspector.go new file mode 100644 index 000000000..e0a639ed0 --- /dev/null +++ b/controllers/operator/inspect/statefulset_inspector.go @@ -0,0 +1,69 @@ +package inspect + +import ( + "fmt" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1 "k8s.io/api/apps/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// StatefulSetState is an entity encapsulating all the information about StatefulSet state +type StatefulSetState struct { + statefulSetKey client.ObjectKey + updated int32 + ready int32 + current int32 + wanted int32 + generation int64 + observedGeneration int64 + updateStrategyType appsv1.StatefulSetUpdateStrategyType +} + +// GetResourcesNotReadyStatus returns the status of dependent resources which have any problems +func (s StatefulSetState) GetResourcesNotReadyStatus() []status.ResourceNotReady { + if s.IsReady() { + return []status.ResourceNotReady{} + } + zap.S().Debugf("StatefulSet %s (wanted: %d, ready: %d, updated: %d, generation: %d, observedGeneration: %d)", s.statefulSetKey.Name, s.wanted, s.ready, s.updated, s.generation, s.observedGeneration) + msg := fmt.Sprintf("Not all the Pods are ready (wanted: %d, updated: %d, ready: %d, current: %d)", s.wanted, s.updated, s.ready, s.current) + return []status.ResourceNotReady{{ + Kind: status.StatefulsetKind, + Name: s.statefulSetKey.Name, + Message: msg, + }} +} + +// GetMessage returns the general message to be shown in status or/and printed in logs +func (s StatefulSetState) GetMessage() string { + if s.IsReady() { + return fmt.Sprintf("StatefulSet %s is ready", s.statefulSetKey) + } + return "StatefulSet not ready" +} + +func (s StatefulSetState) IsReady() bool { + isReady := s.updated == s.ready && + s.ready == s.wanted && + s.observedGeneration == s.generation && + s.current == s.wanted + return isReady || s.updateStrategyType == appsv1.OnDeleteStatefulSetStrategyType +} + +func StatefulSet(set appsv1.StatefulSet) StatefulSetState { + state := StatefulSetState{ + statefulSetKey: types.NamespacedName{Namespace: set.Namespace, Name: set.Name}, + updated: set.Status.UpdatedReplicas, + ready: set.Status.ReadyReplicas, + current: set.Status.Replicas, + wanted: *set.Spec.Replicas, + observedGeneration: set.Status.ObservedGeneration, + generation: set.Generation, + updateStrategyType: set.Spec.UpdateStrategy.Type, + } + return state +} diff --git a/controllers/operator/inspect/statefulset_inspector_test.go b/controllers/operator/inspect/statefulset_inspector_test.go new file mode 100644 index 000000000..6cf0d7ecd --- /dev/null +++ b/controllers/operator/inspect/statefulset_inspector_test.go @@ -0,0 +1,57 @@ +package inspect + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestStatefulSetInspector(t *testing.T) { + statefulSet := appsv1.StatefulSet{ + Spec: appsv1.StatefulSetSpec{ + Replicas: util.Int32Ref(3), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sts", + Namespace: "ns", + Generation: 1, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: 3, + ReadyReplicas: 1, + UpdatedReplicas: 2, + }, + } + + state := StatefulSet(statefulSet) + assert.False(t, state.IsReady()) + assert.Len(t, state.GetResourcesNotReadyStatus(), 1) + assert.Contains(t, state.GetResourcesNotReadyStatus()[0].Message, "Not all the Pods are ready") + assert.Contains(t, state.GetMessage(), "not ready") + assert.Equal(t, state.GetResourcesNotReadyStatus()[0].Kind, status.StatefulsetKind) + assert.Equal(t, state.GetResourcesNotReadyStatus()[0].Name, "sts") + + // StatefulSet "got" to ready state + statefulSet.Status.UpdatedReplicas = 3 + statefulSet.Status.ReadyReplicas = 3 + statefulSet.Status.ObservedGeneration = 1 + + state = StatefulSet(statefulSet) + assert.True(t, state.IsReady()) + assert.Contains(t, state.GetMessage(), "is ready") + assert.Len(t, state.GetResourcesNotReadyStatus(), 0) + + // We "scale" the StatefulSet + // Even though every other properties are the same, we need Spec.Replicas to be equal to Status.Replicas to be ready + statefulSet.Spec.Replicas = util.Int32Ref(5) + + state = StatefulSet(statefulSet) + assert.False(t, state.IsReady()) + assert.Contains(t, state.GetResourcesNotReadyStatus()[0].Message, "Not all the Pods are ready") +} diff --git a/controllers/operator/ldap/ldap_types.go b/controllers/operator/ldap/ldap_types.go new file mode 100644 index 000000000..182a301e0 --- /dev/null +++ b/controllers/operator/ldap/ldap_types.go @@ -0,0 +1,21 @@ +package ldap + +// Ldap holds all the fields required to configure LDAP authentication +type Ldap struct { + AuthzQueryTemplate string `json:"authzQueryTemplate,omitempty"` + BindMethod string `json:"bindMethod"` + BindQueryUser string `json:"bindQueryUser"` + BindSaslMechanisms string `json:"bindSaslMechanisms,omitempty"` + Servers string `json:"servers"` + TransportSecurity string `json:"transportSecurity"` + UserToDnMapping string `json:"userToDNMapping,omitempty"` + ValidateLDAPServerConfig bool `json:"validateLDAPServerConfig"` + BindQueryPassword string `json:"bindQueryPassword"` + TimeoutMS int `json:"timeoutMS,omitempty"` + UserCacheInvalidationInterval int `json:"userCacheInvalidationInterval,omitempty"` + + // Uses an undocumented property from the API used to mount + // a CA file. This is done by the automation agent. + // https://mongodb.slack.com/archives/CN0JB7XT2/p1594229779090700 + CaFileContents string `json:"CAFileContents"` +} diff --git a/controllers/operator/mock/mockedkubeclient.go b/controllers/operator/mock/mockedkubeclient.go new file mode 100644 index 000000000..b62d1144c --- /dev/null +++ b/controllers/operator/mock/mockedkubeclient.go @@ -0,0 +1,298 @@ +package mock + +import ( + "context" + "fmt" + "reflect" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + certsv1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiruntime "k8s.io/apimachinery/pkg/runtime" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/handler" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +const ( + TestProjectConfigMapName = om.TestGroupName + TestCredentialsSecretName = "my-credentials" + TestNamespace = "my-namespace" + TestMongoDBName = "my-mongodb" +) + +// NewDefaultFakeClient initializes returns fake kube client and omConnectionFactory that is suitable for most uses. +// This fake client is initialized with default project resources (project config map and credentials secret, see GetDefaultResources) that are required most of the time to reconcile the resource. +// It automatically adds list of objects to the client. +// +// The reason we couple kube fake client with OM's connection factory is that most tests we rely on the behavior of automatically marking created statefulset to be ready +// along with simulating that hostnames from monitoring are registered to the current OM connecti on. +func NewDefaultFakeClient(objects ...client.Object) (kubernetesClient.Client, *om.CachedOMConnectionFactory) { + omConnectionFactory := om.NewCachedOMConnectionFactory(om.NewEmptyMockedOmConnection) + return NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, objects...), omConnectionFactory +} + +// NewDefaultFakeClientWithOMConnectionFactory is the same as NewDefaultFakeClient, but you can pass omConnectionFactory from outside. +func NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory *om.CachedOMConnectionFactory, objects ...client.Object) kubernetesClient.Client { + return NewEmptyFakeClientWithInterceptor(omConnectionFactory, append(objects, GetDefaultResources()...)...) +} + +// NewEmptyFakeClientWithInterceptor initializes empty fake kube client with interceptor for automatically marking statefulsets as ready. +// It doesn't add any default resources, but adds passed objects if any. +func NewEmptyFakeClientWithInterceptor(omConnectionFactory *om.CachedOMConnectionFactory, objects ...client.Object) kubernetesClient.Client { + fakeClientBuilder := NewEmptyFakeClientBuilder() + if len(objects) > 0 { + fakeClientBuilder.WithObjects(objects...) + } + fakeClientBuilder.WithInterceptorFuncs(interceptor.Funcs{ + Get: GetFakeClientInterceptorGetFunc(omConnectionFactory, true, true), + }) + + return kubernetesClient.NewClient(fakeClientBuilder.Build()) +} + +// NewEmptyFakeClientBuilder return fully prepared fake client builder without any default resources or interceptors. +func NewEmptyFakeClientBuilder() *fake.ClientBuilder { + builder := fake.ClientBuilder{} + s, err := v1.SchemeBuilder.Build() + if err != nil { + return nil + } + err = metav1.AddMetaToScheme(s) + if err != nil { + return nil + } + + err = corev1.AddToScheme(s) + if err != nil { + return nil + } + + err = appsv1.AddToScheme(s) + if err != nil { + return nil + } + + builder.WithStatusSubresource(&mdbv1.MongoDB{}, &mdbmulti.MongoDBMultiCluster{}, &omv1.MongoDBOpsManager{}, &user.MongoDBUser{}) + + ot := testing.NewObjectTracker(s, scheme.Codecs.UniversalDecoder()) + return builder.WithScheme(s).WithObjectTracker(ot) +} + +func GetFakeClientInterceptorGetFunc(omConnectionFactory *om.CachedOMConnectionFactory, markStsAsReady bool, addOMHosts bool) func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + + switch v := obj.(type) { + case *appsv1.StatefulSet: + if markStsAsReady && omConnectionFactory != nil { + markStatefulSetsReady(v, addOMHosts, omConnectionFactory.GetConnectionForResource(v)) + } + } + + return nil + } +} + +func MarkAllStatefulSetsAsReady(ctx context.Context, namespace string, clients ...client.Client) error { + var updatedStsList []string + for _, c := range clients { + stsList := appsv1.StatefulSetList{} + if err := c.List(ctx, &stsList, client.InNamespace(namespace)); err != nil { + return fmt.Errorf("error listing statefulsets in namespace %s in client %+v", namespace, c) + } + + for _, sts := range stsList.Items { + updatedSts := sts + updatedSts.Status.UpdatedReplicas = *sts.Spec.Replicas + updatedSts.Status.ReadyReplicas = *sts.Spec.Replicas + updatedSts.Status.Replicas = *sts.Spec.Replicas + if err := c.Status().Patch(ctx, &updatedSts, client.MergeFrom(&sts)); err != nil { + return fmt.Errorf("error updating sts %s/%s in client %+v", sts.Namespace, sts.Name, c) + } + updatedStsList = append(updatedStsList, sts.Name) + } + } + + zap.S().Debugf("marked fake statefulsets as ready: %+v", updatedStsList) + + return nil +} + +func GetDefaultResources() []client.Object { + return []client.Object{ + GetProjectConfigMap(TestProjectConfigMapName, om.TestGroupName, ""), + GetCredentialsSecret(om.TestUser, om.TestApiKey), + } +} + +func GetProjectConfigMap(configMapName string, projectName string, organizationId string) *corev1.ConfigMap { + cm := configmap.Builder(). + SetName(configMapName). + SetNamespace(TestNamespace). + SetDataField(util.OmBaseUrl, "http://mycompany.example.com:8080"). + SetDataField(util.OmProjectName, projectName). + SetDataField(util.OmOrgId, organizationId). + Build() + return &cm +} + +func GetCredentialsSecret(publicKey string, privateKey string) *corev1.Secret { + stringData := map[string]string{util.OmPublicApiKey: publicKey, util.OmPrivateKey: privateKey} + data := map[string][]byte{} + for s, s2 := range stringData { + data[s] = []byte(s2) + } + credentials := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: TestCredentialsSecretName, Namespace: TestNamespace}, + // we are using Data and not SecretData because our internal secret.Builder only writes information into + // secret.Data not secret.StringData. + Data: data, + } + return credentials +} + +func ApproveAllCSRs(ctx context.Context, m client.Client) { + for _, csrObject := range GetMapForObject(m, &certsv1.CertificateSigningRequest{}) { + csr := csrObject.(*certsv1.CertificateSigningRequest) + approvedCondition := certsv1.CertificateSigningRequestCondition{ + Type: certsv1.CertificateApproved, + } + csr.Status.Conditions = append(csr.Status.Conditions, approvedCondition) + if err := m.Update(ctx, csr); err != nil { + panic(err) + } + } +} + +func CreateOrUpdate(ctx context.Context, m client.Client, obj client.Object) error { + // Determine the object's metadata + metaObj, ok := obj.(metav1.Object) + if !ok { + return fmt.Errorf("object is not a metav1.Object") + } + + namespace := metaObj.GetNamespace() + name := metaObj.GetName() + + existingObj := obj.DeepCopyObject().(client.Object) // Create a deep copy to store the existing object + err := m.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, existingObj) + if err != nil { + if client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to get object: %w", err) + } + + err = m.Create(ctx, obj) + if err != nil { + return fmt.Errorf("failed to create object: %w", err) + } + return nil + } + + err = m.Update(ctx, obj) + if err != nil { + return fmt.Errorf("failed to update object: %w", err) + } + + return nil +} + +func markStatefulSetsReady(set *appsv1.StatefulSet, addOMHosts bool, omConn om.Connection) { + set.Status.UpdatedReplicas = *set.Spec.Replicas + set.Status.ReadyReplicas = *set.Spec.Replicas + set.Status.Replicas = *set.Spec.Replicas + + if addOMHosts { + if mockedOMConnection, ok := omConn.(*om.MockedOmConnection); ok { + // For tests with external domains we set hostnames externally in test, + // as we don't have ExternalAccessConfiguration object in stateful set. + // For tests that don't set hostnames we preserve old behaviour. + hostnames := mockedOMConnection.Hostnames + if hostnames == nil { + if val, ok := set.Annotations[handler.MongoDBMultiResourceAnnotation]; ok { + hostnames = dns.GetMultiClusterProcessHostnames(val, set.Namespace, multicluster.MustGetClusterNumFromMultiStsName(set.Name), int(*set.Spec.Replicas), "cluster.local", nil) + } else { + // We also "register" automation agents. + // So far we don't support custom cluster name + hostnames, _ = dns.GetDnsForStatefulSet(*set, "", nil) + } + } + mockedOMConnection.AddHosts(hostnames) + } + } +} + +func GetMapForObject(m client.Client, obj apiruntime.Object) map[client.ObjectKey]apiruntime.Object { + switch obj.(type) { + case *corev1.Secret: + secrets := &corev1.SecretList{} + if err := m.List(context.TODO(), secrets); err != nil { + return nil + } + secretMap := make(map[client.ObjectKey]apiruntime.Object, len(secrets.Items)) + for _, secret := range secrets.Items { + secretMap[client.ObjectKey{ + Namespace: secret.Namespace, + Name: secret.Name, + }] = &secret + } + return secretMap + case *appsv1.StatefulSet: + statefulSets := &appsv1.StatefulSetList{} + if err := m.List(context.TODO(), statefulSets); err != nil { + return nil + } + statefulSetMap := make(map[client.ObjectKey]apiruntime.Object, len(statefulSets.Items)) + for _, statefulSet := range statefulSets.Items { + statefulSetMap[client.ObjectKey{ + Namespace: statefulSet.Namespace, + Name: statefulSet.Name, + }] = &statefulSet + } + return statefulSetMap + case *corev1.Service: + services := &corev1.ServiceList{} + if err := m.List(context.TODO(), services); err != nil { + return nil + } + serviceMap := make(map[client.ObjectKey]apiruntime.Object, len(services.Items)) + for _, service := range services.Items { + serviceMap[client.ObjectKey{ + Namespace: service.Namespace, + Name: service.Name, + }] = &service + } + return serviceMap + default: + return nil + } +} + +func ObjectKeyFromApiObject(obj interface{}) client.ObjectKey { + ns := reflect.ValueOf(obj).Elem().FieldByName("Namespace").String() + name := reflect.ValueOf(obj).Elem().FieldByName("Name").String() + + return types.NamespacedName{Name: name, Namespace: ns} +} diff --git a/controllers/operator/mock/test_fixtures.go b/controllers/operator/mock/test_fixtures.go new file mode 100644 index 000000000..ad07670d9 --- /dev/null +++ b/controllers/operator/mock/test_fixtures.go @@ -0,0 +1,25 @@ +package mock + +import ( + "os" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// nolint:forbidigo +func InitDefaultEnvVariables() { + _ = os.Setenv(util.NonStaticDatabaseEnterpriseImage, "mongodb-enterprise-database") + _ = os.Setenv(util.AutomationAgentImagePullPolicy, "Never") + _ = os.Setenv(util.OpsManagerImageUrl, "quay.io/mongodb/mongodb-enterprise-ops-manager") + _ = os.Setenv(util.InitOpsManagerImageUrl, "quay.io/mongodb/mongodb-enterprise-init-ops-manager") + _ = os.Setenv(util.InitAppdbImageUrlEnv, "quay.io/mongodb/mongodb-enterprise-init-appdb") + _ = os.Setenv(util.InitDatabaseImageUrlEnv, "quay.io/mongodb/mongodb-enterprise-init-database") + _ = os.Setenv(util.OpsManagerPullPolicy, "Never") + _ = os.Setenv(util.OmOperatorEnv, "test") + _ = os.Setenv(util.PodWaitSecondsEnv, "1") + _ = os.Setenv(util.PodWaitRetriesEnv, "2") + _ = os.Setenv(util.BackupDisableWaitSecondsEnv, "1") + _ = os.Setenv(util.BackupDisableWaitRetriesEnv, "3") + _ = os.Unsetenv(util.ManagedSecurityContextEnv) + _ = os.Unsetenv(util.ImagePullSecrets) +} diff --git a/controllers/operator/mongodbmultireplicaset_controller.go b/controllers/operator/mongodbmultireplicaset_controller.go new file mode 100644 index 000000000..8dee441d2 --- /dev/null +++ b/controllers/operator/mongodbmultireplicaset_controller.go @@ -0,0 +1,1303 @@ +package operator + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbmultiv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + mdbstatus "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/om/process" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + mconstruct "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/multicluster" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/recovery" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + khandler "github.com/10gen/ops-manager-kubernetes/pkg/handler" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + mekoService "github.com/10gen/ops-manager-kubernetes/pkg/kube/service" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster/memberwatch" + enterprisests "github.com/10gen/ops-manager-kubernetes/pkg/statefulset" + "github.com/10gen/ops-manager-kubernetes/pkg/tls" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// ReconcileMongoDbMultiReplicaSet reconciles a MongoDB ReplicaSet across multiple Kubernetes clusters +type ReconcileMongoDbMultiReplicaSet struct { + *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + memberClusterClientsMap map[string]kubernetesClient.Client // holds the client for each of the memberclusters(where the MongoDB ReplicaSet is deployed) + memberClusterSecretClientsMap map[string]secrets.SecretClient + forceEnterprise bool + + imageUrls images.ImageUrls + initDatabaseNonStaticImageVersion string + databaseNonStaticImageVersion string +} + +var _ reconcile.Reconciler = &ReconcileMongoDbMultiReplicaSet{} + +func newMultiClusterReplicaSetReconciler(ctx context.Context, kubeClient client.Client, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, omFunc om.ConnectionFactory, memberClustersMap map[string]client.Client) *ReconcileMongoDbMultiReplicaSet { + clientsMap := make(map[string]kubernetesClient.Client) + secretClientsMap := make(map[string]secrets.SecretClient) + + // extract client from each cluster object. + for k, v := range memberClustersMap { + clientsMap[k] = kubernetesClient.NewClient(v) + secretClientsMap[k] = secrets.SecretClient{ + VaultClient: nil, // Vault is not supported yet on multicluster + KubeClient: clientsMap[k], + } + } + + return &ReconcileMongoDbMultiReplicaSet{ + ReconcileCommonController: NewReconcileCommonController(ctx, kubeClient), + omConnectionFactory: omFunc, + memberClusterClientsMap: clientsMap, + memberClusterSecretClientsMap: secretClientsMap, + forceEnterprise: forceEnterprise, + imageUrls: imageUrls, + initDatabaseNonStaticImageVersion: initDatabaseNonStaticImageVersion, + databaseNonStaticImageVersion: databaseNonStaticImageVersion, + } +} + +// MongoDBMultiCluster Resource +// +kubebuilder:rbac:groups=mongodb.com,resources={mongodbmulticluster,mongodbmulticluster/status,mongodbmulticluster/finalizers},verbs=*,namespace=placeholder + +// Reconcile reads that state of the cluster for a MongoDbMultiReplicaSet object and makes changes based on the state read +// and what is in the MongoDbMultiReplicaSet.Spec +func (r *ReconcileMongoDbMultiReplicaSet) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, e error) { + log := zap.S().With("MultiReplicaSet", request.NamespacedName) + log.Info("-> MultiReplicaSet.Reconcile") + + // Fetch the MongoDBMultiCluster instance + mrs := mdbmultiv1.MongoDBMultiCluster{} + if reconcileResult, err := r.prepareResourceForReconciliation(ctx, request, &mrs, log); err != nil { + if apiErrors.IsNotFound(err) { + return workflow.Invalid("Object for reconciliation not found").ReconcileResult() + } + log.Errorf("error preparing resource for reconciliation: %s", err) + return reconcileResult, err + } + + if !architectures.IsRunningStaticArchitecture(mrs.Annotations) { + agents.UpgradeAllIfNeeded(ctx, agents.ClientSecret{Client: r.client, SecretClient: r.SecretClient}, r.omConnectionFactory, GetWatchedNamespace(), true) + } + + if err := mrs.ProcessValidationsOnReconcile(nil); err != nil { + return r.updateStatus(ctx, &mrs, workflow.Invalid("%s", err.Error()), log) + } + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, &mrs, log) + if err != nil { + return r.updateStatus(ctx, &mrs, workflow.Failed(xerrors.Errorf("Error reading project config and credentials: %w", err)), log) + } + + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, mrs.Namespace, log) + if err != nil { + return r.updateStatus(ctx, &mrs, workflow.Failed(xerrors.Errorf("error establishing connection to Ops Manager: %w", err)), log) + } + + log = log.With("MemberCluster Namespace", mrs.Namespace) + + // check if resource has failedCluster annotation and mark it as failed if automated failover is not enabled + failedClusterNames, err := mrs.GetFailedClusterNames() + if err != nil { + return r.updateStatus(ctx, &mrs, workflow.Failed(err), log) + } + if len(failedClusterNames) > 0 && !multicluster.ShouldPerformFailover() { + return r.updateStatus(ctx, &mrs, workflow.Failed(xerrors.Errorf("resource has failed clusters in the annotation: %+v", failedClusterNames)), log) + } + + r.SetupCommonWatchers(&mrs, nil, nil, mrs.Name) + + publishAutomationConfigFirst, err := r.publishAutomationConfigFirstMultiCluster(ctx, &mrs, log) + if err != nil { + return r.updateStatus(ctx, &mrs, workflow.Failed(err), log) + } + + // Recovery prevents some deadlocks that can occur during reconciliation, e.g. the setting of an incorrect automation + // configuration and a subsequent attempt to overwrite it later, the operator would be stuck in Pending phase. + // See CLOUDP-189433 and CLOUDP-229222 for more details. + if recovery.ShouldTriggerRecovery(mrs.Status.Phase != mdbstatus.PhaseRunning, mrs.Status.LastTransition) { + log.Warnf("Triggering Automatic Recovery. The MongoDB resource %s/%s is in %s state since %s", mrs.Namespace, mrs.Name, mrs.Status.Phase, mrs.Status.LastTransition) + automationConfigError := r.updateOmDeploymentRs(ctx, conn, mrs, true, log) + reconcileStatus := r.reconcileMemberResources(ctx, &mrs, log, conn, projectConfig) + if !reconcileStatus.IsOK() { + log.Errorf("Recovery failed because of reconcile errors, %v", reconcileStatus) + } + if automationConfigError != nil { + log.Errorf("Recovery failed because of Automation Config update errors, %w", automationConfigError) + } + } + + status := workflow.RunInGivenOrder(publishAutomationConfigFirst, + func() workflow.Status { + if err := r.updateOmDeploymentRs(ctx, conn, mrs, false, log); err != nil { + return workflow.Failed(err) + } + return workflow.OK() + }, + func() workflow.Status { + return r.reconcileMemberResources(ctx, &mrs, log, conn, projectConfig) + }) + + if !status.IsOK() { + return r.updateStatus(ctx, &mrs, status, log) + } + + mrs.Status.FeatureCompatibilityVersion = mrs.CalculateFeatureCompatibilityVersion() + if err := r.saveLastAchievedSpec(ctx, mrs); err != nil { + return r.updateStatus(ctx, &mrs, workflow.Failed(xerrors.Errorf("Failed to set annotation: %w", err)), log) + } + + // for purposes of comparison, we don't want to compare entries with 0 members since they will not be present + // as a desired entry. + desiredSpecList := mrs.GetDesiredSpecList() + actualSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return r.updateStatus(ctx, &mrs, workflow.Failed(err), log) + } + + effectiveSpecList := filterClusterSpecItem(actualSpecList, func(item mdb.ClusterSpecItem) bool { + return item.Members > 0 + }) + + // sort both actual and desired to match the effective and desired list version before comparing + sortClusterSpecList(desiredSpecList) + sortClusterSpecList(effectiveSpecList) + + needToRequeue := !clusterSpecListsEqual(effectiveSpecList, desiredSpecList) + if needToRequeue { + return r.updateStatus(ctx, &mrs, workflow.Pending("MongoDBMultiCluster deployment is not yet ready, requeuing reconciliation."), log) + } + + log.Infow("Finished reconciliation for MultiReplicaSet", "Spec", mrs.Spec, "Status", mrs.Status) + return r.updateStatus(ctx, &mrs, workflow.OK(), log, mdbstatus.NewPVCsStatusOptionEmptyStatus()) +} + +// publishAutomationConfigFirstMultiCluster returns a boolean indicating whether Ops Manager +// needs to be updated before the StatefulSets are created for this resource. +func (r *ReconcileMongoDbMultiReplicaSet) publishAutomationConfigFirstMultiCluster(ctx context.Context, mrs *mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger) (bool, error) { + if architectures.IsRunningStaticArchitecture(mrs.Annotations) { + if mrs.IsInChangeVersion() { + return true, nil + } + } + + scalingDown, err := isScalingDown(mrs) + if err != nil { + return false, xerrors.Errorf("failed determining if the resource is scaling down: %w", err) + } + + if scalingDown { + log.Infof("Scaling down in progress, updating Ops Manager state first.") + return true, nil + } + + firstStatefulSet, err := r.firstStatefulSet(ctx, mrs) + if err != nil { + if apiErrors.IsNotFound(err) { + // No need to publish state as this is a new StatefulSet + log.Debugf("New StatefulSet %s", firstStatefulSet.GetName()) + return false, nil + } + return false, err + } + + databaseContainer := container.GetByName(util.DatabaseContainerName, firstStatefulSet.Spec.Template.Spec.Containers) + volumeMounts := databaseContainer.VolumeMounts + if mrs.Spec.Security != nil { + if !mrs.Spec.Security.IsTLSEnabled() && statefulset.VolumeMountWithNameExists(volumeMounts, util.SecretVolumeName) { + log.Debug("About to set `security.tls.enabled` to false. automationConfig needs to be updated first") + return true, nil + } + + if mrs.Spec.Security.TLSConfig.CA == "" && statefulset.VolumeMountWithNameExists(volumeMounts, tls.ConfigMapVolumeCAName) { + log.Debug("About to set `security.tls.CA` to empty. automationConfig needs to be updated first") + return true, nil + } + } + + return false, nil +} + +// isScalingDown returns true if the MongoDBMultiCluster is attempting to scale down. +func isScalingDown(mrs *mdbmultiv1.MongoDBMultiCluster) (bool, error) { + desiredSpec := mrs.Spec.GetClusterSpecList() + + specThisReconciliation, err := mrs.GetClusterSpecItems() + if err != nil { + return false, err + } + + if len(desiredSpec) < len(specThisReconciliation) { + return true, nil + } + + for i := 0; i < len(specThisReconciliation); i++ { + specItem := desiredSpec[i] + reconciliationItem := specThisReconciliation[i] + + if specItem.Members < reconciliationItem.Members { + // when failover is happening, the clusterspec list will always have fewer members + // than the specs for the reconcile. + if _, ok := mdbmultiv1.HasClustersToFailOver(mrs.Annotations); ok { + return false, nil + } + return true, nil + } + + } + + return false, nil +} + +func (r *ReconcileMongoDbMultiReplicaSet) firstStatefulSet(ctx context.Context, mrs *mdbmultiv1.MongoDBMultiCluster) (appsv1.StatefulSet, error) { + // We want to get an existing statefulset, so we should fetch the client from "mrs.Spec.ClusterSpecList.ClusterSpecs" + // instead of mrs.GetClusterSpecItems(), since the later returns the effective clusterspecs, which might return + // clusters which have been removed and do not have a running statefulset. + items := mrs.Spec.ClusterSpecList + var firstMemberClient kubernetesClient.Client + var firstMemberIdx int + foundOne := false + for idx, item := range items { + client, ok := r.memberClusterClientsMap[item.ClusterName] + if ok { + firstMemberClient = client + firstMemberIdx = idx + foundOne = true + break + } + } + if !foundOne { + return appsv1.StatefulSet{}, xerrors.Errorf("was not able to find given member clusters in client map") + } + stsName := kube.ObjectKey(mrs.Namespace, mrs.MultiStatefulsetName(mrs.ClusterNum(items[firstMemberIdx].ClusterName))) + + firstStatefulSet, err := firstMemberClient.GetStatefulSet(ctx, stsName) + if err != nil { + if apiErrors.IsNotFound(err) { + return firstStatefulSet, err + } + return firstStatefulSet, xerrors.Errorf("error getting StatefulSet %s: %w", stsName, err) + } + return firstStatefulSet, err +} + +// reconcileMemberResources handles the synchronization of kubernetes resources, which can be statefulsets, services etc. +// All the resources required in the k8s cluster (as opposed to the automation config) for creating the replicaset +// should be reconciled in this method. +func (r *ReconcileMongoDbMultiReplicaSet) reconcileMemberResources(ctx context.Context, mrs *mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger, conn om.Connection, projectConfig mdb.ProjectConfig) workflow.Status { + err := r.reconcileServices(ctx, log, mrs) + if err != nil { + return workflow.Failed(err) + } + + // create configmap with the hostname-override + err = r.reconcileHostnameOverrideConfigMap(ctx, log, *mrs) + if err != nil { + return workflow.Failed(err) + } + + // Copy over OM CustomCA if specified in project config + if projectConfig.SSLMMSCAConfigMap != "" { + err = r.reconcileOMCAConfigMap(ctx, log, *mrs, projectConfig.SSLMMSCAConfigMap) + if err != nil { + return workflow.Failed(err) + } + } + // Ensure custom roles are created in OM + if status := ensureRoles(mrs.GetSecurity().Roles, conn, log); !status.IsOK() { + return status + } + + return r.reconcileStatefulSets(ctx, mrs, log, conn, projectConfig) +} + +type stsIdentifier struct { + namespace string + name string + client kubernetesClient.Client + clusterName string +} + +func (r *ReconcileMongoDbMultiReplicaSet) reconcileStatefulSets(ctx context.Context, mrs *mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger, conn om.Connection, projectConfig mdb.ProjectConfig) workflow.Status { + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return workflow.Failed(xerrors.Errorf("failed to read cluster spec list: %w", err)) + } + failedClusterNames, err := mrs.GetFailedClusterNames() + if err != nil { + log.Errorf("failed retrieving list of failed clusters: %s", err.Error()) + } + + var stsLocators []stsIdentifier + + for _, item := range clusterSpecList { + if stringutil.Contains(failedClusterNames, item.ClusterName) { + log.Warnf(fmt.Sprintf("failed to reconcile statefulset: cluster %s is marked as failed", item.ClusterName)) + continue + } + + memberClient, ok := r.memberClusterClientsMap[item.ClusterName] + if !ok { + log.Warnf(fmt.Sprintf("failed to reconcile statefulset: cluster %s missing from client map", item.ClusterName)) + continue + } + secretMemberClient := r.memberClusterSecretClientsMap[item.ClusterName] + replicasThisReconciliation, err := getMembersForClusterSpecItemThisReconciliation(mrs, item) + clusterNum := mrs.ClusterNum(item.ClusterName) + if err != nil { + return workflow.Failed(err) + } + + // Copy over the CA config map if it exists on the central cluster + caConfigMapName := mrs.Spec.Security.TLSConfig.CA + if caConfigMapName != "" { + cm, err := r.client.GetConfigMap(ctx, kube.ObjectKey(mrs.Namespace, caConfigMapName)) + if err != nil { + return workflow.Failed(xerrors.Errorf("Expected CA ConfigMap not found on central cluster: %s", caConfigMapName)) + } + + memberCm := configmap.Builder().SetName(caConfigMapName).SetNamespace(mrs.Namespace).SetData(cm.Data).Build() + err = configmap.CreateOrUpdate(ctx, memberClient, memberCm) + + if err != nil && !apiErrors.IsAlreadyExists(err) { + return workflow.Failed(xerrors.Errorf("Failed to sync CA ConfigMap in cluster: %s, err: %w", item.ClusterName, err)) + } + } + + // Ensure TLS for multi-cluster statefulset in each cluster + mrsConfig := certs.MultiReplicaSetConfig(*mrs, clusterNum, item.ClusterName, replicasThisReconciliation) + if status := certs.EnsureSSLCertsForStatefulSet(ctx, r.SecretClient, secretMemberClient, *mrs.Spec.Security, mrsConfig, log); !status.IsOK() { + return status + } + + automationConfig, err := conn.ReadAutomationConfig() + if err != nil { + return workflow.Failed(xerrors.Errorf("Failed to retrieve current automation config in cluster: %s, err: %w", item.ClusterName, err)) + } + + currentAgentAuthMode := automationConfig.GetAgentAuthMode() + + certConfigurator := certs.MongoDBMultiX509CertConfigurator{ + MongoDBMultiCluster: mrs, + ClusterNum: clusterNum, + Replicas: replicasThisReconciliation, + SecretReadClient: r.SecretClient, + SecretWriteClient: secretMemberClient, + } + if status := r.ensureX509SecretAndCheckTLSType(ctx, certConfigurator, currentAgentAuthMode, log); !status.IsOK() { + return status + } + + // copy the agent api key to the member cluster. + apiKeySecretName := agents.ApiKeySecretName(conn.GroupID()) + secretByte, err := secret.ReadByteData(ctx, r.client, types.NamespacedName{Name: apiKeySecretName, Namespace: mrs.Namespace}) + if err != nil { + return workflow.Failed(err) + } + + secretObject := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: apiKeySecretName, + Namespace: mrs.Namespace, + Labels: mongoDBMultiLabels(mrs.Name, mrs.Namespace), + }, + Data: secretByte, + } + + err = secret.CreateOrUpdate(ctx, memberClient, secretObject) + if err != nil { + return workflow.Failed(err) + } + + // get cert hash of tls secret if it exists + certHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, mrs.Namespace, mrsConfig.CertSecretName, "", log) + internalCertHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, mrs.Namespace, mrsConfig.InternalClusterSecretName, "", log) + log.Debugf("Creating StatefulSet %s with %d replicas in cluster: %s", mrs.MultiStatefulsetName(clusterNum), replicasThisReconciliation, item.ClusterName) + + stsOverride := appsv1.StatefulSetSpec{} + if item.StatefulSetConfiguration != nil { + stsOverride = item.StatefulSetConfiguration.SpecWrapper.Spec + } + + var automationAgentVersion string + if architectures.IsRunningStaticArchitecture(mrs.Annotations) { + if !mrs.Spec.IsAgentImageOverridden() { + automationAgentVersion, err = r.getAgentVersion(conn, conn.OpsManagerVersion().VersionString, false, log) + if err != nil { + log.Errorf("Impossible to get agent version, please override the agent image by providing a pod template") + status := workflow.Failed(xerrors.Errorf("Failed to get agent version: %w", err)) + return status + } + } + } + + opts := mconstruct.MultiClusterReplicaSetOptions( + mconstruct.WithClusterNum(clusterNum), + Replicas(replicasThisReconciliation), + mconstruct.WithStsOverride(&stsOverride), + mconstruct.WithAnnotations(mrs.Name, certHash), + mconstruct.WithServiceName(mrs.MultiHeadlessServiceName(clusterNum)), + PodEnvVars(newPodVars(conn, projectConfig, mrs.Spec.LogLevel)), + CurrentAgentAuthMechanism(currentAgentAuthMode), + CertificateHash(certHash), + InternalClusterHash(internalCertHash), + WithLabels(mongoDBMultiLabels(mrs.Name, mrs.Namespace)), + WithAdditionalMongodConfig(mrs.Spec.GetAdditionalMongodConfig()), + WithInitDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.InitDatabaseImageUrlEnv, r.initDatabaseNonStaticImageVersion)), + WithDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.NonStaticDatabaseEnterpriseImage, r.databaseNonStaticImageVersion)), + WithAgentImage(images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, automationAgentVersion)), + WithMongodbImage(images.GetOfficialImage(r.imageUrls, mrs.Spec.Version, mrs.GetAnnotations())), + ) + + sts := mconstruct.MultiClusterStatefulSet(*mrs, opts) + deleteSts, err := shouldDeleteStatefulSet(*mrs, item) + if err != nil { + return workflow.Failed(xerrors.Errorf("failed to create StatefulSet in cluster: %s, err: %w", item.ClusterName, err)) + } + + if deleteSts { + if err := memberClient.Delete(ctx, &sts); err != nil && !apiErrors.IsNotFound(err) { + return workflow.Failed(xerrors.Errorf("failed to delete StatefulSet in cluster: %s, err: %w", item.ClusterName, err)) + } + continue + } + + workflowStatus := create.HandlePVCResize(ctx, memberClient, &sts, log) + if !workflowStatus.IsOK() { + return workflowStatus + } + if err = r.updateStatusFromInnerMethod(ctx, mrs, log, workflowStatus); err != nil { + return workflow.Failed(xerrors.Errorf("%w", err)) + } + + _, err = enterprisests.CreateOrUpdateStatefulset(ctx, memberClient, mrs.Namespace, log, &sts) + if err != nil { + return workflow.Failed(xerrors.Errorf("failed to create/update StatefulSet in cluster: %s, err: %w", item.ClusterName, err)) + } + + processes := automationConfig.Deployment.GetAllProcessNames() + // If we don't have processes defined yet, that means we are in the first deployment, and we can deploy all + // stateful-sets in parallel. + // If we have processes defined, it means we want to wait until all of them are ready. + if len(processes) > 0 { + // We already have processes defined, and therefore we are waiting for each of them + if status := getStatefulSetStatus(ctx, sts.Namespace, sts.Name, memberClient); !status.IsOK() { + return status + } + + log.Infof("Successfully ensured StatefulSet in cluster: %s", item.ClusterName) + } else { + // We create all sts in parallel and wait below for all of them to finish + stsLocators = append(stsLocators, stsIdentifier{ + namespace: sts.Namespace, + name: sts.Name, + client: memberClient, + clusterName: item.ClusterName, + }) + } + } + + // Running into this means we are in the first deployment/don't have processes yet. + // That means we have created them in parallel and now waiting for them to get ready. + for _, locator := range stsLocators { + if status := getStatefulSetStatus(ctx, locator.namespace, locator.name, locator.client); !status.IsOK() { + return status + } + log.Infof("Successfully ensured StatefulSet in cluster: %s", locator.clusterName) + } + + return workflow.OK() +} + +// updateStatusFromInnerMethod ensures to only update the status if it has been updated. +// Since spec.Mapping is just a cache, it would be replaced; therefore, we need to cache it +func (r *ReconcileMongoDbMultiReplicaSet) updateStatusFromInnerMethod(ctx context.Context, mrs *mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger, workflowStatus workflow.Status) error { + // if there are no pvc changes, then we don't need to update the status + if !workflow.ContainsPVCOption(workflowStatus.StatusOptions()) { + return nil + } + tmpMapping := mrs.Spec.Mapping + _, err := r.updateStatus(ctx, mrs, workflow.Pending(""), log, workflowStatus.StatusOptions()...) + mrs.Spec.Mapping = tmpMapping + if err != nil { + return err + } + return nil +} + +// shouldDeleteStatefulSet returns a boolean value indicating whether the StatefulSet associated with +// the given cluster spec item should be deleted or not. +func shouldDeleteStatefulSet(mrs mdbmultiv1.MongoDBMultiCluster, item mdb.ClusterSpecItem) (bool, error) { + for _, specItem := range mrs.Spec.ClusterSpecList { + if item.ClusterName == specItem.ClusterName { + // this spec value has been explicitly defined, don't delete it. + return false, nil + } + } + + items, err := mrs.GetClusterSpecItems() + if err != nil { + return false, err + } + + for _, specItem := range items { + if item.ClusterName == specItem.ClusterName { + // we delete only if we have fully scaled down and are at 0 members + return specItem.Members == 0, nil + } + } + + // we are in the process of scaling down to 0, and should not yet delete the statefulset + return false, nil +} + +// getMembersForClusterSpecItemThisReconciliation returns the value members should have for a given cluster spec item +// for a given reconciliation. This value should increment or decrement in one cluster by one member each reconciliation +// when a scaling operation is taking place. +func getMembersForClusterSpecItemThisReconciliation(mrs *mdbmultiv1.MongoDBMultiCluster, item mdb.ClusterSpecItem) (int, error) { + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return -1, err + } + for _, clusterItem := range clusterSpecList { + if clusterItem.ClusterName == item.ClusterName { + return clusterItem.Members, nil + } + } + return -1, xerrors.Errorf("did not find %s in cluster spec list", item.ClusterName) +} + +// saveLastAchievedSpec updates the MongoDBMultiCluster resource with the spec that was just achieved. +func (r *ReconcileMongoDbMultiReplicaSet) saveLastAchievedSpec(ctx context.Context, mrs mdbmultiv1.MongoDBMultiCluster) error { + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + return err + } + + lastAchievedSpec := mrs.Spec + lastAchievedSpec.ClusterSpecList = clusterSpecs + achievedSpecBytes, err := json.Marshal(lastAchievedSpec) + if err != nil { + return err + } + + if mrs.Annotations == nil { + mrs.Annotations = map[string]string{} + } + + // TODO Find a way to avoid using the spec for this field as we're not writing the information + // back in the resource and the user does not set it. + clusterNumBytes, err := json.Marshal(mrs.Spec.Mapping) + if err != nil { + return err + } + annotationsToAdd := make(map[string]string) + + annotationsToAdd[util.LastAchievedSpec] = string(achievedSpecBytes) + if string(clusterNumBytes) != "null" { + annotationsToAdd[mdbmultiv1.LastClusterNumMapping] = string(clusterNumBytes) + } + + return annotations.SetAnnotations(ctx, &mrs, annotationsToAdd, r.client) +} + +// updateOmDeploymentRs performs OM registration operation for the replicaset. So the changes will be finally propagated +// to automation agents in containers +func (r *ReconcileMongoDbMultiReplicaSet) updateOmDeploymentRs(ctx context.Context, conn om.Connection, mrs mdbmultiv1.MongoDBMultiCluster, isRecovering bool, log *zap.SugaredLogger) error { + reachableHostnames := make([]string, 0) + + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return err + } + failedClusterNames, err := mrs.GetFailedClusterNames() + if err != nil { + // When failing to retrieve the list of failed clusters we proceed assuming there are no failed clusters, + // but log the error as it indicates a malformed annotation. + log.Errorf("failed retrieving list of failed clusters: %s", err.Error()) + } + for _, spec := range clusterSpecList { + hostnamesToAdd := dns.GetMultiClusterProcessHostnames(mrs.Name, mrs.Namespace, mrs.ClusterNum(spec.ClusterName), spec.Members, mrs.Spec.GetClusterDomain(), mrs.Spec.GetExternalDomainForMemberCluster(spec.ClusterName)) + if stringutil.Contains(failedClusterNames, spec.ClusterName) { + log.Debugf("Skipping hostnames %+v as they are part of the failed cluster %s ", hostnamesToAdd, spec.ClusterName) + continue + } + if mrs.GetClusterSpecByName(spec.ClusterName) == nil { + log.Debugf("Skipping hostnames %+v as they are part of a cluster not known by the operator %s ", hostnamesToAdd, spec.ClusterName) + continue + } + reachableHostnames = append(reachableHostnames, hostnamesToAdd...) + } + + err = agents.WaitForRsAgentsToRegisterSpecifiedHostnames(conn, reachableHostnames, log) + if err != nil && !isRecovering { + return err + } + + existingDeployment, err := conn.ReadDeployment() + if err != nil { + return err + } + + processIds := getReplicaSetProcessIdsFromReplicaSets(mrs.Name, existingDeployment) + log.Debugf("Existing process Ids: %+v", processIds) + + certificateFileName := "" + internalClusterPath := "" + + // If tls is enabled we need to configure the "processes" array in opsManager/Cloud Manager with the + // correct certFilePath, with the new tls design, this path has the certHash in it(so that cert can be rotated + // without pod restart), we can get the cert hash from any of the statefulset, here we pick the statefulset in the first cluster. + if mrs.Spec.Security.IsTLSEnabled() { + firstStatefulSet, err := r.firstStatefulSet(ctx, &mrs) + if err != nil { + return err + } + + if hash := firstStatefulSet.Annotations[util.InternalCertAnnotationKey]; hash != "" { + internalClusterPath = fmt.Sprintf("%s%s", util.InternalClusterAuthMountPath, hash) + } + + if certificateHash := firstStatefulSet.Annotations[certs.CertHashAnnotationKey]; certificateHash != "" { + certificateFileName = fmt.Sprintf("%s/%s", util.TLSCertMountPath, certificateHash) + } + } + + processes, err := process.CreateMongodProcessesWithLimitMulti(r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, mrs, certificateFileName) + if err != nil && !isRecovering { + return err + } + + if len(processes) != len(mrs.Spec.GetMemberOptions()) { + log.Warnf("the number of member options is different than the number of mongod processes to be created: %d processes - %d replica set member options", len(processes), len(mrs.Spec.GetMemberOptions())) + } + rs := om.NewMultiClusterReplicaSetWithProcesses(om.NewReplicaSet(mrs.Name, mrs.Spec.Version), processes, mrs.Spec.GetMemberOptions(), processIds, mrs.Spec.Connectivity) + + caFilePath := fmt.Sprintf("%s/ca-pem", util.TLSCaMountPath) + + // We do not provide an agentCertSecretName on purpose because then we will default to the non pem secret on the central cluster. + // Below method has special code handling reading certificates from the central cluster in that case. + status, additionalReconciliationRequired := r.updateOmAuthentication(ctx, conn, rs.GetProcessNames(), &mrs, "", caFilePath, internalClusterPath, isRecovering, log) + if !status.IsOK() && !isRecovering { + return xerrors.Errorf("failed to enable Authentication for MongoDB Multi Replicaset") + } + + lastMongodbConfig := mrs.GetLastAdditionalMongodConfig() + + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + return ReconcileReplicaSetAC(ctx, d, mrs.Spec.DbCommonSpec, lastMongodbConfig, mrs.Name, rs, caFilePath, internalClusterPath, nil, log) + }, + log, + ) + if err != nil && !isRecovering { + return err + } + + reconcileResult, err := ReconcileLogRotateSetting(conn, mrs.Spec.Agent, log) + if !reconcileResult.IsOK() { + return xerrors.Errorf("failed to configure logrotation for MongoDBMultiCluster RS, err: %w", err) + } + + if additionalReconciliationRequired { + return xerrors.Errorf("failed to complete reconciliation") + } + + status = r.ensureBackupConfigurationAndUpdateStatus(ctx, conn, &mrs, r.SecretClient, log) + if !status.IsOK() && !isRecovering { + return xerrors.Errorf("failed to configure backup for MongoDBMultiCluster RS") + } + + reachableProcessNames := make([]string, 0) + for _, proc := range rs.Processes { + if stringutil.Contains(reachableHostnames, proc.HostName()) { + reachableProcessNames = append(reachableProcessNames, proc.Name()) + } + } + if err := om.WaitForReadyState(conn, reachableProcessNames, isRecovering, log); err != nil && !isRecovering { + return err + } + return nil +} + +func getReplicaSetProcessIdsFromReplicaSets(replicaSetName string, deployment om.Deployment) map[string]int { + processIds := map[string]int{} + + replicaSet := deployment.GetReplicaSetByName(replicaSetName) + if replicaSet == nil { + return map[string]int{} + } + + for _, m := range replicaSet.Members() { + processIds[m.Name()] = m.Id() + } + + return processIds +} + +func mongoDBMultiLabels(name, namespace string) map[string]string { + return map[string]string{ + "controller": "mongodb-enterprise-operator", + "mongodbmulticluster": fmt.Sprintf("%s-%s", namespace, name), + } +} + +func getSRVService(mrs *mdbmultiv1.MongoDBMultiCluster) corev1.Service { + svcLabels := mongoDBMultiLabels(mrs.Name, mrs.Namespace) + + additionalConfig := mrs.Spec.GetAdditionalMongodConfig() + port := additionalConfig.GetPortOrDefault() + + svc := service.Builder(). + SetName(fmt.Sprintf("%s-svc", mrs.Name)). + SetNamespace(mrs.Namespace). + SetSelector(mconstruct.PodLabel(mrs.Name)). + SetLabels(svcLabels). + SetPublishNotReadyAddresses(true). + AddPort(&corev1.ServicePort{Port: port, Name: "mongodb"}). + AddPort(&corev1.ServicePort{Port: create.GetNonEphemeralBackupPort(port), Name: "backup", TargetPort: intstr.IntOrString{IntVal: create.GetNonEphemeralBackupPort(port)}}). + Build() + + return svc +} + +func getExternalService(mrs *mdbmultiv1.MongoDBMultiCluster, clusterName string, podNum int) corev1.Service { + clusterNum := mrs.ClusterNum(clusterName) + + svc := getService(mrs, clusterName, podNum) + svc.Name = dns.GetMultiExternalServiceName(mrs.GetName(), clusterNum, podNum) + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + + externalAccessConfig := mrs.Spec.GetExternalAccessConfigurationForMemberCluster(clusterName) + if externalAccessConfig != nil { + // first we override with the Service spec from the root and then from a specific cluster. + if mrs.Spec.ExternalAccessConfiguration != nil { + globalOverrideSpecWrapper := mrs.Spec.ExternalAccessConfiguration.ExternalService.SpecWrapper + if globalOverrideSpecWrapper != nil { + svc.Spec = merge.ServiceSpec(svc.Spec, globalOverrideSpecWrapper.Spec) + } + } + clusterLevelOverrideSpec := externalAccessConfig.ExternalService.SpecWrapper + additionalAnnotations := externalAccessConfig.ExternalService.Annotations + if clusterLevelOverrideSpec != nil { + svc.Spec = merge.ServiceSpec(svc.Spec, clusterLevelOverrideSpec.Spec) + } + svc.Annotations = merge.StringToStringMap(svc.Annotations, additionalAnnotations) + } + + return svc +} + +func getService(mrs *mdbmultiv1.MongoDBMultiCluster, clusterName string, podNum int) corev1.Service { + svcLabels := map[string]string{ + "statefulset.kubernetes.io/pod-name": dns.GetMultiPodName(mrs.Name, mrs.ClusterNum(clusterName), podNum), + "controller": "mongodb-enterprise-operator", + "mongodbmulticluster": fmt.Sprintf("%s-%s", mrs.Namespace, mrs.Name), + } + + labelSelectors := map[string]string{ + "statefulset.kubernetes.io/pod-name": dns.GetMultiPodName(mrs.Name, mrs.ClusterNum(clusterName), podNum), + "controller": "mongodb-enterprise-operator", + } + + additionalConfig := mrs.Spec.GetAdditionalMongodConfig() + port := additionalConfig.GetPortOrDefault() + + svc := service.Builder(). + SetName(dns.GetMultiServiceName(mrs.Name, mrs.ClusterNum(clusterName), podNum)). + SetNamespace(mrs.Namespace). + SetSelector(labelSelectors). + SetLabels(svcLabels). + SetPublishNotReadyAddresses(true). + AddPort(&corev1.ServicePort{Port: port, Name: "mongodb"}). + // Note: in the agent-launcher.sh We explicitly pass an offset of 1. When port N is exposed + // the agent would use port N+1 for the spinning up of the ephemeral mongod process, which is used for backup + AddPort(&corev1.ServicePort{Port: create.GetNonEphemeralBackupPort(port), Name: "backup", TargetPort: intstr.IntOrString{IntVal: create.GetNonEphemeralBackupPort(port)}}). + Build() + + return svc +} + +// reconcileServices makes sure that we have a service object corresponding to each statefulset pod +// in the member clusters +func (r *ReconcileMongoDbMultiReplicaSet) reconcileServices(ctx context.Context, log *zap.SugaredLogger, mrs *mdbmultiv1.MongoDBMultiCluster) error { + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return err + } + failedClusterNames, err := mrs.GetFailedClusterNames() + if err != nil { + log.Errorf("failed retrieving list of failed clusters: %s", err.Error()) + } + + for _, e := range clusterSpecList { + if stringutil.Contains(failedClusterNames, e.ClusterName) { + log.Warnf(fmt.Sprintf("cluster %s is marked as failed", e.ClusterName)) + continue + } + + client, ok := r.memberClusterClientsMap[e.ClusterName] + if !ok { + log.Warnf(fmt.Sprintf("cluster %s missing from client map", e.ClusterName)) + continue + } + if e.Members == 0 { + log.Warnf("skipping services creation: no members assigned to cluster %s", e.ClusterName) + continue + } + + // ensure SRV service + srvService := getSRVService(mrs) + if err := ensureSRVService(ctx, client, srvService, e.ClusterName); err != nil { + return err + } + log.Infof("Successfully created SRV service %s in cluster %s", srvService.Name, e.ClusterName) + + // ensure Headless service + headlessServiceName := mrs.MultiHeadlessServiceName(mrs.ClusterNum(e.ClusterName)) + nameSpacedName := kube.ObjectKey(mrs.Namespace, headlessServiceName) + headlessService := create.BuildService(nameSpacedName, nil, ptr.To(headlessServiceName), nil, mrs.Spec.AdditionalMongodConfig.GetPortOrDefault(), omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeClusterIP}) + if err := ensureHeadlessService(ctx, client, headlessService, e.ClusterName); err != nil { + return err + } + log.Infof("Successfully created headless service %s in cluster: %s", headlessServiceName, e.ClusterName) + } + + // by default, we would create the duplicate services + shouldCreateDuplicates := mrs.Spec.DuplicateServiceObjects == nil || *mrs.Spec.DuplicateServiceObjects + for memberClusterName, memberClusterClient := range r.memberClusterClientsMap { + if stringutil.Contains(failedClusterNames, memberClusterName) { + log.Warnf(fmt.Sprintf("cluster %s is marked as failed, skipping creation of services", memberClusterName)) + continue + } + + for _, clusterSpecItem := range clusterSpecList { + if !shouldCreateDuplicates && clusterSpecItem.ClusterName != memberClusterName { + // skip creating of other cluster's services (duplicates) in the current cluster + continue + } + + if err := ensureServices(ctx, memberClusterClient, memberClusterName, mrs, clusterSpecItem, log); err != nil { + return err + } + } + } + + return nil +} + +func ensureSRVService(ctx context.Context, client service.GetUpdateCreator, svc corev1.Service, clusterName string) error { + err := mekoService.CreateOrUpdateService(ctx, client, svc) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create SRV service: % in cluster: %s, err: %w", svc.Name, clusterName, err) + } + return nil +} + +// ensureServices creates pod services and/or external services. +// If externalAccess is defined (at spec or clusterSpecItem level) then we always create an external service. +// If externalDomain is defined then we DO NOT create pod services (service created for each pod selecting only 1 pod). +// When there are external domains used, we don't use internal pod-service FQDNs as hostnames at all, +// so there is no point in creating pod services. +// But when external domains are not used, then mongod process hostnames use pod service FQDN, and +// at the same time user might want to expose externally using external services. +func ensureServices(ctx context.Context, client service.GetUpdateCreator, clientClusterName string, m *mdbmultiv1.MongoDBMultiCluster, clusterSpecItem mdb.ClusterSpecItem, log *zap.SugaredLogger) error { + for podNum := 0; podNum < clusterSpecItem.Members; podNum++ { + var svc corev1.Service + if m.Spec.GetExternalAccessConfigurationForMemberCluster(clusterSpecItem.ClusterName) != nil { + svc = getExternalService(m, clusterSpecItem.ClusterName, podNum) + externalDomain := m.Spec.GetExternalDomainForMemberCluster(clusterSpecItem.ClusterName) + placeholderReplacer := create.GetMultiClusterMongoDBPlaceholderReplacer(m.Name, m.Name, m.Namespace, clusterSpecItem.ClusterName, m.ClusterNum(clusterSpecItem.ClusterName), externalDomain, m.Spec.ClusterDomain, podNum) + if processedAnnotations, replacedFlag, err := placeholderReplacer.ProcessMap(svc.Annotations); err != nil { + return xerrors.Errorf("failed to process annotations in external service %s in cluster %s: %w", svc.Name, clientClusterName, err) + } else if replacedFlag { + log.Debugf("Replaced placeholders in annotations in external service %s in cluster: %s. Annotations before: %+v, annotations after: %+v", svc.Name, clientClusterName, svc.Annotations, processedAnnotations) + svc.Annotations = processedAnnotations + } + + err := mekoService.CreateOrUpdateService(ctx, client, svc) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, clientClusterName, err) + } + } + + // we create regular pod-services only if we don't use external domains + if m.Spec.GetExternalDomainForMemberCluster(clusterSpecItem.ClusterName) == nil { + svc = getService(m, clusterSpecItem.ClusterName, podNum) + err := mekoService.CreateOrUpdateService(ctx, client, svc) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create pod service %s in cluster: %s, err: %w", svc.Name, clientClusterName, err) + } + } + } + return nil +} + +func ensureHeadlessService(ctx context.Context, client service.GetUpdateCreator, svc corev1.Service, clusterName string) error { + err := mekoService.CreateOrUpdateService(ctx, client, svc) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create headless service: %s in cluster: %s, err: %w", svc.Name, clusterName, err) + } + return nil +} + +func getHostnameOverrideConfigMap(mrs mdbmultiv1.MongoDBMultiCluster, clusterNum int, clusterName string, members int) corev1.ConfigMap { + data := make(map[string]string) + + externalDomain := mrs.Spec.GetExternalDomainForMemberCluster(clusterName) + for podNum := 0; podNum < members; podNum++ { + key := dns.GetMultiPodName(mrs.Name, clusterNum, podNum) + data[key] = dns.GetMultiClusterPodServiceFQDN(mrs.Name, mrs.Namespace, clusterNum, externalDomain, podNum, mrs.Spec.GetClusterDomain()) + } + + cm := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-hostname-override", mrs.Name), + Namespace: mrs.Namespace, + Labels: mongoDBMultiLabels(mrs.Name, mrs.Namespace), + }, + Data: data, + } + return cm +} + +func (r *ReconcileMongoDbMultiReplicaSet) reconcileHostnameOverrideConfigMap(ctx context.Context, log *zap.SugaredLogger, mrs mdbmultiv1.MongoDBMultiCluster) error { + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return err + } + failedClusterNames, err := mrs.GetFailedClusterNames() + if err != nil { + log.Warnf("failed retrieving list of failed clusters: %s", err.Error()) + } + + for i, e := range clusterSpecList { + if stringutil.Contains(failedClusterNames, e.ClusterName) { + log.Warnf(fmt.Sprintf("failed to create configmap: cluster %s is marked as failed", e.ClusterName)) + continue + } + + client, ok := r.memberClusterClientsMap[e.ClusterName] + if !ok { + log.Warnf(fmt.Sprintf("failed to create configmap: cluster %s is missing from client map", e.ClusterName)) + continue + } + cm := getHostnameOverrideConfigMap(mrs, i, e.ClusterName, e.Members) + + err = configmap.CreateOrUpdate(ctx, client, cm) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create configmap: %s in cluster: %s, err: %w", cm.Name, e.ClusterName, err) + } + log.Infof("Successfully ensured configmap: %s in cluster: %s", cm.Name, e.ClusterName) + + } + return nil +} + +func (r *ReconcileMongoDbMultiReplicaSet) reconcileOMCAConfigMap(ctx context.Context, log *zap.SugaredLogger, mrs mdbmultiv1.MongoDBMultiCluster, configMapName string) error { + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return err + } + failedClusterNames, err := mrs.GetFailedClusterNames() + if err != nil { + log.Warnf("failed retrieving list of failed clusters: %s", err.Error()) + } + + cm, err := r.client.GetConfigMap(ctx, kube.ObjectKey(mrs.Namespace, configMapName)) + if err != nil { + return err + } + for _, clusterSpecItem := range clusterSpecList { + if stringutil.Contains(failedClusterNames, clusterSpecItem.ClusterName) { + log.Warnf("failed to create configmap %s: cluster %s is marked as failed", configMapName, clusterSpecItem.ClusterName) + continue + } + client := r.memberClusterClientsMap[clusterSpecItem.ClusterName] + memberCm := configmap.Builder().SetName(configMapName).SetNamespace(mrs.Namespace).SetData(cm.Data).Build() + err := configmap.CreateOrUpdate(ctx, client, memberCm) + if err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create configmap: %s in cluster %s, err: %w", cm.Name, clusterSpecItem.ClusterName, err) + } + log.Infof("Sucessfully ensured configmap: %s in cluster: %s", cm.Name, clusterSpecItem.ClusterName) + } + return nil +} + +// AddMultiReplicaSetController creates a new MongoDbMultiReplicaset Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func AddMultiReplicaSetController(ctx context.Context, mgr manager.Manager, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, memberClustersMap map[string]cluster.Cluster) error { + // Create a new controller + reconciler := newMultiClusterReplicaSetReconciler(ctx, mgr.GetClient(), imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, om.NewOpsManagerConnection, multicluster.ClustersMapToClientMap(memberClustersMap)) + c, err := controller.New(util.MongoDbMultiClusterController, mgr, controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}) // nolint:forbidigo + if err != nil { + return err + } + + eventHandler := ResourceEventHandler{deleter: reconciler} + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &mdbmultiv1.MongoDBMultiCluster{}, &eventHandler, predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldResource := e.ObjectOld.(*mdbmultiv1.MongoDBMultiCluster) + newResource := e.ObjectNew.(*mdbmultiv1.MongoDBMultiCluster) + + oldSpecAnnotation := oldResource.GetAnnotations()[util.LastAchievedSpec] + newSpecAnnotation := newResource.GetAnnotations()[util.LastAchievedSpec] + + // don't handle an update to just the previous spec annotation if they are not the same. + // this prevents the operator triggering reconciliations on resource that it is updating itself. + if !reflect.DeepEqual(oldSpecAnnotation, newSpecAnnotation) { + return false + } + + return reflect.DeepEqual(oldResource.GetStatus(), newResource.GetStatus()) + }, + })) + if err != nil { + return err + } + + err = c.Watch(source.Channel[client.Object](OmUpdateChannel, &handler.EnqueueRequestForObject{})) + if err != nil { + return xerrors.Errorf("not able to setup OmUpdateChannel to listent to update events from OM: %s", err) + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + // register watcher across member clusters + for k, v := range memberClustersMap { + err := c.Watch(source.Kind[client.Object](v.GetCache(), &appsv1.StatefulSet{}, &khandler.EnqueueRequestForOwnerMultiCluster{}, watch.PredicatesForMultiStatefulSet())) + if err != nil { + return xerrors.Errorf("failed to set Watch on member cluster: %s, err: %w", k, err) + } + } + + // the operator watches the member clusters' API servers to determine whether the clusters are healthy or not + eventChannel := make(chan event.GenericEvent) + memberClusterHealthChecker := memberwatch.MemberClusterHealthChecker{Cache: make(map[string]*memberwatch.MemberHeathCheck)} + go memberClusterHealthChecker.WatchMemberClusterHealth(ctx, zap.S(), eventChannel, reconciler.client, memberClustersMap) + + err = c.Watch(source.Channel[client.Object](eventChannel, &handler.EnqueueRequestForObject{})) + if err != nil { + zap.S().Errorf("failed to watch for member cluster healthcheck: %s", err) + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, + watch.ConfigMapEventHandler{ + ConfigMapName: util.MemberListConfigMapName, + ConfigMapNamespace: env.ReadOrPanic(util.CurrentNamespace), // nolint:forbidigo + }, + predicate.ResourceVersionChangedPredicate{}, + )) + if err != nil { + return err + } + + zap.S().Infof("Registered controller %s", util.MongoDbMultiReplicaSetController) + return err +} + +// OnDelete cleans up Ops Manager state and all Kubernetes resources associated with this instance. +func (r *ReconcileMongoDbMultiReplicaSet) OnDelete(ctx context.Context, obj runtime.Object, log *zap.SugaredLogger) error { + mrs := obj.(*mdbmultiv1.MongoDBMultiCluster) + return r.deleteManagedResources(ctx, *mrs, log) +} + +// cleanOpsManagerState removes the project configuration (processes, auth settings etc.) from the corresponding OM project. +func (r *ReconcileMongoDbMultiReplicaSet) cleanOpsManagerState(ctx context.Context, mrs mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger) error { + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, &mrs, log) + if err != nil { + return err + } + + log.Infow("Removing replica set from Ops Manager", "config", mrs.Spec) + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, mrs.Namespace, log) + if err != nil { + return err + } + + processNames := make([]string, 0) + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + processNames = d.GetProcessNames(om.ReplicaSet{}, mrs.Name) + // error means that replica set is not in the deployment - it's ok, and we can proceed (could happen if + // deletion cleanup happened twice and the first one cleaned OM state already) + if e := d.RemoveReplicaSetByName(mrs.Name, log); e != nil { + log.Warnf("Failed to remove replica set from automation config: %s", e) + } + + return nil + }, + log, + ) + if err != nil { + return err + } + + hostsToRemove, err := mrs.GetMultiClusterAgentHostnames() + if err != nil { + return err + } + log.Infow("Stop monitoring removed hosts in Ops Manager", "removedHosts", hostsToRemove) + + if err = host.StopMonitoring(conn, hostsToRemove, log); err != nil { + return err + } + + opts := authentication.Options{ + AuthoritativeSet: false, + ProcessNames: processNames, + } + + if err := authentication.Disable(conn, opts, true, log); err != nil { + return err + } + log.Infof("Removed deployment %s from Ops Manager at %s", mrs.Name, conn.BaseURL()) + return nil +} + +// deleteManagedResources deletes resources across all member clusters that are owned by this MongoDBMultiCluster resource. +func (r *ReconcileMongoDbMultiReplicaSet) deleteManagedResources(ctx context.Context, mrs mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger) error { + var errs error + if err := r.cleanOpsManagerState(ctx, mrs, log); err != nil { + errs = multierror.Append(errs, err) + } + + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + return err + } + + for _, item := range clusterSpecList { + c := r.memberClusterClientsMap[item.ClusterName] + if err := r.deleteClusterResources(ctx, c, mrs, log); err != nil { + errs = multierror.Append(errs, xerrors.Errorf("failed deleting dependant resources in cluster %s: %w", item.ClusterName, err)) + } + } + + return errs +} + +// deleteClusterResources removes all resources that are associated with the given MongoDBMultiCluster resource in a given cluster. +func (r *ReconcileMongoDbMultiReplicaSet) deleteClusterResources(ctx context.Context, c kubernetesClient.Client, mrs mdbmultiv1.MongoDBMultiCluster, log *zap.SugaredLogger) error { + var errs error + + // cleanup resources in the namespace as the MongoDBMultiCluster with the corresponding label. + cleanupOptions := mdb.MongodbCleanUpOptions{ + Namespace: mrs.Namespace, + Labels: mongoDBMultiLabels(mrs.Name, mrs.Namespace), + } + + if err := c.DeleteAllOf(ctx, &corev1.Service{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed Serivces associated with %s/%s", mrs.Namespace, mrs.Name) + } + + if err := c.DeleteAllOf(ctx, &appsv1.StatefulSet{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed StatefulSets associated with %s/%s", mrs.Namespace, mrs.Name) + } + + if err := c.DeleteAllOf(ctx, &corev1.ConfigMap{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed ConfigMaps associated with %s/%s", mrs.Namespace, mrs.Name) + } + + if err := c.DeleteAllOf(ctx, &corev1.Secret{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed Secrets associated with %s/%s", mrs.Namespace, mrs.Name) + } + + r.resourceWatcher.RemoveDependentWatchedResources(kube.ObjectKey(mrs.Namespace, mrs.Name)) + + return errs +} + +// filterClusterSpecItem filters items out of a list based on provided predicate. +func filterClusterSpecItem(items mdb.ClusterSpecList, fn func(item mdb.ClusterSpecItem) bool) mdb.ClusterSpecList { + var result mdb.ClusterSpecList + for _, item := range items { + if fn(item) { + result = append(result, item) + } + } + return result +} + +func sortClusterSpecList(clusterSpecList mdb.ClusterSpecList) { + sort.SliceStable(clusterSpecList, func(i, j int) bool { + return clusterSpecList[i].ClusterName < clusterSpecList[j].ClusterName + }) +} + +func clusterSpecListsEqual(effective, desired mdb.ClusterSpecList) bool { + comparer := cmp.Comparer(func(x, y automationconfig.MemberOptions) bool { + return true + }) + return cmp.Equal(effective, desired, comparer) +} diff --git a/controllers/operator/mongodbmultireplicaset_controller_test.go b/controllers/operator/mongodbmultireplicaset_controller_test.go new file mode 100644 index 000000000..4a3009b3c --- /dev/null +++ b/controllers/operator/mongodbmultireplicaset_controller_test.go @@ -0,0 +1,1475 @@ +package operator + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/status/pvc" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/pkg/agentVersionManagement" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster/failedcluster" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster/memberwatch" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +var clusters = []string{"api1.kube.com", "api2.kube.com", "api3.kube.com"} + +func checkMultiReconcileSuccessful(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, m *mdbmulti.MongoDBMultiCluster, client client.Client, shouldRequeue bool) { + err := client.Update(ctx, m) + assert.NoError(t, err) + + result, e := reconciler.Reconcile(ctx, requestFromObject(m)) + assert.NoError(t, e) + if shouldRequeue { + assert.True(t, result.Requeue || result.RequeueAfter > 0) + } else { + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, result) + } + + // fetch the last updates as the reconciliation loop can update the mdb resource. + err = client.Get(ctx, kube.ObjectKey(m.Namespace, m.Name), m) + assert.NoError(t, err) +} + +func TestChangingFCVMultiCluster(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + reconciler, cl, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, cl, false) + + // Helper function to update and verify FCV + verifyFCV := func(version, expectedFCV string, fcvOverride *string, t *testing.T) { + if fcvOverride != nil { + mrs.Spec.FeatureCompatibilityVersion = fcvOverride + } + + mrs.Spec.Version = version + _ = cl.Update(ctx, mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, cl, false) + assert.Equal(t, expectedFCV, mrs.Status.FeatureCompatibilityVersion) + } + + testFCVsCases(t, verifyFCV) +} + +func TestCreateMultiReplicaSet(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) +} + +func TestMultiReplicaSetClusterReconcileContainerImages(t *testing.T) { + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_1_0_0", util.NonStaticDatabaseEnterpriseImage) + initDatabaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_2_0_0", util.InitDatabaseImageUrlEnv) + + imageUrlsMock := images.ImageUrls{ + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", + initDatabaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", + } + + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).SetVersion("8.0.0").Build() + reconciler, kubeClient, memberClients, _ := defaultMultiReplicaSetReconciler(ctx, imageUrlsMock, "2.0.0", "1.0.0", mrs) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, kubeClient, false) + + clusterSpecs, err := mrs.GetClusterSpecItems() + require.NoError(t, err) + for _, item := range clusterSpecs { + c := memberClients[item.ClusterName] + + t.Run(item.ClusterName, func(t *testing.T) { + sts := appsv1.StatefulSet{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName))), &sts) + require.NoError(t, err) + + require.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", sts.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[0].Image) + }) + } +} + +func TestMultiReplicaSetClusterReconcileContainerImagesWithStaticArchitecture(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0_ubi9", mcoConstruct.MongodbImageEnv) + + imageUrlsMock := images.ImageUrls{ + architectures.MdbAgentImageRepo: "quay.io/mongodb/mongodb-agent-ubi", + mcoConstruct.MongodbImageEnv: "quay.io/mongodb/mongodb-enterprise-server", + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", + } + + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).SetVersion("8.0.0").Build() + reconciler, kubeClient, memberClients, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, imageUrlsMock, "", "", mrs) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + connection.(*om.MockedOmConnection).SetAgentVersion("12.0.30.7791-1", "") + }) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, kubeClient, false) + + clusterSpecs, err := mrs.GetClusterSpecItems() + require.NoError(t, err) + for _, item := range clusterSpecs { + c := memberClients[item.ClusterName] + + t.Run(item.ClusterName, func(t *testing.T) { + sts := appsv1.StatefulSet{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName))), &sts) + require.NoError(t, err) + + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 0) + require.Len(t, sts.Spec.Template.Spec.Containers, 2) + + // Version from OM + operator version + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:12.0.30.7791-1_9.9.9-test", sts.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[1].Image) + }) + } +} + +func TestReconcilePVCResizeMultiCluster(t *testing.T) { + ctx := context.Background() + + configuration := v1.StatefulSetConfiguration{ + SpecWrapper: v1.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To("test"), + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, + }, + }, + }, + } + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + mrs.Spec.StatefulSetConfiguration = &configuration + + reconciler, c, clusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + + // first, we create the shardedCluster with sts and pvc, + // no resize happening, even after running reconcile multiple times + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, c, false) + testNoResizeMulti(t, c, ctx, mrs) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, c, false) + testNoResizeMulti(t, c, ctx, mrs) + + createdConfigPVCs := getPVCsMulti(t, ctx, mrs, clusterMap) + + newSize := "2Gi" + configuration.SpecWrapper.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests = map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse(newSize)} + mrs.Spec.StatefulSetConfiguration = &configuration + err := c.Update(ctx, mrs) + assert.NoError(t, err) + + _, e := reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, e) + + // its only one sts in the pvc status, since we haven't started the next one yet + testMDBStatusMulti(t, c, ctx, mrs, status.PhasePending, status.PVCS{{Phase: pvc.PhasePVCResize, StatefulsetName: "temple-0"}}) + + testPVCSizeHasIncreased(t, createdConfigPVCs[0].client, ctx, newSize, "temple-0") + + // Running the same resize makes no difference, we are still resizing + _, e = reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, e) + + testMDBStatusMulti(t, c, ctx, mrs, status.PhasePending, status.PVCS{{Phase: pvc.PhasePVCResize, StatefulsetName: "temple-0"}}) + + // update the first stses pvc to be ready + for _, claim := range createdConfigPVCs[0].persistentVolumeClaims { + setPVCWithUpdatedResource(ctx, t, createdConfigPVCs[0].client, &claim) + } + + // Running reconcile again should go into orphan + _, e = reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, e) + + // the second pvc is now getting resized + testMDBStatusMulti(t, c, ctx, mrs, status.PhasePending, status.PVCS{ + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: "temple-0"}, + {Phase: pvc.PhasePVCResize, StatefulsetName: "temple-1"}, + }) + // Running reconcile again second pvcState should go into orphan, third one should start + + // update the first stse pvc to be ready + for _, claim := range createdConfigPVCs[1].persistentVolumeClaims { + setPVCWithUpdatedResource(ctx, t, createdConfigPVCs[1].client, &claim) + } + + _, e = reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, e) + + testMDBStatusMulti(t, c, ctx, mrs, status.PhasePending, status.PVCS{ + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: "temple-0"}, + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: "temple-1"}, + {Phase: pvc.PhasePVCResize, StatefulsetName: "temple-2"}, + }) + + // pvc aren't resized. therefore same status expected + _, e = reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, e) + + testMDBStatusMulti(t, c, ctx, mrs, status.PhasePending, status.PVCS{ + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: "temple-0"}, + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: "temple-1"}, + {Phase: pvc.PhasePVCResize, StatefulsetName: "temple-2"}, + }) + + // update the first stse pvc to be ready + for _, claim := range createdConfigPVCs[2].persistentVolumeClaims { + setPVCWithUpdatedResource(ctx, t, createdConfigPVCs[2].client, &claim) + } + + // We move from resize → orphaned and in the final call in the reconciling to running and + // remove the PVCs. + _, err = reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, err) + + // We are now in the running phase, since all statefulsets have finished resizing; therefore, + // no pvc phase is shown anymore + testMDBStatusMulti(t, c, ctx, mrs, status.PhaseRunning, nil) + + for _, item := range mrs.Spec.ClusterSpecList { + c := clusterMap[item.ClusterName] + stsName := mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName)) + testStatefulsetHasAnnotationAndCorrectSize(t, c, ctx, mrs.Namespace, stsName) + } + + _, e = reconciler.Reconcile(ctx, requestFromObject(mrs)) + require.NoError(t, e) + + // We are now in the running phase, since all statefulsets have finished resizing; therefore, + // no pvc phase is shown anymore + testMDBStatusMulti(t, c, ctx, mrs, status.PhaseRunning, nil) +} + +type pvcClient struct { + persistentVolumeClaims []corev1.PersistentVolumeClaim + client client.Client +} + +func getPVCsMulti(t *testing.T, ctx context.Context, mrs *mdbmulti.MongoDBMultiCluster, memberClusterMap map[string]client.Client) []pvcClient { + var createdConfigPVCs []pvcClient + for _, item := range mrs.Spec.ClusterSpecList { + c := memberClusterMap[item.ClusterName] + statefulSetName := mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName)) + sts := appsv1.StatefulSet{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, statefulSetName), &sts) + require.NoError(t, err) + createdConfigPVCs = append(createdConfigPVCs, pvcClient{persistentVolumeClaims: createPVCs(t, sts, c), client: c}) + } + return createdConfigPVCs +} + +func testNoResizeMulti(t *testing.T, c kubernetesClient.Client, ctx context.Context, mrs *mdbmulti.MongoDBMultiCluster) { + m := mdbmulti.MongoDBMultiCluster{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.Name), &m) + assert.NoError(t, err) + assert.Nil(t, m.Status.PVCs) +} + +func testMDBStatusMulti(t *testing.T, c kubernetesClient.Client, ctx context.Context, mrs *mdbmulti.MongoDBMultiCluster, expectedMDBPhase status.Phase, expectedPVCS status.PVCS) { + m := mdbmulti.MongoDBMultiCluster{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.Name), &m) + require.NoError(t, err) + require.Equal(t, expectedMDBPhase, m.Status.Phase) + require.Equal(t, expectedPVCS, m.Status.PVCs) +} + +func TestReconcileFails_WhenProjectConfig_IsNotFound(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + reconciler, _, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + + result, err := reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.Nil(t, err) + assert.True(t, result.RequeueAfter > 0) +} + +func TestMultiClusterConfigMapAndSecretWatched(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, mrs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, mrs.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, mrs.Name)}, + } + + assert.Equal(t, reconciler.resourceWatcher.GetWatchedResources(), expected) +} + +func TestServiceCreation_WithExternalName(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder(). + SetClusterSpecList(clusters). + SetExternalAccess( + mdb.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster-%d.testing"), + }, ptr.To("cluster-%d.testing")). + Build() + reconciler, client, memberClusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + clusterSpecs := clusterSpecList + for _, item := range clusterSpecs { + c := memberClusterMap[item.ClusterName] + for podNum := 0; podNum < item.Members; podNum++ { + externalService := getExternalService(mrs, item.ClusterName, podNum) + + err = c.Get(ctx, kube.ObjectKey(externalService.Namespace, externalService.Name), &corev1.Service{}) + assert.NoError(t, err) + + // ensure that all other clusters do not have this service + for _, otherItem := range clusterSpecs { + if item.ClusterName == otherItem.ClusterName { + continue + } + otherCluster := memberClusterMap[otherItem.ClusterName] + err = otherCluster.Get(ctx, kube.ObjectKey(externalService.Namespace, externalService.Name), &corev1.Service{}) + assert.Error(t, err) + } + } + } +} + +func TestServiceCreation_WithPlaceholders(t *testing.T) { + ctx := context.Background() + annotationsWithPlaceholders := map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + create.PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + create.PlaceholderClusterName: "{clusterName}", + create.PlaceholderClusterIndex: "{clusterIndex}", + } + mrs := mdbmulti.DefaultMultiReplicaSetBuilder(). + SetClusterSpecList(clusters). + SetExternalAccess( + mdb.ExternalAccessConfiguration{ + ExternalService: mdb.ExternalServiceConfiguration{ + Annotations: annotationsWithPlaceholders, + }, + }, nil). + Build() + mrs.Spec.DuplicateServiceObjects = util.BooleanRef(false) + reconciler, client, memberClusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + clusterSpecs := clusterSpecList + for _, item := range clusterSpecs { + c := memberClusterMap[item.ClusterName] + for podNum := 0; podNum < item.Members; podNum++ { + externalServiceName := fmt.Sprintf("%s-%d-%d-svc-external", mrs.Name, mrs.ClusterNum(item.ClusterName), podNum) + + svc := corev1.Service{} + err = c.Get(ctx, kube.ObjectKey(mrs.Namespace, externalServiceName), &svc) + assert.NoError(t, err) + + statefulSetName := fmt.Sprintf("%s-%d", mrs.Name, mrs.ClusterNum(item.ClusterName)) + podName := fmt.Sprintf("%s-%d", statefulSetName, podNum) + mongodProcessDomain := fmt.Sprintf("%s.svc.cluster.local", mrs.Namespace) + expectedAnnotations := map[string]string{ + create.PlaceholderPodIndex: fmt.Sprintf("%d", podNum), + create.PlaceholderNamespace: mrs.Namespace, + create.PlaceholderResourceName: mrs.Name, + create.PlaceholderStatefulSetName: statefulSetName, + create.PlaceholderPodName: podName, + create.PlaceholderExternalServiceName: fmt.Sprintf("%s-svc-external", podName), + create.PlaceholderMongodProcessDomain: mongodProcessDomain, + create.PlaceholderMongodProcessFQDN: fmt.Sprintf("%s-svc.%s", podName, mongodProcessDomain), + create.PlaceholderClusterName: item.ClusterName, + create.PlaceholderClusterIndex: fmt.Sprintf("%d", mrs.ClusterNum(item.ClusterName)), + } + assert.Equal(t, expectedAnnotations, svc.Annotations) + } + } +} + +func TestServiceCreation_WithoutDuplicates(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder(). + SetClusterSpecList(clusters). + Build() + reconciler, client, memberClusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + clusterSpecs := clusterSpecList + for _, item := range clusterSpecs { + c := memberClusterMap[item.ClusterName] + for podNum := 0; podNum < item.Members; podNum++ { + svc := getService(mrs, item.ClusterName, podNum) + + testSvc := corev1.Service{} + err := c.Get(ctx, kube.ObjectKey(svc.Namespace, svc.Name), &testSvc) + assert.NoError(t, err) + + // ensure that all other clusters do not have this service + for _, otherItem := range clusterSpecs { + if item.ClusterName == otherItem.ClusterName { + continue + } + otherCluster := memberClusterMap[otherItem.ClusterName] + err = otherCluster.Get(ctx, kube.ObjectKey(svc.Namespace, svc.Name), &corev1.Service{}) + assert.Error(t, err) + } + } + } +} + +func TestServiceCreation_WithDuplicates(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder(). + SetClusterSpecList(clusters). + Build() + mrs.Spec.DuplicateServiceObjects = util.BooleanRef(true) + + reconciler, client, memberClusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + for _, item := range clusterSpecs { + for podNum := 0; podNum < item.Members; podNum++ { + svc := getService(mrs, item.ClusterName, podNum) + + // ensure that all clusters have all services + for _, otherItem := range clusterSpecs { + otherCluster := memberClusterMap[otherItem.ClusterName] + err := otherCluster.Get(ctx, kube.ObjectKey(svc.Namespace, svc.Name), &corev1.Service{}) + assert.NoError(t, err) + } + } + } +} + +func TestHeadlessServiceCreation(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder(). + SetClusterSpecList(clusters). + Build() + + reconciler, client, memberClusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + + for _, item := range clusterSpecs { + c := memberClusterMap[item.ClusterName] + svcName := mrs.MultiHeadlessServiceName(mrs.ClusterNum(item.ClusterName)) + + svc := &corev1.Service{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, svcName), svc) + assert.NoError(t, err) + + expectedMap := map[string]string{ + "app": mrs.MultiHeadlessServiceName(mrs.ClusterNum(item.ClusterName)), + construct.ControllerLabelName: util.OperatorName, + } + assert.Equal(t, expectedMap, svc.Spec.Selector) + } +} + +func TestResourceDeletion(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + reconciler, client, memberClients, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + t.Run("Resources are created", func(t *testing.T) { + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + for _, item := range clusterSpecs { + c := memberClients[item.ClusterName] + t.Run("Stateful Set in each member cluster has been created", func(t *testing.T) { + ctx := context.Background() + sts := appsv1.StatefulSet{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName))), &sts) + assert.NoError(t, err) + }) + + t.Run("Services in each member cluster have been created", func(t *testing.T) { + ctx := context.Background() + svcList := corev1.ServiceList{} + err := c.List(ctx, &svcList) + assert.NoError(t, err) + assert.Len(t, svcList.Items, item.Members+2) + }) + + t.Run("Configmaps in each member cluster have been created", func(t *testing.T) { + ctx := context.Background() + configMapList := corev1.ConfigMapList{} + err := c.List(ctx, &configMapList) + assert.NoError(t, err) + assert.Len(t, configMapList.Items, 1) + }) + t.Run("Secrets in each member cluster have been created", func(t *testing.T) { + ctx := context.Background() + secretList := corev1.SecretList{} + err := c.List(ctx, &secretList) + assert.NoError(t, err) + assert.Len(t, secretList.Items, 1) + }) + } + }) + + err := reconciler.deleteManagedResources(ctx, *mrs, zap.S()) + assert.NoError(t, err) + + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + for _, item := range clusterSpecs { + c := memberClients[item.ClusterName] + t.Run("Stateful Set in each member cluster has been removed", func(t *testing.T) { + ctx := context.Background() + sts := appsv1.StatefulSet{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName))), &sts) + assert.Error(t, err) + }) + + t.Run("Services in each member cluster have been removed", func(t *testing.T) { + ctx := context.Background() + svcList := corev1.ServiceList{} + err := c.List(ctx, &svcList) + assert.NoError(t, err) + // temple-0-svc is leftover and not deleted since it does not contain the label: mongodbmulticluster -> my-namespace-temple + assert.Len(t, svcList.Items, 1) + }) + + t.Run("Configmaps in each member cluster have been removed", func(t *testing.T) { + ctx := context.Background() + configMapList := corev1.ConfigMapList{} + err := c.List(ctx, &configMapList) + assert.NoError(t, err) + assert.Len(t, configMapList.Items, 0) + }) + + t.Run("Secrets in each member cluster have been removed", func(t *testing.T) { + ctx := context.Background() + secretList := corev1.SecretList{} + err := c.List(ctx, &secretList) + assert.NoError(t, err) + assert.Len(t, secretList.Items, 0) + }) + } + + t.Run("Ops Manager state has been cleaned", func(t *testing.T) { + processes := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses() + assert.Len(t, processes, 0) + + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + + assert.Empty(t, ac.Auth.AutoAuthMechanisms) + assert.Empty(t, ac.Auth.DeploymentAuthMechanisms) + assert.False(t, ac.Auth.IsEnabled()) + }) +} + +func TestGroupSecret_IsCopied_ToEveryMemberCluster(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + reconciler, client, memberClusterMap, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + for _, clusterName := range clusters { + t.Run(fmt.Sprintf("Secret exists in cluster %s", clusterName), func(t *testing.T) { + ctx := context.Background() + c, ok := memberClusterMap[clusterName] + assert.True(t, ok) + + s := corev1.Secret{} + err := c.Get(ctx, kube.ObjectKey(mrs.Namespace, fmt.Sprintf("%s-group-secret", om.TestGroupID)), &s) + assert.NoError(t, err) + assert.Equal(t, mongoDBMultiLabels(mrs.Name, mrs.Namespace), s.Labels) + }) + } +} + +func TestAuthentication_IsEnabledInOM_WhenConfiguredInCR(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetSecurity(&mdb.Security{ + Authentication: &mdb.Authentication{Enabled: true, Modes: []mdb.AuthMode{"SCRAM"}}, + }).SetClusterSpecList(clusters).Build() + + reconciler, client, _, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + + t.Run("Reconciliation is successful when configuring scram", func(t *testing.T) { + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + }) + + t.Run("Automation Config has been updated correctly", func(t *testing.T) { + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + + assert.Contains(t, ac.Auth.AutoAuthMechanism, "SCRAM-SHA-256") + assert.Contains(t, ac.Auth.DeploymentAuthMechanisms, "SCRAM-SHA-256") + assert.True(t, ac.Auth.IsEnabled()) + assert.NotEmpty(t, ac.Auth.AutoPwd) + assert.NotEmpty(t, ac.Auth.Key) + assert.NotEmpty(t, ac.Auth.KeyFile) + assert.NotEmpty(t, ac.Auth.KeyFileWindows) + assert.NotEmpty(t, ac.Auth.AutoUser) + }) +} + +func TestTls_IsEnabledInOM_WhenConfiguredInCR(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).SetSecurity(&mdb.Security{ + TLSConfig: &mdb.TLSConfig{Enabled: true, CA: "some-ca"}, + CertificatesSecretsPrefix: "some-prefix", + }).Build() + + reconciler, client, _, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + createMultiClusterReplicaSetTLSData(t, ctx, client, mrs, "some-ca") + + t.Run("Reconciliation is successful when configuring tls", func(t *testing.T) { + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + }) + + t.Run("Automation Config has been updated correctly", func(t *testing.T) { + processes := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses() + for _, p := range processes { + assert.True(t, p.IsTLSEnabled()) + assert.Equal(t, "requireTLS", p.TLSConfig()["mode"]) + } + }) +} + +func TestSpecIsSavedAsAnnotation_WhenReconciliationIsSuccessful(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + // fetch the resource after reconciliation + err := client.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.Name), mrs) + assert.NoError(t, err) + + expected := mrs.Spec + actual, err := mrs.ReadLastAchievedSpec() + assert.NoError(t, err) + assert.NotNil(t, actual) + + areEqual, err := specsAreEqual(expected, *actual) + + assert.NoError(t, err) + assert.True(t, areEqual) +} + +func TestMultiReplicaSetRace(t *testing.T) { + ctx := context.Background() + rs1, cfgMap1, projectName1 := buildMultiReplicaSetWithCustomProject("my-rs1") + rs2, cfgMap2, projectName2 := buildMultiReplicaSetWithCustomProject("my-rs2") + rs3, cfgMap3, projectName3 := buildMultiReplicaSetWithCustomProject("my-rs3") + + resourceToProjectMapping := map[string]string{ + "my-rs1": projectName1, + "my-rs2": projectName2, + "my-rs3": projectName3, + } + + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(rs1, rs2, rs3). + WithObjects(cfgMap1, cfgMap2, cfgMap3). + WithObjects(mock.GetCredentialsSecret(om.TestUser, om.TestApiKey)). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory().WithResourceToProjectMapping(resourceToProjectMapping) + memberClusterMap := getFakeMultiClusterMapWithConfiguredInterceptor(clusters, omConnectionFactory, true, true) + reconciler := newMultiClusterReplicaSetReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, omConnectionFactory.GetConnectionFunc, memberClusterMap) + + testConcurrentReconciles(ctx, t, fakeClient, reconciler, rs1, rs2, rs3) +} + +func buildMultiReplicaSetWithCustomProject(mcReplicaSetName string) (*mdbmulti.MongoDBMultiCluster, *corev1.ConfigMap, string) { + configMapName := mock.TestProjectConfigMapName + "-" + mcReplicaSetName + projectName := om.TestGroupName + "-" + mcReplicaSetName + + return mdbmulti.DefaultMultiReplicaSetBuilder(). + SetName(mcReplicaSetName). + SetOpsManagerConfigMapName(configMapName). + SetClusterSpecList(clusters). + Build(), mock.GetProjectConfigMap(configMapName, projectName, ""), projectName +} + +func TestScaling(t *testing.T) { + ctx := context.Background() + + t.Run("Can scale to max amount when creating the resource", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + reconciler, client, memberClusters, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + statefulSets := readStatefulSets(ctx, mrs, memberClusters) + assert.Len(t, statefulSets, 3) + + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + for _, item := range clusterSpecs { + sts := statefulSets[item.ClusterName] + assert.Equal(t, item.Members, int(*sts.Spec.Replicas)) + } + }) + + t.Run("Scale one at a time when scaling up", func(t *testing.T) { + stsWrapper := &v1.StatefulSetConfiguration{ + SpecWrapper: v1.StatefulSetSpecWrapper{ + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + }, + }, + } + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + mrs.Spec.ClusterSpecList[0].Members = 1 + mrs.Spec.ClusterSpecList[0].StatefulSetConfiguration = stsWrapper + mrs.Spec.ClusterSpecList[1].Members = 1 + mrs.Spec.ClusterSpecList[2].Members = 1 + reconciler, client, memberClusters, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + statefulSets := readStatefulSets(ctx, mrs, memberClusters) + clusterSpecs, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + for _, item := range clusterSpecs { + sts := statefulSets[item.ClusterName] + assert.Equal(t, 1, int(*sts.Spec.Replicas)) + } + + // make sure we return internal object modifications + assert.Equal(t, clusterSpecs[0].StatefulSetConfiguration, stsWrapper) + + // scale up in two different clusters at once. + mrs.Spec.ClusterSpecList[0].Members = 3 + mrs.Spec.ClusterSpecList[2].Members = 3 + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 2, 1, 1) + assert.Len(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses(), 4) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 1) + assert.Len(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses(), 5) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 2) + assert.Len(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses(), 6) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 3) + assert.Len(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses(), 7) + + clusterSpecs, _ = mrs.GetClusterSpecItems() + // make sure we return internal object modifications + assert.Equal(t, clusterSpecs[0].StatefulSetConfiguration, stsWrapper) + }) + + t.Run("Scale one at a time when scaling down", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + mrs.Spec.ClusterSpecList[0].Members = 3 + mrs.Spec.ClusterSpecList[1].Members = 2 + mrs.Spec.ClusterSpecList[2].Members = 3 + reconciler, client, memberClusters, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + statefulSets := readStatefulSets(ctx, mrs, memberClusters) + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + assert.NoError(t, err) + } + + for _, item := range clusterSpecList { + sts := statefulSets[item.ClusterName] + assert.Equal(t, item.Members, int(*sts.Spec.Replicas)) + } + + mockedOmConnection := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + assert.Len(t, mockedOmConnection.GetProcesses(), 8) + + // scale down in all clusters. + mrs.Spec.ClusterSpecList[0].Members = 1 + mrs.Spec.ClusterSpecList[1].Members = 1 + mrs.Spec.ClusterSpecList[2].Members = 1 + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 2, 2, 3) + assert.Len(t, mockedOmConnection.GetProcesses(), 7) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 2, 3) + assert.Len(t, mockedOmConnection.GetProcesses(), 6) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 1, 3) + assert.Len(t, mockedOmConnection.GetProcesses(), 5) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 1, 2) + assert.Len(t, mockedOmConnection.GetProcesses(), 4) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 1, 1) + assert.Len(t, mockedOmConnection.GetProcesses(), 3) + }) + + t.Run("Added members don't have overlapping replica set member Ids", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + mrs.Spec.ClusterSpecList[0].Members = 1 + mrs.Spec.ClusterSpecList[1].Members = 1 + mrs.Spec.ClusterSpecList[2].Members = 1 + reconciler, client, _, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + assert.Len(t, omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses(), 3) + + dep, err := omConnectionFactory.GetConnection().ReadDeployment() + assert.NoError(t, err) + + replicaSets := dep.ReplicaSets() + + assert.Len(t, replicaSets, 1) + members := replicaSets[0].Members() + assert.Len(t, members, 3) + + assertMemberNameAndId(t, members, fmt.Sprintf("%s-0-0", mrs.Name), 0) + assertMemberNameAndId(t, members, fmt.Sprintf("%s-1-0", mrs.Name), 1) + assertMemberNameAndId(t, members, fmt.Sprintf("%s-2-0", mrs.Name), 2) + + assert.Equal(t, members[0].Id(), 0) + assert.Equal(t, members[1].Id(), 1) + assert.Equal(t, members[2].Id(), 2) + + mrs.Spec.ClusterSpecList[0].Members = 2 + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + dep, err = omConnectionFactory.GetConnection().ReadDeployment() + assert.NoError(t, err) + + replicaSets = dep.ReplicaSets() + + assert.Len(t, replicaSets, 1) + members = replicaSets[0].Members() + assert.Len(t, members, 4) + + assertMemberNameAndId(t, members, fmt.Sprintf("%s-0-0", mrs.Name), 0) + assertMemberNameAndId(t, members, fmt.Sprintf("%s-0-1", mrs.Name), 3) + assertMemberNameAndId(t, members, fmt.Sprintf("%s-1-0", mrs.Name), 1) + assertMemberNameAndId(t, members, fmt.Sprintf("%s-2-0", mrs.Name), 2) + }) + + t.Run("Cluster can be added", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + mrs.Spec.ClusterSpecList = mrs.Spec.ClusterSpecList[:len(mrs.Spec.ClusterSpecList)-1] + + mrs.Spec.ClusterSpecList[0].Members = 1 + mrs.Spec.ClusterSpecList[1].Members = 1 + + reconciler, client, memberClusters, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 1) + + // scale one member and add a new cluster + mrs.Spec.ClusterSpecList[0].Members = 3 + mrs.Spec.ClusterSpecList = append(mrs.Spec.ClusterSpecList, mdb.ClusterSpecItem{ + ClusterName: clusters[2], + Members: 3, + }) + + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 2, 1, 0) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 0) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 1) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 2) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 1, 3) + }) + + t.Run("Cluster can be removed", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + mrs.Spec.ClusterSpecList[0].Members = 3 + mrs.Spec.ClusterSpecList[1].Members = 2 + mrs.Spec.ClusterSpecList[2].Members = 3 + + reconciler, client, memberClusters, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 3, 2, 3) + + mrs.Spec.ClusterSpecList[0].Members = 1 + mrs.Spec.ClusterSpecList = mrs.Spec.ClusterSpecList[:len(mrs.Spec.ClusterSpecList)-1] + + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 2, 2, 3) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 2, 3) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 2, 2) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 2, 1) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 2) + + // can reconcile again and it succeeds. + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 2) + }) + + t.Run("Multiple clusters can be removed", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + mrs.Spec.ClusterSpecList[0].Members = 2 + mrs.Spec.ClusterSpecList[1].Members = 1 + mrs.Spec.ClusterSpecList[2].Members = 2 + + reconciler, client, memberClusters, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 2, 1, 2) + + // remove first and last + mrs.Spec.ClusterSpecList = mdb.ClusterSpecList{mrs.Spec.ClusterSpecList[1]} + + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 1, 1, 2) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 0, 1, 2) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, true) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 0, 1, 1) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + assertStatefulSetReplicas(ctx, t, mrs, memberClusters, 0, 1, 0) + }) +} + +func TestClusterNumbering(t *testing.T) { + ctx := context.Background() + + t.Run("Create MDB CR first time", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterNumMap := getClusterNumMapping(t, mrs) + assertClusterpresent(t, clusterNumMap, mrs.Spec.ClusterSpecList, []int{0, 1, 2}) + }) + + t.Run("Add Cluster", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + mrs.Spec.ClusterSpecList = mrs.Spec.ClusterSpecList[:len(mrs.Spec.ClusterSpecList)-1] + + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterNumMap := getClusterNumMapping(t, mrs) + assertClusterpresent(t, clusterNumMap, mrs.Spec.ClusterSpecList, []int{0, 1}) + + // add cluster + mrs.Spec.ClusterSpecList = append(mrs.Spec.ClusterSpecList, mdb.ClusterSpecItem{ + ClusterName: clusters[2], + Members: 1, + }) + + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + clusterNumMap = getClusterNumMapping(t, mrs) + + assert.Equal(t, 2, clusterNumMap[clusters[2]]) + }) + + t.Run("Remove and Add back cluster", func(t *testing.T) { + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + mrs.Spec.ClusterSpecList[0].Members = 1 + mrs.Spec.ClusterSpecList[1].Members = 1 + mrs.Spec.ClusterSpecList[2].Members = 1 + + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + clusterNumMap := getClusterNumMapping(t, mrs) + assertClusterpresent(t, clusterNumMap, mrs.Spec.ClusterSpecList, []int{0, 1, 2}) + clusterOneIndex := clusterNumMap[clusters[1]] + + // Remove cluster index 1 from the specs + mrs.Spec.ClusterSpecList = mdb.ClusterSpecList{ + { + ClusterName: clusters[0], + Members: 1, + }, + { + ClusterName: clusters[2], + Members: 1, + }, + } + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + // Add cluster index 1 back to the specs + mrs.Spec.ClusterSpecList = append(mrs.Spec.ClusterSpecList, mdb.ClusterSpecItem{ + ClusterName: clusters[1], + Members: 1, + }) + + err = client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + // assert the index corresponsing to cluster 1 is still 1 + clusterNumMap = getClusterNumMapping(t, mrs) + assert.Equal(t, clusterOneIndex, clusterNumMap[clusters[1]]) + }) +} + +func getClusterNumMapping(t *testing.T, m *mdbmulti.MongoDBMultiCluster) map[string]int { + clusterMapping := make(map[string]int) + bytes := m.Annotations[mdbmulti.LastClusterNumMapping] + err := json.Unmarshal([]byte(bytes), &clusterMapping) + assert.NoError(t, err) + + return clusterMapping +} + +// assertMemberNameAndId makes sure that the member with the given name has the given id. +// the processes are sorted and the order in the automation config is not necessarily the order +// in which they appear in the CR. +func assertMemberNameAndId(t *testing.T, members []om.ReplicaSetMember, name string, id int) { + for _, m := range members { + if m.Name() == name { + assert.Equal(t, id, m.Id()) + return + } + } + t.Fatalf("Member with name %s not found in replica set members", name) +} + +func TestBackupConfigurationReplicaSet(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters). + SetConnectionSpec(testConnectionSpec()). + SetBackup(mdb.Backup{ + Mode: "enabled", + }).Build() + + reconciler, client, _, omConnectionFactory := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + uuidStr := uuid.New().String() + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + _, err := connection.UpdateBackupConfig(&backup.Config{ + ClusterId: uuidStr, + Status: backup.Inactive, + }) + if err != nil { + panic(err) + } + + // add the Replicaset cluster to OM + connection.(*om.MockedOmConnection).BackupHostClusters[uuidStr] = &backup.HostCluster{ + ReplicaSetName: mrs.Name, + ClusterName: mrs.Name, + TypeName: "REPLICA_SET", + } + }) + + t.Run("Backup can be started", func(t *testing.T) { + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + configResponse, _ := omConnectionFactory.GetConnection().ReadBackupConfigs() + + assert.Len(t, configResponse.Configs, 1) + config := configResponse.Configs[0] + + assert.Equal(t, backup.Started, config.Status) + assert.Equal(t, uuidStr, config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + }) + + t.Run("Backup snapshot schedule tests", backupSnapshotScheduleTests(mrs, client, reconciler, omConnectionFactory, uuidStr)) + + t.Run("Backup can be stopped", func(t *testing.T) { + mrs.Spec.Backup.Mode = "disabled" + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + configResponse, _ := omConnectionFactory.GetConnection().ReadBackupConfigs() + assert.Len(t, configResponse.Configs, 1) + + config := configResponse.Configs[0] + + assert.Equal(t, backup.Stopped, config.Status) + assert.Equal(t, uuidStr, config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + }) + + t.Run("Backup can be terminated", func(t *testing.T) { + mrs.Spec.Backup.Mode = "terminated" + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + configResponse, _ := omConnectionFactory.GetConnection().ReadBackupConfigs() + assert.Len(t, configResponse.Configs, 1) + + config := configResponse.Configs[0] + + assert.Equal(t, backup.Terminating, config.Status) + assert.Equal(t, uuidStr, config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + }) +} + +func TestMultiClusterFailover(t *testing.T) { + ctx := context.Background() + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + + reconciler, client, memberClusters, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + // trigger failover by adding an annotation to the CR + // read the first cluster from the clusterSpec list and fail it over. + expectedNodeCount := 0 + for _, e := range mrs.Spec.ClusterSpecList { + expectedNodeCount += e.Members + } + + cluster := mrs.Spec.ClusterSpecList[0] + failedClusters := []failedcluster.FailedCluster{{ClusterName: cluster.ClusterName, Members: cluster.Members}} + + clusterSpecBytes, err := json.Marshal(failedClusters) + assert.NoError(t, err) + + mrs.SetAnnotations(map[string]string{failedcluster.FailedClusterAnnotation: string(clusterSpecBytes)}) + + err = client.Update(ctx, mrs) + assert.NoError(t, err) + + t.Setenv("PERFORM_FAILOVER", "true") + + err = memberwatch.AddFailoverAnnotation(ctx, *mrs, cluster.ClusterName, client) + assert.NoError(t, err) + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(mrs), mrs)) + + checkMultiReconcileSuccessful(ctx, t, reconciler, mrs, client, false) + + // assert the statefulset member count in the healthy cluster is same as the initial count + statefulSets := readStatefulSets(ctx, mrs, memberClusters) + currentNodeCount := 0 + + // only 2 clusters' statefulsets should be fetched since the first cluster has been failed-over + assert.Equal(t, 2, len(statefulSets)) + + for _, s := range statefulSets { + currentNodeCount += int(*s.Spec.Replicas) + } + + assert.Equal(t, expectedNodeCount, currentNodeCount) +} + +func TestMultiReplicaSet_AgentVersionMapping(t *testing.T) { + ctx := context.Background() + defaultResource := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).Build() + containers := []corev1.Container{{Name: util.AgentContainerName, Image: "foo"}} + podTemplate := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: containers, + }, + } + overridenResource := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clusters).SetPodSpecTemplate(podTemplate).Build() + nonExistingPath := "/foo/bar/foo" + + t.Run("Static architecture, version retrieving fails, image is overriden, reconciliation should succeeds", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + t.Setenv(agentVersionManagement.MappingFilePathEnv, nonExistingPath) + overridenReconciler, overridenClient, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", overridenResource) + checkMultiReconcileSuccessful(ctx, t, overridenReconciler, overridenResource, overridenClient, false) + }) + + t.Run("Static architecture, version retrieving fails, image is not overriden, reconciliation should fail", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + t.Setenv(agentVersionManagement.MappingFilePathEnv, nonExistingPath) + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", defaultResource) + checkMultiReconcileSuccessful(ctx, t, reconciler, defaultResource, client, true) + }) + + t.Run("Static architecture, version retrieving succeeds, image is not overriden, reconciliation should succeed", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", defaultResource) + checkMultiReconcileSuccessful(ctx, t, reconciler, defaultResource, client, false) + }) + + t.Run("Non-Static architecture, version retrieving fails, image is not overriden, reconciliation should succeed", func(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.NonStatic)) + t.Setenv(agentVersionManagement.MappingFilePathEnv, nonExistingPath) + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", defaultResource) + checkMultiReconcileSuccessful(ctx, t, reconciler, defaultResource, client, false) + }) +} + +func TestValidationsRunOnReconcile(t *testing.T) { + ctx := context.Background() + duplicateName := "duplicate" + clustersWithDuplicate := []string{duplicateName, duplicateName, "cluster-3"} + mrs := mdbmulti.DefaultMultiReplicaSetBuilder().SetClusterSpecList(clustersWithDuplicate).Build() + reconciler, client, _, _ := defaultMultiReplicaSetReconciler(ctx, nil, "", "", mrs) + + // copied + err := client.Update(ctx, mrs) + assert.NoError(t, err) + + _, err = reconciler.Reconcile(ctx, requestFromObject(mrs)) + assert.NoError(t, err) + + // fetch the last updates as the reconciliation loop can update the mdb resource. + err = client.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.Name), mrs) + assert.NoError(t, err) + assert.Equal(t, status.PhaseFailed, mrs.Status.Phase) + assert.Equal(t, fmt.Sprintf("Multiple clusters with the same name (%s) are not allowed", duplicateName), mrs.Status.Message) +} + +func assertClusterpresent(t *testing.T, m map[string]int, specs mdb.ClusterSpecList, arr []int) { + tmp := make([]int, 0) + for _, s := range specs { + tmp = append(tmp, m[s.ClusterName]) + } + + sort.Ints(tmp) + assert.Equal(t, arr, tmp) +} + +func assertStatefulSetReplicas(ctx context.Context, t *testing.T, mrs *mdbmulti.MongoDBMultiCluster, memberClusters map[string]client.Client, expectedReplicas ...int) { + statefulSets := readStatefulSets(ctx, mrs, memberClusters) + + for i := range expectedReplicas { + if val, ok := statefulSets[clusters[i]]; ok { + require.Equal(t, expectedReplicas[i], int(*val.Spec.Replicas)) + } + } +} + +func readStatefulSets(ctx context.Context, mrs *mdbmulti.MongoDBMultiCluster, memberClusters map[string]client.Client) map[string]appsv1.StatefulSet { + allStatefulSets := map[string]appsv1.StatefulSet{} + clusterSpecList, err := mrs.GetClusterSpecItems() + if err != nil { + panic(err) + } + + for _, item := range clusterSpecList { + memberClient := memberClusters[item.ClusterName] + sts := appsv1.StatefulSet{} + err := memberClient.Get(ctx, kube.ObjectKey(mrs.Namespace, mrs.MultiStatefulsetName(mrs.ClusterNum(item.ClusterName))), &sts) + if err == nil { + allStatefulSets[item.ClusterName] = sts + } + } + return allStatefulSets +} + +// specsAreEqual compares two different MongoDBMultiSpec instances and returns true if they are equal. +// the specs need to be marshaled and bytes compared as this ensures that empty slices are converted to nil +// ones and gives an accurate comparison. +// We are unable to use reflect.DeepEqual for this comparision as when deserialization happens, +// some fields on spec2 are nil, while spec1 are empty collections. By converting both to bytes +// we can ensure they are equivalent for our purposes. +func specsAreEqual(spec1, spec2 mdbmulti.MongoDBMultiSpec) (bool, error) { + spec1Bytes, err := json.Marshal(spec1) + if err != nil { + return false, err + } + spec2Bytes, err := json.Marshal(spec2) + if err != nil { + return false, err + } + return bytes.Equal(spec1Bytes, spec2Bytes), nil +} + +func defaultMultiReplicaSetReconciler(ctx context.Context, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, m *mdbmulti.MongoDBMultiCluster) (*ReconcileMongoDbMultiReplicaSet, kubernetesClient.Client, map[string]client.Client, *om.CachedOMConnectionFactory) { + multiReplicaSetController, client, clusterMap, omConnectionFactory := multiReplicaSetReconciler(ctx, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, m) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + connection.(*om.MockedOmConnection).Hostnames = calculateHostNamesForExternalDomains(m) + }) + + return multiReplicaSetController, client, clusterMap, omConnectionFactory +} + +func calculateHostNamesForExternalDomains(m *mdbmulti.MongoDBMultiCluster) []string { + if m.Spec.GetExternalDomain() == nil { + return nil + } + + var expectedHostnames []string + for i, cl := range m.Spec.ClusterSpecList { + for j := 0; j < cl.Members; j++ { + externalDomain := m.Spec.GetExternalDomainForMemberCluster(cl.ClusterName) + if externalDomain == nil { + // we don't have all externalDomains set, so we don't calculate them here at all + // validation should capture invalid external domains configuration, so it must be all or nothing + return nil + } + expectedHostnames = append(expectedHostnames, fmt.Sprintf("%s-%d-%d.%s", m.Name, i, j, *externalDomain)) + } + } + return expectedHostnames +} + +func multiReplicaSetReconciler(ctx context.Context, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, m *mdbmulti.MongoDBMultiCluster) (*ReconcileMongoDbMultiReplicaSet, kubernetesClient.Client, map[string]client.Client, *om.CachedOMConnectionFactory) { + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(m) + memberClusterMap := getFakeMultiClusterMap(omConnectionFactory) + return newMultiClusterReplicaSetReconciler(ctx, kubeClient, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, false, omConnectionFactory.GetConnectionFunc, memberClusterMap), kubeClient, memberClusterMap, omConnectionFactory +} + +func getFakeMultiClusterMap(omConnectionFactory *om.CachedOMConnectionFactory) map[string]client.Client { + return getFakeMultiClusterMapWithClusters(clusters, omConnectionFactory) +} + +func getFakeMultiClusterMapWithClusters(clusters []string, omConnectionFactory *om.CachedOMConnectionFactory) map[string]client.Client { + return getFakeMultiClusterMapWithConfiguredInterceptor(clusters, omConnectionFactory, true, true) +} + +// getAppDBFakeMultiClusterMapWithClusters is used for appdb multi cluster tests +// In AppDB we manually add the hosts in OM by calling the /hosts endpoint. +// Here we set `addOMHosts` to false so that we can test what hostnames the operator adds. +func getAppDBFakeMultiClusterMapWithClusters(clusters []string, omConnectionFactory *om.CachedOMConnectionFactory) map[string]client.Client { + return getFakeMultiClusterMapWithConfiguredInterceptor(clusters, omConnectionFactory, true, false) +} + +func getFakeMultiClusterMapWithConfiguredInterceptor(clusters []string, omConnectionFactory *om.CachedOMConnectionFactory, markStsAsReady bool, addOMHosts bool) map[string]client.Client { + clientMap := make(map[string]client.Client) + + for _, e := range clusters { + fakeClientBuilder := mock.NewEmptyFakeClientBuilder() + fakeClientBuilder.WithInterceptorFuncs(interceptor.Funcs{ + Get: mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, markStsAsReady, addOMHosts), + }) + + clientMap[e] = kubernetesClient.NewClient(fakeClientBuilder.Build()) + } + return clientMap +} + +func getFakeMultiClusterMapWithoutInterceptor(clusters []string) map[string]client.Client { + clientMap := make(map[string]client.Client) + + for _, e := range clusters { + memberCluster := multicluster.New(kubernetesClient.NewClient(mock.NewEmptyFakeClientBuilder().Build())) + clientMap[e] = memberCluster.GetClient() + } + return clientMap +} diff --git a/controllers/operator/mongodbopsmanager_controller.go b/controllers/operator/mongodbopsmanager_controller.go new file mode 100644 index 000000000..bf50de16a --- /dev/null +++ b/controllers/operator/mongodbopsmanager_controller.go @@ -0,0 +1,2184 @@ +package operator + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base32" + "encoding/json" + "errors" + "fmt" + "reflect" + "slices" + "syscall" + + "github.com/blang/semver" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/constants" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + mdbstatus "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/api" + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/identifiable" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" + "github.com/10gen/ops-manager-kubernetes/pkg/vault/vaultwatcher" +) + +var OmUpdateChannel chan event.GenericEvent + +const ( + oldestSupportedOpsManagerVersion = "5.0.0" + programmaticKeyVersion = "5.0.0" +) + +type S3ConfigGetter interface { + GetAuthenticationModes() []string + GetResourceName() string + BuildConnectionString(username, password string, scheme connectionstring.Scheme, connectionParams map[string]string) string +} + +// OpsManagerReconciler is a controller implementation. +// It's Reconciler function is called by Controller Runtime. +// WARNING: do not put any mutable state into this class. Controller runtime uses and shares a single instance of it. +type OpsManagerReconciler struct { + *ReconcileCommonController + omInitializer api.Initializer + omAdminProvider api.AdminProvider + omConnectionFactory om.ConnectionFactory + oldestSupportedVersion semver.Version + programmaticKeyVersion semver.Version + + memberClustersMap map[string]client.Client + + imageUrls images.ImageUrls + initAppdbVersion string + initOpsManagerImageVersion string +} + +var _ reconcile.Reconciler = &OpsManagerReconciler{} + +func NewOpsManagerReconciler(ctx context.Context, kubeClient client.Client, memberClustersMap map[string]client.Client, imageUrls images.ImageUrls, initAppdbVersion, initOpsManagerImageVersion string, omFunc om.ConnectionFactory, initializer api.Initializer, adminProvider api.AdminProvider) *OpsManagerReconciler { + return &OpsManagerReconciler{ + ReconcileCommonController: NewReconcileCommonController(ctx, kubeClient), + omConnectionFactory: omFunc, + omInitializer: initializer, + omAdminProvider: adminProvider, + oldestSupportedVersion: semver.MustParse(oldestSupportedOpsManagerVersion), + programmaticKeyVersion: semver.MustParse(programmaticKeyVersion), + memberClustersMap: memberClustersMap, + imageUrls: imageUrls, + initAppdbVersion: initAppdbVersion, + initOpsManagerImageVersion: initOpsManagerImageVersion, + } +} + +type OMDeploymentState struct { + CommonDeploymentState `json:",inline"` +} + +func NewOMDeploymentState() *OMDeploymentState { + return &OMDeploymentState{CommonDeploymentState{ClusterMapping: map[string]int{}}} +} + +// OpsManagerReconcilerHelper is a type containing the state and logic for a SINGLE reconcile execution. +// This object is NOT shared between multiple reconcile invocations in contrast to OpsManagerReconciler where +// we cannot store state of the current reconcile. +type OpsManagerReconcilerHelper struct { + opsManager *omv1.MongoDBOpsManager + memberClusters []multicluster.MemberCluster + stateStore *StateStore[OMDeploymentState] + deploymentState *OMDeploymentState +} + +func NewOpsManagerReconcilerHelper(ctx context.Context, opsManagerReconciler *OpsManagerReconciler, opsManager *omv1.MongoDBOpsManager, globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger) (*OpsManagerReconcilerHelper, error) { + reconcilerHelper := OpsManagerReconcilerHelper{ + opsManager: opsManager, + } + + if !opsManager.Spec.IsMultiCluster() { + reconcilerHelper.memberClusters = []multicluster.MemberCluster{multicluster.GetLegacyCentralMemberCluster(opsManager.Spec.Replicas, 0, opsManagerReconciler.client, opsManagerReconciler.SecretClient)} + return &reconcilerHelper, nil + } + + if len(globalMemberClustersMap) == 0 { + return nil, xerrors.Errorf("member clusters have to be initialized for MultiCluster OpsManager topology") + } + + // here we access ClusterSpecList directly, as we have to check what's been defined in yaml + if len(opsManager.Spec.ClusterSpecList) == 0 { + return nil, xerrors.Errorf("for MongoDBOpsManager.spec.Topology = MultiCluster, clusterSpecList has to be non empty") + } + + clusterNamesFromClusterSpecList := util.Transform(opsManager.GetClusterSpecList(), func(clusterSpecItem omv1.ClusterSpecOMItem) string { + return clusterSpecItem.ClusterName + }) + if err := reconcilerHelper.initializeStateStore(ctx, opsManagerReconciler, opsManager, globalMemberClustersMap, log, clusterNamesFromClusterSpecList); err != nil { + return nil, xerrors.Errorf("failed to initialize OM state store: %w", err) + } + + for _, clusterSpecItem := range opsManager.GetClusterSpecList() { + var memberClusterKubeClient kubernetesClient.Client + var memberClusterSecretClient secrets.SecretClient + memberClusterClient, ok := globalMemberClustersMap[clusterSpecItem.ClusterName] + if !ok { + var clusterList []string + for m := range globalMemberClustersMap { + clusterList = append(clusterList, m) + } + log.Warnf("Member cluster %s specified in clusterSpecList is not found in the list of operator's member clusters: %+v. "+ + "Assuming the cluster is down. It will be ignored from reconciliation.", clusterSpecItem.ClusterName, clusterList) + } else { + memberClusterKubeClient = kubernetesClient.NewClient(memberClusterClient) + memberClusterSecretClient = secrets.SecretClient{ + VaultClient: nil, // Vault is not supported yet on multicluster + KubeClient: memberClusterKubeClient, + } + } + + reconcilerHelper.memberClusters = append(reconcilerHelper.memberClusters, multicluster.MemberCluster{ + Name: clusterSpecItem.ClusterName, + Index: reconcilerHelper.deploymentState.ClusterMapping[clusterSpecItem.ClusterName], + Client: memberClusterKubeClient, + SecretClient: memberClusterSecretClient, + // TODO should we do lastAppliedMember map like in AppDB? + Replicas: clusterSpecItem.Members, + Active: true, + Healthy: memberClusterKubeClient != nil, + }) + } + + return &reconcilerHelper, nil +} + +func (r *OpsManagerReconcilerHelper) initializeStateStore(ctx context.Context, reconciler *OpsManagerReconciler, opsManager *omv1.MongoDBOpsManager, globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger, clusterNamesFromClusterSpecList []string) error { + r.deploymentState = NewOMDeploymentState() + + r.stateStore = NewStateStore[OMDeploymentState](opsManager.Namespace, opsManager.Name, reconciler.client) + if err := r.stateStore.read(ctx); err != nil { + if apiErrors.IsNotFound(err) { + // If the deployment state config map is missing, then it might be either: + // - fresh deployment + // - existing deployment, but it's a first reconcile on the operator version with the new deployment state + // - existing deployment, but for some reason the deployment state config map has been deleted + // In all cases, the deployment config map will be recreated from the state we're keeping and maintaining in + // the old place (in annotations, spec.status, config maps) in order to allow for the downgrade of the operator. + if err := r.migrateToNewDeploymentState(ctx, opsManager, reconciler.client); err != nil { + return err + } + // Here we don't use saveOMState wrapper, as we don't need to write the legacy state + if err := r.stateStore.WriteState(ctx, r.deploymentState, log); err != nil { + return err + } + } else { + return err + } + } + + if state, err := r.stateStore.ReadState(ctx); err != nil { + return err + } else { + r.deploymentState = state + } + + r.deploymentState.ClusterMapping = multicluster.AssignIndexesForMemberClusterNames(r.deploymentState.ClusterMapping, clusterNamesFromClusterSpecList) + + if err := r.saveOMState(ctx, opsManager, reconciler.client, log); err != nil { + return err + } + + return nil +} + +func (r *OpsManagerReconcilerHelper) saveOMState(ctx context.Context, spec *omv1.MongoDBOpsManager, client kubernetesClient.Client, log *zap.SugaredLogger) error { + if err := r.stateStore.WriteState(ctx, r.deploymentState, log); err != nil { + return err + } + if err := r.writeLegacyStateConfigMap(ctx, spec, client, log); err != nil { + return err + } + return nil +} + +// writeLegacyStateConfigMap converts the DeploymentState to the legacy Config Map and write it to the cluster +func (r *OpsManagerReconcilerHelper) writeLegacyStateConfigMap(ctx context.Context, spec *omv1.MongoDBOpsManager, client kubernetesClient.Client, log *zap.SugaredLogger) error { + // ClusterMapping ConfigMap + mappingConfigMapData := map[string]string{} + for k, v := range r.deploymentState.ClusterMapping { + mappingConfigMapData[k] = fmt.Sprintf("%d", v) + } + mappingConfigMap := configmap.Builder().SetName(spec.ClusterMappingConfigMapName()).SetNamespace(spec.Namespace).SetData(mappingConfigMapData).Build() + if err := configmap.CreateOrUpdate(ctx, client, mappingConfigMap); err != nil { + return xerrors.Errorf("failed to update cluster mapping configmap %s: %w", spec.ClusterMappingConfigMapName(), err) + } + log.Debugf("Saving cluster mapping configmap %s: %v", spec.ClusterMappingConfigMapName(), mappingConfigMapData) + + return nil +} + +func (r *OpsManagerReconcilerHelper) GetMemberClusters() []multicluster.MemberCluster { + return r.memberClusters +} + +func (r *OpsManagerReconcilerHelper) getHealthyMemberClusters() []multicluster.MemberCluster { + var healthyMemberClusters []multicluster.MemberCluster + for i := 0; i < len(r.memberClusters); i++ { + if r.memberClusters[i].Healthy { + healthyMemberClusters = append(healthyMemberClusters, r.memberClusters[i]) + } + } + + return healthyMemberClusters +} + +type backupDaemonFQDN struct { + hostname string + memberClusterName string +} + +// BackupDaemonHeadlessFQDNs returns headless FQDNs for backup daemons for all member clusters. +// It's used primarily for registering backup daemon instances in Ops Manager. +func (r *OpsManagerReconcilerHelper) BackupDaemonHeadlessFQDNs() []backupDaemonFQDN { + var fqdns []backupDaemonFQDN + for _, memberCluster := range r.GetMemberClusters() { + clusterHostnames, _ := dns.GetDNSNames(r.BackupDaemonStatefulSetNameForMemberCluster(memberCluster), r.BackupDaemonHeadlessServiceNameForMemberCluster(memberCluster), + r.opsManager.Namespace, r.opsManager.Spec.GetClusterDomain(), r.BackupDaemonMembersForMemberCluster(memberCluster), nil) + for _, hostname := range clusterHostnames { + fqdns = append(fqdns, backupDaemonFQDN{hostname: hostname, memberClusterName: memberCluster.Name}) + } + + } + + return fqdns +} + +func (r *OpsManagerReconcilerHelper) BackupDaemonMembersForMemberCluster(memberCluster multicluster.MemberCluster) int { + clusterSpecOMItem := r.getClusterSpecOMItem(memberCluster.Name) + if clusterSpecOMItem.Backup != nil { + return clusterSpecOMItem.Backup.Members + } + return 0 +} + +func (r *OpsManagerReconcilerHelper) OpsManagerMembersForMemberCluster(memberCluster multicluster.MemberCluster) int { + clusterSpecOMItem := r.getClusterSpecOMItem(memberCluster.Name) + return clusterSpecOMItem.Members +} + +func (r *OpsManagerReconcilerHelper) getClusterSpecOMItem(clusterName string) omv1.ClusterSpecOMItem { + idx := slices.IndexFunc(r.opsManager.GetClusterSpecList(), func(clusterSpecOMItem omv1.ClusterSpecOMItem) bool { + return clusterSpecOMItem.ClusterName == clusterName + }) + if idx == -1 { + panic(fmt.Errorf("member cluster %s not found in OM's clusterSpecList", clusterName)) + } + + return r.opsManager.GetClusterSpecList()[idx] +} + +func (r *OpsManagerReconcilerHelper) OpsManagerStatefulSetNameForMemberCluster(memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.opsManager.Name + } + + return fmt.Sprintf("%s-%d", r.opsManager.Name, memberCluster.Index) +} + +func (r *OpsManagerReconcilerHelper) BackupDaemonStatefulSetNameForMemberCluster(memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.opsManager.BackupDaemonStatefulSetName() + } + + return r.opsManager.BackupDaemonStatefulSetNameForClusterIndex(memberCluster.Index) +} + +func (r *OpsManagerReconcilerHelper) BackupDaemonHeadlessServiceNameForMemberCluster(memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.opsManager.BackupDaemonServiceName() + } + + return r.opsManager.BackupDaemonHeadlessServiceNameForClusterIndex(memberCluster.Index) +} + +func (r *OpsManagerReconcilerHelper) BackupDaemonPodServiceNameForMemberCluster(memberCluster multicluster.MemberCluster) []string { + if memberCluster.Legacy { + hostnames, _ := dns.GetDNSNames(r.BackupDaemonStatefulSetNameForMemberCluster(memberCluster), r.BackupDaemonHeadlessServiceNameForMemberCluster(memberCluster), + r.opsManager.Namespace, r.opsManager.Spec.GetClusterDomain(), r.BackupDaemonMembersForMemberCluster(memberCluster), nil) + return hostnames + } + + var hostnames []string + for podIdx := 0; podIdx < r.BackupDaemonMembersForMemberCluster(memberCluster); podIdx++ { + hostnames = append(hostnames, fmt.Sprintf("%s-%d-svc", r.BackupDaemonStatefulSetNameForMemberCluster(memberCluster), podIdx)) + } + + return hostnames +} + +func (r *OpsManagerReconcilerHelper) migrateToNewDeploymentState(ctx context.Context, om *omv1.MongoDBOpsManager, centralClient kubernetesClient.Client) error { + legacyMemberClusterMapping, err := getLegacyMemberClusterMapping(ctx, om.Namespace, om.ClusterMappingConfigMapName(), centralClient) + if apiErrors.IsNotFound(err) || !om.Spec.IsMultiCluster() { + legacyMemberClusterMapping = map[string]int{} + } else if err != nil { + return err + } + + r.deploymentState.ClusterMapping = legacyMemberClusterMapping + + return nil +} + +// +kubebuilder:rbac:groups=mongodb.com,resources={opsmanagers,opsmanagers/status,opsmanagers/finalizers},verbs=*,namespace=placeholder + +// Reconcile performs the reconciliation logic for AppDB, Ops Manager and Backup +// AppDB is reconciled first (independent of Ops Manager as the agent is run in headless mode) and +// Ops Manager statefulset is created then. +// Backup daemon statefulset is created/updated and configured optionally if backup is enabled. +// Note, that the pointer to ops manager resource is used in 'Reconcile' method as resource status is mutated +// many times during reconciliation, and It's important to keep updates to avoid status override +func (r *OpsManagerReconciler) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, e error) { + log := zap.S().With("OpsManager", request.NamespacedName) + + opsManager := &omv1.MongoDBOpsManager{} + + opsManagerExtraStatusParams := mdbstatus.NewOMPartOption(mdbstatus.OpsManager) + + if reconcileResult, err := r.readOpsManagerResource(ctx, request, opsManager, log); err != nil { + return reconcileResult, err + } + + log.Info("-> OpsManager.Reconcile") + log.Infow("OpsManager.Spec", "spec", opsManager.Spec) + log.Infow("OpsManager.Status", "status", opsManager.Status) + + // We perform this check here and not inside the validation because we don't want to put OM in failed state + // just log the error and put in the "Unsupported" state + semverVersion, err := versionutil.StringToSemverVersion(opsManager.Spec.Version) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Invalid("%s is not a valid version", opsManager.Spec.Version), log, opsManagerExtraStatusParams) + } + if semverVersion.LT(r.oldestSupportedVersion) { + return r.updateStatus(ctx, opsManager, workflow.Unsupported("Ops Manager Version %s is not supported by this version of the operator. Please upgrade to a version >=%s", opsManager.Spec.Version, oldestSupportedOpsManagerVersion), log, opsManagerExtraStatusParams) + } + + // AppDB Reconciler will be created with a nil OmAdmin, which is set below after initialization + appDbReconciler, err := r.createNewAppDBReconciler(ctx, opsManager, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error initializing AppDB reconciler: %w", err)), log, opsManagerExtraStatusParams) + } + + if part, err := opsManager.ProcessValidationsOnReconcile(); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Invalid("%s", err.Error()), log, mdbstatus.NewOMPartOption(part)) + } + + acClient := appDbReconciler.getMemberCluster(appDbReconciler.getNameOfFirstMemberCluster()).Client + if err := ensureResourcesForArchitectureChange(ctx, acClient, r.SecretClient, opsManager); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error ensuring resources for upgrade from 1 to 3 container AppDB: %w", err)), log, opsManagerExtraStatusParams) + } + + if err := ensureSharedGlobalResources(ctx, r.client, opsManager); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error ensuring shared global resources %w", err)), log, opsManagerExtraStatusParams) + } + + // 1. Reconcile AppDB + emptyResult, _ := workflow.OK().ReconcileResult() + retryResult := reconcile.Result{Requeue: true} + + // TODO: make SetupCommonWatchers support opsmanager watcher setup + // The order matters here, since appDB and opsManager share the same reconcile ObjectKey being opsmanager crd + // That means we need to remove first, which SetupCommonWatchers does, then register additional watches + appDBReplicaSet := opsManager.Spec.AppDB + r.SetupCommonWatchers(&appDBReplicaSet, nil, nil, appDBReplicaSet.GetName()) + + // We need to remove the watches on the top of the reconcile since we might add resources with the same key below. + if opsManager.IsTLSEnabled() { + r.resourceWatcher.RegisterWatchedTLSResources(opsManager.ObjectKey(), opsManager.Spec.GetOpsManagerCA(), []string{opsManager.TLSCertificateSecretName()}) + } + // register backup + r.watchMongoDBResourcesReferencedByBackup(ctx, opsManager, log) + + result, err := appDbReconciler.ReconcileAppDB(ctx, opsManager) + if err != nil || (result != emptyResult && result != retryResult) { + return result, err + } + + appDBPassword, err := appDbReconciler.ensureAppDbPassword(ctx, opsManager, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("Error getting AppDB password: %w", err)), log, opsManagerExtraStatusParams) + } + + opsManagerReconcilerHelper, err := NewOpsManagerReconcilerHelper(ctx, r, opsManager, r.memberClustersMap, log) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(err), log, opsManagerExtraStatusParams) + } + + appDBConnectionString := buildMongoConnectionUrl(opsManager, appDBPassword, appDbReconciler.getCurrentStatefulsetHostnames(opsManager)) + for _, memberCluster := range opsManagerReconcilerHelper.getHealthyMemberClusters() { + if err := r.ensureAppDBConnectionStringInMemberCluster(ctx, opsManager, appDBConnectionString, memberCluster, log); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(xerrors.Errorf("error ensuring AppDB connection string in cluster %s: %w", memberCluster.Name, err)), log, opsManagerExtraStatusParams) + } + } + + initOpsManagerImage := images.ContainerImage(r.imageUrls, util.InitOpsManagerImageUrl, r.initOpsManagerImageVersion) + opsManagerImage := images.ContainerImage(r.imageUrls, util.OpsManagerImageUrl, opsManager.Spec.Version) + + // 2. Reconcile Ops Manager + status, omAdmin := r.reconcileOpsManager(ctx, opsManagerReconcilerHelper, opsManager, appDBConnectionString, initOpsManagerImage, opsManagerImage, log) + if !status.IsOK() { + return r.updateStatus(ctx, opsManager, status, log, opsManagerExtraStatusParams, mdbstatus.NewBaseUrlOption(opsManager.CentralURL())) + } + + // the AppDB still needs to configure monitoring, now that Ops Manager has been created + // we can finish this configuration. + if result.Requeue { + log.Infof("Requeuing reconciliation to configure AppDB monitoring in Ops Manager.") + return result, nil + } + + // 3. Reconcile Backup Daemon + if status := r.reconcileBackupDaemon(ctx, opsManagerReconcilerHelper, opsManager, omAdmin, appDBConnectionString, initOpsManagerImage, opsManagerImage, log); !status.IsOK() { + return r.updateStatus(ctx, opsManager, status, log, mdbstatus.NewOMPartOption(mdbstatus.Backup)) + } + + annotationsToAdd, err := getAnnotationsForOpsManagerResource(opsManager) + if err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(err), log) + } + + if vault.IsVaultSecretBackend() { + vaultMap := make(map[string]string) + for _, s := range opsManager.GetSecretsMountedIntoPod() { + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.OpsManagerSecretMetadataPath(), appDBReplicaSet.Namespace, s) + vaultMap = merge.StringToStringMap(vaultMap, r.VaultClient.GetSecretAnnotation(path)) + } + for _, s := range opsManager.Spec.AppDB.GetSecretsMountedIntoPod() { + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.AppDBSecretMetadataPath(), appDBReplicaSet.Namespace, s) + vaultMap = merge.StringToStringMap(vaultMap, r.VaultClient.GetSecretAnnotation(path)) + } + + for k, val := range vaultMap { + annotationsToAdd[k] = val + } + } + + if err := annotations.SetAnnotations(ctx, opsManager, annotationsToAdd, r.client); err != nil { + return r.updateStatus(ctx, opsManager, workflow.Failed(err), log) + } + // All statuses are updated by now - we don't need to update any others - just return + log.Info("Finished reconciliation for MongoDbOpsManager!") + // success + return workflow.OK().ReconcileResult() +} + +// ensureSharedGlobalResources ensures that resources that are shared across watched namespaces (e.g. secrets) are in sync +func ensureSharedGlobalResources(ctx context.Context, secretGetUpdaterCreator secret.GetUpdateCreator, opsManager *omv1.MongoDBOpsManager) error { + operatorNamespace := env.ReadOrPanic(util.CurrentNamespace) // nolint:forbidigo + if operatorNamespace == opsManager.Namespace { + // nothing to sync, OM runs in the same namespace as the operator + return nil + } + + if imagePullSecretsName, found := env.Read(util.ImagePullSecrets); found { // nolint:forbidigo + imagePullSecrets, err := secretGetUpdaterCreator.GetSecret(ctx, kube.ObjectKey(operatorNamespace, imagePullSecretsName)) + if err != nil { + return err + } + + omNsSecret := secret.Builder(). + SetName(imagePullSecretsName). + SetNamespace(opsManager.Namespace). + SetByteData(imagePullSecrets.Data). + Build() + omNsSecret.Type = imagePullSecrets.Type + if err := createOrUpdateSecretIfNotFound(ctx, secretGetUpdaterCreator, omNsSecret); err != nil { + return err + } + } + + return nil +} + +// ensureResourcesForArchitectureChange ensures that the new resources expected to be present. +func ensureResourcesForArchitectureChange(ctx context.Context, acSecretClient, secretGetUpdaterCreator secret.GetUpdateCreator, opsManager *omv1.MongoDBOpsManager) error { + acSecret, err := acSecretClient.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.AutomationConfigSecretName())) + // if the automation config does not exist, we are not upgrading from an existing deployment. We can create everything from scratch. + if err != nil { + if !secrets.SecretNotExist(err) { + return xerrors.Errorf("error getting existing automation config secret: %w", err) + } + return nil + } + + ac, err := automationconfig.FromBytes(acSecret.Data[automationconfig.ConfigKey]) + if err != nil { + return xerrors.Errorf("error unmarshalling existing automation: %w", err) + } + + // the Ops Manager user should always exist within the automation config. + var omUser automationconfig.MongoDBUser + for _, authUser := range ac.Auth.Users { + if authUser.Username == util.OpsManagerMongoDBUserName { + omUser = authUser + break + } + } + + if omUser.Username == "" { + return xerrors.Errorf("ops manager user not present in the automation config") + } + + err = createOrUpdateSecretIfNotFound(ctx, secretGetUpdaterCreator, secret.Builder(). + SetName(opsManager.Spec.AppDB.OpsManagerUserScramCredentialsName()). + SetNamespace(opsManager.Namespace). + SetField("sha1-salt", omUser.ScramSha1Creds.Salt). + SetField("sha-1-server-key", omUser.ScramSha1Creds.ServerKey). + SetField("sha-1-stored-key", omUser.ScramSha1Creds.StoredKey). + SetField("sha256-salt", omUser.ScramSha256Creds.Salt). + SetField("sha-256-server-key", omUser.ScramSha256Creds.ServerKey). + SetField("sha-256-stored-key", omUser.ScramSha256Creds.StoredKey). + Build()) + if err != nil { + return xerrors.Errorf("failed to create/update scram credentials secret for Ops Manager user: %w", err) + } + + // ensure that the agent password stays consistent with what it was previously + err = createOrUpdateSecretIfNotFound(ctx, secretGetUpdaterCreator, secret.Builder(). + SetName(opsManager.Spec.AppDB.GetAgentPasswordSecretNamespacedName().Name). + SetNamespace(opsManager.Spec.AppDB.GetAgentPasswordSecretNamespacedName().Namespace). + SetField(constants.AgentPasswordKey, ac.Auth.AutoPwd). + Build()) + if err != nil { + return xerrors.Errorf("failed to create/update password secret for agent user: %w", err) + } + + // ensure that the keyfile stays consistent with what it was previously + err = createOrUpdateSecretIfNotFound(ctx, secretGetUpdaterCreator, secret.Builder(). + SetName(opsManager.Spec.AppDB.GetAgentKeyfileSecretNamespacedName().Name). + SetNamespace(opsManager.Spec.AppDB.GetAgentKeyfileSecretNamespacedName().Namespace). + SetField(constants.AgentKeyfileKey, ac.Auth.Key). + Build()) + if err != nil { + return xerrors.Errorf("failed to create/update keyfile secret for agent user: %w", err) + } + + // there was a rename for a specific secret, `om-resource-db-password -> om-resource-db-om-password` + // this was done as now there are multiple secrets associated with the AppDB, and the contents of this old one correspond to the Ops Manager user. + oldOpsManagerUserPasswordSecret, err := secretGetUpdaterCreator.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.Name()+"-password")) + if err != nil { + // if it's not there, we don't want to create it. We only want to create the new secret if it is present. + if secrets.SecretNotExist(err) { + return nil + } + return err + } + + return secret.CreateOrUpdate(ctx, secretGetUpdaterCreator, secret.Builder(). + SetNamespace(opsManager.Namespace). + SetName(opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName()). + SetByteData(oldOpsManagerUserPasswordSecret.Data). + Build(), + ) +} + +// createOrUpdateSecretIfNotFound creates the given secret if it does not exist. +func createOrUpdateSecretIfNotFound(ctx context.Context, secretGetUpdaterCreator secret.GetUpdateCreator, desiredSecret corev1.Secret) error { + _, err := secretGetUpdaterCreator.GetSecret(ctx, kube.ObjectKey(desiredSecret.Namespace, desiredSecret.Name)) + if err != nil { + if secrets.SecretNotExist(err) { + return secret.CreateOrUpdate(ctx, secretGetUpdaterCreator, desiredSecret) + } + return xerrors.Errorf("error getting secret %s/%s: %w", desiredSecret.Namespace, desiredSecret.Name, err) + } + return nil +} + +func (r *OpsManagerReconciler) reconcileOpsManager(ctx context.Context, reconcilerHelper *OpsManagerReconcilerHelper, opsManager *omv1.MongoDBOpsManager, appDBConnectionString, initOpsManagerImage, opsManagerImage string, log *zap.SugaredLogger) (workflow.Status, api.OpsManagerAdmin) { + var genKeySecretMap map[string][]byte + var err error + if genKeySecretMap, err = r.ensureGenKeyInOperatorCluster(ctx, opsManager, log); err != nil { + return workflow.Failed(xerrors.Errorf("error in ensureGenKeyInOperatorCluster: %w", err)), nil + } + + if err := r.replicateGenKeyInMemberClusters(ctx, reconcilerHelper, genKeySecretMap); err != nil { + return workflow.Failed(xerrors.Errorf("error in replicateGenKeyInMemberClusters: %w", err)), nil + } + + if err := r.replicateTLSCAInMemberClusters(ctx, reconcilerHelper); err != nil { + return workflow.Failed(xerrors.Errorf("error in replicateTLSCAInMemberClusters: %w", err)), nil + } + + if err := r.replicateAppDBTLSCAInMemberClusters(ctx, reconcilerHelper); err != nil { + return workflow.Failed(xerrors.Errorf("error in replicateAppDBTLSCAInMemberClusters: %w", err)), nil + } + + if err := r.replicateKMIPCAInMemberClusters(ctx, reconcilerHelper); err != nil { + return workflow.Failed(xerrors.Errorf("error in replicateKMIPCAInMemberClusters: %w", err)), nil + } + + if err := r.replicateQueryableBackupTLSSecretInMemberClusters(ctx, reconcilerHelper); err != nil { + return workflow.Failed(xerrors.Errorf("error in replicateQueryableBackupTLSSecretInMemberClusters: %w", err)), nil + } + + if err := r.replicateLogBackInMemberClusters(ctx, reconcilerHelper); err != nil { + return workflow.Failed(xerrors.Errorf("error in replicateQueryableBackupTLSSecretInMemberClusters: %w", err)), nil + } + + // Prepare Ops Manager StatefulSets in parallel in all member clusters + for _, memberCluster := range reconcilerHelper.getHealthyMemberClusters() { + status := r.createOpsManagerStatefulsetInMemberCluster(ctx, reconcilerHelper, appDBConnectionString, memberCluster, initOpsManagerImage, opsManagerImage, log) + if !status.IsOK() { + return status, nil + } + } + + // wait for all statefulsets to become ready + var statefulSetStatus workflow.Status = workflow.OK() + for _, memberCluster := range reconcilerHelper.getHealthyMemberClusters() { + status := getStatefulSetStatus(ctx, opsManager.Namespace, reconcilerHelper.OpsManagerStatefulSetNameForMemberCluster(memberCluster), memberCluster.Client) + statefulSetStatus = statefulSetStatus.Merge(status) + } + if !statefulSetStatus.IsOK() { + return statefulSetStatus, nil + } + + opsManagerURL := opsManager.CentralURL() + + // 3. Prepare Ops Manager (ensure the first user is created and public API key saved to secret) + var omAdmin api.OpsManagerAdmin + var status workflow.Status + if status, omAdmin = r.prepareOpsManager(ctx, opsManager, opsManagerURL, log); !status.IsOK() { + return status, nil + } + + // 4. Trigger agents upgrade if necessary + if err := triggerOmChangedEventIfNeeded(ctx, opsManager, r.client, log); err != nil { + log.Warn("Not triggering an Ops Manager version changed event: %s", err) + } + + // 5. Stop backup daemon if necessary + if err := r.stopBackupDaemonIfNeeded(ctx, reconcilerHelper); err != nil { + return workflow.Failed(err), nil + } + + statusOptions := []mdbstatus.Option{mdbstatus.NewOMPartOption(mdbstatus.OpsManager), mdbstatus.NewBaseUrlOption(opsManagerURL)} + if _, err := r.updateStatus(ctx, opsManager, workflow.OK(), log, statusOptions...); err != nil { + return workflow.Failed(err), nil + } + + return status, omAdmin +} + +// triggerOmChangedEventIfNeeded triggers upgrade process for all the MongoDB agents in the system if the major/minor version upgrade +// happened for Ops Manager +func triggerOmChangedEventIfNeeded(ctx context.Context, opsManager *omv1.MongoDBOpsManager, c kubernetesClient.Client, log *zap.SugaredLogger) error { + if opsManager.Spec.Version == opsManager.Status.OpsManagerStatus.Version || opsManager.Status.OpsManagerStatus.Version == "" { + return nil + } + newVersion, err := versionutil.StringToSemverVersion(opsManager.Spec.Version) + if err != nil { + return xerrors.Errorf("failed to parse Ops Manager version %s: %w", opsManager.Spec.Version, err) + } + oldVersion, err := versionutil.StringToSemverVersion(opsManager.Status.OpsManagerStatus.Version) + if err != nil { + return xerrors.Errorf("failed to parse Ops Manager status version %s: %w", opsManager.Status.OpsManagerStatus.Version, err) + } + if newVersion.Major != oldVersion.Major || newVersion.Minor != oldVersion.Minor { + log.Infof("Ops Manager version has upgraded from %s to %s - scheduling the upgrade for all the Agents in the system", oldVersion, newVersion) + if architectures.IsRunningStaticArchitecture(opsManager.Annotations) { + mdbList := &mdbv1.MongoDBList{} + err := c.List(ctx, mdbList) + if err != nil { + return err + } + for _, m := range mdbList.Items { + OmUpdateChannel <- event.GenericEvent{Object: &m} + } + + multiList := &mdbmulti.MongoDBMultiClusterList{} + err = c.List(ctx, multiList) + if err != nil { + return err + } + for _, m := range multiList.Items { + OmUpdateChannel <- event.GenericEvent{Object: &m} + } + + } else { + // This is a noop in static-architecture world + agents.ScheduleUpgrade() + } + } + + return nil +} + +// stopBackupDaemonIfNeeded stops the backup daemon when OM is upgraded. +// Otherwise, the backup daemon will remain in a broken state (because of version missmatch between OM and backup daemon) +// due to this STS limitation: https://github.com/kubernetes/kubernetes/issues/67250. +// Later, the normal reconcile process will update the STS and start the backup daemon. +func (r *OpsManagerReconciler) stopBackupDaemonIfNeeded(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper) error { + opsManager := reconcileHelper.opsManager + if opsManager.Spec.Version == opsManager.Status.OpsManagerStatus.Version || opsManager.Status.OpsManagerStatus.Version == "" { + return nil + } + + for _, memberCluster := range reconcileHelper.getHealthyMemberClusters() { + if _, err := r.scaleStatefulSet(ctx, opsManager.Namespace, reconcileHelper.BackupDaemonStatefulSetNameForMemberCluster(memberCluster), 0, memberCluster.Client); client.IgnoreNotFound(err) != nil { + return err + } + + // delete all backup daemon pods, scaling down the statefulSet to 0 does not terminate the pods, + // if the number of pods is greater than 1 and all of them are in an unhealthy state + cleanupOptions := mdbv1.MongodbCleanUpOptions{ + Namespace: opsManager.Namespace, + Labels: map[string]string{ + "app": reconcileHelper.BackupDaemonHeadlessServiceNameForMemberCluster(memberCluster), + }, + } + + if err := memberCluster.Client.DeleteAllOf(ctx, &corev1.Pod{}, &cleanupOptions); client.IgnoreNotFound(err) != nil { + return err + } + } + + return nil +} + +func (r *OpsManagerReconciler) reconcileBackupDaemon(ctx context.Context, reconcilerHelper *OpsManagerReconcilerHelper, opsManager *omv1.MongoDBOpsManager, omAdmin api.OpsManagerAdmin, appDBConnectionString, initOpsManagerImage, opsManagerImage string, log *zap.SugaredLogger) workflow.Status { + backupStatusPartOption := mdbstatus.NewOMPartOption(mdbstatus.Backup) + + // If backup is not enabled, we check whether it is still configured in OM to update the status. + if !opsManager.Spec.Backup.Enabled { + var backupStatus workflow.Status + backupStatus = workflow.Disabled() + + for _, fqdn := range reconcilerHelper.BackupDaemonHeadlessFQDNs() { + // In case there is a backup daemon running still, while backup is not enabled, we check whether it is still configured in OM. + // We're keeping the `Running` status if still configured and marking it as `Disabled` otherwise. + backupStatus = workflow.OK() + _, err := omAdmin.ReadDaemonConfig(fqdn.hostname, util.PvcMountPathHeadDb) + if apierror.NewNonNil(err).ErrorBackupDaemonConfigIsNotFound() || errors.Is(err, syscall.ECONNREFUSED) { + backupStatus = workflow.Disabled() + break + } + } + + _, err := r.updateStatus(ctx, opsManager, backupStatus, log, backupStatusPartOption) + if err != nil { + return workflow.Failed(err) + } + return backupStatus + } + + for _, memberCluster := range reconcilerHelper.getHealthyMemberClusters() { + // Prepare Backup Daemon StatefulSet (create and wait) + if status := r.createBackupDaemonStatefulset(ctx, reconcilerHelper, appDBConnectionString, memberCluster, initOpsManagerImage, opsManagerImage, log); !status.IsOK() { + return status + } + } + + // Configure Backup using API + if status := r.prepareBackupInOpsManager(ctx, reconcilerHelper, opsManager, omAdmin, appDBConnectionString, log); !status.IsOK() { + return status + } + + // StatefulSet will reach ready state eventually once backup has been configured in Ops Manager. + + // wait for all statefulsets to become ready + for _, memberCluster := range reconcilerHelper.getHealthyMemberClusters() { + if status := getStatefulSetStatus(ctx, opsManager.Namespace, reconcilerHelper.BackupDaemonStatefulSetNameForMemberCluster(memberCluster), memberCluster.Client); !status.IsOK() { + return status + } + } + + if _, err := r.updateStatus(ctx, opsManager, workflow.OK(), log, backupStatusPartOption); err != nil { + return workflow.Failed(err) + } + + return workflow.OK() +} + +// readOpsManagerResource reads Ops Manager Custom resource into pointer provided +func (r *OpsManagerReconciler) readOpsManagerResource(ctx context.Context, request reconcile.Request, ref *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (reconcile.Result, error) { + if result, err := r.getResource(ctx, request, ref, log); err != nil { + return result, err + } + // Reset warnings so that they are not stale, will populate accurate warnings in reconciliation + ref.SetWarnings([]mdbstatus.Warning{}, mdbstatus.NewOMPartOption(mdbstatus.OpsManager), mdbstatus.NewOMPartOption(mdbstatus.AppDb), mdbstatus.NewOMPartOption(mdbstatus.Backup)) + return reconcile.Result{}, nil +} + +// ensureAppDBConnectionString ensures that the AppDB Connection String exists in a secret. +func (r *OpsManagerReconciler) ensureAppDBConnectionStringInMemberCluster(ctx context.Context, opsManager *omv1.MongoDBOpsManager, computedConnectionString string, memberCluster multicluster.MemberCluster, log *zap.SugaredLogger) error { + var opsManagerSecretPath string + if r.VaultClient != nil { + opsManagerSecretPath = r.VaultClient.OpsManagerSecretPath() + } + + _, err := memberCluster.SecretClient.ReadSecret(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.AppDBMongoConnectionStringSecretName()), opsManagerSecretPath) + if err != nil { + if secrets.SecretNotExist(err) { + log.Debugf("AppDB connection string secret was not found in cluster %s, creating %s now", memberCluster.Name, kube.ObjectKey(opsManager.Namespace, opsManager.AppDBMongoConnectionStringSecretName())) + // assume the secret was not found, need to create it + + connectionStringSecret := secret.Builder(). + SetName(opsManager.AppDBMongoConnectionStringSecretName()). + SetNamespace(opsManager.Namespace). + SetField(util.AppDbConnectionStringKey, computedConnectionString). + Build() + + return memberCluster.SecretClient.PutSecret(ctx, connectionStringSecret, opsManagerSecretPath) + } + log.Warnf("Error getting connection string secret: %s", err) + return err + } + + connectionStringSecretData := map[string]string{ + util.AppDbConnectionStringKey: computedConnectionString, + } + connectionStringSecret := secret.Builder(). + SetName(opsManager.AppDBMongoConnectionStringSecretName()). + SetNamespace(opsManager.Namespace). + SetStringMapToData(connectionStringSecretData).Build() + log.Debugf("Connection string secret already exists, updating %s", kube.ObjectKey(opsManager.Namespace, opsManager.AppDBMongoConnectionStringSecretName())) + return memberCluster.SecretClient.PutSecret(ctx, connectionStringSecret, opsManagerSecretPath) +} + +func hashConnectionString(connectionString string) string { + bytes := []byte(connectionString) + hashBytes := sha256.Sum256(bytes) + return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hashBytes[:]) +} + +func (r *OpsManagerReconciler) createOpsManagerStatefulsetInMemberCluster(ctx context.Context, reconcilerHelper *OpsManagerReconcilerHelper, appDBConnectionString string, memberCluster multicluster.MemberCluster, initOpsManagerImage, opsManagerImage string, log *zap.SugaredLogger) workflow.Status { + opsManager := reconcilerHelper.opsManager + + r.ensureConfiguration(reconcilerHelper, log) + + var vaultConfig vault.VaultConfiguration + if r.VaultClient != nil { + vaultConfig = r.VaultClient.VaultConfig + } + + debugPort, err := opsManager.Spec.DebugPort() + if err != nil { + log.Debugf("Error while retrieving debug port for Ops Manager: %s", err) + } + + clusterSpecItem := reconcilerHelper.getClusterSpecOMItem(memberCluster.Name) + sts, err := construct.OpsManagerStatefulSet(ctx, r.SecretClient, opsManager, memberCluster, log, + construct.WithInitOpsManagerImage(initOpsManagerImage), + construct.WithOpsManagerImage(opsManagerImage), + construct.WithConnectionStringHash(hashConnectionString(appDBConnectionString)), + construct.WithVaultConfig(vaultConfig), + construct.WithKmipConfig(ctx, opsManager, memberCluster.Client, log), + construct.WithStsOverride(clusterSpecItem.GetStatefulSetSpecOverride()), + construct.WithReplicas(reconcilerHelper.OpsManagerMembersForMemberCluster(memberCluster)), + construct.WithDebugPort(debugPort), + ) + if err != nil { + return workflow.Failed(xerrors.Errorf("error building OpsManager stateful set: %w", err)) + } + + if err := create.OpsManagerInKubernetes(ctx, memberCluster, opsManager, sts, log); err != nil { + return workflow.Failed(err) + } + + return workflow.OK() +} + +func AddOpsManagerController(ctx context.Context, mgr manager.Manager, memberClustersMap map[string]cluster.Cluster, imageUrls images.ImageUrls, initAppdbVersion, initOpsManagerImageVersion string) error { + reconciler := NewOpsManagerReconciler(ctx, mgr.GetClient(), multicluster.ClustersMapToClientMap(memberClustersMap), imageUrls, initAppdbVersion, initOpsManagerImageVersion, om.NewOpsManagerConnection, &api.DefaultInitializer{}, api.NewOmAdmin) + c, err := controller.New(util.MongoDbOpsManagerController, mgr, controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}) // nolint:forbidigo + if err != nil { + return err + } + + // watch for changes to the Ops Manager resources + eventHandler := MongoDBOpsManagerEventHandler{reconciler: reconciler} + + if err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &omv1.MongoDBOpsManager{}, &eventHandler, watch.PredicatesForOpsManager())); err != nil { + return err + } + + // watch the secret with the Ops Manager user password + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &mdbv1.MongoDB{}, + &watch.ResourcesHandler{ResourceType: watch.MongoDB, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + // if vault secret backend is enabled watch for Vault secret change and trigger reconcile + if vault.IsVaultSecretBackend() { + eventChannel := make(chan event.GenericEvent) + go vaultwatcher.WatchSecretChangeForOM(ctx, zap.S(), eventChannel, reconciler.client, reconciler.VaultClient) + + err = c.Watch(source.Channel[client.Object](eventChannel, &handler.EnqueueRequestForObject{})) + if err != nil { + zap.S().Errorf("Failed to watch for vault secret changes: %v", err) + } + } + zap.S().Infof("Registered controller %s", util.MongoDbOpsManagerController) + return nil +} + +// ensureConfiguration makes sure the mandatory configuration is specified. +func (r *OpsManagerReconciler) ensureConfiguration(reconcilerHelper *OpsManagerReconcilerHelper, log *zap.SugaredLogger) { + // update the central URL + setConfigProperty(reconcilerHelper.opsManager, util.MmsCentralUrlPropKey, reconcilerHelper.opsManager.CentralURL(), log) + + if reconcilerHelper.opsManager.Spec.AppDB.Security.IsTLSEnabled() { + setConfigProperty(reconcilerHelper.opsManager, util.MmsMongoSSL, "true", log) + } + if reconcilerHelper.opsManager.Spec.AppDB.GetCAConfigMapName() != "" { + setConfigProperty(reconcilerHelper.opsManager, util.MmsMongoCA, omv1.GetAppDBCaPemPath(), log) + } + + // override the versions directory (defaults to "/opt/mongodb/mms/mongodb-releases/") + setConfigProperty(reconcilerHelper.opsManager, util.MmsVersionsDirectory, "/mongodb-ops-manager/mongodb-releases/", log) + + // feature controls will always be enabled + setConfigProperty(reconcilerHelper.opsManager, util.MmsFeatureControls, "true", log) + + if reconcilerHelper.opsManager.Spec.Backup.QueryableBackupSecretRef.Name != "" { + setConfigProperty(reconcilerHelper.opsManager, util.BrsQueryablePem, "/certs/queryable.pem", log) + } +} + +// createBackupDaemonStatefulset creates a StatefulSet for backup daemon and waits shortly until it's started +// Note, that the idea of creating two statefulsets for Ops Manager and Backup Daemon in parallel hasn't worked out +// as the daemon in this case just hangs silently (in practice it's ok to start it in ~1 min after start of OM though +// we will just start them sequentially) +func (r *OpsManagerReconciler) createBackupDaemonStatefulset(ctx context.Context, reconcilerHelper *OpsManagerReconcilerHelper, appDBConnectionString string, memberCluster multicluster.MemberCluster, initOpsManagerImage, opsManagerImage string, log *zap.SugaredLogger) workflow.Status { + if !reconcilerHelper.opsManager.Spec.Backup.Enabled { + return workflow.OK() + } + + if err := r.ensureAppDBConnectionStringInMemberCluster(ctx, reconcilerHelper.opsManager, appDBConnectionString, memberCluster, log); err != nil { + return workflow.Failed(err) + } + + r.ensureConfiguration(reconcilerHelper, log) + + var vaultConfig vault.VaultConfiguration + if r.VaultClient != nil { + vaultConfig = r.VaultClient.VaultConfig + } + clusterSpecItem := reconcilerHelper.getClusterSpecOMItem(memberCluster.Name) + sts, err := construct.BackupDaemonStatefulSet(ctx, r.SecretClient, reconcilerHelper.opsManager, memberCluster, log, + construct.WithInitOpsManagerImage(initOpsManagerImage), + construct.WithOpsManagerImage(opsManagerImage), + construct.WithConnectionStringHash(hashConnectionString(appDBConnectionString)), + construct.WithVaultConfig(vaultConfig), + // TODO KMIP support will not work across clusters + construct.WithKmipConfig(ctx, reconcilerHelper.opsManager, memberCluster.Client, log), + construct.WithStsOverride(clusterSpecItem.GetBackupStatefulSetSpecOverride()), + construct.WithReplicas(reconcilerHelper.BackupDaemonMembersForMemberCluster(memberCluster)), + ) + if err != nil { + return workflow.Failed(xerrors.Errorf("error building stateful set: %w", err)) + } + + needToRequeue, err := create.BackupDaemonInKubernetes(ctx, memberCluster.Client, reconcilerHelper.opsManager, sts, log) + if err != nil { + return workflow.Failed(err) + } + if needToRequeue { + return workflow.OK().Requeue() + } + return workflow.OK() +} + +func (r *OpsManagerReconciler) watchMongoDBResourcesReferencedByKmip(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) { + if !opsManager.Spec.IsKmipEnabled() { + return + } + + mdbList := &mdbv1.MongoDBList{} + err := r.client.List(ctx, mdbList) + if err != nil { + log.Warnf("failed to fetch MongoDBList from Kubernetes: %v", err) + } + + for _, m := range mdbList.Items { + if m.Spec.Backup != nil && m.Spec.Backup.IsKmipEnabled() { + r.resourceWatcher.AddWatchedResourceIfNotAdded( + m.Name, + m.Namespace, + watch.MongoDB, + kube.ObjectKeyFromApiObject(opsManager)) + + r.resourceWatcher.AddWatchedResourceIfNotAdded( + m.Spec.Backup.Encryption.Kmip.Client.ClientCertificateSecretName(m.GetName()), + opsManager.Namespace, + watch.Secret, + kube.ObjectKeyFromApiObject(opsManager)) + + r.resourceWatcher.AddWatchedResourceIfNotAdded( + m.Spec.Backup.Encryption.Kmip.Client.ClientCertificatePasswordSecretName(m.GetName()), + opsManager.Namespace, + watch.Secret, + kube.ObjectKeyFromApiObject(opsManager)) + } + } +} + +func (r *OpsManagerReconciler) watchCaReferencedByKmip(opsManager *omv1.MongoDBOpsManager) { + if !opsManager.Spec.IsKmipEnabled() { + return + } + + r.resourceWatcher.AddWatchedResourceIfNotAdded( + opsManager.Spec.Backup.Encryption.Kmip.Server.CA, + opsManager.Namespace, + watch.ConfigMap, + kube.ObjectKeyFromApiObject(opsManager)) +} + +func (r *OpsManagerReconciler) watchMongoDBResourcesReferencedByBackup(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) { + if !opsManager.Spec.Backup.Enabled { + return + } + + // watch mongodb resources for oplog + oplogs := opsManager.Spec.Backup.OplogStoreConfigs + for _, oplogConfig := range oplogs { + r.resourceWatcher.AddWatchedResourceIfNotAdded( + oplogConfig.MongoDBResourceRef.Name, + opsManager.Namespace, + watch.MongoDB, + kube.ObjectKeyFromApiObject(opsManager), + ) + } + + // watch mongodb resources for block stores + blockstores := opsManager.Spec.Backup.BlockStoreConfigs + for _, blockStoreConfig := range blockstores { + r.resourceWatcher.AddWatchedResourceIfNotAdded( + blockStoreConfig.MongoDBResourceRef.Name, + opsManager.Namespace, + watch.MongoDB, + kube.ObjectKeyFromApiObject(opsManager), + ) + } + + // watch mongodb resources for s3 stores + s3Stores := opsManager.Spec.Backup.S3Configs + for _, s3StoreConfig := range s3Stores { + // If S3StoreConfig doesn't have mongodb resource reference, skip it (appdb will be used) + if s3StoreConfig.MongoDBResourceRef != nil { + r.resourceWatcher.AddWatchedResourceIfNotAdded( + s3StoreConfig.MongoDBResourceRef.Name, + opsManager.Namespace, + watch.MongoDB, + kube.ObjectKeyFromApiObject(opsManager), + ) + } + } + + r.watchMongoDBResourcesReferencedByKmip(ctx, opsManager, log) + r.watchCaReferencedByKmip(opsManager) +} + +// buildMongoConnectionUrl returns a connection URL to the appdb. +// +// Note, that it overrides the default authMechanism (which internally depends +// on the mongodb version). +func buildMongoConnectionUrl(opsManager *omv1.MongoDBOpsManager, password string, multiClusterHostnames []string) string { + connectionString := opsManager.Spec.AppDB.BuildConnectionURL( + util.OpsManagerMongoDBUserName, + password, + connectionstring.SchemeMongoDB, + map[string]string{"authMechanism": "SCRAM-SHA-256"}, + multiClusterHostnames) + + return connectionString +} + +func setConfigProperty(opsManager *omv1.MongoDBOpsManager, key, value string, log *zap.SugaredLogger) { + if opsManager.AddConfigIfDoesntExist(key, value) { + if key == util.MmsMongoUri { + log.Debugw("Configured property", key, util.RedactMongoURI(value)) + } else { + log.Debugw("Configured property", key, value) + } + } +} + +func (r *OpsManagerReconciler) ensureGenKeyInOperatorCluster(ctx context.Context, om *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (map[string][]byte, error) { + objectKey := kube.ObjectKey(om.Namespace, om.Name+"-gen-key") + var opsManagerSecretPath string + if r.VaultClient != nil { + opsManagerSecretPath = r.VaultClient.OpsManagerSecretPath() + } + genKeySecretMap, err := r.ReadBinarySecret(ctx, objectKey, opsManagerSecretPath) + if err == nil { + return genKeySecretMap, nil + } + + if secrets.SecretNotExist(err) { + // todo if the key is not found but the AppDB is initialized - OM will fail to start as preflight + // check will complain that keys are different - we need to validate against this here + + // the length must be equal to 'EncryptionUtils.DES3_KEY_LENGTH' (24) from mms + token := make([]byte, 24) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + keyMap := map[string][]byte{"gen.key": token} + + log.Infof("Creating secret %s", objectKey) + + genKeySecret := secret.Builder(). + SetName(objectKey.Name). + SetNamespace(objectKey.Namespace). + SetLabels(map[string]string{}). + SetByteData(keyMap). + Build() + + return keyMap, r.PutBinarySecret(ctx, genKeySecret, opsManagerSecretPath) + } + + return nil, xerrors.Errorf("error reading secret %v: %w", objectKey, err) +} + +func (r *OpsManagerReconciler) replicateGenKeyInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper, secretMap map[string][]byte) error { + if !reconcileHelper.opsManager.Spec.IsMultiCluster() { + return nil + } + objectKey := kube.ObjectKey(reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Name+"-gen-key") + var opsManagerSecretPath string + if r.VaultClient != nil { + opsManagerSecretPath = r.VaultClient.OpsManagerSecretPath() + } + + genKeySecret := secret.Builder(). + SetName(objectKey.Name). + SetNamespace(objectKey.Namespace). + SetLabels(map[string]string{}). + SetByteData(secretMap). + Build() + + for _, memberCluster := range reconcileHelper.getHealthyMemberClusters() { + if err := memberCluster.SecretClient.PutBinarySecret(ctx, genKeySecret, opsManagerSecretPath); err != nil { + return xerrors.Errorf("error replicating %v secret to cluster %s: %w", objectKey, memberCluster.Name, err) + } + } + + return nil +} + +func (r *OpsManagerReconciler) replicateQueryableBackupTLSSecretInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper) error { + if !reconcileHelper.opsManager.Spec.IsMultiCluster() { + return nil + } + + if reconcileHelper.opsManager.Spec.Backup == nil || reconcileHelper.opsManager.Spec.Backup.QueryableBackupSecretRef.Name == "" { + return nil + } + return r.replicateSecretInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.Backup.QueryableBackupSecretRef.Name) +} + +func (r *OpsManagerReconciler) replicateSecretInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper, namespace string, secretName string) error { + objectKey := kube.ObjectKey(namespace, secretName) + var opsManagerSecretPath string + if r.VaultClient != nil { + opsManagerSecretPath = r.VaultClient.OpsManagerSecretPath() + } + secretMap, err := r.ReadSecret(ctx, objectKey, opsManagerSecretPath) + if err != nil { + return xerrors.Errorf("failed to read secret %s: %w", secretName, err) + } + + newSecret := secret.Builder(). + SetName(objectKey.Name). + SetNamespace(objectKey.Namespace). + SetLabels(map[string]string{}). + SetStringMapToData(secretMap). + Build() + + for _, memberCluster := range reconcileHelper.getHealthyMemberClusters() { + if err := memberCluster.SecretClient.PutSecretIfChanged(ctx, newSecret, opsManagerSecretPath); err != nil { + return xerrors.Errorf("error replicating secret %v to cluster %s: %w", objectKey, memberCluster.Name, err) + } + } + + return nil +} + +func (r *OpsManagerReconciler) replicateTLSCAInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper) error { + if !reconcileHelper.opsManager.Spec.IsMultiCluster() || reconcileHelper.opsManager.Spec.GetOpsManagerCA() == "" { + return nil + } + + return r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.GetOpsManagerCA()) +} + +func (r *OpsManagerReconciler) replicateLogBackInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper) error { + if !reconcileHelper.opsManager.Spec.IsMultiCluster() || reconcileHelper.opsManager.Spec.Logging == nil { + return nil + } + + if reconcileHelper.opsManager.Spec.Logging.LogBackRef != nil { + if err := r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.Logging.LogBackRef.Name); err != nil { + return err + } + } + if reconcileHelper.opsManager.Spec.Logging.LogBackAccessRef != nil { + if err := r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.Logging.LogBackAccessRef.Name); err != nil { + return err + } + } + + if !reconcileHelper.opsManager.Spec.Backup.Enabled { + return nil + } + + if reconcileHelper.opsManager.Spec.Backup.Logging.LogBackRef != nil { + if err := r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.Backup.Logging.LogBackRef.Name); err != nil { + return err + } + } + if reconcileHelper.opsManager.Spec.Backup.Logging.LogBackAccessRef != nil { + if err := r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.Backup.Logging.LogBackAccessRef.Name); err != nil { + return err + } + } + return nil +} + +func (r *OpsManagerReconciler) replicateAppDBTLSCAInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper) error { + if !reconcileHelper.opsManager.Spec.IsMultiCluster() || reconcileHelper.opsManager.Spec.GetAppDbCA() == "" { + return nil + } + + return r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.GetAppDbCA()) +} + +func (r *OpsManagerReconciler) replicateKMIPCAInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper) error { + if !reconcileHelper.opsManager.Spec.IsKmipEnabled() { + return nil + } + + return r.replicateConfigMapInMemberClusters(ctx, reconcileHelper, reconcileHelper.opsManager.Namespace, reconcileHelper.opsManager.Spec.Backup.Encryption.Kmip.Server.CA) +} + +func (r *OpsManagerReconciler) replicateConfigMapInMemberClusters(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper, namespace string, configMapName string) error { + if !reconcileHelper.opsManager.Spec.IsMultiCluster() || configMapName == "" { + return nil + } + + configMapNSName := types.NamespacedName{Name: configMapName, Namespace: namespace} + caConfigMapData, err := configmap.ReadData(ctx, r.client, configMapNSName) + if err != nil { + return xerrors.Errorf("failed to read config map %+v from central cluster: %w", configMapNSName, err) + } + + caConfigMap := configmap.Builder(). + SetName(configMapNSName.Name). + SetNamespace(configMapNSName.Namespace). + SetData(caConfigMapData). + Build() + + for _, memberCluster := range reconcileHelper.getHealthyMemberClusters() { + if err := configmap.CreateOrUpdate(ctx, memberCluster.Client, caConfigMap); err != nil && !apiErrors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create or update config map %+v in cluster %s: %w", configMapNSName, memberCluster.Name, err) + } + } + + return nil +} + +func (r *OpsManagerReconciler) getOpsManagerAPIKeySecretName(ctx context.Context, opsManager *omv1.MongoDBOpsManager) (string, workflow.Status) { + var operatorVaultSecretPath string + if r.VaultClient != nil { + operatorVaultSecretPath = r.VaultClient.OperatorSecretPath() + } + APISecretName, err := opsManager.APIKeySecretName(ctx, r.SecretClient, operatorVaultSecretPath) + if err != nil { + return "", workflow.Failed(xerrors.Errorf("failed to get ops-manager API key secret name: %w", err)).WithRetry(10) + } + return APISecretName, workflow.OK() +} + +func detailedAPIErrorMsg(adminKeySecretName types.NamespacedName) string { + return fmt.Sprintf("This is a fatal error, as the"+ + " Operator requires public API key for the admin user to exist. Please create the GLOBAL_ADMIN user in "+ + "Ops Manager manually and create a secret '%s' with fields '%s' and '%s'", adminKeySecretName, util.OmPublicApiKey, + util.OmPrivateKey) +} + +// prepareOpsManager ensures the admin user is created and the admin public key exists. It returns the instance of +// api.OpsManagerAdmin to perform future Ops Manager configuration +// Note the exception handling logic - if the controller fails to save the public API key secret - it cannot fix this +// manually (the first OM user can be created only once) - so the resource goes to Failed state and shows the message +// asking the user to fix this manually. +// Theoretically, the Operator could remove the appdb StatefulSet (as the OM must be empty without any user data) and +// allow the db to get recreated, but this is a quite radical operation. +func (r *OpsManagerReconciler) prepareOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, centralURL string, log *zap.SugaredLogger) (workflow.Status, api.OpsManagerAdmin) { + // We won't support cross-namespace secrets until CLOUDP-46636 is resolved + adminObjectKey := kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AdminSecret) + + var operatorVaultPath string + if r.VaultClient != nil { + operatorVaultPath = r.VaultClient.OperatorSecretPath() + } + + // 1. Read the admin secret + userData, err := r.ReadSecret(ctx, adminObjectKey, operatorVaultPath) + + if secrets.SecretNotExist(err) { + // This requires user actions - let's wait a bit longer than 10 seconds + return workflow.Failed(xerrors.Errorf("the secret %s doesn't exist - you need to create it to finish Ops Manager initialization", adminObjectKey)).WithRetry(60), nil + } else if err != nil { + return workflow.Failed(err), nil + } + + newUser, err := newUserFromSecret(userData) + if err != nil { + return workflow.Failed(xerrors.Errorf("failed to read user data from the secret %s: %w", adminObjectKey, err)), nil + } + + var ca *string + if opsManager.IsTLSEnabled() { + log.Debug("TLS is enabled, creating the first user with the mms-ca.crt") + opsManagerCA := opsManager.Spec.GetOpsManagerCA() + cm, err := r.client.GetConfigMap(ctx, kube.ObjectKey(opsManager.Namespace, opsManagerCA)) + if err != nil { + return workflow.Failed(xerrors.Errorf("failed to retrieve om ca certificate to create the initial user: %w", err)).WithRetry(30), nil + } + ca = ptr.To(cm.Data["mms-ca.crt"]) + } + + APISecretName, status := r.getOpsManagerAPIKeySecretName(ctx, opsManager) + if !status.IsOK() { + return status, nil + } + adminKeySecretName := kube.ObjectKey(operatorNamespace(), APISecretName) + // 2. Create a user in Ops Manager if necessary. Note, that we don't send the request if the API key secret exists. + // This is because of the weird Ops Manager /unauth endpoint logic: it allows to create any number of users though only + // the first one will have GLOBAL_ADMIN permission. So we should avoid the situation when the admin changes the + // user secret and reconciles OM resource and the new user (non admin one) is created overriding the previous API secret + _, err = r.ReadSecret(ctx, adminKeySecretName, operatorVaultPath) + if secrets.SecretNotExist(err) { + apiKey, err := r.omInitializer.TryCreateUser(centralURL, opsManager.Spec.Version, newUser, ca) + if err != nil { + // Will wait more than usual (10 seconds) as most of all the problem needs to get fixed by the user + // by modifying the credentials secret + return workflow.Failed(xerrors.Errorf("failed to create an admin user in Ops Manager: %w", err)).WithRetry(30), nil + } + + // Recreate an admin key secret in the Operator namespace if the user was created + if apiKey.PublicKey != "" { + log.Infof("Created an admin user %s with GLOBAL_ADMIN role", newUser.Username) + + // The structure matches the structure of a credentials secret used by normal mongodb resources + secretData := map[string]string{util.OmPublicApiKey: apiKey.PublicKey, util.OmPrivateKey: apiKey.PrivateKey} + + if err = r.client.DeleteSecret(ctx, adminKeySecretName); err != nil && !secrets.SecretNotExist(err) { + // TODO our desired behavior is not to fail but just append the warning to the status (CLOUDP-51340) + return workflow.Failed(xerrors.Errorf("failed to replace a secret for admin public api key. %s. The error : %w", + detailedAPIErrorMsg(adminKeySecretName), err)).WithRetry(300), nil + } + + adminSecret := secret.Builder(). + SetNamespace(adminKeySecretName.Namespace). + SetName(adminKeySecretName.Name). + SetStringMapToData(secretData). + SetLabels(map[string]string{}).Build() + + if err := r.PutSecret(ctx, adminSecret, operatorVaultPath); err != nil { + return workflow.Failed(xerrors.Errorf("failed to create a secret for admin public api key. %s. The error : %w", + detailedAPIErrorMsg(adminKeySecretName), err)).WithRetry(30), nil + } + log.Infof("Created a secret for admin public api key %s", adminKeySecretName) + } else { + log.Debug("Ops Manager did not return a valid User object.") + } + } + + // 3. Final validation of admin secret. We must ensure it's refreshed by the informers as ReadCredentials + // is going to fail otherwise. + readAdminKeySecretFunc := func() (string, bool) { + if _, err = r.ReadSecret(ctx, adminKeySecretName, operatorVaultPath); err != nil { + return fmt.Sprintf("%v", err), false + } + return "", true + } + if found, msg := util.DoAndRetry(readAdminKeySecretFunc, log, 10, 5); !found { + return workflow.Failed(xerrors.Errorf("admin API key secret for Ops Manager doesn't exist - was it removed accidentally? %s. The error: %s", + detailedAPIErrorMsg(adminKeySecretName), msg)).WithRetry(30), nil + } + + // Ops Manager api key Secret has the same structure as the MongoDB credentials secret + APIKeySecretName, err := opsManager.APIKeySecretName(ctx, r.SecretClient, operatorVaultPath) + if err != nil { + return workflow.Failed(err), nil + } + + cred, err := project.ReadCredentials(ctx, r.SecretClient, kube.ObjectKey(operatorNamespace(), APIKeySecretName), log) + if err != nil { + return workflow.Failed(xerrors.Errorf("failed to locate the api key secret. The error : %w", err)), nil + } + + admin := r.omAdminProvider(centralURL, cred.PublicAPIKey, cred.PrivateAPIKey, ca) + return workflow.OK(), admin +} + +// prepareBackupInOpsManager makes the changes to backup admin configuration based on the Ops Manager spec +func (r *OpsManagerReconciler) prepareBackupInOpsManager(ctx context.Context, reconcileHelper *OpsManagerReconcilerHelper, opsManager *omv1.MongoDBOpsManager, omAdmin api.OpsManagerAdmin, appDBConnectionString string, log *zap.SugaredLogger) workflow.Status { + if !opsManager.Spec.Backup.Enabled { + return workflow.OK() + } + + // 1. Enabling Daemon Config if necessary + backupFQDNs := reconcileHelper.BackupDaemonHeadlessFQDNs() + for _, fqdn := range backupFQDNs { + dc, err := omAdmin.ReadDaemonConfig(fqdn.hostname, util.PvcMountPathHeadDb) + if apierror.NewNonNil(err).ErrorBackupDaemonConfigIsNotFound() { + log.Infow("Backup Daemon is not configured, enabling it", "hostname", fqdn.hostname, "headDB", util.PvcMountPathHeadDb) + + err = omAdmin.CreateDaemonConfig(fqdn.hostname, util.PvcMountPathHeadDb, opsManager.GetMemberClusterBackupAssignmentLabels(fqdn.memberClusterName)) + if apierror.NewNonNil(err).ErrorBackupDaemonConfigIsNotFound() { + // Unfortunately by this time backup daemon may not have been started yet, and we don't have proper + // mechanism to ensure this using readiness probe, so we just retry + return workflow.Pending("BackupDaemon hasn't started yet") + } else if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } else if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } else { + // The Assignment Labels are the only thing that can change at the moment. + // If we add new features for controlling the Backup Daemons, we may want + // to compare the whole backup.DaemonConfig objects. + assignmentLabels := opsManager.GetMemberClusterBackupAssignmentLabels(fqdn.memberClusterName) + if !reflect.DeepEqual(assignmentLabels, dc.Labels) { + dc.Labels = assignmentLabels + err = omAdmin.UpdateDaemonConfig(dc) + if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + } + } + + // 2. Oplog store configs + status := r.ensureOplogStoresInOpsManager(ctx, opsManager, omAdmin, log) + + // 3. S3 Oplog Configs + status = status.Merge(r.ensureS3OplogStoresInOpsManager(ctx, opsManager, omAdmin, appDBConnectionString, log)) + + // 4. S3 Configs + status = status.Merge(r.ensureS3ConfigurationInOpsManager(ctx, opsManager, omAdmin, appDBConnectionString, log)) + + // 5. Block store configs + status = status.Merge(r.ensureBlockStoresInOpsManager(ctx, opsManager, omAdmin, log)) + + // 6. FileSystem store configs + status = status.Merge(r.ensureFileSystemStoreConfigurationInOpsManager(opsManager, omAdmin)) + if len(opsManager.Spec.Backup.S3Configs) == 0 && len(opsManager.Spec.Backup.BlockStoreConfigs) == 0 && len(opsManager.Spec.Backup.FileSystemStoreConfigs) == 0 { + return status.Merge(workflow.Invalid("Either S3 or Blockstore or FileSystem Snapshot configuration is required for backup").WithTargetPhase(mdbstatus.PhasePending)) + } + + return status +} + +// ensureOplogStoresInOpsManager aligns the oplog stores in Ops Manager with the Operator state. So it adds the new configs +// and removes the non-existing ones. Note that there's no update operation as so far the Operator manages only one field +// 'path'. This will allow users to make any additional changes to the file system stores using Ops Manager UI and the +// Operator won't override them +func (r *OpsManagerReconciler) ensureOplogStoresInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, omAdmin api.OplogStoreAdmin, log *zap.SugaredLogger) workflow.Status { + if !opsManager.Spec.Backup.Enabled { + return workflow.OK() + } + + opsManagerOplogConfigs, err := omAdmin.ReadOplogStoreConfigs() + if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + + // Creating new configs + operatorOplogConfigs := opsManager.Spec.Backup.OplogStoreConfigs + configsToCreate := identifiable.SetDifferenceGeneric(operatorOplogConfigs, opsManagerOplogConfigs) + for _, v := range configsToCreate { + omConfig, status := r.buildOMDatastoreConfig(ctx, opsManager, v.(omv1.DataStoreConfig)) + if !status.IsOK() { + return status + } + log.Debugw("Creating Oplog Store in Ops Manager", "config", omConfig) + if err = omAdmin.CreateOplogStoreConfig(omConfig); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Updating existing configs. It intersects the OM API configs with Operator spec configs and returns pairs + //["omConfig", "operatorConfig"]. + configsToUpdate := identifiable.SetIntersectionGeneric(opsManagerOplogConfigs, operatorOplogConfigs) + for _, v := range configsToUpdate { + omConfig := v[0].(backup.DataStoreConfig) + operatorConfig := v[1].(omv1.DataStoreConfig) + operatorView, status := r.buildOMDatastoreConfig(ctx, opsManager, operatorConfig) + if !status.IsOK() { + return status + } + + // Now we need to merge the Operator version into the OM one overriding only the fields that the Operator + // "owns" + configToUpdate := operatorView.MergeIntoOpsManagerConfig(omConfig) + log.Debugw("Updating Oplog Store in Ops Manager", "config", configToUpdate) + if err = omAdmin.UpdateOplogStoreConfig(configToUpdate); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Removing non-existing configs + configsToRemove := identifiable.SetDifferenceGeneric(opsManagerOplogConfigs, opsManager.Spec.Backup.OplogStoreConfigs) + for _, v := range configsToRemove { + log.Debugf("Removing Oplog Store %s from Ops Manager", v.Identifier()) + if err = omAdmin.DeleteOplogStoreConfig(v.Identifier().(string)); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + operatorS3OplogConfigs := opsManager.Spec.Backup.S3OplogStoreConfigs + if len(operatorOplogConfigs) == 0 && len(operatorS3OplogConfigs) == 0 { + return workflow.Invalid("Oplog Store configuration is required for backup").WithTargetPhase(mdbstatus.PhasePending) + } + return workflow.OK() +} + +func (r *OpsManagerReconciler) ensureS3OplogStoresInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, s3OplogAdmin api.S3OplogStoreAdmin, appDBConnectionString string, log *zap.SugaredLogger) workflow.Status { + if !opsManager.Spec.Backup.Enabled { + return workflow.OK() + } + + opsManagerS3OpLogConfigs, err := s3OplogAdmin.ReadS3OplogStoreConfigs() + if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + + // Creating new configs + s3OperatorOplogConfigs := opsManager.Spec.Backup.S3OplogStoreConfigs + configsToCreate := identifiable.SetDifferenceGeneric(s3OperatorOplogConfigs, opsManagerS3OpLogConfigs) + for _, v := range configsToCreate { + omConfig, status := r.buildOMS3Config(ctx, opsManager, v.(omv1.S3Config), true, appDBConnectionString) + if !status.IsOK() { + return status + } + log.Infow("Creating S3 Oplog Store in Ops Manager", "config", omConfig) + if err = s3OplogAdmin.CreateS3OplogStoreConfig(omConfig); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Updating existing configs. It intersects the OM API configs with Operator spec configs and returns pairs + //["omConfig", "operatorConfig"]. + configsToUpdate := identifiable.SetIntersectionGeneric(opsManagerS3OpLogConfigs, s3OperatorOplogConfigs) + for _, v := range configsToUpdate { + omConfig := v[0].(backup.S3Config) + operatorConfig := v[1].(omv1.S3Config) + operatorView, status := r.buildOMS3Config(ctx, opsManager, operatorConfig, true, appDBConnectionString) + if !status.IsOK() { + return status + } + + // Now we need to merge the Operator version into the OM one overriding only the fields that the Operator + // "owns" + configToUpdate := operatorView.MergeIntoOpsManagerConfig(omConfig) + log.Infow("Updating S3 Oplog Store in Ops Manager", "config", configToUpdate) + if err = s3OplogAdmin.UpdateS3OplogConfig(configToUpdate); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Removing non-existing configs + configsToRemove := identifiable.SetDifferenceGeneric(opsManagerS3OpLogConfigs, opsManager.Spec.Backup.S3OplogStoreConfigs) + for _, v := range configsToRemove { + log.Infof("Removing Oplog Store %s from Ops Manager", v.Identifier()) + if err = s3OplogAdmin.DeleteS3OplogStoreConfig(v.Identifier().(string)); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + operatorOplogConfigs := opsManager.Spec.Backup.OplogStoreConfigs + if len(operatorOplogConfigs) == 0 && len(s3OperatorOplogConfigs) == 0 { + return workflow.Invalid("Oplog Store configuration is required for backup").WithTargetPhase(mdbstatus.PhasePending) + } + return workflow.OK() +} + +// ensureBlockStoresInOpsManager aligns the blockStore configs in Ops Manager with the Operator state. So it adds the new configs +// and removes the non-existing ones. Note that there's no update operation as so far the Operator manages only one field +// 'path'. This will allow users to make any additional changes to the file system stores using Ops Manager UI and the +// Operator won't override them +func (r *OpsManagerReconciler) ensureBlockStoresInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, omAdmin api.BlockStoreAdmin, log *zap.SugaredLogger) workflow.Status { + if !opsManager.Spec.Backup.Enabled { + return workflow.OK() + } + + opsManagerBlockStoreConfigs, err := omAdmin.ReadBlockStoreConfigs() + if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + + // Creating new configs + operatorBlockStoreConfigs := opsManager.Spec.Backup.BlockStoreConfigs + configsToCreate := identifiable.SetDifferenceGeneric(operatorBlockStoreConfigs, opsManagerBlockStoreConfigs) + for _, v := range configsToCreate { + omConfig, status := r.buildOMDatastoreConfig(ctx, opsManager, v.(omv1.DataStoreConfig)) + if !status.IsOK() { + return status + } + log.Debugw("Creating Block Store in Ops Manager", "config", omConfig) + if err = omAdmin.CreateBlockStoreConfig(omConfig); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Updating existing configs. It intersects the OM API configs with Operator spec configs and returns pairs + //["omConfig", "operatorConfig"]. + configsToUpdate := identifiable.SetIntersectionGeneric(opsManagerBlockStoreConfigs, operatorBlockStoreConfigs) + for _, v := range configsToUpdate { + omConfig := v[0].(backup.DataStoreConfig) + operatorConfig := v[1].(omv1.DataStoreConfig) + operatorView, status := r.buildOMDatastoreConfig(ctx, opsManager, operatorConfig) + if !status.IsOK() { + return status + } + + // Now we need to merge the Operator version into the OM one overriding only the fields that the Operator + // "owns" + configToUpdate := operatorView.MergeIntoOpsManagerConfig(omConfig) + log.Debugw("Updating Block Store in Ops Manager", "config", configToUpdate) + if err = omAdmin.UpdateBlockStoreConfig(configToUpdate); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Removing non-existing configs + configsToRemove := identifiable.SetDifferenceGeneric(opsManagerBlockStoreConfigs, opsManager.Spec.Backup.BlockStoreConfigs) + for _, v := range configsToRemove { + log.Debugf("Removing Block Store %s from Ops Manager", v.Identifier()) + if err = omAdmin.DeleteBlockStoreConfig(v.Identifier().(string)); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + return workflow.OK() +} + +func (r *OpsManagerReconciler) ensureS3ConfigurationInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, omAdmin api.S3StoreBlockStoreAdmin, appDBConnectionString string, log *zap.SugaredLogger) workflow.Status { + if !opsManager.Spec.Backup.Enabled { + return workflow.OK() + } + + opsManagerS3Configs, err := omAdmin.ReadS3Configs() + if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + + operatorS3Configs := opsManager.Spec.Backup.S3Configs + configsToCreate := identifiable.SetDifferenceGeneric(operatorS3Configs, opsManagerS3Configs) + for _, config := range configsToCreate { + omConfig, status := r.buildOMS3Config(ctx, opsManager, config.(omv1.S3Config), false, appDBConnectionString) + if !status.IsOK() { + return status + } + + log.Infow("Creating S3Config in Ops Manager", "config", omConfig) + if err := omAdmin.CreateS3Config(omConfig); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + // Updating existing configs. It intersects the OM API configs with Operator spec configs and returns pairs + //["omConfig", "operatorConfig"]. + configsToUpdate := identifiable.SetIntersectionGeneric(opsManagerS3Configs, operatorS3Configs) + for _, v := range configsToUpdate { + omConfig := v[0].(backup.S3Config) + operatorConfig := v[1].(omv1.S3Config) + operatorView, status := r.buildOMS3Config(ctx, opsManager, operatorConfig, false, appDBConnectionString) + if !status.IsOK() { + return status + } + + // Now we need to merge the Operator version into the OM one overriding only the fields that the Operator + // "owns" + configToUpdate := operatorView.MergeIntoOpsManagerConfig(omConfig) + log.Infow("Updating S3Config in Ops Manager", "config", configToUpdate) + if err = omAdmin.UpdateS3Config(configToUpdate); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + configsToRemove := identifiable.SetDifferenceGeneric(opsManagerS3Configs, operatorS3Configs) + for _, config := range configsToRemove { + log.Infof("Removing S3Config %s from Ops Manager", config.Identifier()) + if err := omAdmin.DeleteS3Config(config.Identifier().(string)); err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + } + + return workflow.OK() +} + +// readS3Credentials reads the access and secret keys from the awsCredentials secret specified +// in the resource +func (r *OpsManagerReconciler) readS3Credentials(ctx context.Context, s3SecretName, namespace string) (*backup.S3Credentials, error) { + var operatorSecretPath string + if r.VaultClient != nil { + operatorSecretPath = r.VaultClient.OperatorSecretPath() + } + + s3SecretData, err := r.ReadSecret(ctx, kube.ObjectKey(namespace, s3SecretName), operatorSecretPath) + if err != nil { + return nil, xerrors.New(err.Error()) + } + + s3Creds := &backup.S3Credentials{} + if accessKey, ok := s3SecretData[util.S3AccessKey]; !ok { + return nil, xerrors.Errorf("key %s was not present in the secret %s", util.S3AccessKey, s3SecretName) + } else { + s3Creds.AccessKey = accessKey + } + + if secretKey, ok := s3SecretData[util.S3SecretKey]; !ok { + return nil, xerrors.Errorf("key %s was not present in the secret %s", util.S3SecretKey, s3SecretName) + } else { + s3Creds.SecretKey = secretKey + } + + return s3Creds, nil +} + +// ensureFileSystemStoreConfigurationInOpsManage makes sure that the FileSystem snapshot stores specified in the +// MongoDB CR are configured correctly in OpsManager. +func (r *OpsManagerReconciler) ensureFileSystemStoreConfigurationInOpsManager(opsManager *omv1.MongoDBOpsManager, omAdmin api.OpsManagerAdmin) workflow.Status { + opsManagerFSStoreConfigs, err := omAdmin.ReadFileSystemStoreConfigs() + if err != nil { + return workflow.Failed(xerrors.New(err.Error())) + } + + fsStoreNames := make(map[string]struct{}) + for _, e := range opsManager.Spec.Backup.FileSystemStoreConfigs { + fsStoreNames[e.Name] = struct{}{} + } + // count the number of FS snapshots configured in OM and match them with the one in CR. + countFS := 0 + + for _, e := range opsManagerFSStoreConfigs { + if _, ok := fsStoreNames[e.Id]; ok { + countFS++ + } + } + + if countFS != len(opsManager.Spec.Backup.FileSystemStoreConfigs) { + return workflow.Failed(xerrors.Errorf("Not all fileSystem snapshots have been configured in OM.")) + } + return workflow.OK() +} + +// shouldUseAppDb accepts an S3Config and returns true if the AppDB should be used +// for this S3 configuration. Otherwise, a MongoDB resource is configured for use. +func shouldUseAppDb(config omv1.S3Config) bool { + return config.MongoDBResourceRef == nil +} + +// buildAppDbOMS3Config creates a backup.S3Config which is configured to use The AppDb. +func (r *OpsManagerReconciler) buildAppDbOMS3Config(ctx context.Context, om *omv1.MongoDBOpsManager, config omv1.S3Config, isOpLog bool, appDBConnectionString string) (backup.S3Config, workflow.Status) { + var s3Creds *backup.S3Credentials + + if !config.IRSAEnabled { + var err error + s3Creds, err = r.readS3Credentials(ctx, config.S3SecretRef.Name, om.Namespace) + if err != nil { + return backup.S3Config{}, workflow.Failed(xerrors.New(err.Error())) + } + } + + bucket := backup.S3Bucket{ + Endpoint: config.S3BucketEndpoint, + Name: config.S3BucketName, + } + + customCAOpts, err := r.readCustomCAFilePathsAndContents(ctx, om, isOpLog) + if err != nil { + return backup.S3Config{}, workflow.Failed(xerrors.New(err.Error())) + } + + return backup.NewS3Config(om, config, appDBConnectionString, customCAOpts, bucket, s3Creds), workflow.OK() +} + +// buildMongoDbOMS3Config creates a backup.S3Config which is configured to use a referenced +// MongoDB resource. +func (r *OpsManagerReconciler) buildMongoDbOMS3Config(ctx context.Context, opsManager *omv1.MongoDBOpsManager, config omv1.S3Config, isOpLog bool) (backup.S3Config, workflow.Status) { + mongodb, status := r.getMongoDbForS3Config(ctx, opsManager, config) + if !status.IsOK() { + return backup.S3Config{}, status + } + + if status := validateS3Config(mongodb.GetAuthenticationModes(), mongodb.GetResourceName(), config); !status.IsOK() { + return backup.S3Config{}, status + } + + userName, password, status := r.getS3MongoDbUserNameAndPassword(ctx, mongodb.GetAuthenticationModes(), opsManager.Namespace, config) + if !status.IsOK() { + return backup.S3Config{}, status + } + + var s3Creds *backup.S3Credentials + var err error + + if !config.IRSAEnabled { + s3Creds, err = r.readS3Credentials(ctx, config.S3SecretRef.Name, opsManager.Namespace) + if err != nil { + return backup.S3Config{}, workflow.Failed(err) + } + } + + uri := mongodb.BuildConnectionString(userName, password, connectionstring.SchemeMongoDB, map[string]string{}) + + bucket := backup.S3Bucket{ + Endpoint: config.S3BucketEndpoint, + Name: config.S3BucketName, + } + + customCAOpts, err := r.readCustomCAFilePathsAndContents(ctx, opsManager, isOpLog) + if err != nil { + return backup.S3Config{}, workflow.Failed(err) + } + + return backup.NewS3Config(opsManager, config, uri, customCAOpts, bucket, s3Creds), workflow.OK() +} + +// readCustomCAFilePathsAndContents returns the filepath and contents of the custom CA which is used to configure +// the S3Store. +func (r *OpsManagerReconciler) readCustomCAFilePathsAndContents(ctx context.Context, opsManager *omv1.MongoDBOpsManager, isOpLog bool) ([]backup.S3CustomCertificate, error) { + var customCertificates []backup.S3CustomCertificate + var err error + + if isOpLog { + customCertificates, err = getCAs(ctx, opsManager.Spec.Backup.S3OplogStoreConfigs, opsManager.Namespace, r.client) + } else { + customCertificates, err = getCAs(ctx, opsManager.Spec.Backup.S3Configs, opsManager.Namespace, r.client) + } + + if err != nil { + return customCertificates, err + } + + if opsManager.Spec.GetAppDbCA() != "" { + cmContents, err := configmap.ReadKey(ctx, r.client, "ca-pem", kube.ObjectKey(opsManager.Namespace, opsManager.Spec.GetAppDbCA())) + if err != nil { + return []backup.S3CustomCertificate{}, xerrors.New(err.Error()) + } + customCertificates = append(customCertificates, backup.S3CustomCertificate{ + Filename: omv1.GetAppDBCaPemPath(), + CertString: cmContents, + }) + } + + return customCertificates, nil +} + +func getCAs(ctx context.Context, s3Config []omv1.S3Config, ns string, client secret.Getter) ([]backup.S3CustomCertificate, error) { + var certificates []backup.S3CustomCertificate + for _, config := range s3Config { + for _, backupCert := range config.CustomCertificateSecretRefs { + if backupCert.Name != "" { + aliasName := backupCert.Name + "/" + backupCert.Key + if cmContents, err := secret.ReadKey(ctx, client, backupCert.Key, kube.ObjectKey(ns, backupCert.Name)); err != nil { + return []backup.S3CustomCertificate{}, xerrors.New(err.Error()) + } else { + certificates = append(certificates, backup.S3CustomCertificate{ + Filename: aliasName, + CertString: cmContents, + }) + } + } + } + } + return certificates, nil +} + +// buildOMS3Config builds the OM API S3 config from the Operator OM CR configuration. This involves some logic to +// get the mongo URI, which points to either the external resource or to the AppDB +func (r *OpsManagerReconciler) buildOMS3Config(ctx context.Context, opsManager *omv1.MongoDBOpsManager, config omv1.S3Config, isOpLog bool, appDBConnectionString string) (backup.S3Config, workflow.Status) { + if shouldUseAppDb(config) { + return r.buildAppDbOMS3Config(ctx, opsManager, config, isOpLog, appDBConnectionString) + } + return r.buildMongoDbOMS3Config(ctx, opsManager, config, isOpLog) +} + +// getMongoDbForS3Config returns the referenced MongoDB resource which should be used when configuring the backup config. +func (r *OpsManagerReconciler) getMongoDbForS3Config(ctx context.Context, opsManager *omv1.MongoDBOpsManager, config omv1.S3Config) (S3ConfigGetter, workflow.Status) { + mongodb, mongodbMulti := &mdbv1.MongoDB{}, &mdbmulti.MongoDBMultiCluster{} + mongodbObjectKey := config.MongodbResourceObjectKey(opsManager) + + err := r.client.Get(ctx, mongodbObjectKey, mongodb) + if err != nil { + if secrets.SecretNotExist(err) { + + // try to fetch mongodbMulti if it exists + err = r.client.Get(ctx, mongodbObjectKey, mongodbMulti) + if err != nil { + if secrets.SecretNotExist(err) { + // Returning pending as the user may create the mongodb resource soon + return nil, workflow.Pending("The MongoDB object %s doesn't exist", mongodbObjectKey) + } + return nil, workflow.Failed(xerrors.New(err.Error())) + } + return mongodbMulti, workflow.OK() + } + + return nil, workflow.Failed(err) + } + + return mongodb, workflow.OK() +} + +// getS3MongoDbUserNameAndPassword returns userName and password if MongoDB resource has scram-sha enabled. +// Note, that we don't worry if the 'mongodbUserRef' is specified but SCRAM-SHA is not enabled - we just ignore the +// user. +func (r *OpsManagerReconciler) getS3MongoDbUserNameAndPassword(ctx context.Context, modes []string, namespace string, config omv1.S3Config) (string, string, workflow.Status) { + if !stringutil.Contains(modes, util.SCRAM) { + return "", "", workflow.OK() + } + mongodbUser := &user.MongoDBUser{} + mongodbUserObjectKey := config.MongodbUserObjectKey(namespace) + err := r.client.Get(ctx, mongodbUserObjectKey, mongodbUser) + if secrets.SecretNotExist(err) { + return "", "", workflow.Pending("The MongoDBUser object %s doesn't exist", mongodbUserObjectKey) + } + if err != nil { + return "", "", workflow.Failed(xerrors.Errorf("Failed to fetch the user %s: %w", mongodbUserObjectKey, err)) + } + userName := mongodbUser.Spec.Username + password, err := mongodbUser.GetPassword(ctx, r.SecretClient) + if err != nil { + return "", "", workflow.Failed(xerrors.Errorf("Failed to read password for the user %s: %w", mongodbUserObjectKey, err)) + } + return userName, password, workflow.OK() +} + +// buildOMDatastoreConfig builds the OM API datastore config based on the Kubernetes OM resource one. +// To do this it may need to read the Mongodb User and its password to build mongodb url correctly +func (r *OpsManagerReconciler) buildOMDatastoreConfig(ctx context.Context, opsManager *omv1.MongoDBOpsManager, operatorConfig omv1.DataStoreConfig) (backup.DataStoreConfig, workflow.Status) { + mongodb := &mdbv1.MongoDB{} + mongodbObjectKey := operatorConfig.MongodbResourceObjectKey(opsManager.Namespace) + + err := r.client.Get(ctx, mongodbObjectKey, mongodb) + if err != nil { + if secrets.SecretNotExist(err) { + // Returning pending as the user may create the mongodb resource soon + return backup.DataStoreConfig{}, workflow.Pending("The MongoDB object %s doesn't exist", mongodbObjectKey) + } + return backup.DataStoreConfig{}, workflow.Failed(xerrors.New(err.Error())) + } + + status := validateDataStoreConfig(mongodb.Spec.Security.Authentication.GetModes(), mongodb.Name, operatorConfig) + if !status.IsOK() { + return backup.DataStoreConfig{}, status + } + + // If MongoDB resource has scram-sha enabled then we need to read the username and the password. + // Note, that we don't worry if the 'mongodbUserRef' is specified but SCRAM-SHA is not enabled - we just ignore the + // user + var userName, password string + if stringutil.Contains(mongodb.Spec.Security.Authentication.GetModes(), util.SCRAM) { + mongodbUser := &user.MongoDBUser{} + mongodbUserObjectKey := operatorConfig.MongodbUserObjectKey(opsManager.Namespace) + err := r.client.Get(ctx, mongodbUserObjectKey, mongodbUser) + if secrets.SecretNotExist(err) { + return backup.DataStoreConfig{}, workflow.Pending("The MongoDBUser object %s doesn't exist", operatorConfig.MongodbResourceObjectKey(opsManager.Namespace)) + } + if err != nil { + return backup.DataStoreConfig{}, workflow.Failed(xerrors.Errorf("Failed to fetch the user %s: %w", operatorConfig.MongodbResourceObjectKey(opsManager.Namespace), err)) + } + userName = mongodbUser.Spec.Username + password, err = mongodbUser.GetPassword(ctx, r.SecretClient) + if err != nil { + return backup.DataStoreConfig{}, workflow.Failed(xerrors.Errorf("Failed to read password for the user %s: %w", mongodbUserObjectKey, err)) + } + } + + tls := mongodb.Spec.Security.TLSConfig.Enabled + mongoUri := mongodb.BuildConnectionString(userName, password, connectionstring.SchemeMongoDB, map[string]string{}) + return backup.NewDataStoreConfig(operatorConfig.Name, mongoUri, tls, operatorConfig.AssignmentLabels), workflow.OK() +} + +func validateS3Config(modes []string, mdbName string, s3Config omv1.S3Config) workflow.Status { + return validateConfig(modes, mdbName, s3Config.MongoDBUserRef, "S3 metadata database") +} + +func validateDataStoreConfig(modes []string, mdbName string, dataStoreConfig omv1.DataStoreConfig) workflow.Status { + return validateConfig(modes, mdbName, dataStoreConfig.MongoDBUserRef, "Oplog/Blockstore databases") +} + +func validateConfig(modes []string, mdbName string, userRef *omv1.MongoDBUserRef, description string) workflow.Status { + // validate + if !stringutil.Contains(modes, util.SCRAM) && + len(modes) > 0 { + return workflow.Failed(xerrors.Errorf("The only authentication mode supported for the %s is SCRAM-SHA", description)) + } + if stringutil.Contains(modes, util.SCRAM) && + (userRef == nil || userRef.Name == "") { + return workflow.Failed(xerrors.Errorf("MongoDB resource %s is configured to use SCRAM-SHA authentication mode, the user must be"+ + " specified using 'mongodbUserRef'", mdbName)) + } + + return workflow.OK() +} + +func newUserFromSecret(data map[string]string) (api.User, error) { + // validate + for _, v := range []string{"Username", "Password", "FirstName", "LastName"} { + if _, ok := data[v]; !ok { + return api.User{}, xerrors.Errorf("%s property is missing in the admin secret", v) + } + } + newUser := api.User{ + Username: data["Username"], + Password: data["Password"], + FirstName: data["FirstName"], + LastName: data["LastName"], + } + return newUser, nil +} + +// OnDelete cleans up Ops Manager related resources on CR removal. +// it's used in MongoDBOpsManagerEventHandler +func (r *OpsManagerReconciler) OnDelete(ctx context.Context, obj interface{}, log *zap.SugaredLogger) { + opsManager := obj.(*omv1.MongoDBOpsManager) + helper, err := NewOpsManagerReconcilerHelper(ctx, r, opsManager, r.memberClustersMap, log) + if err != nil { + log.Errorf("Error initializing OM reconciler helper: %s", err) + return + } + + // r.resourceWatcher.RemoveAllDependentWatchedResources(opsManager.Namespace, kube.ObjectKeyFromApiObject(opsManager)) + for _, memberCluster := range helper.getHealthyMemberClusters() { + stsName := helper.OpsManagerStatefulSetNameForMemberCluster(memberCluster) + r.resourceWatcher.RemoveAllDependentWatchedResources(opsManager.Namespace, kube.ObjectKey(opsManager.Namespace, stsName)) + } + + for _, memberCluster := range helper.getHealthyMemberClusters() { + memberClient := memberCluster.Client + stsName := helper.OpsManagerStatefulSetNameForMemberCluster(memberCluster) + err := memberClient.DeleteStatefulSet(ctx, kube.ObjectKey(opsManager.Namespace, stsName)) + if err != nil { + log.Warnf("Failed to delete statefulset: %s in cluster: %s", stsName, memberCluster.Name) + } + } + + for _, memberCluster := range helper.getHealthyMemberClusters() { + stsName := helper.BackupDaemonStatefulSetNameForMemberCluster(memberCluster) + r.resourceWatcher.RemoveAllDependentWatchedResources(opsManager.Namespace, kube.ObjectKey(opsManager.Namespace, stsName)) + } + + for _, memberCluster := range helper.getHealthyMemberClusters() { + memberClient := memberCluster.Client + stsName := helper.BackupDaemonStatefulSetNameForMemberCluster(memberCluster) + err := memberClient.DeleteStatefulSet(ctx, kube.ObjectKey(opsManager.Namespace, stsName)) + if err != nil { + log.Warnf("Failed to delete statefulset: %s in cluster: %s", stsName, memberCluster.Name) + } + } + + appDbReconciler, err := r.createNewAppDBReconciler(ctx, opsManager, log) + if err != nil { + log.Errorf("Error initializing AppDB reconciler: %s", err) + return + } + + // remove AppDB from each of the member clusters(or the same cluster as OM in case of single cluster ) + for _, memberCluster := range appDbReconciler.GetHealthyMemberClusters() { + // fetch the clusterNum for a given clusterName + r.resourceWatcher.RemoveAllDependentWatchedResources(opsManager.Namespace, opsManager.AppDBStatefulSetObjectKey(appDbReconciler.getMemberClusterIndex(memberCluster.Name))) + } + + // delete the AppDB statefulset form each of the member cluster. We need to delete the + // resource explicitly in case of multi-cluster because we can't set owner reference cross cluster + for _, memberCluster := range appDbReconciler.GetHealthyMemberClusters() { + memberClient := memberCluster.Client + stsNamespacedName := opsManager.AppDBStatefulSetObjectKey(appDbReconciler.getMemberClusterIndex(memberCluster.Name)) + + err := memberClient.DeleteStatefulSet(ctx, stsNamespacedName) + if err != nil { + log.Warnf("Failed to delete statefulset: %s in cluster: %s", stsNamespacedName, memberCluster.Name) + } + } + log.Info("Cleaned up Ops Manager related resources.") +} + +func (r *OpsManagerReconciler) createNewAppDBReconciler(ctx context.Context, opsManager *omv1.MongoDBOpsManager, log *zap.SugaredLogger) (*ReconcileAppDbReplicaSet, error) { + return NewAppDBReplicaSetReconciler(ctx, r.imageUrls, r.initAppdbVersion, opsManager.Spec.AppDB, r.ReconcileCommonController, r.omConnectionFactory, opsManager.Annotations, r.memberClustersMap, log) +} + +// getAnnotationsForOpsManagerResource returns all the annotations that should be applied to the resource +// at the end of the reconciliation. +func getAnnotationsForOpsManagerResource(opsManager *omv1.MongoDBOpsManager) (map[string]string, error) { + finalAnnotations := make(map[string]string) + specBytes, err := json.Marshal(opsManager.Spec) + if err != nil { + return nil, err + } + finalAnnotations[util.LastAchievedSpec] = string(specBytes) + return finalAnnotations, nil +} diff --git a/controllers/operator/mongodbopsmanager_controller_multi_test.go b/controllers/operator/mongodbopsmanager_controller_multi_test.go new file mode 100644 index 000000000..18a57760f --- /dev/null +++ b/controllers/operator/mongodbopsmanager_controller_multi_test.go @@ -0,0 +1,400 @@ +package operator + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" +) + +func omStsName(name string, clusterIdx int) string { + return fmt.Sprintf("%s-%d", name, clusterIdx) +} + +func genKeySecretName(omName string) string { + return fmt.Sprintf("%s-gen-key", omName) +} + +func connectionStringSecretName(omName string) string { + return fmt.Sprintf("%s-db-connection-string", omName) +} + +func agentPasswordSecretName(omName string) string { + return fmt.Sprintf("%s-db-agent-password", omName) +} + +func omPasswordSecretName(omName string) string { + return fmt.Sprintf("%s-db-om-password", omName) +} + +func omUserScramCredentialsSecretName(omName string) string { + return fmt.Sprintf("%s-db-om-user-scram-credentials", omName) +} + +type omMemberClusterChecks struct { + ctx context.Context + t *testing.T + namespace string + clusterName string + kubeClient client.Client + clusterIndex int + om *omv1.MongoDBOpsManager +} + +func newOMMemberClusterChecks(ctx context.Context, t *testing.T, opsManager *omv1.MongoDBOpsManager, clusterName string, kubeClient client.Client, clusterIndex int) *omMemberClusterChecks { + result := omMemberClusterChecks{ + ctx: ctx, + t: t, + namespace: opsManager.Namespace, + om: opsManager, + clusterName: clusterName, + kubeClient: kubeClient, + clusterIndex: clusterIndex, + } + + return &result +} + +func createOMCAConfigMap(ctx context.Context, t *testing.T, kubeClient client.Client, opsManager *omv1.MongoDBOpsManager) string { + cert, _ := createMockCertAndKeyBytes() + cm := configmap.Builder(). + SetName(opsManager.Spec.GetOpsManagerCA()). + SetNamespace(opsManager.GetNamespace()). + SetDataField("mms-ca.crt", string(cert)). + Build() + + err := kubeClient.Create(ctx, &cm) + require.NoError(t, err) + + return opsManager.Spec.GetOpsManagerCA() +} + +func createOMTLSCert(ctx context.Context, t *testing.T, kubeClient client.Client, opsManager *omv1.MongoDBOpsManager) (string, string) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: opsManager.TLSCertificateSecretName(), + Namespace: opsManager.GetNamespace(), + }, + Type: corev1.SecretTypeTLS, + } + + certs := map[string][]byte{} + certs["tls.crt"], certs["tls.key"] = createMockCertAndKeyBytes() + + secret.Data = certs + err := kubeClient.Create(ctx, secret) + require.NoError(t, err) + + pemHash := enterprisepem.ReadHashFromData(secrets.DataToStringData(secret.Data), zap.S()) + require.NotEmpty(t, pemHash) + + return secret.Name, pemHash +} + +func (c *omMemberClusterChecks) checkStatefulSetExists() { + sts := appsv1.StatefulSet{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.om.Namespace, omStsName(c.om.Name, c.clusterIndex)), &sts) + assert.NoError(c.t, err) +} + +func (c *omMemberClusterChecks) checkSecretNotFound(secretName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, secretName), &sec) + assert.Error(c.t, err, "clusterName: %s", c.clusterName) + assert.True(c.t, apiErrors.IsNotFound(err)) +} + +func (c *omMemberClusterChecks) checkGenKeySecret(omName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, genKeySecretName(omName)), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "gen.key", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkClusterMapping(omName string, expectedClusterMapping map[string]int) { + checkClusterMapping(c.ctx, c.t, c.kubeClient, c.namespace, omName, expectedClusterMapping) + checkLegacyClusterMapping(c.ctx, c.t, c.kubeClient, c.namespace, omName, expectedClusterMapping) +} + +func (c *omMemberClusterChecks) checkConnectionStringSecret(omName string) { + sec := corev1.Secret{} + secretName := connectionStringSecretName(omName) + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, secretName), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "connectionString", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkAgentPasswordSecret(omName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, agentPasswordSecretName(omName)), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "password", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkOmPasswordSecret(omName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, omPasswordSecretName(omName)), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "password", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkPEMSecret(secretName string, pemHash string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, secretName), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + assert.Contains(c.t, sec.Data, pemHash, "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkAppDBCAConfigMap(configMapName string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, cm.Data, "ca-pem", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkOMCAConfigMap(configMapName string) { + cm := corev1.ConfigMap{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, configMapName), &cm) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, cm.Data, "mms-ca.crt", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) checkOmUserScramCredentialsSecretName(omName string) { + sec := corev1.Secret{} + err := c.kubeClient.Get(c.ctx, kube.ObjectKey(c.namespace, omUserScramCredentialsSecretName(omName)), &sec) + require.NoError(c.t, err, "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "sha-1-server-key", "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "sha-1-stored-key", "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "sha-256-server-key", "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "sha-256-stored-key", "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "sha1-salt", "clusterName: %s", c.clusterName) + require.Contains(c.t, sec.Data, "sha256-salt", "clusterName: %s", c.clusterName) +} + +func (c *omMemberClusterChecks) reconcileAndCheck(reconciler reconcile.Reconciler, expectedRequeue bool) { + res, err := reconciler.Reconcile(c.ctx, requestFromObject(c.om)) + if expectedRequeue { + assert.True(c.t, res.Requeue || res.RequeueAfter > 0, "result=%+v", res) + } else { + assert.True(c.t, !res.Requeue && res.RequeueAfter > 0) + } + assert.NoError(c.t, err) +} + +func TestOpsManagerMultiCluster(t *testing.T) { + ctx := context.Background() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName := "kind-e2e-cluster-1" + memberClusterName2 := "kind-e2e-cluster-2" + clusters := []string{memberClusterName, memberClusterName2} + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters, omConnectionFactory) + + appDBClusterSpecItems := mdbv1.ClusterSpecList{ + { + ClusterName: memberClusterName, + Members: 1, + }, + { + ClusterName: memberClusterName2, + Members: 2, + }, + } + clusterSpecItems := []omv1.ClusterSpecOMItem{ + { + ClusterName: memberClusterName, + Members: 1, + Backup: &omv1.MongoDBOpsManagerBackupClusterSpecItem{ + Members: 1, + }, + }, + { + ClusterName: memberClusterName2, + Members: 1, + }, + } + + builder := DefaultOpsManagerBuilder(). + SetOpsManagerTopology(mdbv1.ClusterTopologyMultiCluster). + SetOpsManagerClusterSpecList(clusterSpecItems). + SetTLSConfig(omv1.MongoDBOpsManagerTLS{ + CA: "om-ca", + }). + SetAppDBTopology(mdbv1.ClusterTopologyMultiCluster). + SetAppDbMembers(0). + SetAppDBClusterSpecList(appDBClusterSpecItems). + SetAppDBTLSConfig(mdbv1.TLSConfig{ + Enabled: true, + AdditionalCertificateDomains: nil, + CA: "appdb-ca", + }) + + opsManager := builder.Build() + opsManager.Spec.Security.CertificatesSecretsPrefix = "om-prefix" + appDB := opsManager.Spec.AppDB + + reconciler, omClient, _ := defaultTestOmReconciler(ctx, t, nil, "", "", opsManager, memberClusterMap, omConnectionFactory) + + // prepare TLS certificates and CA in central cluster + + appDbCAConfigMapName := createAppDbCAConfigMap(ctx, t, omClient, appDB) + appDbTLSCertSecret, appDbTLSSecretPemHash := createAppDBTLSCert(ctx, t, omClient, appDB) + appDbPemSecretName := appDbTLSCertSecret + "-pem" + + /* omCAConfigMapName */ + _ = createOMCAConfigMap(ctx, t, omClient, opsManager) + omTLSCertSecret, omTLSSecretPemHash := createOMTLSCert(ctx, t, omClient, opsManager) + omPemSecretName := omTLSCertSecret + "-pem" + + /* checkOMReconciliationSuccessful(t, reconciler, opsManager) */ + + centralClusterChecks := newOMMemberClusterChecks(ctx, t, opsManager, centralClusterName, omClient, -1) + centralClusterChecks.reconcileAndCheck(reconciler, true) + // secrets and config maps created in the central cluster + centralClusterChecks.checkClusterMapping(opsManager.Name, map[string]int{ + memberClusterName: 0, + memberClusterName2: 1, + }) + centralClusterChecks.checkGenKeySecret(opsManager.Name) + centralClusterChecks.checkAgentPasswordSecret(opsManager.Name) + centralClusterChecks.checkOmPasswordSecret(opsManager.Name) + centralClusterChecks.checkOmUserScramCredentialsSecretName(opsManager.Name) + centralClusterChecks.checkSecretNotFound(appDbPemSecretName) + centralClusterChecks.checkSecretNotFound(omPemSecretName) + centralClusterChecks.checkOMCAConfigMap(opsManager.Spec.GetOpsManagerCA()) + + for clusterIdx, clusterSpecItem := range clusterSpecItems { + memberClusterClient := memberClusterMap[clusterSpecItem.ClusterName] + memberClusterChecks := newOMMemberClusterChecks(ctx, t, opsManager, clusterSpecItem.ClusterName, memberClusterClient, clusterIdx) + memberClusterChecks.checkStatefulSetExists() + memberClusterChecks.checkGenKeySecret(opsManager.Name) + memberClusterChecks.checkConnectionStringSecret(opsManager.Name) + memberClusterChecks.checkPEMSecret(appDbPemSecretName, appDbTLSSecretPemHash) + memberClusterChecks.checkPEMSecret(omPemSecretName, omTLSSecretPemHash) + memberClusterChecks.checkAppDBCAConfigMap(appDbCAConfigMapName) + memberClusterChecks.checkOMCAConfigMap(opsManager.Spec.GetOpsManagerCA()) + } +} + +func TestOpsManagerMultiClusterUnreachableNoPanic(t *testing.T) { + ctx := context.Background() + centralClusterName := multicluster.LegacyCentralClusterName + memberClusterName := "kind-e2e-cluster-1" + memberClusterName2 := "kind-e2e-cluster-2" + memberClusterNameUnreachable := "kind-e2e-cluster-unreachable" + clusters := []string{memberClusterName, memberClusterName2} + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + memberClusterMap := getFakeMultiClusterMapWithClusters(clusters, omConnectionFactory) + + appDBClusterSpecItems := []mdbv1.ClusterSpecItem{ + { + ClusterName: memberClusterName, + Members: 1, + }, + { + ClusterName: memberClusterName2, + Members: 2, + }, + } + clusterSpecItems := []omv1.ClusterSpecOMItem{ + { + ClusterName: memberClusterName, + Members: 1, + Backup: &omv1.MongoDBOpsManagerBackupClusterSpecItem{ + Members: 1, + }, + }, + { + ClusterName: memberClusterName2, + Members: 1, + }, + { + ClusterName: memberClusterNameUnreachable, + Members: 1, + }, + } + + builder := DefaultOpsManagerBuilder(). + SetOpsManagerTopology(omv1.ClusterTopologyMultiCluster). + SetOpsManagerClusterSpecList(clusterSpecItems). + SetTLSConfig(omv1.MongoDBOpsManagerTLS{ + CA: "om-ca", + }). + SetAppDBTopology(omv1.ClusterTopologyMultiCluster). + SetAppDbMembers(0). + SetAppDBClusterSpecList(appDBClusterSpecItems). + SetAppDBTLSConfig(mdbv1.TLSConfig{ + Enabled: true, + AdditionalCertificateDomains: nil, + CA: "appdb-ca", + }) + + opsManager := builder.Build() + opsManager.Spec.Security.CertificatesSecretsPrefix = "om-prefix" + appDB := opsManager.Spec.AppDB + + reconciler, omClient, _ := defaultTestOmReconciler(ctx, t, nil, "", "", opsManager, memberClusterMap, omConnectionFactory) + + // prepare TLS certificates and CA in central cluster + + appDbCAConfigMapName := createAppDbCAConfigMap(ctx, t, omClient, appDB) + appDbTLSCertSecret, appDbTLSSecretPemHash := createAppDBTLSCert(ctx, t, omClient, appDB) + appDbPemSecretName := appDbTLSCertSecret + "-pem" + + /* omCAConfigMapName */ + _ = createOMCAConfigMap(ctx, t, omClient, opsManager) + omTLSCertSecret, omTLSSecretPemHash := createOMTLSCert(ctx, t, omClient, opsManager) + omPemSecretName := omTLSCertSecret + "-pem" + + /* checkOMReconciliationSuccessful(t, reconciler, opsManager) */ + + centralClusterChecks := newOMMemberClusterChecks(ctx, t, opsManager, centralClusterName, omClient, -1) + require.NotPanics(t, func() { + centralClusterChecks.reconcileAndCheck(reconciler, true) + }) + + // secrets and config maps created in the central cluster + centralClusterChecks.checkGenKeySecret(opsManager.Name) + centralClusterChecks.checkAgentPasswordSecret(opsManager.Name) + centralClusterChecks.checkOmPasswordSecret(opsManager.Name) + centralClusterChecks.checkOmUserScramCredentialsSecretName(opsManager.Name) + centralClusterChecks.checkSecretNotFound(appDbPemSecretName) + centralClusterChecks.checkSecretNotFound(omPemSecretName) + centralClusterChecks.checkOMCAConfigMap(opsManager.Spec.GetOpsManagerCA()) + + for clusterIdx, clusterSpecItem := range clusterSpecItems { + if clusterSpecItem.ClusterName == memberClusterNameUnreachable { + continue + } + + memberClusterClient := memberClusterMap[clusterSpecItem.ClusterName] + memberClusterChecks := newOMMemberClusterChecks(ctx, t, opsManager, clusterSpecItem.ClusterName, memberClusterClient, clusterIdx) + memberClusterChecks.checkStatefulSetExists() + memberClusterChecks.checkGenKeySecret(opsManager.Name) + memberClusterChecks.checkConnectionStringSecret(opsManager.Name) + memberClusterChecks.checkPEMSecret(appDbPemSecretName, appDbTLSSecretPemHash) + memberClusterChecks.checkPEMSecret(omPemSecretName, omTLSSecretPemHash) + memberClusterChecks.checkAppDBCAConfigMap(appDbCAConfigMapName) + memberClusterChecks.checkOMCAConfigMap(opsManager.Spec.GetOpsManagerCA()) + } +} diff --git a/controllers/operator/mongodbopsmanager_controller_test.go b/controllers/operator/mongodbopsmanager_controller_test.go new file mode 100644 index 000000000..0753d1598 --- /dev/null +++ b/controllers/operator/mongodbopsmanager_controller_test.go @@ -0,0 +1,1392 @@ +package operator + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/constants" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/api" + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + operatorConstruct "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" +) + +func TestOpsManagerReconciler_watchedResources(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().Build() + otherTestOm := DefaultOpsManagerBuilder().Build() + otherTestOm.Name = "otherOM" + + otherTestOm.Spec.Backup.Enabled = true + testOm.Spec.Backup.Enabled = true + otherTestOm.Spec.Backup.OplogStoreConfigs = []omv1.DataStoreConfig{{MongoDBResourceRef: userv1.MongoDBResourceRef{Name: "oplog1"}}} + testOm.Spec.Backup.OplogStoreConfigs = []omv1.DataStoreConfig{{MongoDBResourceRef: userv1.MongoDBResourceRef{Name: "oplog1"}}} + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, _, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + reconciler.watchMongoDBResourcesReferencedByBackup(ctx, testOm, zap.S()) + reconciler.watchMongoDBResourcesReferencedByBackup(ctx, otherTestOm, zap.S()) + + key := watch.Object{ + ResourceType: watch.MongoDB, + Resource: types.NamespacedName{ + Name: "oplog1", + Namespace: testOm.Namespace, + }, + } + + // om watches oplog MDB resource + assert.Contains(t, reconciler.resourceWatcher.GetWatchedResources(), key) + assert.Contains(t, reconciler.resourceWatcher.GetWatchedResources()[key], mock.ObjectKeyFromApiObject(testOm)) + assert.Contains(t, reconciler.resourceWatcher.GetWatchedResources()[key], mock.ObjectKeyFromApiObject(otherTestOm)) +} + +// TestOMTLSResourcesAreWatchedAndUnwatched verifies that TLS config map and secret are added to the internal +// map that allows to watch them for changes +func TestOMTLSResourcesAreWatchedAndUnwatched(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: true, + }).SetAppDBTLSConfig(mdbv1.TLSConfig{ + Enabled: true, + CA: "custom-ca-appdb", + }).SetTLSConfig(omv1.MongoDBOpsManagerTLS{ + SecretRef: omv1.TLSSecretRef{ + Name: "om-tls-secret", + }, + CA: "custom-ca", + }). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + Build() + + testOm.Spec.Backup.Encryption = &omv1.Encryption{ + Kmip: &omv1.KmipConfig{ + Server: v1.KmipServerConfig{ + CA: "custom-kmip-ca", + URL: "kmip:8080", + }, + }, + } + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + addOMTLSResources(ctx, client, "om-tls-secret") + addAppDBTLSResources(ctx, client, testOm.Spec.AppDB.GetTlsCertificatesSecretName()) + addKMIPTestResources(ctx, client, testOm, "test-mdb", "test-prefix") + addOmCACm(ctx, t, testOm, reconciler) + + configureBackupResources(ctx, client, testOm) + + checkOMReconciliationSuccessful(ctx, t, reconciler, testOm, reconciler.client) + + ns := testOm.Namespace + KmipCaKey := getWatch(ns, "custom-kmip-ca", watch.ConfigMap) + omCAKey := getWatch(ns, "custom-ca", watch.ConfigMap) + appDBCAKey := getWatch(ns, "custom-ca-appdb", watch.ConfigMap) + KmipMongoDBKey := getWatch(ns, "test-prefix-test-mdb-kmip-client", watch.Secret) + KmipMongoDBPasswordKey := getWatch(ns, "test-prefix-test-mdb-kmip-client-password", watch.Secret) + omTLSSecretKey := getWatch(ns, "om-tls-secret", watch.Secret) + appdbTLSecretCert := getWatch(ns, "test-om-db-cert", watch.Secret) + + expectedWatchedResources := []watch.Object{ + getWatch("testNS", "test-mdb", watch.MongoDB), + getWatch(ns, "config-0-mdb", watch.MongoDB), + KmipCaKey, + omCAKey, + appDBCAKey, + KmipMongoDBKey, + KmipMongoDBPasswordKey, + omTLSSecretKey, + appdbTLSecretCert, + } + + var actual []watch.Object + for obj := range reconciler.resourceWatcher.GetWatchedResources() { + actual = append(actual, obj) + } + + assert.ElementsMatch(t, expectedWatchedResources, actual) + testOm.Spec.Security.TLS.SecretRef.Name = "" + testOm.Spec.Backup.Enabled = false + + err := client.Update(ctx, testOm) + assert.NoError(t, err) + + res, err := reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, res) + assert.NoError(t, err) + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), omTLSSecretKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), omCAKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), KmipMongoDBKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), KmipMongoDBPasswordKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), KmipCaKey) + + testOm.Spec.AppDB.Security.TLSConfig.Enabled = false + testOm.Spec.Backup.Enabled = true + testOm.Spec.Backup.Encryption.Kmip = nil + err = client.Update(ctx, testOm) + require.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, res) + assert.NoError(t, err) + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), appDBCAKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), appdbTLSecretCert) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), KmipMongoDBKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), KmipMongoDBPasswordKey) + assert.NotContains(t, reconciler.resourceWatcher.GetWatchedResources(), KmipCaKey) +} + +func TestOpsManagerPrefixForTLSSecret(t *testing.T) { + testOm := DefaultOpsManagerBuilder().SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: false, + }).SetTLSConfig(omv1.MongoDBOpsManagerTLS{ + CA: "custom-ca", + }).Build() + + testOm.Spec.Security.CertificatesSecretsPrefix = "prefix" + assert.Equal(t, fmt.Sprintf("prefix-%s-cert", testOm.Name), testOm.TLSCertificateSecretName()) + + testOm.Spec.Security.TLS.SecretRef.Name = "om-tls-secret" + assert.Equal(t, "om-tls-secret", testOm.TLSCertificateSecretName()) +} + +func TestOpsManagerReconciler_removeWatchedResources(t *testing.T) { + ctx := context.Background() + resourceName := "oplog1" + testOm := DefaultOpsManagerBuilder().Build() + testOm.Spec.Backup.Enabled = true + testOm.Spec.Backup.OplogStoreConfigs = []omv1.DataStoreConfig{{MongoDBResourceRef: userv1.MongoDBResourceRef{Name: resourceName}}} + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, _, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + reconciler.watchMongoDBResourcesReferencedByBackup(ctx, testOm, zap.S()) + + key := watch.Object{ + ResourceType: watch.MongoDB, + Resource: types.NamespacedName{Name: resourceName, Namespace: testOm.Namespace}, + } + + // om watches oplog MDB resource + assert.Contains(t, reconciler.resourceWatcher.GetWatchedResources(), key) + assert.Contains(t, reconciler.resourceWatcher.GetWatchedResources()[key], mock.ObjectKeyFromApiObject(testOm)) + + // watched resources list is cleared when CR is deleted + reconciler.OnDelete(ctx, testOm, zap.S()) + assert.Zero(t, len(reconciler.resourceWatcher.GetWatchedResources())) +} + +func TestOpsManagerReconciler_prepareOpsManager(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, initializer := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + reconcileStatus, _ := reconciler.prepareOpsManager(ctx, testOm, testOm.CentralURL(), zap.S()) + + assert.Equal(t, workflow.OK(), reconcileStatus) + assert.Equal(t, "jane.doe@g.com", api.CurrMockedAdmin.PublicKey) + + // the user "created" in Ops Manager + assert.Len(t, initializer.currentUsers, 1) + assert.Equal(t, "Jane", initializer.currentUsers[0].FirstName) + assert.Equal(t, "Doe", initializer.currentUsers[0].LastName) + assert.Equal(t, "pwd", initializer.currentUsers[0].Password) + assert.Equal(t, "jane.doe@g.com", initializer.currentUsers[0].Username) + + // One secret was created by the user, another one - by the Operator for the user public key + assert.Len(t, mock.GetMapForObject(client, &corev1.Secret{}), 2) + expectedSecretData := map[string]string{"publicKey": "jane.doe@g.com", "privateKey": "jane.doe@g.com-key"} + + APIKeySecretName, err := testOm.APIKeySecretName(ctx, secrets.SecretClient{KubeClient: client}, "") + assert.NoError(t, err) + + existingSecretData, _ := secret.ReadStringData(ctx, client, kube.ObjectKey(OperatorNamespace, APIKeySecretName)) + assert.Equal(t, expectedSecretData, existingSecretData) +} + +func TestOpsManagerReconcilerPrepareOpsManagerWithTLS(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().SetTLSConfig(omv1.MongoDBOpsManagerTLS{ + SecretRef: omv1.TLSSecretRef{ + Name: "om-tls-secret", + }, + CA: "custom-ca", + }).Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, _, initializer := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + initializer.expectedCaContent = ptr.To("abc") + + addOmCACm(ctx, t, testOm, reconciler) + + reconcileStatus, _ := reconciler.prepareOpsManager(ctx, testOm, testOm.CentralURL(), zap.S()) + + assert.Equal(t, workflow.OK(), reconcileStatus) +} + +func addOmCACm(ctx context.Context, t *testing.T, testOm *omv1.MongoDBOpsManager, reconciler *OpsManagerReconciler) { + cm := configmap.Builder(). + SetName(testOm.Spec.GetOpsManagerCA()). + SetNamespace(testOm.Namespace). + SetData(map[string]string{"mms-ca.crt": "abc"}). + SetOwnerReferences(kube.BaseOwnerReference(testOm)). + Build() + assert.NoError(t, reconciler.client.CreateConfigMap(ctx, cm)) +} + +// TestOpsManagerReconciler_prepareOpsManagerTwoCalls checks that second call to 'prepareOpsManager' doesn't call +// OM api to create a user as the API secret already exists +func TestOpsManagerReconciler_prepareOpsManagerTwoCalls(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, initializer := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + reconciler.prepareOpsManager(ctx, testOm, testOm.CentralURL(), zap.S()) + + APIKeySecretName, err := testOm.APIKeySecretName(ctx, secrets.SecretClient{KubeClient: client}, "") + assert.NoError(t, err) + + // let's "update" the user admin secret - this must not affect anything + s, _ := client.GetSecret(ctx, kube.ObjectKey(OperatorNamespace, APIKeySecretName)) + s.Data["Username"] = []byte("this-is-not-expected@g.com") + err = client.UpdateSecret(ctx, s) + assert.NoError(t, err) + + // second call is ok - we just don't create the admin user in OM and don't add new secrets + reconcileStatus, _ := reconciler.prepareOpsManager(ctx, testOm, testOm.CentralURL(), zap.S()) + assert.Equal(t, workflow.OK(), reconcileStatus) + assert.Equal(t, "jane.doe@g.com-key", api.CurrMockedAdmin.PrivateKey) + + // the call to the api didn't happen + assert.Equal(t, 1, initializer.numberOfCalls) + assert.Len(t, initializer.currentUsers, 1) + assert.Equal(t, "jane.doe@g.com", initializer.currentUsers[0].Username) + + assert.Len(t, mock.GetMapForObject(client, &corev1.Secret{}), 2) + + data, _ := secret.ReadStringData(ctx, client, kube.ObjectKey(OperatorNamespace, APIKeySecretName)) + assert.Equal(t, "jane.doe@g.com", data["publicKey"]) +} + +// TestOpsManagerReconciler_prepareOpsManagerDuplicatedUser checks that if the public API key secret is removed by the +// user - the Operator will try to create a user again and this will result in UserAlreadyExists error +func TestOpsManagerReconciler_prepareOpsManagerDuplicatedUser(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, initializer := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + reconciler.prepareOpsManager(ctx, testOm, testOm.CentralURL(), zap.S()) + + APIKeySecretName, err := testOm.APIKeySecretName(ctx, secrets.SecretClient{KubeClient: client}, "") + assert.NoError(t, err) + + // for some reason the admin removed the public Api key secret so the call will be done to OM to create a user - + // it will fail as the user already exists + _ = client.Delete(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: OperatorNamespace, Name: APIKeySecretName}, + }) + + reconcileStatus, admin := reconciler.prepareOpsManager(ctx, testOm, testOm.CentralURL(), zap.S()) + assert.Equal(t, status.PhaseFailed, reconcileStatus.Phase()) + + option, exists := status.GetOption(reconcileStatus.StatusOptions(), status.MessageOption{}) + assert.True(t, exists) + assert.Contains(t, option.(status.MessageOption).Message, "USER_ALREADY_EXISTS") + reconcileStatus.StatusOptions() + assert.Nil(t, admin) + + // the call to the api happened, but the user wasn't added + assert.Equal(t, 2, initializer.numberOfCalls) + assert.Len(t, initializer.currentUsers, 1) + assert.Equal(t, "jane.doe@g.com", initializer.currentUsers[0].Username) + + // api secret wasn't created + assert.Len(t, mock.GetMapForObject(client, &corev1.Secret{}), 1) + + assert.NotContains(t, mock.GetMapForObject(client, &corev1.Secret{}), kube.ObjectKey(OperatorNamespace, APIKeySecretName)) +} + +func TestOpsManagerGeneratesAppDBPassword_IfNotProvided(t *testing.T) { + ctx := context.Background() + + testOm := DefaultOpsManagerBuilder().Build() + kubeManager, omConnectionFactory := mock.NewDefaultFakeClient(testOm) + appDBReconciler, err := newAppDbReconciler(ctx, kubeManager, testOm, omConnectionFactory.GetConnectionFunc, zap.S()) + require.NoError(t, err) + + password, err := appDBReconciler.ensureAppDbPassword(ctx, testOm, zap.S()) + assert.NoError(t, err) + assert.Len(t, password, 12, "auto generated password should have a size of 12") +} + +func TestOpsManagerUsersPassword_SpecifiedInSpec(t *testing.T) { + ctx := context.Background() + log := zap.S() + testOm := DefaultOpsManagerBuilder().SetAppDBPassword("my-secret", "password").Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + s := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: testOm.Spec.AppDB.PasswordSecretKeyRef.Name, Namespace: testOm.Namespace}, + Data: map[string][]byte{ + testOm.Spec.AppDB.PasswordSecretKeyRef.Key: []byte("my-password"), // create the secret with the password + }, + } + err := client.CreateSecret(ctx, s) + + require.NoError(t, err) + err = client.UpdateSecret(ctx, s) + + require.NoError(t, err) + + appDBReconciler, err := reconciler.createNewAppDBReconciler(ctx, testOm, log) + require.NoError(t, err) + password, err := appDBReconciler.ensureAppDbPassword(ctx, testOm, zap.S()) + + assert.NoError(t, err) + assert.Equal(t, password, "my-password", "the password specified by the SecretRef should have been returned when specified") +} + +func TestBackupStatefulSetIsNotRemoved_WhenDisabled(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: true, + }).Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + checkOMReconciliationInvalid(ctx, t, reconciler, testOm, client) + + backupSts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.BackupDaemonStatefulSetName()), &backupSts) + assert.NoError(t, err, "Backup StatefulSet should have been created when backup is enabled") + + testOm.Spec.Backup.Enabled = false + err = client.Update(ctx, testOm) + assert.NoError(t, err) + + res, err := reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, res) + assert.NoError(t, err) + require.NoError(t, client.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + + backupSts = appsv1.StatefulSet{} + err = client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.BackupDaemonStatefulSetName()), &backupSts) + assert.NoError(t, err, "Backup StatefulSet should not be removed when backup is disabled") +} + +func TestOpsManagerPodTemplateSpec_IsAnnotatedWithHash(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: false, + }).Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + s := secret.Builder(). + SetName(testOm.Spec.AppDB.GetOpsManagerUserPasswordSecretName()). + SetNamespace(testOm.Namespace). + SetOwnerReferences(kube.BaseOwnerReference(testOm)). + SetByteData(map[string][]byte{ + "password": []byte("password"), + }).Build() + + err := reconciler.client.CreateSecret(ctx, s) + assert.NoError(t, err) + + checkOMReconciliationSuccessful(ctx, t, reconciler, testOm, reconciler.client) + + connectionString, err := secret.ReadKey(ctx, reconciler.client, util.AppDbConnectionStringKey, kube.ObjectKey(testOm.Namespace, testOm.AppDBMongoConnectionStringSecretName())) + assert.NoError(t, err) + assert.NotEmpty(t, connectionString) + + sts := appsv1.StatefulSet{} + err = client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.Name), &sts) + assert.NoError(t, err) + + podTemplate := sts.Spec.Template + + assert.Contains(t, podTemplate.Annotations, "connectionStringHash") + assert.Equal(t, podTemplate.Annotations["connectionStringHash"], hashConnectionString(buildMongoConnectionUrl(testOm, "password", nil))) + testOm.Spec.AppDB.Members = 5 + assert.NotEqual(t, podTemplate.Annotations["connectionStringHash"], hashConnectionString(buildMongoConnectionUrl(testOm, "password", nil)), + "Changing the number of members should result in a different Connection String and different hash") + testOm.Spec.AppDB.Members = 3 + testOm.Spec.AppDB.Version = "4.2.0" + assert.Equal(t, podTemplate.Annotations["connectionStringHash"], hashConnectionString(buildMongoConnectionUrl(testOm, "password", nil)), + "Changing version should not change connection string and so the hash should stay the same") +} + +func TestOpsManagerReconcileContainerImages(t *testing.T) { + initOpsManagerRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_1_2_3", util.InitOpsManagerImageUrl) + opsManagerRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0", util.OpsManagerImageUrl) + mongodbRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0", mcoConstruct.MongodbImageEnv) + initAppdbRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_3_4_5", util.InitAppdbImageUrlEnv) + + imageUrlsMock := images.ImageUrls{ + // Ops manager & backup deamon images + initOpsManagerRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-init-ops-manager:@sha256:MONGODB_INIT_APPDB", + opsManagerRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-ops-manager:@sha256:MONGODB_OPS_MANAGER", + + // AppDB images + mcoConstruct.AgentImageEnv: "quay.io/mongodb/mongodb-agent@sha256:AGENT_SHA", // In non-static architecture, this env var holds full container image uri + mongodbRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi@sha256:MONGODB_SHA", + initAppdbRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-init-appdb@sha256:INIT_APPDB_SHA", + } + + ctx := context.Background() + testOm := DefaultOpsManagerBuilder(). + SetVersion("8.0.0"). + SetAppDbVersion("8.0.0"). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, imageUrlsMock, "3.4.5", "1.2.3", testOm, nil, omConnectionFactory) + configureBackupResources(ctx, client, testOm) + + checkOMReconciliationSuccessful(ctx, t, reconciler, testOm, reconciler.client) + + for stsAlias, stsName := range map[string]string{ + "opsManagerSts": testOm.Name, + "backupSts": testOm.BackupDaemonStatefulSetName(), + } { + t.Run(stsAlias, func(t *testing.T) { + sts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, stsName), &sts) + require.NoError(t, err) + + require.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-ops-manager:@sha256:MONGODB_INIT_APPDB", sts.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-ops-manager:@sha256:MONGODB_OPS_MANAGER", sts.Spec.Template.Spec.Containers[0].Image) + }) + } + + appDBSts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.Spec.AppDB.Name()), &appDBSts) + require.NoError(t, err) + + require.Len(t, appDBSts.Spec.Template.Spec.InitContainers, 1) + require.Len(t, appDBSts.Spec.Template.Spec.Containers, 3) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-appdb@sha256:INIT_APPDB_SHA", appDBSts.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-agent@sha256:AGENT_SHA", appDBSts.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi@sha256:MONGODB_SHA", appDBSts.Spec.Template.Spec.Containers[1].Image) + // Version from the mapping file (agent version + operator version) + assert.Contains(t, appDBSts.Spec.Template.Spec.Containers[2].Image, "_9.9.9-test") +} + +func TestOpsManagerReconcileContainerImagesWithStaticArchitecture(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + opsManagerRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0", util.OpsManagerImageUrl) + mongodbRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0", mcoConstruct.MongodbImageEnv) + + imageUrlsMock := images.ImageUrls{ + // Ops manager & backup deamon images + opsManagerRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-ops-manager:@sha256:MONGODB_OPS_MANAGER", + + // AppDB images + mongodbRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi@sha256:MONGODB_SHA", + architectures.MdbAgentImageRepo: "quay.io/mongodb/mongodb-agent-ubi", + } + + ctx := context.Background() + testOm := DefaultOpsManagerBuilder(). + SetVersion("8.0.0"). + SetAppDbVersion("8.0.0"). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, imageUrlsMock, "", "", testOm, nil, omConnectionFactory) + configureBackupResources(ctx, client, testOm) + + checkOMReconciliationSuccessful(ctx, t, reconciler, testOm, reconciler.client) + + for stsAlias, stsName := range map[string]string{ + "opsManagerSts": testOm.Name, + "backupSts": testOm.BackupDaemonStatefulSetName(), + } { + t.Run(stsAlias, func(t *testing.T) { + sts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, stsName), &sts) + require.NoError(t, err) + + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 0) + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-ops-manager:@sha256:MONGODB_OPS_MANAGER", sts.Spec.Template.Spec.Containers[0].Image) + }) + } + + appDBSts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.Spec.AppDB.Name()), &appDBSts) + require.NoError(t, err) + + require.Len(t, appDBSts.Spec.Template.Spec.InitContainers, 0) + require.Len(t, appDBSts.Spec.Template.Spec.Containers, 3) + + // Version from the mapping file (agent version + operator version) + assert.Contains(t, appDBSts.Spec.Template.Spec.Containers[0].Image, "-1_9.9.9-test") + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi@sha256:MONGODB_SHA", appDBSts.Spec.Template.Spec.Containers[1].Image) + // In static architecture this container is a copy of agent container + assert.Equal(t, appDBSts.Spec.Template.Spec.Containers[0].Image, appDBSts.Spec.Template.Spec.Containers[2].Image) +} + +func TestOpsManagerConnectionString_IsPassedAsSecretRef(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().SetBackup(omv1.MongoDBOpsManagerBackup{ + Enabled: false, + }).Build() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + checkOMReconciliationSuccessful(ctx, t, reconciler, testOm, reconciler.client) + + sts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.Name), &sts) + assert.NoError(t, err) + + var uriVol corev1.Volume + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.Name == operatorConstruct.AppDBConnectionStringVolume { + uriVol = v + break + } + } + assert.NotEmpty(t, uriVol.Name, "MmsMongoUri volume should have been present!") + assert.NotNil(t, uriVol.VolumeSource) + assert.NotNil(t, uriVol.Secret) + assert.Equal(t, uriVol.Secret.SecretName, testOm.AppDBMongoConnectionStringSecretName()) +} + +func TestOpsManagerWithKMIP(t *testing.T) { + ctx := context.Background() + // given + kmipURL := "kmip.mongodb.com:5696" + kmipCAConfigMapName := "kmip-ca" + mdbName := "test-mdb" + + clientCertificatePrefix := "test-prefix" + expectedClientCertificateSecretName := clientCertificatePrefix + "-" + mdbName + "-kmip-client" + + testOm := DefaultOpsManagerBuilder(). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + Build() + + testOm.Spec.Backup.Encryption = &omv1.Encryption{ + Kmip: &omv1.KmipConfig{ + Server: v1.KmipServerConfig{ + CA: kmipCAConfigMapName, + URL: kmipURL, + }, + }, + } + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + addKMIPTestResources(ctx, client, testOm, mdbName, clientCertificatePrefix) + configureBackupResources(ctx, client, testOm) + + // when + checkOMReconciliationSuccessful(ctx, t, reconciler, testOm, reconciler.client) + sts := appsv1.StatefulSet{} + err := client.Get(ctx, kube.ObjectKey(testOm.Namespace, testOm.Name), &sts) + envs := sts.Spec.Template.Spec.Containers[0].Env + volumes := sts.Spec.Template.Spec.Volumes + volumeMounts := sts.Spec.Template.Spec.Containers[0].VolumeMounts + + // then + assert.NoError(t, err) + host, port, _ := net.SplitHostPort(kmipURL) + + expectedVars := []corev1.EnvVar{ + {Name: "OM_PROP_backup_kmip_server_host", Value: host}, + {Name: "OM_PROP_backup_kmip_server_port", Value: port}, + {Name: "OM_PROP_backup_kmip_server_ca_file", Value: util.KMIPCAFileInContainer}, + } + assert.Subset(t, envs, expectedVars) + + expectedCAMount := corev1.VolumeMount{ + Name: util.KMIPServerCAName, + MountPath: util.KMIPServerCAHome, + ReadOnly: true, + } + assert.Contains(t, volumeMounts, expectedCAMount) + expectedClientCertMount := corev1.VolumeMount{ + Name: util.KMIPClientSecretNamePrefix + expectedClientCertificateSecretName, + MountPath: util.KMIPClientSecretsHome + "/" + expectedClientCertificateSecretName, + ReadOnly: true, + } + assert.Contains(t, volumeMounts, expectedClientCertMount) + + expectedCAVolume := statefulset.CreateVolumeFromConfigMap(util.KMIPServerCAName, kmipCAConfigMapName) + assert.Contains(t, volumes, expectedCAVolume) + expectedClientCertVolume := statefulset.CreateVolumeFromSecret(util.KMIPClientSecretNamePrefix+expectedClientCertificateSecretName, expectedClientCertificateSecretName) + assert.Contains(t, volumes, expectedClientCertVolume) +} + +// TODO move this test to 'opsmanager_types_test.go' when the builder is moved to 'apis' package +func TestOpsManagerCentralUrl(t *testing.T) { + assert.Equal(t, "http://test-om-svc.my-namespace.svc.cluster.local:8080", + DefaultOpsManagerBuilder().Build().CentralURL()) + assert.Equal(t, "http://test-om-svc.my-namespace.svc.some.domain:8080", + DefaultOpsManagerBuilder().SetClusterDomain("some.domain").Build().CentralURL()) +} + +// TODO move this test to 'opsmanager_types_test.go' when the builder is moved to 'apis' package +func TestOpsManagerBackupDaemonHostName(t *testing.T) { + assert.Equal(t, []string{"test-om-backup-daemon-0.test-om-backup-daemon-svc.my-namespace.svc.cluster.local"}, + DefaultOpsManagerBuilder().Build().BackupDaemonFQDNs()) + // The host name doesn't depend on cluster domain + assert.Equal(t, []string{"test-om-backup-daemon-0.test-om-backup-daemon-svc.my-namespace.svc.some.domain"}, + DefaultOpsManagerBuilder().SetClusterDomain("some.domain").Build().BackupDaemonFQDNs()) + + assert.Equal(t, []string{"test-om-backup-daemon-0.test-om-backup-daemon-svc.my-namespace.svc.cluster.local", "test-om-backup-daemon-1.test-om-backup-daemon-svc.my-namespace.svc.cluster.local", "test-om-backup-daemon-2.test-om-backup-daemon-svc.my-namespace.svc.cluster.local"}, + DefaultOpsManagerBuilder().SetBackupMembers(3).Build().BackupDaemonFQDNs()) +} + +func TestOpsManagerBackupAssignmentLabels(t *testing.T) { + ctx := context.Background() + // given + assignmentLabels := []string{"test"} + + testOm := DefaultOpsManagerBuilder(). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddS3Config("s3-config", "s3-secret"). + Build() + + testOm.Spec.Backup.AssignmentLabels = assignmentLabels + testOm.Spec.Backup.OplogStoreConfigs[0].AssignmentLabels = assignmentLabels + testOm.Spec.Backup.BlockStoreConfigs[0].AssignmentLabels = assignmentLabels + testOm.Spec.Backup.S3Configs[0].AssignmentLabels = assignmentLabels + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, client, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + configureBackupResources(ctx, client, testOm) + + mockedAdmin := api.NewMockedAdminProvider("testUrl", "publicApiKey", "privateApiKey", true) + defer mockedAdmin.(*api.MockedOmAdmin).Reset() + + reconcilerHelper, err := NewOpsManagerReconcilerHelper(ctx, reconciler, testOm, nil, zap.S()) + require.NoError(t, err) + + // when + reconciler.prepareBackupInOpsManager(ctx, reconcilerHelper, testOm, mockedAdmin, "", zap.S()) + blockStoreConfigs, _ := mockedAdmin.ReadBlockStoreConfigs() + oplogConfigs, _ := mockedAdmin.ReadOplogStoreConfigs() + s3Configs, _ := mockedAdmin.ReadS3Configs() + daemonConfigs, _ := mockedAdmin.(*api.MockedOmAdmin).ReadDaemonConfigs() + + // then + assert.Equal(t, assignmentLabels, blockStoreConfigs[0].Labels) + assert.Equal(t, assignmentLabels, oplogConfigs[0].Labels) + assert.Equal(t, assignmentLabels, s3Configs[0].Labels) + assert.Equal(t, assignmentLabels, daemonConfigs[0].Labels) +} + +func TestTriggerOmChangedEventIfNeeded(t *testing.T) { + ctx := context.Background() + t.Run("Om changed event got triggered, major version update", func(t *testing.T) { + nextScheduledTime := agents.NextScheduledUpgradeTime() + assert.NoError(t, triggerOmChangedEventIfNeeded(ctx, omv1.NewOpsManagerBuilder().SetVersion("5.2.13").SetOMStatusVersion("4.2.13").Build(), nil, zap.S())) + assert.NotEqual(t, nextScheduledTime, agents.NextScheduledUpgradeTime()) + }) + t.Run("Om changed event got triggered, minor version update", func(t *testing.T) { + nextScheduledTime := agents.NextScheduledUpgradeTime() + assert.NoError(t, triggerOmChangedEventIfNeeded(ctx, omv1.NewOpsManagerBuilder().SetVersion("4.4.0").SetOMStatusVersion("4.2.13").Build(), nil, zap.S())) + assert.NotEqual(t, nextScheduledTime, agents.NextScheduledUpgradeTime()) + }) + t.Run("Om changed event got triggered, minor version update, candidate version", func(t *testing.T) { + nextScheduledTime := agents.NextScheduledUpgradeTime() + assert.NoError(t, triggerOmChangedEventIfNeeded(ctx, omv1.NewOpsManagerBuilder().SetVersion("4.4.0-rc2").SetOMStatusVersion("4.2.13").Build(), nil, zap.S())) + assert.NotEqual(t, nextScheduledTime, agents.NextScheduledUpgradeTime()) + }) + t.Run("Om changed event not triggered, patch version update", func(t *testing.T) { + nextScheduledTime := agents.NextScheduledUpgradeTime() + assert.NoError(t, triggerOmChangedEventIfNeeded(ctx, omv1.NewOpsManagerBuilder().SetVersion("4.4.10").SetOMStatusVersion("4.4.0").Build(), nil, zap.S())) + assert.Equal(t, nextScheduledTime, agents.NextScheduledUpgradeTime()) + }) +} + +func TestBackupIsStillConfigured_WhenAppDBIsConfigured_WithTls(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder().AddS3Config("s3-config", "s3-secret"). + AddOplogStoreConfig("oplog-store-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + SetAppDBTLSConfig(mdbv1.TLSConfig{Enabled: true}). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, mockedClient, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + addAppDBTLSResources(ctx, mockedClient, fmt.Sprintf("%s-cert", testOm.Spec.AppDB.Name())) + configureBackupResources(ctx, mockedClient, testOm) + + // initially requeued as monitoring needs to be configured + res, err := reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + assert.Equal(t, true, res.Requeue) + + // monitoring is configured successfully + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + + assert.NoError(t, err) + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) +} + +func TestBackupConfig_ChangingName_ResultsIn_DeleteAndAdd(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder(). + AddOplogStoreConfig("oplog-store", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddS3Config("s3-config-0", "s3-secret"). + AddS3Config("s3-config-1", "s3-secret"). + AddS3Config("s3-config-2", "s3-secret"). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, mockedClient, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + configureBackupResources(ctx, mockedClient, testOm) + + // initially requeued as monitoring needs to be configured + res, err := reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + assert.Equal(t, true, res.Requeue) + + // monitoring is configured successfully + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + + t.Run("Configs are created successfully", func(t *testing.T) { + s3Configs, err := api.CurrMockedAdmin.ReadS3Configs() + assert.NoError(t, err) + assert.Len(t, s3Configs, 3) + }) + + require.NoError(t, mockedClient.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + testOm.Spec.Backup.S3Configs[0].Name = "new-name" + err = mockedClient.Update(ctx, testOm) + assert.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + + t.Run("Name change resulted in a different config being created", func(t *testing.T) { + s3Configs, err := api.CurrMockedAdmin.ReadS3Configs() + assert.NoError(t, err) + assert.Len(t, s3Configs, 3) + + assert.Equal(t, "new-name", s3Configs[0].Id) + assert.Equal(t, "s3-config-1", s3Configs[1].Id) + assert.Equal(t, "s3-config-2", s3Configs[2].Id) + }) +} + +func TestOpsManagerRace(t *testing.T) { + ctx := context.Background() + opsManager1 := DefaultOpsManagerBuilder().SetName("om1"). + AddOplogStoreConfig("oplog-store-1", "my-user", types.NamespacedName{Name: "config-1-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-1", "my-user", types.NamespacedName{Name: "config-1-mdb", Namespace: mock.TestNamespace}).Build() + opsManager2 := DefaultOpsManagerBuilder().SetName("om2"). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-2-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-2", "my-user", types.NamespacedName{Name: "config-2-mdb", Namespace: mock.TestNamespace}).Build() + opsManager3 := DefaultOpsManagerBuilder().SetName("om3"). + AddOplogStoreConfig("oplog-store-3", "my-user", types.NamespacedName{Name: "config-3-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-3", "my-user", types.NamespacedName{Name: "config-3-mdb", Namespace: mock.TestNamespace}).Build() + + resourceToProjectMapping := map[string]string{ + "om1": opsManager1.Spec.AppDB.GetName(), + "om2": opsManager2.Spec.AppDB.GetName(), + "om3": opsManager3.Spec.AppDB.GetName(), + } + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory().WithResourceToProjectMapping(resourceToProjectMapping) + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(opsManager1, opsManager2, opsManager3). + WithInterceptorFuncs(interceptor.Funcs{ + Get: mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, true, true), + }).Build() + + kubeClient := kubernetesClient.NewClient(fakeClient) + // create an admin user secret + data := map[string]string{"Username": "jane.doe@g.com", "Password": "pwd", "FirstName": "Jane", "LastName": "Doe"} + + // All instances can use the same secret + s := secret.Builder(). + SetName(opsManager1.Spec.AdminSecret). + SetNamespace(opsManager1.Namespace). + SetStringMapToData(data). + SetLabels(map[string]string{}). + SetOwnerReferences(kube.BaseOwnerReference(opsManager1)). + Build() + + configureBackupResources(ctx, kubeClient, opsManager1) + configureBackupResources(ctx, kubeClient, opsManager2) + configureBackupResources(ctx, kubeClient, opsManager3) + + initializer := &MockedInitializer{expectedOmURL: opsManager1.CentralURL(), t: t, skipChecks: true} + + reconciler := NewOpsManagerReconciler(ctx, kubeClient, nil, nil, "fake-initAppdbVersion", "fake-initOpsManagerImageVersion", omConnectionFactory.GetConnectionFunc, initializer, func(baseUrl string, user string, publicApiKey string, ca *string) api.OpsManagerAdmin { + return api.NewMockedAdminProvider(baseUrl, user, publicApiKey, false).(*api.MockedOmAdmin) + }) + + assert.NoError(t, reconciler.client.CreateSecret(ctx, s)) + + testConcurrentReconciles(ctx, t, kubeClient, reconciler, opsManager1, opsManager2, opsManager3) +} + +func TestBackupConfigs_AreRemoved_WhenRemovedFromCR(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder(). + AddS3Config("s3-config-0", "s3-secret"). + AddS3Config("s3-config-1", "s3-secret"). + AddS3Config("s3-config-2", "s3-secret"). + AddOplogStoreConfig("oplog-store-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddOplogStoreConfig("oplog-store-1", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-1", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-2", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, mockedClient, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + configureBackupResources(ctx, mockedClient, testOm) + + // initially requeued as monitoring needs to be configured + res, err := reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + assert.Equal(t, true, res.Requeue) + + // monitoring is configured successfully + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + + assert.NoError(t, err) + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) + + t.Run("Configs are created successfully", func(t *testing.T) { + configs, err := api.CurrMockedAdmin.ReadOplogStoreConfigs() + assert.NoError(t, err) + assert.Len(t, configs, 3) + + s3Configs, err := api.CurrMockedAdmin.ReadS3Configs() + assert.NoError(t, err) + assert.Len(t, s3Configs, 3) + + blockstores, err := api.CurrMockedAdmin.ReadBlockStoreConfigs() + assert.NoError(t, err) + assert.Len(t, blockstores, 3) + }) + + require.NoError(t, mockedClient.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + + // remove the first entry + testOm.Spec.Backup.OplogStoreConfigs = testOm.Spec.Backup.OplogStoreConfigs[1:] + + // remove middle element + testOm.Spec.Backup.S3Configs = []omv1.S3Config{testOm.Spec.Backup.S3Configs[0], testOm.Spec.Backup.S3Configs[2]} + + // remove first and last + testOm.Spec.Backup.BlockStoreConfigs = []omv1.DataStoreConfig{testOm.Spec.Backup.BlockStoreConfigs[1]} + + err = mockedClient.Update(ctx, testOm) + assert.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + + t.Run("Configs are removed successfully", func(t *testing.T) { + configs, err := api.CurrMockedAdmin.ReadOplogStoreConfigs() + assert.NoError(t, err) + assert.Len(t, configs, 2) + + assert.Equal(t, "oplog-store-1", configs[0].Id) + assert.Equal(t, "oplog-store-2", configs[1].Id) + + s3Configs, err := api.CurrMockedAdmin.ReadS3Configs() + assert.NoError(t, err) + assert.Len(t, s3Configs, 2) + + assert.Equal(t, "s3-config-0", s3Configs[0].Id) + assert.Equal(t, "s3-config-2", s3Configs[1].Id) + + blockstores, err := api.CurrMockedAdmin.ReadBlockStoreConfigs() + assert.NoError(t, err) + assert.Len(t, blockstores, 1) + assert.Equal(t, "block-store-config-1", blockstores[0].Id) + }) +} + +func TestEnsureResourcesForArchitectureChange(t *testing.T) { + ctx := context.Background() + opsManager := DefaultOpsManagerBuilder().Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + t.Run("When no automation config is present, there is no error", func(t *testing.T) { + client := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory) + err := ensureResourcesForArchitectureChange(ctx, client, client, opsManager) + assert.NoError(t, err) + }) + + t.Run("If User is not present, there is an error", func(t *testing.T) { + client := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory) + ac, err := automationconfig.NewBuilder().SetAuth(automationconfig.Auth{ + Users: []automationconfig.MongoDBUser{ + { + Username: "not-ops-manager-user", + }, + }, + }).Build() + + assert.NoError(t, err) + + acBytes, err := json.Marshal(ac) + assert.NoError(t, err) + + // create the automation config secret + err = client.CreateSecret(ctx, secret.Builder().SetNamespace(opsManager.Namespace).SetName(opsManager.Spec.AppDB.AutomationConfigSecretName()).SetField(automationconfig.ConfigKey, string(acBytes)).Build()) + assert.NoError(t, err) + + err = ensureResourcesForArchitectureChange(ctx, client, client, opsManager) + assert.Error(t, err) + }) + + t.Run("If an automation config is present, all secrets are created with the correct values", func(t *testing.T) { + client := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory) + ac, err := automationconfig.NewBuilder().SetAuth(automationconfig.Auth{ + AutoPwd: "VrBQgsUZJJs", + Key: "Z8PSBtvvjnvds4zcI6iZ", + Users: []automationconfig.MongoDBUser{ + { + Username: util.OpsManagerMongoDBUserName, + ScramSha256Creds: &scramcredentials.ScramCreds{ + Salt: "sha256-salt-value", + ServerKey: "sha256-serverkey-value", + StoredKey: "sha256-storedkey-value", + }, + ScramSha1Creds: &scramcredentials.ScramCreds{ + Salt: "sha1-salt-value", + ServerKey: "sha1-serverkey-value", + StoredKey: "sha1-storedkey-value", + }, + }, + }, + }).Build() + + assert.NoError(t, err) + + acBytes, err := json.Marshal(ac) + assert.NoError(t, err) + + // create the automation config secret + err = client.CreateSecret(ctx, secret.Builder().SetNamespace(opsManager.Namespace).SetName(opsManager.Spec.AppDB.AutomationConfigSecretName()).SetField(automationconfig.ConfigKey, string(acBytes)).Build()) + assert.NoError(t, err) + + // create the old ops manager user password + err = client.CreateSecret(ctx, secret.Builder().SetNamespace(opsManager.Namespace).SetName(opsManager.Spec.AppDB.Name()+"-password").SetField("my-password", "jrJP7eUeyn").Build()) + assert.NoError(t, err) + + err = ensureResourcesForArchitectureChange(ctx, client, client, opsManager) + assert.NoError(t, err) + + t.Run("Scram credentials have been created", func(t *testing.T) { + ctx := context.Background() + scramCreds, err := client.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.OpsManagerUserScramCredentialsName())) + assert.NoError(t, err) + + assert.Equal(t, ac.Auth.Users[0].ScramSha256Creds.Salt, string(scramCreds.Data["sha256-salt"])) + assert.Equal(t, ac.Auth.Users[0].ScramSha256Creds.StoredKey, string(scramCreds.Data["sha-256-stored-key"])) + assert.Equal(t, ac.Auth.Users[0].ScramSha256Creds.ServerKey, string(scramCreds.Data["sha-256-server-key"])) + + assert.Equal(t, ac.Auth.Users[0].ScramSha1Creds.Salt, string(scramCreds.Data["sha1-salt"])) + assert.Equal(t, ac.Auth.Users[0].ScramSha1Creds.StoredKey, string(scramCreds.Data["sha-1-stored-key"])) + assert.Equal(t, ac.Auth.Users[0].ScramSha1Creds.ServerKey, string(scramCreds.Data["sha-1-server-key"])) + }) + + t.Run("Ops Manager user password has been copied", func(t *testing.T) { + ctx := context.Background() + newOpsManagerUserPassword, err := client.GetSecret(ctx, kube.ObjectKey(opsManager.Namespace, opsManager.Spec.AppDB.GetOpsManagerUserPasswordSecretName())) + assert.NoError(t, err) + assert.Equal(t, string(newOpsManagerUserPassword.Data["my-password"]), "jrJP7eUeyn") + }) + + t.Run("Agent password has been created", func(t *testing.T) { + ctx := context.Background() + agentPasswordSecret, err := client.GetSecret(ctx, opsManager.Spec.AppDB.GetAgentPasswordSecretNamespacedName()) + assert.NoError(t, err) + assert.Equal(t, ac.Auth.AutoPwd, string(agentPasswordSecret.Data[constants.AgentPasswordKey])) + }) + + t.Run("Keyfile has been created", func(t *testing.T) { + ctx := context.Background() + keyFileSecret, err := client.GetSecret(ctx, opsManager.Spec.AppDB.GetAgentKeyfileSecretNamespacedName()) + assert.NoError(t, err) + assert.Equal(t, ac.Auth.Key, string(keyFileSecret.Data[constants.AgentKeyfileKey])) + }) + }) +} + +func TestDependentResources_AreRemoved_WhenBackupIsDisabled(t *testing.T) { + ctx := context.Background() + testOm := DefaultOpsManagerBuilder(). + AddS3Config("s3-config-0", "s3-secret"). + AddS3Config("s3-config-1", "s3-secret"). + AddS3Config("s3-config-2", "s3-secret"). + AddOplogStoreConfig("oplog-store-0", "my-user", types.NamespacedName{Name: "config-0-mdb", Namespace: mock.TestNamespace}). + AddOplogStoreConfig("oplog-store-1", "my-user", types.NamespacedName{Name: "config-1-mdb", Namespace: mock.TestNamespace}). + AddOplogStoreConfig("oplog-store-2", "my-user", types.NamespacedName{Name: "config-2-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-0", "my-user", types.NamespacedName{Name: "block-store-config-0-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-1", "my-user", types.NamespacedName{Name: "block-store-config-1-mdb", Namespace: mock.TestNamespace}). + AddBlockStoreConfig("block-store-config-2", "my-user", types.NamespacedName{Name: "block-store-config-2-mdb", Namespace: mock.TestNamespace}). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + reconciler, mockedClient, _ := defaultTestOmReconciler(ctx, t, nil, "", "", testOm, nil, omConnectionFactory) + + configureBackupResources(ctx, mockedClient, testOm) + + // initially requeued as monitoring needs to be configured + res, err := reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + assert.Equal(t, true, res.Requeue) + + // monitoring is configured successfully + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + + t.Run("All MongoDB resource should be watched.", func(t *testing.T) { + assert.Len(t, reconciler.resourceWatcher.GetWatchedResourcesOfType(watch.MongoDB, testOm.Namespace), 6, "All non S3 configs should have a corresponding MongoDB resource and should be watched.") + }) + + t.Run("Removing backup configs causes the resource no longer be watched", func(t *testing.T) { + // remove last + require.NoError(t, mockedClient.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + testOm.Spec.Backup.BlockStoreConfigs = testOm.Spec.Backup.BlockStoreConfigs[0:2] + // remove first + testOm.Spec.Backup.OplogStoreConfigs = testOm.Spec.Backup.OplogStoreConfigs[1:3] + err = mockedClient.Update(ctx, testOm) + assert.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + + watchedResources := reconciler.resourceWatcher.GetWatchedResourcesOfType(watch.MongoDB, testOm.Namespace) + assert.Len(t, watchedResources, 4, "The two configs that were removed should no longer be watched.") + + assert.True(t, containsName("block-store-config-0-mdb", watchedResources)) + assert.True(t, containsName("block-store-config-1-mdb", watchedResources)) + assert.True(t, containsName("config-1-mdb", watchedResources)) + assert.True(t, containsName("config-2-mdb", watchedResources)) + }) + + t.Run("Disabling backup should cause all resources to no longer be watched.", func(t *testing.T) { + require.NoError(t, mockedClient.Get(ctx, kube.ObjectKeyFromApiObject(testOm), testOm)) + testOm.Spec.Backup.Enabled = false + err = mockedClient.Update(ctx, testOm) + assert.NoError(t, err) + + res, err = reconciler.Reconcile(ctx, requestFromObject(testOm)) + assert.NoError(t, err) + assert.Len(t, reconciler.resourceWatcher.GetWatchedResourcesOfType(watch.MongoDB, testOm.Namespace), 0, "Backup has been disabled, none of the resources should be watched anymore.") + }) +} + +func TestUniqueClusterNames(t *testing.T) { + testOm := DefaultOpsManagerBuilder().Build() + testOm.Spec.AppDB.Topology = "MultiCluster" + testOm.Spec.AppDB.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: "abc", + Members: 2, + }, + { + ClusterName: "def", + Members: 1, + }, + { + ClusterName: "abc", + Members: 1, + }, + } + + _, err := testOm.ValidateCreate() + require.Error(t, err) + assert.Equal(t, "Multiple clusters with the same name (abc) are not allowed", err.Error()) +} + +func containsName(name string, nsNames []types.NamespacedName) bool { + for _, nsName := range nsNames { + if nsName.Name == name { + return true + } + } + return false +} + +// configureBackupResources ensures all the dependent resources for the Backup configuration +// are created in the mocked client. This includes MongoDB resources for OplogStores, S3 credentials secrets +// MongodbUsers and their credentials secrets. +func configureBackupResources(ctx context.Context, m kubernetesClient.Client, testOm *omv1.MongoDBOpsManager) { + // configure S3 Secret + for _, s3Config := range testOm.Spec.Backup.S3Configs { + // skip if no secret ref is provided (valid for IRSA enabled deployments) + if s3Config.S3SecretRef == nil { + continue + } + + s3Creds := secret.Builder(). + SetName(s3Config.S3SecretRef.Name). + SetNamespace(testOm.Namespace). + SetField(util.S3AccessKey, "s3AccessKey"). + SetField(util.S3SecretKey, "s3SecretKey"). + Build() + _ = m.CreateSecret(ctx, s3Creds) + } + + // create MDB resource for oplog configs + for _, oplogConfig := range append(testOm.Spec.Backup.OplogStoreConfigs, testOm.Spec.Backup.BlockStoreConfigs...) { + oplogStoreResource := mdbv1.NewReplicaSetBuilder(). + SetName(oplogConfig.MongoDBResourceRef.Name). + SetNamespace(testOm.Namespace). + SetVersion("3.6.9"). + SetMembers(3). + EnableAuth([]mdbv1.AuthMode{util.SCRAM}). + Build() + + _ = m.Create(ctx, oplogStoreResource) + + // create user for mdb resource + oplogStoreUser := DefaultMongoDBUserBuilder(). + SetResourceName(oplogConfig.MongoDBUserRef.Name). + SetNamespace(testOm.Namespace). + Build() + + _ = m.Create(ctx, oplogStoreUser) + + // create secret for user + userPasswordSecret := secret.Builder(). + SetNamespace(testOm.Namespace). + SetName(oplogStoreUser.Spec.PasswordSecretKeyRef.Name). + SetField(oplogStoreUser.Spec.PasswordSecretKeyRef.Key, "KeJfV1ucQ_vZl"). + Build() + + _ = m.CreateSecret(ctx, userPasswordSecret) + } +} + +func defaultTestOmReconciler(ctx context.Context, t *testing.T, imageUrls images.ImageUrls, initAppdbVersion, initOpsManagerImageVersion string, opsManager *omv1.MongoDBOpsManager, globalMemberClustersMap map[string]client.Client, omConnectionFactory *om.CachedOMConnectionFactory) (*OpsManagerReconciler, kubernetesClient.Client, *MockedInitializer) { + kubeClient := mock.NewEmptyFakeClientWithInterceptor(omConnectionFactory, opsManager.DeepCopy()) + + // create an admin user secret + data := map[string]string{"Username": "jane.doe@g.com", "Password": "pwd", "FirstName": "Jane", "LastName": "Doe"} + + s := secret.Builder(). + SetName(opsManager.Spec.AdminSecret). + SetNamespace(opsManager.Namespace). + SetStringMapToData(data). + SetLabels(map[string]string{}). + SetOwnerReferences(kube.BaseOwnerReference(opsManager)). + Build() + + initializer := &MockedInitializer{expectedOmURL: opsManager.CentralURL(), t: t} + + reconciler := NewOpsManagerReconciler(ctx, kubeClient, globalMemberClustersMap, imageUrls, initAppdbVersion, initOpsManagerImageVersion, omConnectionFactory.GetConnectionFunc, initializer, func(baseUrl string, user string, publicApiKey string, ca *string) api.OpsManagerAdmin { + if api.CurrMockedAdmin == nil { + api.CurrMockedAdmin = api.NewMockedAdminProvider(baseUrl, user, publicApiKey, true).(*api.MockedOmAdmin) + } + return api.CurrMockedAdmin + }) + + assert.NoError(t, reconciler.client.CreateSecret(ctx, s)) + return reconciler, kubeClient, initializer +} + +func DefaultOpsManagerBuilder() *omv1.OpsManagerBuilder { + spec := omv1.MongoDBOpsManagerSpec{ + Version: "7.0.0", + AppDB: *omv1.DefaultAppDbBuilder().Build(), + AdminSecret: "om-admin", + } + resource := omv1.MongoDBOpsManager{Spec: spec, ObjectMeta: metav1.ObjectMeta{Name: "test-om", Namespace: mock.TestNamespace}} + return omv1.NewOpsManagerBuilderFromResource(resource) +} + +type MockedInitializer struct { + mu sync.RWMutex + currentUsers []api.User + expectedAPIError *apierror.Error + expectedOmURL string + expectedCaContent *string + t *testing.T + numberOfCalls int + skipChecks bool +} + +func (o *MockedInitializer) TryCreateUser(omUrl string, _ string, user api.User, ca *string) (api.OpsManagerKeyPair, error) { + o.mu.Lock() + defer o.mu.Unlock() + + o.numberOfCalls++ + if !o.skipChecks { + assert.Equal(o.t, o.expectedOmURL, omUrl) + } + + if o.expectedCaContent != nil { + assert.Equal(o.t, *o.expectedCaContent, *ca) + } + if o.expectedAPIError != nil { + return api.OpsManagerKeyPair{}, o.expectedAPIError + } + // OM logic: any number of users is created. But we cannot of course create the user with the same name + if !o.skipChecks { + for _, v := range o.currentUsers { + if v.Username == user.Username { + return api.OpsManagerKeyPair{}, apierror.NewErrorWithCode(apierror.UserAlreadyExists) + } + } + } + o.currentUsers = append(o.currentUsers, user) + + return api.OpsManagerKeyPair{ + PublicKey: user.Username, + PrivateKey: user.Username + "-key", + }, nil +} + +func addKMIPTestResources(ctx context.Context, client client.Client, om *omv1.MongoDBOpsManager, mdbName, clientCertificatePrefixName string) { + mdb := mdbv1.NewReplicaSetBuilder().SetBackup(mdbv1.Backup{ + Mode: "enabled", + Encryption: &mdbv1.Encryption{ + Kmip: &mdbv1.KmipConfig{ + Client: v1.KmipClientConfig{ + ClientCertificatePrefix: clientCertificatePrefixName, + }, + }, + }, + }).SetName(mdbName).Build() + _ = client.Create(ctx, mdb) + + mockCert, mockKey := createMockCertAndKeyBytes() + + ca := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: om.Spec.Backup.Encryption.Kmip.Server.CA, + Namespace: om.Namespace, + }, + } + ca.Data = map[string]string{} + ca.Data["ca.pem"] = string(mockCert) + _ = client.Create(ctx, ca) + + clientCertificate := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: mdb.GetBackupSpec().Encryption.Kmip.Client.ClientCertificateSecretName(mdb.GetName()), + Namespace: om.Namespace, + }, + } + clientCertificate.Data = map[string][]byte{} + clientCertificate.Data["tls.key"] = mockKey + clientCertificate.Data["tls.crt"] = mockCert + _ = client.Create(ctx, clientCertificate) + + clientCertificatePassword := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: mdb.GetBackupSpec().Encryption.Kmip.Client.ClientCertificatePasswordSecretName(mdb.GetName()), + Namespace: om.Namespace, + }, + } + clientCertificatePassword.Data = map[string]string{ + mdb.GetBackupSpec().Encryption.Kmip.Client.ClientCertificatePasswordKeyName(): "test", + } + _ = client.Create(ctx, clientCertificatePassword) +} + +func addAppDBTLSResources(ctx context.Context, client client.Client, secretName string) { + // Let's create a secret with Certificates and private keys! + certSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: mock.TestNamespace, + }, + } + + certs := map[string][]byte{} + certs["tls.crt"], certs["tls.key"] = createMockCertAndKeyBytes() + + certSecret.Data = certs + _ = client.Create(ctx, certSecret) +} + +func addOMTLSResources(ctx context.Context, client client.Client, secretName string) { + // Let's create a secret with Certificates and private keys! + certSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: mock.TestNamespace, + }, + } + + certs := map[string][]byte{} + certs["tls.crt"], certs["tls.key"] = createMockCertAndKeyBytes() + + certSecret.Data = certs + _ = client.Create(ctx, certSecret) +} diff --git a/controllers/operator/mongodbopsmanager_event_handler.go b/controllers/operator/mongodbopsmanager_event_handler.go new file mode 100644 index 000000000..0f63be82c --- /dev/null +++ b/controllers/operator/mongodbopsmanager_event_handler.go @@ -0,0 +1,33 @@ +package operator + +import ( + "context" + + "go.uber.org/zap" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + + "github.com/10gen/ops-manager-kubernetes/pkg/kube" +) + +// MongoDBOpsManagerEventHandler extends handler.EnqueueRequestForObject (from controller-runtime) +// which enqueues a Request containing the Name and Namespace of the object that is the source of the Event. +// It is used by the OpsManagerReconciler to reconcile OpsManager resource. +type MongoDBOpsManagerEventHandler struct { + *handler.EnqueueRequestForObject + reconciler interface { + OnDelete(ctx context.Context, obj interface{}, log *zap.SugaredLogger) + } +} + +// Delete implements EventHandler and it is called when the CR is removed +func (eh *MongoDBOpsManagerEventHandler) Delete(ctx context.Context, e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + objectKey := kube.ObjectKey(e.Object.GetNamespace(), e.Object.GetName()) + logger := zap.S().With("resource", objectKey) + + zap.S().Infow("Cleaning up OpsManager resource", "resource", e.Object) + eh.reconciler.OnDelete(ctx, e.Object, logger) + + logger.Info("Removed Ops Manager resource") +} diff --git a/controllers/operator/mongodbreplicaset_controller.go b/controllers/operator/mongodbreplicaset_controller.go new file mode 100644 index 000000000..08df40e02 --- /dev/null +++ b/controllers/operator/mongodbreplicaset_controller.go @@ -0,0 +1,599 @@ +package operator + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbstatus "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/om/deployment" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/om/replicaset" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/recovery" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + util_int "github.com/10gen/ops-manager-kubernetes/pkg/util/int" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" + "github.com/10gen/ops-manager-kubernetes/pkg/vault/vaultwatcher" +) + +// ReconcileMongoDbReplicaSet reconciles a MongoDB with a type of ReplicaSet +type ReconcileMongoDbReplicaSet struct { + *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + imageUrls images.ImageUrls + forceEnterprise bool + + initDatabaseNonStaticImageVersion string + databaseNonStaticImageVersion string +} + +var _ reconcile.Reconciler = &ReconcileMongoDbReplicaSet{} + +func newReplicaSetReconciler(ctx context.Context, kubeClient client.Client, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, omFunc om.ConnectionFactory) *ReconcileMongoDbReplicaSet { + return &ReconcileMongoDbReplicaSet{ + ReconcileCommonController: NewReconcileCommonController(ctx, kubeClient), + omConnectionFactory: omFunc, + imageUrls: imageUrls, + forceEnterprise: forceEnterprise, + + initDatabaseNonStaticImageVersion: initDatabaseNonStaticImageVersion, + databaseNonStaticImageVersion: databaseNonStaticImageVersion, + } +} + +// Generic Kubernetes Resources +// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=list;watch,namespace=placeholder +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch,namespace=placeholder +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update,namespace=placeholder +// +kubebuilder:rbac:groups=core,resources={secrets,configmaps},verbs=get;list;watch;create;delete;update,namespace=placeholder +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=create;get;list;watch;delete;update,namespace=placeholder + +// MongoDB Resource +// +kubebuilder:rbac:groups=mongodb.com,resources={mongodb,mongodb/status,mongodb/finalizers},verbs=*,namespace=placeholder + +// Setting up a webhook +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;create;update;delete + +// Certificate generation +// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=get;create;list;watch + +// Reconcile reads that state of the cluster for a MongoDbReplicaSet object and makes changes based on the state read +// and what is in the MongoDbReplicaSet.Spec +func (r *ReconcileMongoDbReplicaSet) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, e error) { + log := zap.S().With("ReplicaSet", request.NamespacedName) + rs := &mdbv1.MongoDB{} + + if reconcileResult, err := r.prepareResourceForReconciliation(ctx, request, rs, log); err != nil { + if errors.IsNotFound(err) { + return workflow.Invalid("Object for reconciliation not found").ReconcileResult() + } + return reconcileResult, err + } + + if !architectures.IsRunningStaticArchitecture(rs.Annotations) { + agents.UpgradeAllIfNeeded(ctx, agents.ClientSecret{Client: r.client, SecretClient: r.SecretClient}, r.omConnectionFactory, GetWatchedNamespace(), false) + } + + log.Info("-> ReplicaSet.Reconcile") + log.Infow("ReplicaSet.Spec", "spec", rs.Spec, "desiredReplicas", scale.ReplicasThisReconciliation(rs), "isScaling", scale.IsStillScaling(rs)) + log.Infow("ReplicaSet.Status", "status", rs.Status) + + if err := rs.ProcessValidationsOnReconcile(nil); err != nil { + return r.updateStatus(ctx, rs, workflow.Invalid("%s", err.Error()), log) + } + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, rs, log) + if err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(err), log) + } + + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, rs.Namespace, log) + if err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(xerrors.Errorf("Failed to prepare Ops Manager connection: %w", err)), log) + } + + if status := ensureSupportedOpsManagerVersion(conn); status.Phase() != mdbstatus.PhaseRunning { + return r.updateStatus(ctx, rs, status, log) + } + + r.SetupCommonWatchers(rs, nil, nil, rs.Name) + + reconcileResult := checkIfHasExcessProcesses(conn, rs.Name, log) + if !reconcileResult.IsOK() { + return r.updateStatus(ctx, rs, reconcileResult, log) + } + + if status := validateMongoDBResource(rs, conn); !status.IsOK() { + return r.updateStatus(ctx, rs, status, log) + } + + status := certs.EnsureSSLCertsForStatefulSet(ctx, r.SecretClient, r.SecretClient, *rs.Spec.Security, certs.ReplicaSetConfig(*rs), log) + if !status.IsOK() { + return r.updateStatus(ctx, rs, status, log) + } + + prometheusCertHash, err := certs.EnsureTLSCertsForPrometheus(ctx, r.SecretClient, rs.GetNamespace(), rs.GetPrometheus(), certs.Database, log) + if err != nil { + log.Infof("Could not generate certificates for Prometheus: %s", err) + return r.updateStatus(ctx, rs, workflow.Pending("%s", err.Error()), log) + } + + if status := controlledfeature.EnsureFeatureControls(*rs, conn, conn.OpsManagerVersion(), log); !status.IsOK() { + return r.updateStatus(ctx, rs, status, log) + } + + currentAgentAuthMode, err := conn.GetAgentAuthMode() + if err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(err), log) + } + + certConfigurator := certs.ReplicaSetX509CertConfigurator{MongoDB: rs, SecretClient: r.SecretClient} + status = r.ensureX509SecretAndCheckTLSType(ctx, certConfigurator, currentAgentAuthMode, log) + if !status.IsOK() { + return r.updateStatus(ctx, rs, status, log) + } + + rsCertsConfig := certs.ReplicaSetConfig(*rs) + + var vaultConfig vault.VaultConfiguration + var databaseSecretPath string + if r.VaultClient != nil { + vaultConfig = r.VaultClient.VaultConfig + databaseSecretPath = r.VaultClient.DatabaseSecretPath() + } + + var automationAgentVersion string + if architectures.IsRunningStaticArchitecture(rs.Annotations) { + // In case the Agent *is* overridden, its version will be merged into the StatefulSet. The merging process + // happens after creating the StatefulSet definition. + if !rs.IsAgentImageOverridden() { + automationAgentVersion, err = r.getAgentVersion(conn, conn.OpsManagerVersion().VersionString, false, log) + if err != nil { + log.Errorf("Impossible to get agent version, please override the agent image by providing a pod template") + status := workflow.Failed(xerrors.Errorf("Failed to get agent version: %w", err)) + return r.updateStatus(ctx, rs, status, log) + } + } + } + + rsConfig := construct.ReplicaSetOptions( + PodEnvVars(newPodVars(conn, projectConfig, rs.Spec.LogLevel)), + CurrentAgentAuthMechanism(currentAgentAuthMode), + CertificateHash(enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, rs.Namespace, rsCertsConfig.CertSecretName, databaseSecretPath, log)), + InternalClusterHash(enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, rs.Namespace, rsCertsConfig.InternalClusterSecretName, databaseSecretPath, log)), + PrometheusTLSCertHash(prometheusCertHash), + WithVaultConfig(vaultConfig), + WithLabels(rs.Labels), + WithAdditionalMongodConfig(rs.Spec.GetAdditionalMongodConfig()), + WithInitDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.InitDatabaseImageUrlEnv, r.initDatabaseNonStaticImageVersion)), + WithDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.NonStaticDatabaseEnterpriseImage, r.databaseNonStaticImageVersion)), + WithAgentImage(images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, automationAgentVersion)), + WithMongodbImage(images.GetOfficialImage(r.imageUrls, rs.Spec.Version, rs.GetAnnotations())), + ) + + caFilePath := fmt.Sprintf("%s/ca-pem", util.TLSCaMountPath) + + if err := r.reconcileHostnameOverrideConfigMap(ctx, log, r.client, *rs); err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(xerrors.Errorf("Failed to reconcileHostnameOverrideConfigMap: %w", err)), log) + } + + sts := construct.DatabaseStatefulSet(*rs, rsConfig, log) + if status := ensureRoles(rs.Spec.GetSecurity().Roles, conn, log); !status.IsOK() { + return r.updateStatus(ctx, rs, status, log) + } + + if scale.ReplicasThisReconciliation(rs) < rs.Status.Members { + if err := replicaset.PrepareScaleDownFromStatefulSet(conn, sts, rs, log); err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(xerrors.Errorf("Failed to prepare Replica Set for scaling down using Ops Manager: %w", err)), log) + } + } + + agentCertSecretName := rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name + agentCertSecretName += certs.OperatorGeneratedCertSuffix + + // Recovery prevents some deadlocks that can occur during reconciliation, e.g. the setting of an incorrect automation + // configuration and a subsequent attempt to overwrite it later, the operator would be stuck in Pending phase. + // See CLOUDP-189433 and CLOUDP-229222 for more details. + if recovery.ShouldTriggerRecovery(rs.Status.Phase != mdbstatus.PhaseRunning, rs.Status.LastTransition) { + log.Warnf("Triggering Automatic Recovery. The MongoDB resource %s/%s is in %s state since %s", rs.Namespace, rs.Name, rs.Status.Phase, rs.Status.LastTransition) + automationConfigStatus := r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, agentCertSecretName, prometheusCertHash, true).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + deploymentError := create.DatabaseInKubernetes(ctx, r.client, *rs, sts, rsConfig, log) + if deploymentError != nil { + log.Errorf("Recovery failed because of deployment errors, %w", deploymentError) + } + if !automationConfigStatus.IsOK() { + log.Errorf("Recovery failed because of Automation Config update errors, %v", automationConfigStatus) + } + } + + lastSpec, err := rs.GetLastSpec() + if err != nil { + lastSpec = &mdbv1.MongoDbSpec{} + } + status = workflow.RunInGivenOrder(publishAutomationConfigFirst(ctx, r.client, *rs, lastSpec, rsConfig, log), + func() workflow.Status { + return r.updateOmDeploymentRs(ctx, conn, rs.Status.Members, rs, sts, log, caFilePath, agentCertSecretName, prometheusCertHash, false).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + }, + func() workflow.Status { + workflowStatus := create.HandlePVCResize(ctx, r.client, &sts, log) + if !workflowStatus.IsOK() { + return workflowStatus + } + if workflow.ContainsPVCOption(workflowStatus.StatusOptions()) { + _, _ = r.updateStatus(ctx, rs, workflow.Pending(""), log, workflowStatus.StatusOptions()...) + } + + if err := create.DatabaseInKubernetes(ctx, r.client, *rs, sts, rsConfig, log); err != nil { + return workflow.Failed(xerrors.Errorf("Failed to create/update (Kubernetes reconciliation phase): %w", err)) + } + + if status := getStatefulSetStatus(ctx, rs.Namespace, rs.Name, r.client); !status.IsOK() { + return status + } + + log.Info("Updated StatefulSet for replica set") + return workflow.OK() + }) + + if !status.IsOK() { + return r.updateStatus(ctx, rs, status, log) + } + + if scale.IsStillScaling(rs) { + return r.updateStatus(ctx, rs, workflow.Pending("Continuing scaling operation for ReplicaSet %s, desiredMembers=%d, currentMembers=%d", rs.ObjectKey(), rs.DesiredReplicas(), scale.ReplicasThisReconciliation(rs)), log, mdbstatus.MembersOption(rs)) + } + + annotationsToAdd, err := getAnnotationsForResource(rs) + if err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(err), log) + } + + if vault.IsVaultSecretBackend() { + secrets := rs.GetSecretsMountedIntoDBPod() + vaultMap := make(map[string]string) + for _, s := range secrets { + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.DatabaseSecretMetadataPath(), rs.Namespace, s) + vaultMap = merge.StringToStringMap(vaultMap, r.VaultClient.GetSecretAnnotation(path)) + } + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.OperatorScretMetadataPath(), rs.Namespace, rs.Spec.Credentials) + vaultMap = merge.StringToStringMap(vaultMap, r.VaultClient.GetSecretAnnotation(path)) + for k, val := range vaultMap { + annotationsToAdd[k] = val + } + } + + if err := annotations.SetAnnotations(ctx, rs, annotationsToAdd, r.client); err != nil { + return r.updateStatus(ctx, rs, workflow.Failed(err), log) + } + + log.Infof("Finished reconciliation for MongoDbReplicaSet! %s", completionMessage(conn.BaseURL(), conn.GroupID())) + return r.updateStatus(ctx, rs, workflow.OK(), log, mdbstatus.NewBaseUrlOption(deployment.Link(conn.BaseURL(), conn.GroupID())), mdbstatus.MembersOption(rs), mdbstatus.NewPVCsStatusOptionEmptyStatus()) +} + +func getHostnameOverrideConfigMapForReplicaset(mdb mdbv1.MongoDB) corev1.ConfigMap { + data := make(map[string]string) + + if mdb.Spec.DbCommonSpec.GetExternalDomain() != nil { + hostnames, names := dns.GetDNSNames(mdb.Name, "", mdb.GetObjectMeta().GetNamespace(), mdb.Spec.GetClusterDomain(), mdb.Spec.Members, mdb.Spec.DbCommonSpec.GetExternalDomain()) + for i := range hostnames { + data[names[i]] = hostnames[i] + } + } + + cm := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-hostname-override", mdb.Name), + Namespace: mdb.Namespace, + }, + Data: data, + } + return cm +} + +func (r *ReconcileMongoDbReplicaSet) reconcileHostnameOverrideConfigMap(ctx context.Context, log *zap.SugaredLogger, getUpdateCreator configmap.GetUpdateCreator, mdb mdbv1.MongoDB) error { + if mdb.Spec.DbCommonSpec.GetExternalDomain() == nil { + return nil + } + + cm := getHostnameOverrideConfigMapForReplicaset(mdb) + err := configmap.CreateOrUpdate(ctx, getUpdateCreator, cm) + if err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create configmap: %s, err: %w", cm.Name, err) + } + log.Infof("Successfully ensured configmap: %s", cm.Name) + + return nil +} + +// AddReplicaSetController creates a new MongoDbReplicaset Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func AddReplicaSetController(ctx context.Context, mgr manager.Manager, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool) error { + // Create a new controller + reconciler := newReplicaSetReconciler(ctx, mgr.GetClient(), imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, om.NewOpsManagerConnection) + c, err := controller.New(util.MongoDbReplicaSetController, mgr, controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}) // nolint:forbidigo + if err != nil { + return err + } + + // watch for changes to replica set MongoDB resources + eventHandler := ResourceEventHandler{deleter: reconciler} + // Watch for changes to primary resource MongoDbReplicaSet + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &mdbv1.MongoDB{}, &eventHandler, watch.PredicatesForMongoDB(mdbv1.ReplicaSet))) + if err != nil { + return err + } + + err = c.Watch(source.Channel[client.Object](OmUpdateChannel, &handler.EnqueueRequestForObject{}, source.WithPredicates(watch.PredicatesForMongoDB(mdbv1.ReplicaSet)))) + if err != nil { + return xerrors.Errorf("not able to setup OmUpdateChannel to listent to update events from OM: %s", err) + } + + err = c.Watch( + source.Kind[client.Object](mgr.GetCache(), &appsv1.StatefulSet{}, + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &mdbv1.MongoDB{}, handler.OnlyControllerOwner()), + watch.PredicatesForStatefulSet())) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, + &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + // if vault secret backend is enabled watch for Vault secret change and trigger reconcile + if vault.IsVaultSecretBackend() { + eventChannel := make(chan event.GenericEvent) + go vaultwatcher.WatchSecretChangeForMDB(ctx, zap.S(), eventChannel, reconciler.client, reconciler.VaultClient, mdbv1.ReplicaSet) + + err = c.Watch(source.Channel[client.Object](eventChannel, &handler.EnqueueRequestForObject{})) + if err != nil { + zap.S().Errorf("Failed to watch for vault secret changes: %w", err) + } + } + zap.S().Infof("Registered controller %s", util.MongoDbReplicaSetController) + + return nil +} + +// updateOmDeploymentRs performs OM registration operation for the replicaset. So the changes will be finally propagated +// to automation agents in containers +func (r *ReconcileMongoDbReplicaSet) updateOmDeploymentRs(ctx context.Context, conn om.Connection, membersNumberBefore int, rs *mdbv1.MongoDB, set appsv1.StatefulSet, log *zap.SugaredLogger, caFilePath string, agentCertSecretName string, prometheusCertHash string, isRecovering bool) workflow.Status { + log.Debug("Entering UpdateOMDeployments") + // Only "concrete" RS members should be observed + // - if scaling down, let's observe only members that will remain after scale-down operation + // - if scaling up, observe only current members, because new ones might not exist yet + err := agents.WaitForRsAgentsToRegister(set, util_int.Min(membersNumberBefore, int(*set.Spec.Replicas)), rs.Spec.GetClusterDomain(), conn, log, rs) + if err != nil && !isRecovering { + return workflow.Failed(err) + } + + // If current operation is to Disable TLS, then we should the current members of the Replica Set, + // this is, do not scale them up or down util TLS disabling has completed. + shouldLockMembers, err := updateOmDeploymentDisableTLSConfiguration(conn, r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, membersNumberBefore, rs, set, log, caFilePath) + if err != nil && !isRecovering { + return workflow.Failed(err) + } + + var updatedMembers int + if shouldLockMembers { + // We should not add or remove members during this run, we'll wait for + // TLS to be completely disabled first. + updatedMembers = membersNumberBefore + } else { + updatedMembers = int(*set.Spec.Replicas) + } + + replicaSet := replicaset.BuildFromStatefulSetWithReplicas(r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, set, rs.GetSpec(), updatedMembers, rs.CalculateFeatureCompatibilityVersion()) + processNames := replicaSet.GetProcessNames() + + internalClusterPath := "" + if hash := set.Annotations[util.InternalCertAnnotationKey]; hash != "" { + internalClusterPath = fmt.Sprintf("%s%s", util.InternalClusterAuthMountPath, hash) + } + + status, additionalReconciliationRequired := r.updateOmAuthentication(ctx, conn, processNames, rs, agentCertSecretName, caFilePath, internalClusterPath, isRecovering, log) + if !status.IsOK() && !isRecovering { + return status + } + + lastRsConfig, err := rs.GetLastAdditionalMongodConfigByType(mdbv1.ReplicaSetConfig) + if err != nil && !isRecovering { + return workflow.Failed(err) + } + + p := PrometheusConfiguration{ + prometheus: rs.GetPrometheus(), + conn: conn, + secretsClient: r.SecretClient, + namespace: rs.GetNamespace(), + prometheusCertHash: prometheusCertHash, + } + + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + return ReconcileReplicaSetAC(ctx, d, rs.Spec.DbCommonSpec, lastRsConfig.ToMap(), rs.Name, replicaSet, caFilePath, internalClusterPath, &p, log) + }, + log, + ) + + if err != nil && !isRecovering { + return workflow.Failed(err) + } + + if err := om.WaitForReadyState(conn, processNames, isRecovering, log); err != nil { + return workflow.Failed(err) + } + + reconcileResult, _ := ReconcileLogRotateSetting(conn, rs.Spec.Agent, log) + if !reconcileResult.IsOK() { + return reconcileResult + } + + if additionalReconciliationRequired { + return workflow.Pending("Performing multi stage reconciliation") + } + + externalDomain := rs.Spec.DbCommonSpec.GetExternalDomain() + hostsBefore := getAllHostsRs(set, rs.Spec.GetClusterDomain(), membersNumberBefore, externalDomain) + hostsAfter := getAllHostsRs(set, rs.Spec.GetClusterDomain(), scale.ReplicasThisReconciliation(rs), externalDomain) + + if err := host.CalculateDiffAndStopMonitoring(conn, hostsBefore, hostsAfter, log); err != nil && !isRecovering { + return workflow.Failed(err) + } + + if status := r.ensureBackupConfigurationAndUpdateStatus(ctx, conn, rs, r.SecretClient, log); !status.IsOK() && !isRecovering { + return status + } + + log.Info("Updated Ops Manager for replica set") + return workflow.OK() +} + +// updateOmDeploymentDisableTLSConfiguration checks if TLS configuration needs +// to be disabled. In which case it will disable it and inform to the calling +// function. +func updateOmDeploymentDisableTLSConfiguration(conn om.Connection, mongoDBImage string, forceEnterprise bool, membersNumberBefore int, rs *mdbv1.MongoDB, set appsv1.StatefulSet, log *zap.SugaredLogger, caFilePath string) (bool, error) { + tlsConfigWasDisabled := false + + err := conn.ReadUpdateDeployment( + func(d om.Deployment) error { + if !d.TLSConfigurationWillBeDisabled(rs.Spec.GetSecurity()) { + return nil + } + + tlsConfigWasDisabled = true + d.ConfigureTLS(rs.Spec.GetSecurity(), caFilePath) + + // configure as many agents/Pods as we currently have, no more (in case + // there's a scale up change at the same time). + replicaSet := replicaset.BuildFromStatefulSetWithReplicas(mongoDBImage, forceEnterprise, set, rs.GetSpec(), membersNumberBefore, rs.CalculateFeatureCompatibilityVersion()) + + lastConfig, err := rs.GetLastAdditionalMongodConfigByType(mdbv1.ReplicaSetConfig) + if err != nil { + return err + } + + d.MergeReplicaSet(replicaSet, rs.Spec.AdditionalMongodConfig.ToMap(), lastConfig.ToMap(), log) + + return nil + }, + log, + ) + + return tlsConfigWasDisabled, err +} + +func (r *ReconcileMongoDbReplicaSet) OnDelete(ctx context.Context, obj runtime.Object, log *zap.SugaredLogger) error { + rs := obj.(*mdbv1.MongoDB) + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, rs, log) + if err != nil { + return err + } + + log.Infow("Removing replica set from Ops Manager", "config", rs.Spec) + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, rs.Namespace, log) + if err != nil { + return err + } + processNames := make([]string, 0) + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + processNames = d.GetProcessNames(om.ReplicaSet{}, rs.Name) + // error means that replica set is not in the deployment - it's ok, and we can proceed (could happen if + // deletion cleanup happened twice and the first one cleaned OM state already) + if e := d.RemoveReplicaSetByName(rs.Name, log); e != nil { + log.Warnf("Failed to remove replica set from automation config: %s", e) + } + + return nil + }, + log, + ) + if err != nil { + return err + } + + if err := om.WaitForReadyState(conn, processNames, false, log); err != nil { + return err + } + + if rs.Spec.Backup != nil && rs.Spec.Backup.AutoTerminateOnDeletion { + if err := backup.StopBackupIfEnabled(conn, conn, rs.Name, backup.ReplicaSetType, log); err != nil { + return err + } + } + + hostsToRemove, _ := dns.GetDNSNames(rs.Name, rs.ServiceName(), rs.Namespace, rs.Spec.GetClusterDomain(), util.MaxInt(rs.Status.Members, rs.Spec.Members), nil) + log.Infow("Stop monitoring removed hosts in Ops Manager", "removedHosts", hostsToRemove) + + if err = host.StopMonitoring(conn, hostsToRemove, log); err != nil { + return err + } + + if err := r.clearProjectAuthenticationSettings(ctx, conn, rs, processNames, log); err != nil { + return err + } + + r.resourceWatcher.RemoveDependentWatchedResources(rs.ObjectKey()) + + log.Infow("Clear feature control for group: %s", "groupID", conn.GroupID()) + if result := controlledfeature.ClearFeatureControls(conn, conn.OpsManagerVersion(), log); !result.IsOK() { + result.Log(log) + log.Warnf("Failed to clear feature control from group: %s", conn.GroupID()) + } + + log.Info("Removed replica set from Ops Manager!") + return nil +} + +func getAllHostsRs(set appsv1.StatefulSet, clusterName string, membersCount int, externalDomain *string) []string { + hostnames, _ := dns.GetDnsForStatefulSetReplicasSpecified(set, clusterName, membersCount, externalDomain) + return hostnames +} diff --git a/controllers/operator/mongodbreplicaset_controller_test.go b/controllers/operator/mongodbreplicaset_controller_test.go new file mode 100644 index 000000000..ca91aead0 --- /dev/null +++ b/controllers/operator/mongodbreplicaset_controller_test.go @@ -0,0 +1,1186 @@ +package operator + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/status/pvc" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/om/deployment" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" +) + +type ReplicaSetBuilder struct { + *mdbv1.MongoDB +} + +func TestCreateReplicaSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().Build() + + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + assert.Len(t, mock.GetMapForObject(client, &corev1.Service{}), 1) + assert.Len(t, mock.GetMapForObject(client, &appsv1.StatefulSet{}), 1) + assert.Len(t, mock.GetMapForObject(client, &corev1.Secret{}), 2) + + sts, err := client.GetStatefulSet(ctx, rs.ObjectKey()) + assert.NoError(t, err) + assert.Equal(t, *sts.Spec.Replicas, int32(3)) + + connection := omConnectionFactory.GetConnection() + connection.(*om.MockedOmConnection).CheckDeployment(t, deployment.CreateFromReplicaSet("fake-mongoDBImage", false, rs), "auth", "ssl") + connection.(*om.MockedOmConnection).CheckNumberOfUpdateRequests(t, 2) +} + +func TestReplicaSetRace(t *testing.T) { + ctx := context.Background() + rs, cfgMap, projectName := buildReplicaSetWithCustomProjectName("my-rs") + rs2, cfgMap2, projectName2 := buildReplicaSetWithCustomProjectName("my-rs2") + rs3, cfgMap3, projectName3 := buildReplicaSetWithCustomProjectName("my-rs3") + + resourceToProjectMapping := map[string]string{ + "my-rs": projectName, + "my-rs2": projectName2, + "my-rs3": projectName3, + } + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory().WithResourceToProjectMapping(resourceToProjectMapping) + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(rs, rs2, rs3). + WithObjects(cfgMap, cfgMap2, cfgMap3). + WithObjects(mock.GetCredentialsSecret(om.TestUser, om.TestApiKey)). + WithInterceptorFuncs(interceptor.Funcs{ + Get: mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, true, true), + }).Build() + + reconciler := newReplicaSetReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, omConnectionFactory.GetConnectionFunc) + + testConcurrentReconciles(ctx, t, fakeClient, reconciler, rs, rs2, rs3) +} + +func TestReplicaSetClusterReconcileContainerImages(t *testing.T) { + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_1_0_0", util.NonStaticDatabaseEnterpriseImage) + initDatabaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_2_0_0", util.InitDatabaseImageUrlEnv) + + imageUrlsMock := images.ImageUrls{ + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", + initDatabaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", + } + + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetVersion("8.0.0").Build() + reconciler, kubeClient, _ := defaultReplicaSetReconciler(ctx, imageUrlsMock, "2.0.0", "1.0.0", rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + sts := &appsv1.StatefulSet{} + err := kubeClient.Get(ctx, kube.ObjectKey(rs.Namespace, rs.Name), sts) + assert.NoError(t, err) + + require.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", sts.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[0].Image) +} + +func TestReplicaSetClusterReconcileContainerImagesWithStaticArchitecture(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0_ubi9", mcoConstruct.MongodbImageEnv) + + imageUrlsMock := images.ImageUrls{ + architectures.MdbAgentImageRepo: "quay.io/mongodb/mongodb-agent-ubi", + mcoConstruct.MongodbImageEnv: "quay.io/mongodb/mongodb-enterprise-server", + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", + } + + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetVersion("8.0.0").Build() + reconciler, kubeClient, omConnectionFactory := defaultReplicaSetReconciler(ctx, imageUrlsMock, "", "", rs) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + connection.(*om.MockedOmConnection).SetAgentVersion("12.0.30.7791-1", "") + }) + + checkReconcileSuccessful(ctx, t, reconciler, rs, kubeClient) + + sts := &appsv1.StatefulSet{} + err := kubeClient.Get(ctx, kube.ObjectKey(rs.Namespace, rs.Name), sts) + assert.NoError(t, err) + + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 0) + require.Len(t, sts.Spec.Template.Spec.Containers, 2) + + // Version from OM + operator version + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:12.0.30.7791-1_9.9.9-test", sts.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[1].Image) +} + +func buildReplicaSetWithCustomProjectName(rsName string) (*mdbv1.MongoDB, *corev1.ConfigMap, string) { + configMapName := mock.TestProjectConfigMapName + "-" + rsName + projectName := om.TestGroupName + "-" + rsName + return DefaultReplicaSetBuilder().SetName(rsName).SetOpsManagerConfigMapName(configMapName).Build(), + mock.GetProjectConfigMap(configMapName, projectName, ""), projectName +} + +func TestReplicaSetServiceName(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetService("rs-svc").Build() + rs.Spec.StatefulSetConfiguration = &mdbcv1.StatefulSetConfiguration{} + rs.Spec.StatefulSetConfiguration.SpecWrapper.Spec.ServiceName = "foo" + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + assert.Equal(t, "foo", rs.ServiceName()) + _, err := client.GetService(ctx, kube.ObjectKey(rs.Namespace, rs.ServiceName())) + assert.NoError(t, err) +} + +func TestHorizonVerificationTLS(t *testing.T) { + ctx := context.Background() + replicaSetHorizons := []mdbv1.MongoDBHorizonConfig{ + {"my-horizon": "my-db.com:12345"}, + {"my-horizon": "my-db.com:12346"}, + {"my-horizon": "my-db.com:12347"}, + } + rs := DefaultReplicaSetBuilder().SetReplicaSetHorizons(replicaSetHorizons).Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + msg := "TLS must be enabled in order to use replica set horizons" + checkReconcileFailed(ctx, t, reconciler, rs, false, msg, client) +} + +func TestHorizonVerificationCount(t *testing.T) { + ctx := context.Background() + replicaSetHorizons := []mdbv1.MongoDBHorizonConfig{ + {"my-horizon": "my-db.com:12345"}, + {"my-horizon": "my-db.com:12346"}, + } + rs := DefaultReplicaSetBuilder(). + EnableTLS(). + SetReplicaSetHorizons(replicaSetHorizons). + Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + msg := "Number of horizons must be equal to number of members in replica set" + checkReconcileFailed(ctx, t, reconciler, rs, false, msg, client) +} + +// This test is broken as it recreates the object entirely which doesn't keep last members and scales immediately up to 5 +// We already have unit tests that checks scaling one member at a time: TestScalingScalesOneMemberAtATime* +// TestScaleUpReplicaSet verifies scaling up for replica set. Statefulset and OM Deployment must be changed accordingly +//func TestScaleUpReplicaSet(t *testing.T) { +// ctx := context.Background() +// rs := DefaultReplicaSetBuilder().SetMembers(3).Build() +// +// reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) +// +// checkReconcileSuccessful(ctx, t, reconciler, rs, client) +// set := &appsv1.StatefulSet{} +// _ = client.Get(ctx, mock.ObjectKeyFromApiObject(rs), set) +// +// // Now scale up to 5 nodes +// rs = DefaultReplicaSetBuilder().SetMembers(5).Build() +// _ = client.Update(ctx, rs) +// +// checkReconcileSuccessful(ctx, t, reconciler, rs, client) +// +// updatedSet := &appsv1.StatefulSet{} +// _ = client.Get(ctx, mock.ObjectKeyFromApiObject(rs), updatedSet) +// +// // Statefulset is expected to be the same - only number of replicas changed +// set.Spec.Replicas = util.Int32Ref(int32(5)) +// assert.Equal(t, set.Spec, updatedSet.Spec) +// +// connection := omConnectionFactory.GetConnection() +// connection.(*om.MockedOmConnection).CheckDeployment(t, deployment.CreateFromReplicaSet(imageUrlsMock, false, rs), "auth", "tls") +// connection.(*om.MockedOmConnection).CheckNumberOfUpdateRequests(t, 4) +//} + +func TestExposedExternallyReplicaSet(t *testing.T) { + ctx := context.Background() + // given + rs := DefaultReplicaSetBuilder().SetMembers(3).ExposedExternally(nil, nil, nil).Build() + + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + // when + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + // then + // We removed support for single external service named -svc-external (round-robin to all pods). + externalService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{}, + } + err := client.Get(ctx, types.NamespacedName{Name: rs.Name + "-svc-external", Namespace: rs.Namespace}, externalService) + assert.Error(t, err) + + for podNum := 0; podNum < 3; podNum++ { + err := client.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%d-svc-external", rs.Name, podNum), Namespace: rs.Namespace}, externalService) + assert.NoError(t, err) + + assert.NoError(t, err) + assert.Equal(t, corev1.ServiceTypeLoadBalancer, externalService.Spec.Type) + assert.Len(t, externalService.Spec.Ports, 1) + assert.Equal(t, "mongodb", externalService.Spec.Ports[0].Name) + assert.Equal(t, 27017, externalService.Spec.Ports[0].TargetPort.IntValue()) + } + + processes := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses() + require.Len(t, processes, 3) + // check hostnames are pod's headless service FQDNs + for i, process := range processes { + assert.Equal(t, fmt.Sprintf("%s-%d.%s-svc.%s.svc.cluster.local", rs.Name, i, rs.Name, rs.Namespace), process.HostName()) + } +} + +func TestExposedExternallyReplicaSetExternalDomainInHostnames(t *testing.T) { + ctx := context.Background() + externalDomain := "example.com" + memberCount := 3 + replicaSetName := "rs" + var expectedHostnames []string + for i := 0; i < memberCount; i++ { + expectedHostnames = append(expectedHostnames, fmt.Sprintf("%s-%d.%s", replicaSetName, i, externalDomain)) + } + + testExposedExternallyReplicaSetExternalDomainInHostnames(ctx, t, replicaSetName, memberCount, externalDomain, expectedHostnames) +} + +func testExposedExternallyReplicaSetExternalDomainInHostnames(ctx context.Context, t *testing.T, replicaSetName string, memberCount int, externalDomain string, expectedHostnames []string) { + rs := DefaultReplicaSetBuilder().SetName(replicaSetName).SetMembers(memberCount).ExposedExternally(nil, nil, &externalDomain).Build() + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + // We set this to mock processes that agents are registering in OM, otherwise reconcile will hang on agent.WaitForRsAgentsToRegister. + // hostnames are already mocked in markStatefulSetsReady, + // but we don't have externalDomain information in statefulset alone there, hence we're setting them here + connection.(*om.MockedOmConnection).Hostnames = expectedHostnames + }) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + processes := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses() + require.Len(t, processes, memberCount) + // check hostnames are external domain + for i, process := range processes { + // process.HostName is created when building automation config using resource spec + assert.Equal(t, expectedHostnames[i], process.HostName()) + } +} + +func TestExposedExternallyReplicaSetWithNodePort(t *testing.T) { + ctx := context.Background() + // given + rs := DefaultReplicaSetBuilder(). + SetMembers(3). + ExposedExternally( + &corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + }, + map[string]string{"test": "test"}, + nil). + Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + // when + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + externalService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{}, + } + + // then + for podNum := 0; podNum < 3; podNum++ { + err := client.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-%d-svc-external", rs.Name, podNum), Namespace: rs.Namespace}, externalService) + assert.NoError(t, err) + + assert.NoError(t, err) + assert.Equal(t, corev1.ServiceTypeNodePort, externalService.Spec.Type) + assert.Len(t, externalService.Spec.Ports, 1) + assert.Equal(t, "mongodb", externalService.Spec.Ports[0].Name) + assert.Equal(t, 27017, externalService.Spec.Ports[0].TargetPort.IntValue()) + } +} + +func TestCreateReplicaSet_TLS(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetMembers(3).EnableTLS().SetTLSCA("custom-ca").Build() + + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + addKubernetesTlsResources(ctx, client, rs) + mock.ApproveAllCSRs(ctx, client) + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + processes := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetProcesses() + assert.Len(t, processes, 3) + for _, v := range processes { + assert.NotNil(t, v.TLSConfig()) + assert.Len(t, v.TLSConfig(), 2) + assert.Equal(t, fmt.Sprintf("%s/%s", util.TLSCertMountPath, pem.ReadHashFromSecret(ctx, reconciler.SecretClient, rs.Namespace, fmt.Sprintf("%s-cert", rs.Name), "", zap.S())), v.TLSConfig()["certificateKeyFile"]) + assert.Equal(t, "requireTLS", v.TLSConfig()["mode"]) + } + + sslConfig := omConnectionFactory.GetConnection().(*om.MockedOmConnection).GetTLS() + assert.Equal(t, fmt.Sprintf("%s/%s", util.TLSCaMountPath, "ca-pem"), sslConfig["CAFilePath"]) + assert.Equal(t, "OPTIONAL", sslConfig["clientCertificateMode"]) +} + +// TestUpdateDeploymentTLSConfiguration a combination of tests checking that: +// +// TLS Disabled -> TLS Disabled: should not lock members +// TLS Disabled -> TLS Enabled: should not lock members +// TLS Enabled -> TLS Enabled: should not lock members +// TLS Enabled -> TLS Disabled: *should lock members* +func TestUpdateDeploymentTLSConfiguration(t *testing.T) { + rsWithTLS := mdbv1.NewReplicaSetBuilder().SetSecurityTLSEnabled().Build() + rsNoTLS := mdbv1.NewReplicaSetBuilder().Build() + deploymentWithTLS := deployment.CreateFromReplicaSet("fake-mongoDBImage", false, rsWithTLS) + deploymentNoTLS := deployment.CreateFromReplicaSet("fake-mongoDBImage", false, rsNoTLS) + stsWithTLS := construct.DatabaseStatefulSet(*rsWithTLS, construct.ReplicaSetOptions(construct.GetPodEnvOptions()), zap.S()) + stsNoTLS := construct.DatabaseStatefulSet(*rsNoTLS, construct.ReplicaSetOptions(construct.GetPodEnvOptions()), zap.S()) + + // TLS Disabled -> TLS Disabled + shouldLockMembers, err := updateOmDeploymentDisableTLSConfiguration(om.NewMockedOmConnection(deploymentNoTLS), "fake-mongoDBImage", false, 3, rsNoTLS, stsNoTLS, zap.S(), util.CAFilePathInContainer) + assert.NoError(t, err) + assert.False(t, shouldLockMembers) + + // TLS Disabled -> TLS Enabled + shouldLockMembers, err = updateOmDeploymentDisableTLSConfiguration(om.NewMockedOmConnection(deploymentNoTLS), "fake-mongoDBImage", false, 3, rsWithTLS, stsWithTLS, zap.S(), util.CAFilePathInContainer) + assert.NoError(t, err) + assert.False(t, shouldLockMembers) + + // TLS Enabled -> TLS Enabled + shouldLockMembers, err = updateOmDeploymentDisableTLSConfiguration(om.NewMockedOmConnection(deploymentWithTLS), "fake-mongoDBImage", false, 3, rsWithTLS, stsWithTLS, zap.S(), util.CAFilePathInContainer) + assert.NoError(t, err) + assert.False(t, shouldLockMembers) + + // TLS Enabled -> TLS Disabled + shouldLockMembers, err = updateOmDeploymentDisableTLSConfiguration(om.NewMockedOmConnection(deploymentWithTLS), "fake-mongoDBImage", false, 3, rsNoTLS, stsNoTLS, zap.S(), util.CAFilePathInContainer) + assert.NoError(t, err) + assert.True(t, shouldLockMembers) +} + +// TestCreateDeleteReplicaSet checks that no state is left in OpsManager on removal of the replicaset +func TestCreateDeleteReplicaSet(t *testing.T) { + ctx := context.Background() + // First we need to create a replicaset + rs := DefaultReplicaSetBuilder().Build() + + omConnectionFactory := om.NewCachedOMConnectionFactory(omConnectionFactoryFuncSettingVersion()) + fakeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + reconciler := newReplicaSetReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, fakeClient) + omConn := omConnectionFactory.GetConnection() + mockedOmConn := omConn.(*om.MockedOmConnection) + mockedOmConn.CleanHistory() + + // Now delete it + assert.NoError(t, reconciler.OnDelete(ctx, rs, zap.S())) + + // Operator doesn't mutate K8s state, so we don't check its changes, only OM + mockedOmConn.CheckResourcesDeleted(t) + + mockedOmConn.CheckOrderOfOperations(t, + reflect.ValueOf(mockedOmConn.ReadUpdateDeployment), reflect.ValueOf(mockedOmConn.ReadAutomationStatus), + reflect.ValueOf(mockedOmConn.GetHosts), reflect.ValueOf(mockedOmConn.RemoveHost)) +} + +func TestReplicaSetScramUpgradeDowngrade(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetVersion("4.0.0").EnableAuth().SetAuthModes([]mdbv1.AuthMode{"SCRAM"}).Build() + + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.Contains(t, ac.Auth.AutoAuthMechanisms, string(authentication.ScramSha256)) + + // downgrade to version that will not use SCRAM-SHA-256 + rs.Spec.Version = "3.6.9" + + _ = client.Update(ctx, rs) + + checkReconcileFailed(ctx, t, reconciler, rs, false, "Unable to downgrade to SCRAM-SHA-1 when SCRAM-SHA-256 has been enabled", client) +} + +func TestChangingFCVReplicaSet(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetVersion("4.0.0").Build() + reconciler, cl, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + // Helper function to update and verify FCV + verifyFCV := func(version, expectedFCV string, fcvOverride *string, t *testing.T) { + if fcvOverride != nil { + rs.Spec.FeatureCompatibilityVersion = fcvOverride + } + + rs.Spec.Version = version + _ = cl.Update(ctx, rs) + checkReconcileSuccessful(ctx, t, reconciler, rs, cl) + assert.Equal(t, expectedFCV, rs.Status.FeatureCompatibilityVersion) + } + + testFCVsCases(t, verifyFCV) +} + +func TestReplicaSetCustomPodSpecTemplate(t *testing.T) { + ctx := context.Background() + podSpec := corev1.PodSpec{ + NodeName: "some-node-name", + Hostname: "some-host-name", + Containers: []corev1.Container{{ + Name: "my-custom-container", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + } + + rs := DefaultReplicaSetBuilder().EnableTLS().SetTLSCA("custom-ca").SetPodSpecTemplate(corev1.PodTemplateSpec{ + Spec: podSpec, + }).Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + addKubernetesTlsResources(ctx, client, rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + // read the stateful set that was created by the operator + statefulSet, err := client.GetStatefulSet(ctx, mock.ObjectKeyFromApiObject(rs)) + assert.NoError(t, err) + + assertPodSpecSts(t, &statefulSet, podSpec.NodeName, podSpec.Hostname, podSpec.RestartPolicy) + + podSpecTemplate := statefulSet.Spec.Template.Spec + assert.Len(t, podSpecTemplate.Containers, 2, "Should have 2 containers now") + assert.Equal(t, util.DatabaseContainerName, podSpecTemplate.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container", podSpecTemplate.Containers[1].Name, "Custom container should be second") +} + +func TestReplicaSetCustomPodSpecTemplateStatic(t *testing.T) { + ctx := context.Background() + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + podSpec := corev1.PodSpec{ + NodeName: "some-node-name", + Hostname: "some-host-name", + Containers: []corev1.Container{{ + Name: "my-custom-container", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + } + + rs := DefaultReplicaSetBuilder().EnableTLS().SetTLSCA("custom-ca").SetPodSpecTemplate(corev1.PodTemplateSpec{ + Spec: podSpec, + }).Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + addKubernetesTlsResources(ctx, client, rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + // read the stateful set that was created by the operator + statefulSet, err := client.GetStatefulSet(ctx, mock.ObjectKeyFromApiObject(rs)) + assert.NoError(t, err) + + assertPodSpecSts(t, &statefulSet, podSpec.NodeName, podSpec.Hostname, podSpec.RestartPolicy) + + podSpecTemplate := statefulSet.Spec.Template.Spec + assert.Len(t, podSpecTemplate.Containers, 3, "Should have 3 containers now") + assert.Equal(t, util.AgentContainerName, podSpecTemplate.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container", podSpecTemplate.Containers[2].Name, "Custom container should be second") +} + +func TestFeatureControlPolicyAndTagAddedWithNewerOpsManager(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().Build() + + omConnectionFactory := om.NewCachedOMConnectionFactory(omConnectionFactoryFuncSettingVersion()) + fakeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + reconciler := newReplicaSetReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, fakeClient) + + mockedConn := omConnectionFactory.GetConnection() + cf, _ := mockedConn.GetControlledFeature() + + assert.Len(t, cf.Policies, 3) + assert.Equal(t, cf.ManagementSystem.Version, util.OperatorVersion) + assert.Equal(t, cf.ManagementSystem.Name, util.OperatorName) + + project := mockedConn.(*om.MockedOmConnection).FindGroup("my-project") + assert.Contains(t, project.Tags, util.OmGroupExternallyManagedTag) +} + +func TestFeatureControlPolicyNoAuthNewerOpsManager(t *testing.T) { + ctx := context.Background() + rsBuilder := DefaultReplicaSetBuilder() + rsBuilder.Spec.Security = nil + + rs := rsBuilder.Build() + + omConnectionFactory := om.NewCachedOMConnectionFactory(omConnectionFactoryFuncSettingVersion()) + fakeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + reconciler := newReplicaSetReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, rs, fakeClient) + + mockedConn := omConnectionFactory.GetConnection() + cf, _ := mockedConn.GetControlledFeature() + + assert.Len(t, cf.Policies, 2) + assert.Equal(t, cf.ManagementSystem.Version, util.OperatorVersion) + assert.Equal(t, cf.ManagementSystem.Name, util.OperatorName) + assert.Equal(t, cf.Policies[0].PolicyType, controlledfeature.ExternallyManaged) + assert.Equal(t, cf.Policies[1].PolicyType, controlledfeature.DisableMongodVersion) + assert.Len(t, cf.Policies[0].DisabledParams, 0) +} + +func TestScalingScalesOneMemberAtATime_WhenScalingDown(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetMembers(5).Build() + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + // perform initial reconciliation so we are not creating a new resource + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + // scale down from 5 to 3 members + rs.Spec.Members = 3 + + err := client.Update(ctx, rs) + assert.NoError(t, err) + + res, err := reconciler.Reconcile(ctx, requestFromObject(rs)) + + assert.NoError(t, err) + assert.Equal(t, time.Duration(10000000000), res.RequeueAfter, "Scaling from 5 -> 4 should enqueue another reconciliation") + + assertCorrectNumberOfMembersAndProcesses(ctx, t, 4, rs, client, omConnectionFactory.GetConnection(), "We should have updated the status with the intermediate value of 4") + + res, err = reconciler.Reconcile(ctx, requestFromObject(rs)) + assert.NoError(t, err) + assert.Equal(t, util.TWENTY_FOUR_HOURS, res.RequeueAfter, "Once we reach the target value, we should not scale anymore") + + assertCorrectNumberOfMembersAndProcesses(ctx, t, 3, rs, client, omConnectionFactory.GetConnection(), "The members should now be set to the final desired value") +} + +func TestScalingScalesOneMemberAtATime_WhenScalingUp(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().SetMembers(1).Build() + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + // perform initial reconciliation so we are not creating a new resource + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + // scale up from 1 to 3 members + rs.Spec.Members = 3 + + err := client.Update(ctx, rs) + assert.NoError(t, err) + + res, err := reconciler.Reconcile(ctx, requestFromObject(rs)) + assert.NoError(t, err) + + assert.Equal(t, time.Duration(10000000000), res.RequeueAfter, "Scaling from 1 -> 3 should enqueue another reconciliation") + + assertCorrectNumberOfMembersAndProcesses(ctx, t, 2, rs, client, omConnectionFactory.GetConnection(), "We should have updated the status with the intermediate value of 2") + + res, err = reconciler.Reconcile(ctx, requestFromObject(rs)) + assert.NoError(t, err) + + assertCorrectNumberOfMembersAndProcesses(ctx, t, 3, rs, client, omConnectionFactory.GetConnection(), "Once we reach the target value, we should not scale anymore") +} + +func TestReplicaSetPortIsConfigurable_WithAdditionalMongoConfig(t *testing.T) { + ctx := context.Background() + config := mdbv1.NewAdditionalMongodConfig("net.port", 30000) + rs := mdbv1.NewReplicaSetBuilder(). + SetNamespace(mock.TestNamespace). + SetAdditionalConfig(config). + SetConnectionSpec(testConnectionSpec()). + Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + svc, err := client.GetService(ctx, kube.ObjectKey(rs.Namespace, rs.ServiceName())) + assert.NoError(t, err) + assert.Equal(t, int32(30000), svc.Spec.Ports[0].Port) +} + +// TestReplicaSet_ConfigMapAndSecretWatched verifies that config map and secret are added to the internal +// map that allows to watch them for changes +func TestReplicaSet_ConfigMapAndSecretWatched(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, rs.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + } + + assert.Equal(t, reconciler.resourceWatcher.GetWatchedResources(), expected) +} + +// TestTLSResourcesAreWatchedAndUnwatched verifies that TLS config map and secret are added to the internal +// map that allows to watch them for changes +func TestTLSResourcesAreWatchedAndUnwatched(t *testing.T) { + ctx := context.Background() + rs := DefaultReplicaSetBuilder().EnableTLS().SetTLSCA("custom-ca").Build() + + reconciler, client, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + + addKubernetesTlsResources(ctx, client, rs) + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, rs.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, "custom-ca")}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, rs.GetName()+"-cert")}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + } + + assert.Equal(t, reconciler.resourceWatcher.GetWatchedResources(), expected) + + rs.Spec.Security.TLSConfig.Enabled = false + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + expected = map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, rs.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, rs.Name)}, + } + + assert.Equal(t, reconciler.resourceWatcher.GetWatchedResources(), expected) +} + +func TestBackupConfiguration_ReplicaSet(t *testing.T) { + ctx := context.Background() + rs := mdbv1.NewReplicaSetBuilder(). + SetNamespace(mock.TestNamespace). + SetConnectionSpec(testConnectionSpec()). + SetBackup(mdbv1.Backup{ + Mode: "enabled", + }). + Build() + + reconciler, client, omConnectionFactory := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + uuidStr := uuid.New().String() + // configure backup for this project in Ops Manager in the mocked connection + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + _, err := connection.UpdateBackupConfig(&backup.Config{ + ClusterId: uuidStr, + Status: backup.Inactive, + }) + assert.NoError(t, err) + + // add corresponding host cluster. + connection.(*om.MockedOmConnection).BackupHostClusters[uuidStr] = &backup.HostCluster{ + ReplicaSetName: rs.Name, + ClusterName: rs.Name, + TypeName: "REPLICA_SET", + } + }) + + t.Run("Backup can be started", func(t *testing.T) { + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + configResponse, _ := omConnectionFactory.GetConnection().ReadBackupConfigs() + assert.Len(t, configResponse.Configs, 1) + + config := configResponse.Configs[0] + + assert.Equal(t, backup.Started, config.Status) + assert.Equal(t, uuidStr, config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + }) + + t.Run("Backup snapshot schedule tests", backupSnapshotScheduleTests(rs, client, reconciler, omConnectionFactory, uuidStr)) + + t.Run("Backup can be stopped", func(t *testing.T) { + rs.Spec.Backup.Mode = "disabled" + err := client.Update(ctx, rs) + assert.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + configResponse, _ := omConnectionFactory.GetConnection().ReadBackupConfigs() + assert.Len(t, configResponse.Configs, 1) + + config := configResponse.Configs[0] + + assert.Equal(t, backup.Stopped, config.Status) + assert.Equal(t, uuidStr, config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + }) + + t.Run("Backup can be terminated", func(t *testing.T) { + rs.Spec.Backup.Mode = "terminated" + err := client.Update(ctx, rs) + assert.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, rs, client) + + configResponse, _ := omConnectionFactory.GetConnection().ReadBackupConfigs() + assert.Len(t, configResponse.Configs, 1) + + config := configResponse.Configs[0] + + assert.Equal(t, backup.Terminating, config.Status) + assert.Equal(t, uuidStr, config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + }) +} + +func TestReplicaSetAgentVersionMapping(t *testing.T) { + ctx := context.Background() + defaultResource := DefaultReplicaSetBuilder().Build() + // Go couldn't infer correctly that *ReconcileMongoDbReplicaset implemented *reconciler.Reconciler interface + // without this anonymous function + reconcilerFactory := func(rs *mdbv1.MongoDB) (reconcile.Reconciler, kubernetesClient.Client) { + // Call the original defaultReplicaSetReconciler, which returns a *ReconcileMongoDbReplicaSet that implements reconcile.Reconciler + reconciler, mockClient, _ := defaultReplicaSetReconciler(ctx, nil, "", "", rs) + // Return the reconciler as is, because it implements the reconcile.Reconciler interface + return reconciler, mockClient + } + defaultResources := testReconciliationResources{ + Resource: defaultResource, + ReconcilerFactory: reconcilerFactory, + } + + containers := []corev1.Container{{Name: util.AgentContainerName, Image: "foo"}} + podTemplate := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: containers, + }, + } + + overridenResource := DefaultReplicaSetBuilder().SetPodSpecTemplate(podTemplate).Build() + overridenResources := testReconciliationResources{ + Resource: overridenResource, + ReconcilerFactory: reconcilerFactory, + } + + agentVersionMappingTest(ctx, t, defaultResources, overridenResources) +} + +func TestHandlePVCResize(t *testing.T) { + statefulSet := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-sts", + Namespace: "default", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(3)), + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data-pvc", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } + + p := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "data-pvc-example-sts-0", + Namespace: "default", + }, + Spec: corev1.PersistentVolumeClaimSpec{Resources: corev1.VolumeResourceRequirements{Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("1Gi")}}}, + Status: corev1.PersistentVolumeClaimStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + } + + ctx := context.TODO() + logger := zap.NewExample().Sugar() + reconciledResource := DefaultReplicaSetBuilder().SetName("temple").Build() + + memberClient, _ := setupClients(t, ctx, p, statefulSet, reconciledResource) + + // *** "PVC Phase: No Action" *** + testPhaseNoActionRequired(t, ctx, memberClient, statefulSet, logger) + + // *** "PVC Phase: No Action, Storage Increase Detected" *** + testPVCResizeStarted(t, ctx, memberClient, reconciledResource, statefulSet, logger) + + // *** "PVC Phase: PVCResize, Still Resizing" *** + testPVCStillResizing(t, ctx, memberClient, reconciledResource, statefulSet, logger) + + // *** "PVC Phase: PVCResize, Finished Resizing *** + testPVCFinishedResizing(t, ctx, memberClient, p, reconciledResource, statefulSet, logger) +} + +func testPVCFinishedResizing(t *testing.T, ctx context.Context, memberClient kubernetesClient.Client, p *corev1.PersistentVolumeClaim, reconciledResource *mdbv1.MongoDB, statefulSet *appsv1.StatefulSet, logger *zap.SugaredLogger) { + // Simulate that the PVC has finished resizing + setPVCWithUpdatedResource(ctx, t, memberClient, p) + + st := create.HandlePVCResize(ctx, memberClient, statefulSet, logger) + + assert.Equal(t, status.PhaseRunning, st.Phase()) + assert.Equal(t, &status.PVC{Phase: pvc.PhaseSTSOrphaned, StatefulsetName: "example-sts"}, getPVCOption(st)) + // mirror reconciler.updateStatus() action, a new reconcile starts and the next step runs + reconciledResource.UpdateStatus(status.PhaseRunning, st.StatusOptions()...) + + // *** "No Storage Change, No Action Required" *** + statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("1Gi") + st = create.HandlePVCResize(ctx, memberClient, statefulSet, logger) + + assert.Equal(t, status.PhaseRunning, st.Phase()) + + // Make sure the statefulset does not exist + stsToRetrieve := appsv1.StatefulSet{} + err := memberClient.Get(ctx, kube.ObjectKey(statefulSet.Namespace, statefulSet.Name), &stsToRetrieve) + if !apiErrors.IsNotFound(err) { + t.Fatal("STS should not be around anymore!") + } +} + +func testPVCStillResizing(t *testing.T, ctx context.Context, memberClient kubernetesClient.Client, reconciledResource *mdbv1.MongoDB, statefulSet *appsv1.StatefulSet, logger *zap.SugaredLogger) { + // Simulate that the PVC is still resizing by not updating the Capacity in the PVC status + + // Call the HandlePVCResize function + st := create.HandlePVCResize(ctx, memberClient, statefulSet, logger) + + // Verify the function returns Pending + assert.Equal(t, status.PhasePending, st.Phase()) + // Verify that the PVC resize is still ongoing + assert.Equal(t, &status.PVC{Phase: pvc.PhasePVCResize, StatefulsetName: "example-sts"}, getPVCOption(st)) + + // mirror reconciler.updateStatus() action, a new reconcile starts and the next step runs + reconciledResource.UpdateStatus(status.PhasePending, status.NewPVCsStatusOption(getPVCOption(st))) +} + +func testPVCResizeStarted(t *testing.T, ctx context.Context, memberClient kubernetesClient.Client, reconciledResource *mdbv1.MongoDB, statefulSet *appsv1.StatefulSet, logger *zap.SugaredLogger) { + // Update the StatefulSet to request more storage + statefulSet.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("2Gi") + + // Call the HandlePVCResize function + st := create.HandlePVCResize(ctx, memberClient, statefulSet, logger) + + // Verify the function returns Pending + assert.Equal(t, status.PhasePending, st.Phase()) + assert.Equal(t, &status.PVC{Phase: pvc.PhasePVCResize, StatefulsetName: "example-sts"}, getPVCOption(st)) + + // mirror reconciler.updateStatus() action, a new reconcile starts and the next step runs + reconciledResource.UpdateStatus(status.PhasePending, st.StatusOptions()...) +} + +func testPhaseNoActionRequired(t *testing.T, ctx context.Context, memberClient kubernetesClient.Client, statefulSet *appsv1.StatefulSet, logger *zap.SugaredLogger) { + st := create.HandlePVCResize(ctx, memberClient, statefulSet, logger) + // Verify the function returns Pending + assert.Equal(t, status.PhaseRunning, st.Phase()) + require.Nil(t, getPVCOption(st)) +} + +func setPVCWithUpdatedResource(ctx context.Context, t *testing.T, c client.Client, p *corev1.PersistentVolumeClaim) { + var updatedPVC corev1.PersistentVolumeClaim + err := c.Get(ctx, client.ObjectKey{Name: p.Name, Namespace: p.Namespace}, &updatedPVC) + assert.NoError(t, err) + updatedPVC.Status.Capacity = map[corev1.ResourceName]resource.Quantity{} + updatedPVC.Status.Capacity[corev1.ResourceStorage] = resource.MustParse("2Gi") + updatedPVC.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("2Gi") + + // This is required, otherwise the status subResource is reset to the initial field + err = c.SubResource("status").Update(ctx, &updatedPVC) + assert.NoError(t, err) +} + +func setupClients(t *testing.T, ctx context.Context, p *corev1.PersistentVolumeClaim, statefulSet *appsv1.StatefulSet, reconciledResource *mdbv1.MongoDB) (kubernetesClient.Client, *kubernetesClient.Client) { + memberClient, _ := mock.NewDefaultFakeClient(reconciledResource) + + err := memberClient.Create(ctx, p) + assert.NoError(t, err) + err = memberClient.Create(ctx, statefulSet) + assert.NoError(t, err) + + return memberClient, nil +} + +func getPVCOption(st workflow.Status) *status.PVC { + s, exists := status.GetOption(st.StatusOptions(), status.PVCStatusOption{}) + if !exists { + return nil + } + + return s.(status.PVCStatusOption).PVC +} + +// assertCorrectNumberOfMembersAndProcesses ensures that both the mongodb resource and the Ops Manager deployment +// have the correct number of processes/replicas at each stage of the scaling operation +func assertCorrectNumberOfMembersAndProcesses(ctx context.Context, t *testing.T, expected int, mdb *mdbv1.MongoDB, client client.Client, omConnection om.Connection, msg string) { + err := client.Get(ctx, mdb.ObjectKey(), mdb) + assert.NoError(t, err) + assert.Equal(t, expected, mdb.Status.Members, msg) + dep, err := omConnection.ReadDeployment() + assert.NoError(t, err) + assert.Len(t, dep.ProcessesCopy(), expected) +} + +func defaultReplicaSetReconciler(ctx context.Context, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, rs *mdbv1.MongoDB) (*ReconcileMongoDbReplicaSet, kubernetesClient.Client, *om.CachedOMConnectionFactory) { + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(rs) + return newReplicaSetReconciler(ctx, kubeClient, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, false, omConnectionFactory.GetConnectionFunc), kubeClient, omConnectionFactory +} + +// newDefaultPodSpec creates pod spec with default values,sets only the topology key and persistence sizes, +// seems we shouldn't set CPU and Memory if they are not provided by user +func newDefaultPodSpec() mdbv1.MongoDbPodSpec { + podSpecWrapper := mdbv1.NewEmptyPodSpecWrapperBuilder(). + SetSinglePersistence(mdbv1.NewPersistenceBuilder(util.DefaultMongodStorageSize)). + SetMultiplePersistence(mdbv1.NewPersistenceBuilder(util.DefaultMongodStorageSize), + mdbv1.NewPersistenceBuilder(util.DefaultJournalStorageSize), + mdbv1.NewPersistenceBuilder(util.DefaultLogsStorageSize)). + Build() + + return podSpecWrapper.MongoDbPodSpec +} + +// TODO remove in favor of '/api/mongodbbuilder.go' +func DefaultReplicaSetBuilder() *ReplicaSetBuilder { + podSpec := newDefaultPodSpec() + spec := mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + Version: "4.0.0", + Persistent: util.BooleanRef(false), + ConnectionSpec: mdbv1.ConnectionSpec{ + SharedConnectionSpec: mdbv1.SharedConnectionSpec{ + OpsManagerConfig: &mdbv1.PrivateCloudConfig{ + ConfigMapRef: mdbv1.ConfigMapRef{ + Name: mock.TestProjectConfigMapName, + }, + }, + }, + Credentials: mock.TestCredentialsSecretName, + }, + ResourceType: mdbv1.ReplicaSet, + Security: &mdbv1.Security{ + TLSConfig: &mdbv1.TLSConfig{}, + Authentication: &mdbv1.Authentication{}, + Roles: []mdbv1.MongoDbRole{}, + }, + }, + Members: 3, + PodSpec: &podSpec, + } + rs := &mdbv1.MongoDB{Spec: spec, ObjectMeta: metav1.ObjectMeta{Name: "temple", Namespace: mock.TestNamespace}} + return &ReplicaSetBuilder{rs} +} + +func (b *ReplicaSetBuilder) SetName(name string) *ReplicaSetBuilder { + b.Name = name + return b +} + +func (b *ReplicaSetBuilder) SetVersion(version string) *ReplicaSetBuilder { + b.Spec.Version = version + return b +} + +func (b *ReplicaSetBuilder) SetPersistent(p *bool) *ReplicaSetBuilder { + b.Spec.Persistent = p + return b +} + +func (b *ReplicaSetBuilder) SetPodSpec(podSpec *mdbv1.MongoDbPodSpec) *ReplicaSetBuilder { + b.Spec.PodSpec = podSpec + return b +} + +func (b *ReplicaSetBuilder) SetMembers(m int) *ReplicaSetBuilder { + b.Spec.Members = m + return b +} + +func (b *ReplicaSetBuilder) SetSecurity(security mdbv1.Security) *ReplicaSetBuilder { + b.Spec.Security = &security + return b +} + +func (b *ReplicaSetBuilder) SetService(name string) *ReplicaSetBuilder { + b.Spec.Service = name + return b +} + +func (b *ReplicaSetBuilder) SetAuthentication(auth *mdbv1.Authentication) *ReplicaSetBuilder { + if b.Spec.Security == nil { + b.Spec.Security = &mdbv1.Security{} + } + b.Spec.Security.Authentication = auth + return b +} + +func (b *ReplicaSetBuilder) SetRoles(roles []mdbv1.MongoDbRole) *ReplicaSetBuilder { + if b.Spec.Security == nil { + b.Spec.Security = &mdbv1.Security{} + } + b.Spec.Security.Roles = roles + return b +} + +func (b *ReplicaSetBuilder) EnableAuth() *ReplicaSetBuilder { + b.Spec.Security.Authentication.Enabled = true + return b +} + +func (b *ReplicaSetBuilder) AgentAuthMode(agentMode string) *ReplicaSetBuilder { + if b.Spec.Security == nil { + b.Spec.Security = &mdbv1.Security{} + } + + if b.Spec.Security.Authentication == nil { + b.Spec.Security.Authentication = &mdbv1.Authentication{} + } + b.Spec.Security.Authentication.Agents = mdbv1.AgentAuthentication{Mode: agentMode} + return b +} + +func (b *ReplicaSetBuilder) LDAP(ldap mdbv1.Ldap) *ReplicaSetBuilder { + b.Spec.Security.Authentication.Ldap = &ldap + return b +} + +func (b *ReplicaSetBuilder) SetAuthModes(modes []mdbv1.AuthMode) *ReplicaSetBuilder { + b.Spec.Security.Authentication.Modes = modes + return b +} + +func (b *ReplicaSetBuilder) EnableX509InternalClusterAuth() *ReplicaSetBuilder { + b.Spec.Security.Authentication.InternalCluster = util.X509 + return b +} + +func (b *ReplicaSetBuilder) SetReplicaSetHorizons(replicaSetHorizons []mdbv1.MongoDBHorizonConfig) *ReplicaSetBuilder { + if b.Spec.Connectivity == nil { + b.Spec.Connectivity = &mdbv1.MongoDBConnectivity{} + } + b.Spec.Connectivity.ReplicaSetHorizons = replicaSetHorizons + return b +} + +func (b *ReplicaSetBuilder) EnableTLS() *ReplicaSetBuilder { + if b.Spec.Security == nil || b.Spec.Security.TLSConfig == nil { + b.SetSecurity(mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{}}) + } + b.Spec.Security.TLSConfig.Enabled = true + return b +} + +func (b *ReplicaSetBuilder) SetTLSCA(ca string) *ReplicaSetBuilder { + if b.Spec.Security == nil || b.Spec.Security.TLSConfig == nil { + b.SetSecurity(mdbv1.Security{TLSConfig: &mdbv1.TLSConfig{}}) + } + b.Spec.Security.TLSConfig.CA = ca + return b +} + +func (b *ReplicaSetBuilder) EnableX509() *ReplicaSetBuilder { + b.Spec.Security.Authentication.Enabled = true + b.Spec.Security.Authentication.Modes = append(b.Spec.Security.Authentication.Modes, util.X509) + return b +} + +func (b *ReplicaSetBuilder) EnableSCRAM() *ReplicaSetBuilder { + b.Spec.Security.Authentication.Enabled = true + b.Spec.Security.Authentication.Modes = append(b.Spec.Security.Authentication.Modes, util.SCRAM) + return b +} + +func (b *ReplicaSetBuilder) EnableLDAP() *ReplicaSetBuilder { + b.Spec.Security.Authentication.Enabled = true + b.Spec.Security.Authentication.Modes = append(b.Spec.Security.Authentication.Modes, util.LDAP) + return b +} + +func (b *ReplicaSetBuilder) SetPodSpecTemplate(spec corev1.PodTemplateSpec) *ReplicaSetBuilder { + if b.Spec.PodSpec == nil { + b.Spec.PodSpec = &mdbv1.MongoDbPodSpec{} + } + b.Spec.PodSpec.PodTemplateWrapper.PodTemplate = &spec + return b +} + +func (b *ReplicaSetBuilder) SetOpsManagerConfigMapName(opsManagerConfigMapName string) *ReplicaSetBuilder { + b.Spec.OpsManagerConfig.ConfigMapRef.Name = opsManagerConfigMapName + return b +} + +func (b *ReplicaSetBuilder) ExposedExternally(specOverride *corev1.ServiceSpec, annotationsOverride map[string]string, externalDomain *string) *ReplicaSetBuilder { + b.Spec.ExternalAccessConfiguration = &mdbv1.ExternalAccessConfiguration{} + b.Spec.ExternalAccessConfiguration.ExternalDomain = externalDomain + if specOverride != nil { + b.Spec.ExternalAccessConfiguration.ExternalService.SpecWrapper = &mdbv1.ServiceSpecWrapper{Spec: *specOverride} + } + if len(annotationsOverride) > 0 { + b.Spec.ExternalAccessConfiguration.ExternalService.Annotations = annotationsOverride + } + return b +} + +func (b *ReplicaSetBuilder) Build() *mdbv1.MongoDB { + b.InitDefaults() + return b.DeepCopy() +} diff --git a/controllers/operator/mongodbresource_event_handler.go b/controllers/operator/mongodbresource_event_handler.go new file mode 100644 index 000000000..9da2b0e88 --- /dev/null +++ b/controllers/operator/mongodbresource_event_handler.go @@ -0,0 +1,40 @@ +package operator + +import ( + "context" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + + "github.com/10gen/ops-manager-kubernetes/pkg/kube" +) + +// Deleter cleans up any state required upon deletion of a resource. +type Deleter interface { + OnDelete(ctx context.Context, obj runtime.Object, log *zap.SugaredLogger) error +} + +// ResourceEventHandler is a custom event handler that extends the +// handler.EnqueueRequestForObject event handler. It overrides the Delete +// method used to clean up custom resources when a deletion event happens. +// This results in a single, synchronous attempt to clean up the resource +// rather than an asynchronous one. +type ResourceEventHandler struct { + *handler.EnqueueRequestForObject + deleter Deleter +} + +func (h *ResourceEventHandler) Delete(ctx context.Context, e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + objectKey := kube.ObjectKey(e.Object.GetNamespace(), e.Object.GetName()) + logger := zap.S().With("resource", objectKey) + + zap.S().Infow("Cleaning up Resource", "resource", e.Object) + if err := h.deleter.OnDelete(ctx, e.Object, logger); err != nil { + logger.Errorf("Resource removed from Kubernetes, but failed to clean some state in Ops Manager: %s", err) + return + } + logger.Info("Removed Resource from Kubernetes and Ops Manager") +} diff --git a/controllers/operator/mongodbshardedcluster_controller.go b/controllers/operator/mongodbshardedcluster_controller.go new file mode 100644 index 000000000..6940bfb54 --- /dev/null +++ b/controllers/operator/mongodbshardedcluster_controller.go @@ -0,0 +1,2990 @@ +package operator + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale" + + mdbcv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1" + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + mdbstatus "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/om/deployment" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/om/replicaset" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct/scalers/interfaces" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + enterprisepem "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/recovery" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + mekoService "github.com/10gen/ops-manager-kubernetes/pkg/kube/service" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" + "github.com/10gen/ops-manager-kubernetes/pkg/vault/vaultwatcher" +) + +// ReconcileMongoDbShardedCluster is the reconciler for the sharded cluster +type ReconcileMongoDbShardedCluster struct { + *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + memberClustersMap map[string]client.Client + imageUrls images.ImageUrls + forceEnterprise bool + + initDatabaseNonStaticImageVersion string + databaseNonStaticImageVersion string +} + +func newShardedClusterReconciler(ctx context.Context, kubeClient client.Client, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, memberClusterMap map[string]client.Client, omFunc om.ConnectionFactory) *ReconcileMongoDbShardedCluster { + return &ReconcileMongoDbShardedCluster{ + ReconcileCommonController: NewReconcileCommonController(ctx, kubeClient), + omConnectionFactory: omFunc, + memberClustersMap: memberClusterMap, + forceEnterprise: forceEnterprise, + imageUrls: imageUrls, + + initDatabaseNonStaticImageVersion: initDatabaseNonStaticImageVersion, + databaseNonStaticImageVersion: databaseNonStaticImageVersion, + } +} + +type ShardedClusterDeploymentState struct { + CommonDeploymentState `json:",inline"` + LastAchievedSpec *mdbv1.MongoDbSpec `json:"lastAchievedSpec"` + Status *mdbv1.MongoDbStatus `json:"status"` +} + +// updateStatusFromResourceStatus updates the status in the deployment state with values from the resource status with additional ensurance that no data is accidentally lost. +// In a rare situation when we're performing an upgrade of the operator from non-deployment state version (<=1.27) the migrateToNewDeploymentState +// function correctly migrates the sizes of the cluster, but then, in case of an early return (in case of any error or waiting too long for the sts/agents) +// the updateStatus might clear the migrated data. +// This function ensures we're copying the status, but at the same time we're not losing those sizes from the deployment state. +// The logic of updateStatus in the reconciler works on options. If the option is not passed, the value is not updated, but it's also not cleared if the option is not passed. +// Early returns with updateStatus don't pass any options, so the calculated status shouldn't clear the sizes we've just calculated into the deployment state. +func (s *ShardedClusterDeploymentState) updateStatusFromResourceStatus(statusFromResource mdbv1.MongoDbStatus) { + resultStatus := statusFromResource.DeepCopy() + if resultStatus.SizeStatusInClusters == nil && s.Status.SizeStatusInClusters != nil { + resultStatus.SizeStatusInClusters = s.Status.SizeStatusInClusters.DeepCopy() + } + s.Status = resultStatus +} + +func NewShardedClusterDeploymentState() *ShardedClusterDeploymentState { + return &ShardedClusterDeploymentState{ + CommonDeploymentState: CommonDeploymentState{ClusterMapping: map[string]int{}}, + LastAchievedSpec: &mdbv1.MongoDbSpec{}, + Status: &mdbv1.MongoDbStatus{}, + } +} + +func (r *ShardedClusterReconcileHelper) initializeMemberClusters(globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger) error { + mongoDB := r.sc + shardsMap := r.desiredShardsConfiguration + if mongoDB.Spec.IsMultiCluster() { + if !multicluster.IsMemberClusterMapInitializedForMultiCluster(globalMemberClustersMap) { + return xerrors.Errorf("member clusters have to be initialized for MultiCluster Sharded Cluster topology") + } + + allReferencedClusterNamesMap := map[string]struct{}{} + for _, clusterSpecItem := range r.getConfigSrvClusterSpecList() { + allReferencedClusterNamesMap[clusterSpecItem.ClusterName] = struct{}{} + } + for _, clusterSpecItem := range r.getMongosClusterSpecList() { + allReferencedClusterNamesMap[clusterSpecItem.ClusterName] = struct{}{} + } + for _, shardComponentSpec := range shardsMap { + for _, clusterSpecItem := range shardComponentSpec.ClusterSpecList { + allReferencedClusterNamesMap[clusterSpecItem.ClusterName] = struct{}{} + } + } + var allReferencedClusterNames []string + for clusterName := range allReferencedClusterNamesMap { + allReferencedClusterNames = append(allReferencedClusterNames, clusterName) + } + slices.Sort(allReferencedClusterNames) + + r.deploymentState.ClusterMapping = multicluster.AssignIndexesForMemberClusterNames(r.deploymentState.ClusterMapping, allReferencedClusterNames) + + configSrvGetLastAppliedMembersFunc := func(memberClusterName string) int { + if count, ok := r.deploymentState.Status.SizeStatusInClusters.ConfigServerMongodsInClusters[memberClusterName]; ok { + return count + } else { + return 0 + } + } + r.configSrvMemberClusters = createMemberClusterListFromClusterSpecList(r.getConfigSrvClusterSpecList(), globalMemberClustersMap, log, r.deploymentState.ClusterMapping, configSrvGetLastAppliedMembersFunc, false) + + mongosGetLastAppliedMembersFunc := func(memberClusterName string) int { + if count, ok := r.deploymentState.Status.SizeStatusInClusters.MongosCountInClusters[memberClusterName]; ok { + return count + } else { + return 0 + } + } + r.mongosMemberClusters = createMemberClusterListFromClusterSpecList(r.getMongosClusterSpecList(), globalMemberClustersMap, log, r.deploymentState.ClusterMapping, mongosGetLastAppliedMembersFunc, false) + r.shardsMemberClustersMap, r.allShardsMemberClusters = r.createShardsMemberClusterLists(shardsMap, globalMemberClustersMap, log, r.deploymentState, false) + } else { + r.shardsMemberClustersMap, r.allShardsMemberClusters = r.createShardsMemberClusterLists(shardsMap, globalMemberClustersMap, log, r.deploymentState, true) + + // SizeStatusInClusters is the primary struct for storing state, designed with multi-cluster support in mind. + // For a single-cluster setup, we first attempt to read from the fields in SizeStatusInClusters, + // falling back to the legacy structure (MongodbShardedClusterSizeConfig) if they are unavailable. The fallback + // is to be defensive but with the migration performed at the beginning of the reconcile (if necessary), there + // should be no case of having only the legacy fields populated in the state. + configSrvCount, configSrvCountExists := r.deploymentState.Status.SizeStatusInClusters.ConfigServerMongodsInClusters[multicluster.LegacyCentralClusterName] + if !configSrvCountExists { + configSrvCount = r.deploymentState.Status.ConfigServerCount + } + r.configSrvMemberClusters = []multicluster.MemberCluster{multicluster.GetLegacyCentralMemberCluster(configSrvCount, 0, r.commonController.client, r.commonController.SecretClient)} + + mongosCount, mongosCountExists := r.deploymentState.Status.SizeStatusInClusters.MongosCountInClusters[multicluster.LegacyCentralClusterName] + if !mongosCountExists { + mongosCount = r.deploymentState.Status.MongosCount + } + r.mongosMemberClusters = []multicluster.MemberCluster{multicluster.GetLegacyCentralMemberCluster(mongosCount, 0, r.commonController.client, r.commonController.SecretClient)} + } + + r.allMemberClusters = r.createAllMemberClustersList() + + log.Debugf("Initialized shards member cluster list: %+v", util.Transform(r.allShardsMemberClusters, func(m multicluster.MemberCluster) string { + // TODO Replicas is not relevant when iterating over allShardsMemberClusters; construct full list by iterating over shardsMemberClustersMap + return fmt.Sprintf("{Name: %s, Index: %d, Replicas: %d, Active: %t, Healthy: %t}", m.Name, m.Index, m.Replicas, m.Active, m.Healthy) + })) + log.Debugf("Initialized mongos member cluster list: %+v", util.Transform(r.mongosMemberClusters, func(m multicluster.MemberCluster) string { + return fmt.Sprintf("{Name: %s, Index: %d, Replicas: %d, Active: %t, Healthy: %t}", m.Name, m.Index, m.Replicas, m.Active, m.Healthy) + })) + log.Debugf("Initialized config servers member cluster list: %+v", util.Transform(r.configSrvMemberClusters, func(m multicluster.MemberCluster) string { + return fmt.Sprintf("{Name: %s, Index: %d, Replicas: %d, Active: %t, Healthy: %t}", m.Name, m.Index, m.Replicas, m.Active, m.Healthy) + })) + return nil +} + +// createAllMemberClustersList is returning a list of all unique member clusters used across all clusterSpecLists. +func (r *ShardedClusterReconcileHelper) createAllMemberClustersList() []multicluster.MemberCluster { + var allClusters []multicluster.MemberCluster + allClusters = append(allClusters, r.allShardsMemberClusters...) + allClusters = append(allClusters, r.mongosMemberClusters...) + allClusters = append(allClusters, r.configSrvMemberClusters...) + allClustersMap := map[string]multicluster.MemberCluster{} + for _, memberCluster := range allClusters { + // we deliberately reset replicas to not use it accidentally + // allClustersMap contains unique cluster names across all clusterSpecLists, but replicas part will be invalid + memberCluster.Replicas = 0 + allClustersMap[memberCluster.Name] = memberCluster + } + + allClusters = nil + for _, memberCluster := range allClustersMap { + allClusters = append(allClusters, memberCluster) + } + return allClusters +} + +// createShardsMemberClusterLists creates a list of member clusters from the current desired shards configuration. +// legacyMemberCluster parameter is used to indicate the member cluster should be marked as Legacy for reusing this function also in single-cluster mode. +func (r *ShardedClusterReconcileHelper) createShardsMemberClusterLists(shardsMap map[int]*mdbv1.ShardedClusterComponentSpec, globalMemberClustersMap map[string]client.Client, log *zap.SugaredLogger, deploymentState *ShardedClusterDeploymentState, legacyMemberCluster bool) (map[int][]multicluster.MemberCluster, []multicluster.MemberCluster) { + shardMemberClustersMap := map[int][]multicluster.MemberCluster{} + var allShardsMemberClusters []multicluster.MemberCluster + alreadyAdded := map[string]struct{}{} + // Shards can have different member clusters specified in spec.ShardSpec.ClusterSpecList and in shard overrides. + // Here we construct a unique list of member clusters on which shards are deployed + for shardIdx, shardSpec := range shardsMap { + shardGetLastAppliedMembersFunc := func(memberClusterName string) int { + shardOverridesInClusters := deploymentState.Status.SizeStatusInClusters.ShardOverridesInClusters + if _, ok := shardOverridesInClusters[r.sc.ShardRsName(shardIdx)]; ok { + if count, ok := shardOverridesInClusters[r.sc.ShardRsName(shardIdx)][memberClusterName]; ok { + // If we stored an override for this shard in the status, get the member count from it + return count + } + } + // Because we store one common distribution for all shards in ShardMongodsInClusters, we need to make sure + // we assign a size of 0 to newly created shards, as they haven't scaled yet. + if shardIdx >= deploymentState.Status.ShardCount { + return 0 + } + if count, ok := deploymentState.Status.SizeStatusInClusters.ShardMongodsInClusters[memberClusterName]; ok { + // Otherwise get the default one ShardMongodsInClusters + // ShardMongodsInClusters is not correct in the edge case where all shards are overridden + // but we won't enter this branch as we check for override in the branch above + // This edge case is tested in e2e_multi_cluster_sharded_scaling_all_shard_overrides + return count + } + + return 0 + } + // we use here shardSpec.ClusterSpecList directly as it's already a "processed" one from shardMap + shardMemberClustersMap[shardIdx] = createMemberClusterListFromClusterSpecList(shardSpec.ClusterSpecList, globalMemberClustersMap, log, deploymentState.ClusterMapping, shardGetLastAppliedMembersFunc, legacyMemberCluster) + + for _, shardMemberCluster := range shardMemberClustersMap[shardIdx] { + if _, ok := alreadyAdded[shardMemberCluster.Name]; !ok { + // We don't care from which shard we use memberCluster for this list; + // we deliberately reset Replicas to not accidentally use it + shardMemberCluster.Replicas = 0 + allShardsMemberClusters = append(allShardsMemberClusters, shardMemberCluster) + alreadyAdded[shardMemberCluster.Name] = struct{}{} + } + } + } + + return shardMemberClustersMap, allShardsMemberClusters +} + +func (r *ShardedClusterReconcileHelper) getShardNameToShardIdxMap() map[string]int { + mapping := map[string]int{} + for shardIdx := 0; shardIdx < max(r.sc.Spec.ShardCount, r.deploymentState.Status.ShardCount); shardIdx++ { + mapping[r.sc.ShardRsName(shardIdx)] = shardIdx + } + + return mapping +} + +func (r *ShardedClusterReconcileHelper) getShardClusterSpecList() mdbv1.ClusterSpecList { + spec := r.sc.Spec + if spec.IsMultiCluster() { + return spec.ShardSpec.ClusterSpecList + } else { + return mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: spec.MongodsPerShardCount, + MemberConfig: spec.MemberConfig, + }, + } + } +} + +func (r *ShardedClusterReconcileHelper) getMongosClusterSpecList() mdbv1.ClusterSpecList { + spec := r.sc.Spec + if spec.IsMultiCluster() { + // TODO return merged, desired mongos configuration + return spec.MongosSpec.ClusterSpecList + } else { + return mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: spec.MongosCount, + }, + } + } +} + +func (r *ShardedClusterReconcileHelper) getConfigSrvClusterSpecList() mdbv1.ClusterSpecList { + spec := r.sc.Spec + if spec.IsMultiCluster() { + return spec.ConfigSrvSpec.ClusterSpecList + } else { + return mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: spec.ConfigServerCount, + MemberConfig: spec.MemberConfig, + }, + } + } +} + +// prepareDesiredShardsConfiguration calculates full expected configuration of sharded cluster spec resource. +// It returns map of each shard (by index) with its configuration over all clusters and applying all pods spec overrides. +// In other words, this function is rendering final configuration of each shard over all member clusters applying all override logic. +// The reconciler implementation should refer to this structure only without taking into consideration complexities of MongoDbSpec wrt sharded clusters. +func (r *ShardedClusterReconcileHelper) prepareDesiredShardsConfiguration() map[int]*mdbv1.ShardedClusterComponentSpec { + spec := r.sc.Spec.DeepCopy() + // We initialize ClusterSpecList to contain a single legacy cluster in case of SingleCluster mode. + if spec.ShardSpec == nil { + spec.ShardSpec = &mdbv1.ShardedClusterComponentSpec{} + } + spec.ShardSpec.ClusterSpecList = r.getShardClusterSpecList() + // We don't need to do the same for shardOverrides for single-cluster as shardOverrides[].ClusterSpecList can be set only for Multi-Cluster mode. + // And we don't need that artificial legacy cluster as for single-cluster all necessary configuration is defined top-level. + + // We create here a collapsed structure of each shard configuration with all overrides applied to make the final configuration. + // For single cluster deployment it will be a single-element ClusterSpecList for each shard. + // For multiple clusters, each shard will have configuration specified for each member cluster. + shardComponentSpecs := map[int]*mdbv1.ShardedClusterComponentSpec{} + + for shardIdx := 0; shardIdx < max(spec.ShardCount, r.deploymentState.Status.ShardCount); shardIdx++ { + topLevelPersistenceOverride, topLevelPodSpecOverride := getShardTopLevelOverrides(spec, shardIdx) + shardComponentSpec := *spec.ShardSpec.DeepCopy() + shardComponentSpec.ClusterSpecList = processClusterSpecList(shardComponentSpec.ClusterSpecList, topLevelPodSpecOverride, topLevelPersistenceOverride) + shardComponentSpecs[shardIdx] = &shardComponentSpec + } + + for _, shardOverride := range expandShardOverrides(spec.ShardOverrides) { + // guaranteed to have one shard name in expandedShardOverrides + shardName := shardOverride.ShardNames[0] + shardIndex := r.getShardNameToShardIdxMap()[shardName] + // here we copy the whole element and overwrite at the end of every iteration + defaultShardConfiguration := shardComponentSpecs[shardIndex].DeepCopy() + topLevelPersistenceOverride, topLevelPodSpecOverride := getShardTopLevelOverrides(spec, shardIndex) + shardComponentSpecs[shardIndex] = processShardOverride(spec, shardOverride, defaultShardConfiguration, topLevelPodSpecOverride, topLevelPersistenceOverride) + } + return shardComponentSpecs +} + +func getShardTopLevelOverrides(spec *mdbv1.MongoDbSpec, shardIdx int) (*mdbv1.Persistence, *corev1.PodTemplateSpec) { + topLevelPodSpecOverride, topLevelPersistenceOverride := extractOverridesFromPodSpec(spec.ShardPodSpec) + + // specific shard level sts and persistence override + // TODO: as of 1.30 we deprecated ShardSpecificPodSpec, we should completely get rid of it in a few releases + if shardIdx < len(spec.ShardSpecificPodSpec) { + shardSpecificPodSpec := spec.ShardSpecificPodSpec[shardIdx] + if shardSpecificPodSpec.PodTemplateWrapper.PodTemplate != nil { + // We replace the override instead of merging it, because in single-cluster the override wasn't merging + // those specs; we keep the same behavior for backwards compatibility + topLevelPodSpecOverride = shardSpecificPodSpec.PodTemplateWrapper.PodTemplate.DeepCopy() + } + // ShardSpecificPodSpec applies to both template and persistence + if shardSpecificPodSpec.Persistence != nil { + topLevelPersistenceOverride = shardSpecificPodSpec.Persistence.DeepCopy() + } + } + return topLevelPersistenceOverride, topLevelPodSpecOverride +} + +func mergeOverrideClusterSpecList(shardOverride mdbv1.ShardOverride, defaultShardConfiguration *mdbv1.ShardedClusterComponentSpec, topLevelPodSpecOverride *corev1.PodTemplateSpec, topLevelPersistenceOverride *mdbv1.Persistence) *mdbv1.ShardedClusterComponentSpec { + finalShardConfiguration := defaultShardConfiguration.DeepCopy() + // We override here all elements of ClusterSpecList, but statefulset overrides if provided here + // will be merged on top of previous sts overrides. + for shardOverrideClusterSpecIdx := range shardOverride.ClusterSpecList { + shardOverrideClusterSpecItem := &shardOverride.ClusterSpecList[shardOverrideClusterSpecIdx] + foundIdx := slices.IndexFunc(defaultShardConfiguration.ClusterSpecList, func(item mdbv1.ClusterSpecItem) bool { + return item.ClusterName == shardOverrideClusterSpecItem.ClusterName + }) + // If the cluster is not found, it means this ShardOverride adds a new cluster that was not in ClusterSpecList + // We need to propagate top level specs, from e.g ShardPodSpec or ShardSpecificPodSpec, and apply a merge + if foundIdx == -1 { + if shardOverrideClusterSpecItem.StatefulSetConfiguration == nil { + shardOverrideClusterSpecItem.StatefulSetConfiguration = &mdbcv1.StatefulSetConfiguration{} + } + // We only need to perform a merge if there is a top level override, otherwise we keep an empty sts configuration + if topLevelPodSpecOverride != nil { + shardOverrideClusterSpecItem.StatefulSetConfiguration.SpecWrapper.Spec.Template = merge.PodTemplateSpecs(*topLevelPodSpecOverride, shardOverrideClusterSpecItem.StatefulSetConfiguration.SpecWrapper.Spec.Template) + } + if (shardOverrideClusterSpecItem.PodSpec == nil || shardOverrideClusterSpecItem.PodSpec.Persistence == nil) && + topLevelPersistenceOverride != nil { + shardOverrideClusterSpecItem.PodSpec = &mdbv1.MongoDbPodSpec{ + Persistence: topLevelPersistenceOverride.DeepCopy(), + } + } + continue + } + finalShardConfigurationClusterSpecItem := finalShardConfiguration.ClusterSpecList[foundIdx] + if finalShardConfigurationClusterSpecItem.StatefulSetConfiguration != nil { + if shardOverrideClusterSpecItem.StatefulSetConfiguration == nil { + shardOverrideClusterSpecItem.StatefulSetConfiguration = finalShardConfigurationClusterSpecItem.StatefulSetConfiguration + } else { + shardOverrideClusterSpecItem.StatefulSetConfiguration.SpecWrapper.Spec = merge.StatefulSetSpecs(finalShardConfigurationClusterSpecItem.StatefulSetConfiguration.SpecWrapper.Spec, shardOverrideClusterSpecItem.StatefulSetConfiguration.SpecWrapper.Spec) + } + } + + if shardOverrideClusterSpecItem.Members == nil { + shardOverrideClusterSpecItem.Members = ptr.To(finalShardConfigurationClusterSpecItem.Members) + } + + if shardOverrideClusterSpecItem.MemberConfig == nil { + shardOverrideClusterSpecItem.MemberConfig = finalShardConfigurationClusterSpecItem.MemberConfig + } + + // The two if blocks below make sure that PodSpec (for persistence) defined at the override level applies to all + // clusters by default, except if it is set at shardOverride.ClusterSpecList.PodSpec level + if shardOverride.PodSpec != nil { + finalShardConfigurationClusterSpecItem.PodSpec = shardOverride.PodSpec + } + if shardOverrideClusterSpecItem.PodSpec == nil { + shardOverrideClusterSpecItem.PodSpec = finalShardConfigurationClusterSpecItem.PodSpec + } + } + + // we reconstruct clusterSpecList from shardOverride list + finalShardConfiguration.ClusterSpecList = nil + for i := range shardOverride.ClusterSpecList { + so := shardOverride.ClusterSpecList[i].DeepCopy() + // guaranteed to be non-nil here + members := *shardOverride.ClusterSpecList[i].Members + + // We need to retrieve the original ExternalAccessConfiguration because shardOverride struct doesn't contain + // the field + var externalAccessConfiguration *mdbv1.ExternalAccessConfiguration + foundIdx := slices.IndexFunc(defaultShardConfiguration.ClusterSpecList, func(item mdbv1.ClusterSpecItem) bool { + return item.ClusterName == so.ClusterName + }) + if foundIdx != -1 { + externalAccessConfiguration = defaultShardConfiguration.ClusterSpecList[foundIdx].ExternalAccessConfiguration + } + + finalShardConfiguration.ClusterSpecList = append(finalShardConfiguration.ClusterSpecList, mdbv1.ClusterSpecItem{ + ClusterName: so.ClusterName, + ExternalAccessConfiguration: externalAccessConfiguration, + Members: members, + MemberConfig: so.MemberConfig, + StatefulSetConfiguration: so.StatefulSetConfiguration, + PodSpec: so.PodSpec, + }) + } + + return finalShardConfiguration +} + +// ShardOverrides can apply to multiple shard (e.g shardNames: ["sh-0", "sh-2"]) +// we expand overrides to get a list with each entry applying to a single shard +func expandShardOverrides(initialOverrides []mdbv1.ShardOverride) []mdbv1.ShardOverride { + var expandedShardOverrides []mdbv1.ShardOverride + for _, shardOverride := range initialOverrides { + for _, shardName := range shardOverride.ShardNames { + shardOverrideCopy := shardOverride.DeepCopy() + shardOverrideCopy.ShardNames = []string{shardName} + expandedShardOverrides = append(expandedShardOverrides, *shardOverrideCopy) + } + } + return expandedShardOverrides +} + +func processShardOverride(spec *mdbv1.MongoDbSpec, shardOverride mdbv1.ShardOverride, defaultShardConfiguration *mdbv1.ShardedClusterComponentSpec, topLevelPodSpecOverride *corev1.PodTemplateSpec, topLevelPersistenceOverride *mdbv1.Persistence) *mdbv1.ShardedClusterComponentSpec { + if shardOverride.Agent != nil { + defaultShardConfiguration.Agent = *shardOverride.Agent + } + if shardOverride.AdditionalMongodConfig != nil { + defaultShardConfiguration.AdditionalMongodConfig = shardOverride.AdditionalMongodConfig.DeepCopy() + } + // in single cluster, we put members override in a legacy cluster + if shardOverride.Members != nil && !spec.IsMultiCluster() { + // it's guaranteed it will have 1 element + defaultShardConfiguration.ClusterSpecList[0].Members = *shardOverride.Members + } + + if shardOverride.MemberConfig != nil && !spec.IsMultiCluster() { + defaultShardConfiguration.ClusterSpecList[0].MemberConfig = shardOverride.MemberConfig + } + + // in single-cluster we need to override podspec of the first dummy member cluster, as we won't go into shardOverride.ClusterSpecList iteration below + if shardOverride.PodSpec != nil && !spec.IsMultiCluster() { + defaultShardConfiguration.ClusterSpecList[0].PodSpec = shardOverride.PodSpec + } + + // The below loop makes the field ShardOverrides.StatefulSetConfiguration the default configuration for + // stateful sets in all clusters for that shard. The merge priority order is the following (lowest to highest): + // ShardSpec.ClusterSpecList.StatefulSetConfiguration -> ShardOverrides.StatefulSetConfiguration -> ShardOverrides.ClusterSpecList.StatefulSetConfiguration + if shardOverride.StatefulSetConfiguration != nil { + for idx := range defaultShardConfiguration.ClusterSpecList { + // Handle case where defaultShardConfiguration.ClusterSpecList[idx].StatefulSetConfiguration is nil + if defaultShardConfiguration.ClusterSpecList[idx].StatefulSetConfiguration == nil { + defaultShardConfiguration.ClusterSpecList[idx].StatefulSetConfiguration = &mdbcv1.StatefulSetConfiguration{} + } + defaultShardConfiguration.ClusterSpecList[idx].StatefulSetConfiguration.SpecWrapper.Spec = merge.StatefulSetSpecs(defaultShardConfiguration.ClusterSpecList[idx].StatefulSetConfiguration.SpecWrapper.Spec, shardOverride.StatefulSetConfiguration.SpecWrapper.Spec) + } + } + + // Merge existing clusterSpecList with clusterSpecList from a specific shard override. + // In single-cluster shardOverride cannot have any ClusterSpecList elements. + if shardOverride.ClusterSpecList != nil { + return mergeOverrideClusterSpecList(shardOverride, defaultShardConfiguration, topLevelPodSpecOverride, topLevelPersistenceOverride) + } else { + return defaultShardConfiguration + } +} + +func extractOverridesFromPodSpec(podSpec *mdbv1.MongoDbPodSpec) (*corev1.PodTemplateSpec, *mdbv1.Persistence) { + var podTemplateOverride *corev1.PodTemplateSpec + var persistenceOverride *mdbv1.Persistence + if podSpec != nil { + if podSpec.PodTemplateWrapper.PodTemplate != nil { + podTemplateOverride = podSpec.PodTemplateWrapper.PodTemplate + } + if podSpec.Persistence != nil { + persistenceOverride = podSpec.Persistence + } + } + return podTemplateOverride, persistenceOverride +} + +// prepareDesiredMongosConfiguration calculates full expected configuration of mongos resource. +// It returns a configuration for all clusters and applying all pods spec overrides. +// In other words, this function is rendering final configuration for the mongos over all member clusters applying all override logic. +// The reconciler implementation should refer to this structure only without taking into consideration complexities of MongoDbSpec wrt mongos. +// We share the same logic and data structures used for Config Server, although some fields are not relevant for mongos +// e.g MemberConfig. They will simply be ignored when the database is constructed +func (r *ShardedClusterReconcileHelper) prepareDesiredMongosConfiguration() *mdbv1.ShardedClusterComponentSpec { + // We initialize ClusterSpecList to contain a single legacy cluster in case of SingleCluster mode. + spec := r.sc.Spec.DeepCopy() + if spec.MongosSpec == nil { + spec.MongosSpec = &mdbv1.ShardedClusterComponentSpec{} + } + spec.MongosSpec.ClusterSpecList = r.getMongosClusterSpecList() + topLevelPodSpecOverride, topLevelPersistenceOverride := extractOverridesFromPodSpec(spec.MongosPodSpec) + mongosComponentSpec := spec.MongosSpec.DeepCopy() + mongosComponentSpec.ClusterSpecList = processClusterSpecList(mongosComponentSpec.ClusterSpecList, topLevelPodSpecOverride, topLevelPersistenceOverride) + return mongosComponentSpec +} + +// prepareDesiredConfigServerConfiguration works the same way as prepareDesiredMongosConfiguration, but for config server +func (r *ShardedClusterReconcileHelper) prepareDesiredConfigServerConfiguration() *mdbv1.ShardedClusterComponentSpec { + // We initialize ClusterSpecList to contain a single legacy cluster in case of SingleCluster mode. + spec := r.sc.Spec.DeepCopy() + if spec.ConfigSrvSpec == nil { + spec.ConfigSrvSpec = &mdbv1.ShardedClusterComponentSpec{} + } + spec.ConfigSrvSpec.ClusterSpecList = r.getConfigSrvClusterSpecList() + topLevelPodSpecOverride, topLevelPersistenceOverride := extractOverridesFromPodSpec(spec.ConfigSrvPodSpec) + configSrvComponentSpec := spec.ConfigSrvSpec.DeepCopy() + configSrvComponentSpec.ClusterSpecList = processClusterSpecList(configSrvComponentSpec.ClusterSpecList, topLevelPodSpecOverride, topLevelPersistenceOverride) + return configSrvComponentSpec +} + +// processClusterSpecList is a function shared by prepare desired configuration functions for shards, mongos and config servers +// it iterates through currently defined clusterSpecLists and set the correct STS configurations and persistence values, +// depending on top level overrides +func processClusterSpecList( + clusterSpecList []mdbv1.ClusterSpecItem, + topLevelPodSpecOverride *corev1.PodTemplateSpec, + topLevelPersistenceOverride *mdbv1.Persistence, +) []mdbv1.ClusterSpecItem { + for i := range clusterSpecList { + // we will store final sts overrides for each cluster in clusterSpecItem.StatefulSetOverride + // therefore we initialize it here and merge into it in case there is anything to override in the first place + // in case higher level overrides are empty, we just use whatever is specified in clusterSpecItem (maybe nothing as well) + if topLevelPodSpecOverride != nil { + if clusterSpecList[i].StatefulSetConfiguration == nil { + clusterSpecList[i].StatefulSetConfiguration = &mdbcv1.StatefulSetConfiguration{} + } + clusterSpecList[i].StatefulSetConfiguration.SpecWrapper.Spec.Template = merge.PodTemplateSpecs(*topLevelPodSpecOverride.DeepCopy(), clusterSpecList[i].StatefulSetConfiguration.SpecWrapper.Spec.Template) + } + if clusterSpecList[i].PodSpec == nil { + clusterSpecList[i].PodSpec = &mdbv1.MongoDbPodSpec{} + } + if topLevelPersistenceOverride != nil { + if clusterSpecList[i].PodSpec.Persistence == nil { + clusterSpecList[i].PodSpec.Persistence = topLevelPersistenceOverride.DeepCopy() + } + } + // If the MemberConfigs count is smaller than the number of numbers, append default values + for j := range clusterSpecList[i].Members { + if j >= len(clusterSpecList[i].MemberConfig) { + clusterSpecList[i].MemberConfig = append(clusterSpecList[i].MemberConfig, automationconfig.MemberOptions{ + Votes: ptr.To(1), + Priority: ptr.To("1"), + Tags: nil, + }) + } + } + // Explicitly set PodTemplate field to nil, as the pod template configuration is stored in StatefulSetConfiguration + // in the processed ShardedClusterComponentSpec structures. + // PodSpec should only be used for Persistence + clusterSpecList[i].PodSpec.PodTemplateWrapper.PodTemplate = nil + } + return clusterSpecList +} + +type ShardedClusterReconcileHelper struct { + commonController *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + imageUrls images.ImageUrls + forceEnterprise bool + automationAgentVersion string + + initDatabaseNonStaticImageVersion string + databaseNonStaticImageVersion string + + // sc is the resource being reconciled + sc *mdbv1.MongoDB + + // desired Configurations structs contain the target state - they reflect applying all the override rules to render the final, desired configuration + desiredShardsConfiguration map[int]*mdbv1.ShardedClusterComponentSpec + desiredConfigServerConfiguration *mdbv1.ShardedClusterComponentSpec + desiredMongosConfiguration *mdbv1.ShardedClusterComponentSpec + + // all member clusters here contain the number of members set to the current state read from deployment state + shardsMemberClustersMap map[int][]multicluster.MemberCluster + allShardsMemberClusters []multicluster.MemberCluster + configSrvMemberClusters []multicluster.MemberCluster + mongosMemberClusters []multicluster.MemberCluster + allMemberClusters []multicluster.MemberCluster + + // deploymentState is a helper structure containing the current deployment state + // It's initialized at the beginning of the reconcile and stored whenever we need to save changes to the deployment state. + // Also, deploymentState is always persisted in updateStatus method. + deploymentState *ShardedClusterDeploymentState + + stateStore *StateStore[ShardedClusterDeploymentState] +} + +func NewShardedClusterReconcilerHelper(ctx context.Context, reconciler *ReconcileCommonController, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, sc *mdbv1.MongoDB, globalMemberClustersMap map[string]client.Client, omConnectionFactory om.ConnectionFactory, log *zap.SugaredLogger) (*ShardedClusterReconcileHelper, error) { + // It's a workaround for single cluster topology to add there __default cluster. + // With the multi-cluster sharded refactor, we went so far with the multi-cluster first approach so we have very few places with conditional single/multi logic. + // Therefore, some parts of the reconciler logic uses that globalMemberClusterMap even in single-cluster mode (look for usages of createShardsMemberClusterLists) and expect + // to have __default member cluster defined in the globalMemberClustersMap as the __default member cluster is artificially added in initializeMemberClusters to clusterSpecList + // in single-cluster mode to simulate it's a special case of multi-cluster run. + globalMemberClustersMap = multicluster.InitializeGlobalMemberClusterMapForSingleCluster(globalMemberClustersMap, reconciler.client) + + helper := &ShardedClusterReconcileHelper{ + commonController: reconciler, + omConnectionFactory: omConnectionFactory, + imageUrls: imageUrls, + forceEnterprise: forceEnterprise, + + initDatabaseNonStaticImageVersion: initDatabaseNonStaticImageVersion, + databaseNonStaticImageVersion: databaseNonStaticImageVersion, + } + + helper.sc = sc + helper.deploymentState = NewShardedClusterDeploymentState() + if err := helper.initializeStateStore(ctx, reconciler, sc, log); err != nil { + return nil, xerrors.Errorf("failed to initialize sharded cluster state store: %w", err) + } + + helper.desiredShardsConfiguration = helper.prepareDesiredShardsConfiguration() + helper.desiredConfigServerConfiguration = helper.prepareDesiredConfigServerConfiguration() + helper.desiredMongosConfiguration = helper.prepareDesiredMongosConfiguration() + + if err := helper.initializeMemberClusters(globalMemberClustersMap, log); err != nil { + return nil, xerrors.Errorf("failed to initialize sharded cluster controller: %w", err) + } + + if err := helper.stateStore.WriteState(ctx, helper.deploymentState, log); err != nil { + return nil, err + } + + if helper.deploymentState.Status != nil { + // If we have the status in the deployment state, we make sure that status in the CR is the same. + // Status in the deployment state takes precedence. E.g. in case of restoring CR from yaml/git, the user-facing Status field will be restored + // from the deployment state. + // Most of the operations should mutate only deployment state, but some parts of Sharded Cluster implementation still updates the status directly in the CR. + // Having Status in CR synced with the deployment state allows to copy CR's Status into deployment state in updateStatus method. + sc.Status = *helper.deploymentState.Status + } + + return helper, nil +} + +func blockScalingBothWays(desiredReplicasScalers []interfaces.MultiClusterReplicaSetScaler) error { + scalingUp := false + scalingDown := false + var scalingUpLogs []string + var scalingDownLogs []string + + // We have one scaler instance per component per cluster. That means we block scaling both ways across components, + // but also within a single component + // For example, if a component (e.g the config server) tries to scale up on member cluster 1 and scale down on + // member cluster 2, reconciliation will be blocked, even if the total number of replicas for this component stays + // the same. + for _, mcScaler := range desiredReplicasScalers { + desired := mcScaler.TargetReplicas() + current := mcScaler.CurrentReplicas() + logMessage := fmt.Sprintf("Component=%s, Cluster=%s, Current=%d, Desired=%d;", mcScaler.ScalerDescription(), mcScaler.MemberClusterName(), current, desired) + if desired > current { + scalingUp = true + scalingUpLogs = append(scalingUpLogs, logMessage) + } + if desired < current { + scalingDown = true + scalingDownLogs = append(scalingDownLogs, logMessage) + } + } + + if scalingUp && scalingDown { + return xerrors.Errorf( + "Cannot perform scale up and scale down operations at the same time. Scaling Up: %v, Scaling Down: %v", + scalingUpLogs, scalingDownLogs, + ) + } + + return nil +} + +func (r *ShardedClusterReconcileHelper) initializeStateStore(ctx context.Context, reconciler *ReconcileCommonController, sc *mdbv1.MongoDB, log *zap.SugaredLogger) error { + r.deploymentState = NewShardedClusterDeploymentState() + + r.stateStore = NewStateStore[ShardedClusterDeploymentState](sc.GetNamespace(), sc.Name, reconciler.client) + if state, err := r.stateStore.ReadState(ctx); err != nil { + if errors.IsNotFound(err) { + // If the deployment state config map is missing, then it might be either: + // - fresh deployment + // - existing deployment, but it's a first reconcile on the operator version with the new deployment state + // - existing deployment, but for some reason the deployment state config map has been deleted + // In all cases, the deployment config map will be recreated from the state we're keeping and maintaining in + // the old place (in annotations, spec.status, config maps) in order to allow for the downgrade of the operator. + log.Infof("Migrating deployment state from annotations and status to the configmap based deployment state") + if err := r.migrateToNewDeploymentState(sc); err != nil { + return err + } + // This will migrate the deployment state to the new structure and this branch of code won't be executed again. + if err := r.stateStore.WriteState(ctx, r.deploymentState, log); err != nil { + return err + } + } else { + return err + } + } else { + r.deploymentState = state + if r.deploymentState.Status.SizeStatusInClusters == nil { + r.deploymentState.Status.SizeStatusInClusters = &mdbstatus.MongodbShardedSizeStatusInClusters{} + } + if r.deploymentState.Status.SizeStatusInClusters.MongosCountInClusters == nil { + r.deploymentState.Status.SizeStatusInClusters.MongosCountInClusters = map[string]int{} + } + if r.deploymentState.Status.SizeStatusInClusters.ConfigServerMongodsInClusters == nil { + r.deploymentState.Status.SizeStatusInClusters.ConfigServerMongodsInClusters = map[string]int{} + } + if r.deploymentState.Status.SizeStatusInClusters.ShardMongodsInClusters == nil { + r.deploymentState.Status.SizeStatusInClusters.ShardMongodsInClusters = map[string]int{} + } + if r.deploymentState.Status.SizeStatusInClusters.ShardOverridesInClusters == nil { + r.deploymentState.Status.SizeStatusInClusters.ShardOverridesInClusters = map[string]map[string]int{} + } + } + + return nil +} + +func (r *ReconcileMongoDbShardedCluster) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, e error) { + log := zap.S().With("ShardedCluster", request.NamespacedName) + sc := &mdbv1.MongoDB{} + reconcileResult, err := r.prepareResourceForReconciliation(ctx, request, sc, log) + if err != nil { + if errors.IsNotFound(err) { + return workflow.Invalid("Object for reconciliation not found").ReconcileResult() + } + return reconcileResult, err + } + + reconcilerHelper, err := NewShardedClusterReconcilerHelper(ctx, r.ReconcileCommonController, r.imageUrls, r.initDatabaseNonStaticImageVersion, r.databaseNonStaticImageVersion, r.forceEnterprise, sc, r.memberClustersMap, r.omConnectionFactory, log) + if err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(xerrors.Errorf("Failed to initialize sharded cluster reconciler: %w", err)), log) + } + return reconcilerHelper.Reconcile(ctx, log) +} + +// OnDelete tries to complete a Deletion reconciliation event +func (r *ReconcileMongoDbShardedCluster) OnDelete(ctx context.Context, obj runtime.Object, log *zap.SugaredLogger) error { + reconcilerHelper, err := NewShardedClusterReconcilerHelper(ctx, r.ReconcileCommonController, r.imageUrls, r.initDatabaseNonStaticImageVersion, r.databaseNonStaticImageVersion, r.forceEnterprise, obj.(*mdbv1.MongoDB), r.memberClustersMap, r.omConnectionFactory, log) + if err != nil { + return err + } + return reconcilerHelper.OnDelete(ctx, obj, log) +} + +func (r *ShardedClusterReconcileHelper) Reconcile(ctx context.Context, log *zap.SugaredLogger) (res reconcile.Result, e error) { + sc := r.sc + if err := sc.ProcessValidationsOnReconcile(nil); err != nil { + return r.commonController.updateStatus(ctx, sc, workflow.Invalid("%s", err.Error()), log) + } + + log.Info("-> ShardedCluster.Reconcile") + log.Infow("ShardedCluster.Spec", "spec", sc.Spec) + log.Infow("ShardedCluster.Status", "status", r.deploymentState.Status) + log.Infow("ShardedCluster.deploymentState", "sizeStatus", r.deploymentState.Status.MongodbShardedClusterSizeConfig, "sizeStatusInClusters", r.deploymentState.Status.SizeStatusInClusters) + + r.logAllScalers(log) + + // After processing normal validations, we check for conflicting scale-up and scale-down operations within the same + // reconciliation cycle. If both scaling directions are detected, we block the reconciliation. + // This is not currently possible to do it safely with the operator. We check direction of scaling to decide for + // global operations like publishing AC first. + // Therefore, we can obtain inconsistent behaviour in case scaling goes in both directions. + if err := blockScalingBothWays(r.getAllScalers()); err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + + if !architectures.IsRunningStaticArchitecture(sc.Annotations) { + agents.UpgradeAllIfNeeded(ctx, agents.ClientSecret{Client: r.commonController.client, SecretClient: r.commonController.SecretClient}, r.omConnectionFactory, GetWatchedNamespace(), false) + } + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.commonController.client, r.commonController.SecretClient, sc, log) + if err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + + conn, agentAPIKey, err := connection.PrepareOpsManagerConnection(ctx, r.commonController.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, sc.Namespace, log) + if err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + + if err := r.replicateAgentKeySecret(ctx, conn, agentAPIKey, log); err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + if err := r.reconcileHostnameOverrideConfigMap(ctx, log); err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + if err := r.replicateSSLMMSCAConfigMap(ctx, projectConfig, log); err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + + var automationAgentVersion string + if architectures.IsRunningStaticArchitecture(sc.Annotations) { + // In case the Agent *is* overridden, its version will be merged into the StatefulSet. The merging process + // happens after creating the StatefulSet definition. + if !sc.IsAgentImageOverridden() { + automationAgentVersion, err = r.commonController.getAgentVersion(conn, conn.OpsManagerVersion().VersionString, false, log) + if err != nil { + log.Errorf("Impossible to get agent version, please override the agent image by providing a pod template") + return r.updateStatus(ctx, sc, workflow.Failed(xerrors.Errorf("Failed to get agent version: %w", err)), log) + } + } + } + + r.automationAgentVersion = automationAgentVersion + + workflowStatus := r.doShardedClusterProcessing(ctx, sc, conn, projectConfig, log) + if !workflowStatus.IsOK() || workflowStatus.Phase() == mdbstatus.PhaseUnsupported { + return r.updateStatus(ctx, sc, workflowStatus, log) + } + + // note: we don't calculate shardCount in calculateSizeStatus + // shardCount is only updated at the last updateStatus in the reconcile + sizeStatusInClusters, sizeStatus := r.calculateSizeStatus(r.sc) + + // We will continue scaling here if any of the components (i.e. statefulsets in any cluster of any component - mongos, cs, shards) + // are not yet on the target (defined in the spec) levels. + // It's important to understand the flow here: + // - we reach to this point only if we have all the statefulsets reporting ready state (statefulsets ready and agents are in goal state) + // - reaching here means, the all the statefulsets must be on the sizes reported by ReplicasThisReconciliation + // - ReplicasThisReconciliation is always taking into account what's written into the sizes in the deployment state and returns +1 if not at the target level + // - so reaching here means, we're done scaling by the increment (+1 for existing RS, or to the final size if this is a new replicaset scaled from zero) + // Returning true, means we've done the scaling, "one by one" step and it's time to save the current (incremented) size to the deployment state. + // Saving the inremented sizes into the deployment state and requeuing will make ReplicasThisReconciliation to report again +1 and will perform another scaling one by one. + // Returning false here means, we've finished scaling. In this case the sizes will be updated as the last step of the reconcile when reporting Running state. + if r.shouldContinueScalingOneByOne() { + return r.updateStatus(ctx, sc, workflow.Pending("Continuing scaling operation for ShardedCluster %s mongodsPerShardCount ... %+v, mongosCount %+v, configServerCount %+v", + sc.ObjectKey(), + sizeStatus.MongodsPerShardCount, + sizeStatus.MongosCount, + sizeStatus.ConfigServerCount, + ), log, mdbstatus.ShardedClusterSizeConfigOption{SizeConfig: sizeStatus}, mdbstatus.ShardedClusterSizeStatusInClustersOption{SizeConfigInClusters: sizeStatusInClusters}) + } + + // Only remove any stateful sets if we are scaling down. + // This is executed only after the replicaset which are going to be removed are properly drained and + // all the processes in the cluster reports ready state. At this point the statefulsets are + // no longer part of the replicaset and are safe to remove - all the data from them is migrated (drained) to other shards. + if sc.Spec.ShardCount < r.deploymentState.Status.ShardCount { + r.removeUnusedStatefulsets(ctx, sc, log) + } + + annotationsToAdd, err := getAnnotationsForResource(sc) + if err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + + if vault.IsVaultSecretBackend() { + secrets := sc.GetSecretsMountedIntoDBPod() + vaultMap := make(map[string]string) + for _, s := range secrets { + path := fmt.Sprintf("%s/%s/%s", r.commonController.VaultClient.DatabaseSecretMetadataPath(), sc.Namespace, s) + vaultMap = merge.StringToStringMap(vaultMap, r.commonController.VaultClient.GetSecretAnnotation(path)) + } + path := fmt.Sprintf("%s/%s/%s", r.commonController.VaultClient.OperatorScretMetadataPath(), sc.Namespace, sc.Spec.Credentials) + vaultMap = merge.StringToStringMap(vaultMap, r.commonController.VaultClient.GetSecretAnnotation(path)) + for k, val := range vaultMap { + annotationsToAdd[k] = val + } + } + // Set annotations that should be saved at the end of the reconciliation, e.g lastAchievedSpec + if err := annotations.SetAnnotations(ctx, sc, annotationsToAdd, r.commonController.client); err != nil { + return r.updateStatus(ctx, sc, workflow.Failed(err), log) + } + + // Save last achieved spec in state + r.deploymentState.LastAchievedSpec = &sc.Spec + log.Infof("Finished reconciliation for Sharded Cluster! %s", completionMessage(conn.BaseURL(), conn.GroupID())) + // It's the second place in the reconcile logic we're updating sizes of all the components + // We're also updating the shardCount here - it's the only place we're doing that. + return r.updateStatus(ctx, sc, workflowStatus, log, + mdbstatus.NewBaseUrlOption(deployment.Link(conn.BaseURL(), conn.GroupID())), + mdbstatus.ShardedClusterSizeConfigOption{SizeConfig: sizeStatus}, + mdbstatus.ShardedClusterSizeStatusInClustersOption{SizeConfigInClusters: sizeStatusInClusters}, + mdbstatus.ShardedClusterMongodsPerShardCountOption{Members: r.sc.Spec.ShardCount}, + mdbstatus.NewPVCsStatusOptionEmptyStatus(), + ) +} + +func (r *ShardedClusterReconcileHelper) logAllScalers(log *zap.SugaredLogger) { + for _, s := range r.getAllScalers() { + log.Debugf("%+v", s) + } +} + +func (r *ShardedClusterReconcileHelper) doShardedClusterProcessing(ctx context.Context, obj interface{}, conn om.Connection, projectConfig mdbv1.ProjectConfig, log *zap.SugaredLogger) workflow.Status { + log.Info("ShardedCluster.doShardedClusterProcessing") + sc := obj.(*mdbv1.MongoDB) + + if workflowStatus := ensureSupportedOpsManagerVersion(conn); workflowStatus.Phase() != mdbstatus.PhaseRunning { + return workflowStatus + } + + r.commonController.SetupCommonWatchers(sc, getTLSSecretNames(sc), getInternalAuthSecretNames(sc), sc.Name) + + reconcileResult := checkIfHasExcessProcesses(conn, sc.Name, log) + if !reconcileResult.IsOK() { + return reconcileResult + } + + security := sc.Spec.Security + if security.Authentication != nil && security.Authentication.Enabled && security.Authentication.IsX509Enabled() && !sc.Spec.GetSecurity().IsTLSEnabled() { + return workflow.Invalid("cannot have a non-tls deployment when x509 authentication is enabled") + } + + currentAgentAuthMode, err := conn.GetAgentAuthMode() + if err != nil { + return workflow.Failed(err) + } + + podEnvVars := newPodVars(conn, projectConfig, sc.Spec.LogLevel) + + workflowStatus, certSecretTypesForSTS := r.ensureSSLCertificates(ctx, sc, log) + if !workflowStatus.IsOK() { + return workflowStatus + } + + prometheusCertHash, err := certs.EnsureTLSCertsForPrometheus(ctx, r.commonController.SecretClient, sc.GetNamespace(), sc.GetPrometheus(), certs.Database, log) + if err != nil { + return workflow.Failed(xerrors.Errorf("Could not generate certificates for Prometheus: %w", err)) + } + + opts := deploymentOptions{ + podEnvVars: podEnvVars, + currentAgentAuthMode: currentAgentAuthMode, + certTLSType: certSecretTypesForSTS, + } + + if err = r.prepareScaleDownShardedCluster(conn, log); err != nil { + return workflow.Failed(xerrors.Errorf("failed to perform scale down preliminary actions: %w", err)) + } + + if workflowStatus := validateMongoDBResource(sc, conn); !workflowStatus.IsOK() { + return workflowStatus + } + + // Ensures that all sharded cluster certificates are either of Opaque type (old design) + // or are all of kubernetes.io/tls type + // and save the value for future use + allCertsType, err := getCertTypeForAllShardedClusterCertificates(certSecretTypesForSTS) + if err != nil { + return workflow.Failed(err) + } + + caFilePath := util.CAFilePathInContainer + if allCertsType == corev1.SecretTypeTLS { + caFilePath = fmt.Sprintf("%s/ca-pem", util.TLSCaMountPath) + } + + if workflowStatus := controlledfeature.EnsureFeatureControls(*sc, conn, conn.OpsManagerVersion(), log); !workflowStatus.IsOK() { + return workflowStatus + } + + for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) { + certConfigurator := r.prepareX509CertConfigurator(memberCluster) + if workflowStatus := r.commonController.ensureX509SecretAndCheckTLSType(ctx, certConfigurator, currentAgentAuthMode, log); !workflowStatus.IsOK() { + return workflowStatus + } + } + + if workflowStatus := ensureRoles(sc.Spec.GetSecurity().Roles, conn, log); !workflowStatus.IsOK() { + return workflowStatus + } + + agentCertSecretName := sc.GetSecurity().AgentClientCertificateSecretName(sc.Name).Name + + opts = deploymentOptions{ + podEnvVars: podEnvVars, + currentAgentAuthMode: currentAgentAuthMode, + caFilePath: caFilePath, + agentCertSecretName: agentCertSecretName, + prometheusCertHash: prometheusCertHash, + } + allConfigs := r.getAllConfigs(ctx, *sc, opts, log) + + // Recovery prevents some deadlocks that can occur during reconciliation, e.g. the setting of an incorrect automation + // configuration and a subsequent attempt to overwrite it later, the operator would be stuck in Pending phase. + // See CLOUDP-189433 and CLOUDP-229222 for more details. + if recovery.ShouldTriggerRecovery(r.deploymentState.Status.Phase != mdbstatus.PhaseRunning, r.deploymentState.Status.LastTransition) { + log.Warnf("Triggering Automatic Recovery. The MongoDB resource %s/%s is in %s state since %s", sc.Namespace, sc.Name, r.deploymentState.Status.Phase, r.deploymentState.Status.LastTransition) + automationConfigStatus := r.updateOmDeploymentShardedCluster(ctx, conn, sc, opts, true, log).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + deploymentStatus := r.createKubernetesResources(ctx, sc, opts, log) + if !deploymentStatus.IsOK() { + log.Errorf("Recovery failed because of deployment errors, %v", deploymentStatus) + } + if !automationConfigStatus.IsOK() { + log.Errorf("Recovery failed because of Automation Config update errors, %v", automationConfigStatus) + } + } + + workflowStatus = workflow.RunInGivenOrder(anyStatefulSetNeedsToPublishStateToOM(ctx, *sc, r.commonController.client, r.deploymentState.LastAchievedSpec, allConfigs, log), + func() workflow.Status { + return r.updateOmDeploymentShardedCluster(ctx, conn, sc, opts, false, log).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + }, + func() workflow.Status { + return r.createKubernetesResources(ctx, sc, opts, log).OnErrorPrepend("Failed to create/update (Kubernetes reconciliation phase):") + }) + + if !workflowStatus.IsOK() { + return workflowStatus + } + return reconcileResult +} + +// prepareX509CertConfigurator returns x509 configurator for the specified memberCluster. +func (r *ShardedClusterReconcileHelper) prepareX509CertConfigurator(memberCluster multicluster.MemberCluster) certs.ShardedSetX509CertConfigurator { + var opts []certs.Options + + // we don't have inverted mapping of memberCluster -> shard/configSrv/mongos configuration, so we need to find the specified member cluster first + for shardIdx := range r.desiredShardsConfiguration { + for _, shardMemberCluster := range r.shardsMemberClustersMap[shardIdx] { + if shardMemberCluster.Name == memberCluster.Name { + opts = append(opts, certs.ShardConfig(*r.sc, shardIdx, r.sc.Spec.GetExternalDomain(), r.GetShardScaler(shardIdx, shardMemberCluster))) + } + } + } + + for _, configSrvMemberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + if memberCluster.Name == configSrvMemberCluster.Name { + opts = append(opts, certs.ConfigSrvConfig(*r.sc, r.sc.Spec.GetExternalDomain(), r.GetConfigSrvScaler(configSrvMemberCluster))) + } + } + + for _, mongosMemberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + if memberCluster.Name == mongosMemberCluster.Name { + opts = append(opts, certs.MongosConfig(*r.sc, r.sc.Spec.GetExternalDomain(), r.GetMongosScaler(mongosMemberCluster))) + } + } + + certConfigurator := certs.ShardedSetX509CertConfigurator{ + MongoDB: r.sc, + SecretReadClient: r.commonController.SecretClient, + MemberCluster: memberCluster, + CertOptions: opts, + } + + return certConfigurator +} + +func getTLSSecretNames(sc *mdbv1.MongoDB) func() []string { + return func() []string { + var secretNames []string + secretNames = append(secretNames, + sc.GetSecurity().MemberCertificateSecretName(sc.MongosRsName()), + sc.GetSecurity().MemberCertificateSecretName(sc.ConfigRsName()), + ) + for i := 0; i < sc.Spec.ShardCount; i++ { + secretNames = append(secretNames, sc.GetSecurity().MemberCertificateSecretName(sc.ShardRsName(i))) + } + return secretNames + } +} + +func getInternalAuthSecretNames(sc *mdbv1.MongoDB) func() []string { + return func() []string { + var secretNames []string + secretNames = append(secretNames, + sc.GetSecurity().InternalClusterAuthSecretName(sc.MongosRsName()), + sc.GetSecurity().InternalClusterAuthSecretName(sc.ConfigRsName()), + ) + for i := 0; i < sc.Spec.ShardCount; i++ { + secretNames = append(secretNames, sc.GetSecurity().InternalClusterAuthSecretName(sc.ShardRsName(i))) + } + return secretNames + } +} + +// getCertTypeForAllShardedClusterCertificates checks whether all certificates secret are of the same type and returns it. +func getCertTypeForAllShardedClusterCertificates(certTypes map[string]bool) (corev1.SecretType, error) { + if len(certTypes) == 0 { + return corev1.SecretTypeTLS, nil + } + valueSlice := make([]bool, 0, len(certTypes)) + for _, v := range certTypes { + valueSlice = append(valueSlice, v) + } + curTypeIsTLS := valueSlice[0] + for i := 1; i < len(valueSlice); i++ { + if valueSlice[i] != curTypeIsTLS { + return corev1.SecretTypeOpaque, xerrors.Errorf("TLS Certificates for Sharded cluster must all be of the same type - either kubernetes.io/tls or secrets containing a concatenated pem file") + } + } + if curTypeIsTLS { + return corev1.SecretTypeTLS, nil + } + return corev1.SecretTypeOpaque, nil +} + +// anyStatefulSetNeedsToPublishStateToOM checks to see if any stateful set +// of the given sharded cluster needs to publish state to Ops Manager before updating Kubernetes resources +func anyStatefulSetNeedsToPublishStateToOM(ctx context.Context, sc mdbv1.MongoDB, getter ConfigMapStatefulSetSecretGetter, lastSpec *mdbv1.MongoDbSpec, configs []func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions, log *zap.SugaredLogger) bool { + for _, cf := range configs { + if publishAutomationConfigFirst(ctx, getter, sc, lastSpec, cf, log) { + return true + } + } + return false +} + +// getAllConfigs returns a list of all the configuration functions associated with the Sharded Cluster. +// This includes the Mongos, the Config Server and all Shards +func (r *ShardedClusterReconcileHelper) getAllConfigs(ctx context.Context, sc mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger) []func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions { + allConfigs := make([]func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions, 0) + for shardIdx, shardSpec := range r.desiredShardsConfiguration { + for _, memberCluster := range getHealthyMemberClusters(r.shardsMemberClustersMap[shardIdx]) { + allConfigs = append(allConfigs, r.getShardOptions(ctx, sc, shardIdx, opts, log, shardSpec, memberCluster)) + } + } + for _, memberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + allConfigs = append(allConfigs, r.getConfigServerOptions(ctx, sc, opts, log, r.desiredConfigServerConfiguration, memberCluster)) + } + for _, memberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + allConfigs = append(allConfigs, r.getMongosOptions(ctx, sc, opts, log, r.desiredMongosConfiguration, memberCluster)) + } + return allConfigs +} + +func getHealthyMemberClusters(memberClusters []multicluster.MemberCluster) []multicluster.MemberCluster { + var result []multicluster.MemberCluster + for i := range memberClusters { + if memberClusters[i].Healthy { + result = append(result, memberClusters[i]) + } + } + + return result +} + +func (r *ShardedClusterReconcileHelper) removeUnusedStatefulsets(ctx context.Context, sc *mdbv1.MongoDB, log *zap.SugaredLogger) { + statefulsetsToRemove := r.deploymentState.Status.ShardCount - sc.Spec.ShardCount + shardsCount := r.deploymentState.Status.ShardCount + + // we iterate over last 'statefulsetsToRemove' shards if any + for i := shardsCount - statefulsetsToRemove; i < shardsCount; i++ { + for _, memberCluster := range r.shardsMemberClustersMap[i] { + key := kube.ObjectKey(sc.Namespace, r.GetShardStsName(i, memberCluster)) + err := memberCluster.Client.DeleteStatefulSet(ctx, key) + if err != nil { + // Most of all the error won't be recoverable, also our sharded cluster is in good shape - we can just warn + // the error and leave the cleanup work for the admins + log.Warnf("Failed to delete the statefulset %s in cluster %s: %s", key, memberCluster.Name, err) + } + log.Infof("Removed statefulset %s in cluster %s as it's was removed from sharded cluster", key, memberCluster.Name) + } + } +} + +func (r *ShardedClusterReconcileHelper) ensureSSLCertificates(ctx context.Context, s *mdbv1.MongoDB, log *zap.SugaredLogger) (workflow.Status, map[string]bool) { + tlsConfig := s.Spec.GetTLSConfig() + + certSecretTypes := map[string]bool{} + if tlsConfig == nil || !s.Spec.GetSecurity().IsTLSEnabled() { + return workflow.OK(), certSecretTypes + } + + if err := r.replicateTLSCAConfigMap(ctx, log); err != nil { + return workflow.Failed(err), nil + } + + var workflowStatus workflow.Status = workflow.OK() + for _, memberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + mongosCert := certs.MongosConfig(*s, r.sc.Spec.GetExternalDomain(), r.GetMongosScaler(memberCluster)) + tStatus := certs.EnsureSSLCertsForStatefulSet(ctx, r.commonController.SecretClient, memberCluster.SecretClient, *s.Spec.Security, mongosCert, log) + certSecretTypes[mongosCert.CertSecretName] = true + workflowStatus = workflowStatus.Merge(tStatus) + } + + for _, memberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + configSrvCert := certs.ConfigSrvConfig(*s, r.sc.Spec.DbCommonSpec.GetExternalDomain(), r.GetConfigSrvScaler(memberCluster)) + tStatus := certs.EnsureSSLCertsForStatefulSet(ctx, r.commonController.SecretClient, memberCluster.SecretClient, *s.Spec.Security, configSrvCert, log) + certSecretTypes[configSrvCert.CertSecretName] = true + workflowStatus = workflowStatus.Merge(tStatus) + } + + for i := 0; i < s.Spec.ShardCount; i++ { + for _, memberCluster := range getHealthyMemberClusters(r.shardsMemberClustersMap[i]) { + shardCert := certs.ShardConfig(*s, i, r.sc.Spec.DbCommonSpec.GetExternalDomain(), r.GetShardScaler(i, memberCluster)) + tStatus := certs.EnsureSSLCertsForStatefulSet(ctx, r.commonController.SecretClient, memberCluster.SecretClient, *s.Spec.Security, shardCert, log) + certSecretTypes[shardCert.CertSecretName] = true + workflowStatus = workflowStatus.Merge(tStatus) + } + } + + return workflowStatus, certSecretTypes +} + +// createKubernetesResources creates all Kubernetes objects that are specified in 'state' parameter. +// This function returns errorStatus if any errors occurred or pendingStatus if the statefulsets are not +// ready yet +// Note, that it doesn't remove any existing shards - this will be done later +func (r *ShardedClusterReconcileHelper) createKubernetesResources(ctx context.Context, s *mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger) workflow.Status { + if r.sc.Spec.IsMultiCluster() { + // for multi-cluster deployment we should create pod-services first, as doing it after is a bit too late + // statefulset creation loops and waits for sts to become ready, and it's easier for the replica set to be ready if + // it can connect to other members in the clusters + // TODO the same should be considered for external services, we should always create them before sts; now external services are created inside DatabaseInKubernetes function; + if err := r.reconcileServices(ctx, log); err != nil { + return workflow.Failed(xerrors.Errorf("Failed to create Config Server Stateful Set: %w", err)) + } + } + + lastSpec := r.deploymentState.LastAchievedSpec + // In static containers, the operator controls the order of up and downgrades. + // For sharded clusters, we need to reverse the order of downgrades vs. upgrades. + // See more here: https://www.mongodb.com/docs/manual/release-notes/6.0-downgrade-sharded-cluster/ + if lastSpec != nil && architectures.IsRunningStaticArchitecture(s.Annotations) && versionutil.IsDowngrade(lastSpec.Version, s.Spec.Version) { + if mongosStatus := r.createOrUpdateMongos(ctx, s, opts, log); !mongosStatus.IsOK() { + return mongosStatus + } + + if shardsStatus := r.createOrUpdateShards(ctx, s, opts, log); !shardsStatus.IsOK() { + return shardsStatus + } + + if configStatus := r.createOrUpdateConfigServers(ctx, s, opts, log); !configStatus.IsOK() { + return configStatus + } + } else { + if configStatus := r.createOrUpdateConfigServers(ctx, s, opts, log); !configStatus.IsOK() { + return configStatus + } + + if shardsStatus := r.createOrUpdateShards(ctx, s, opts, log); !shardsStatus.IsOK() { + return shardsStatus + } + + if mongosStatus := r.createOrUpdateMongos(ctx, s, opts, log); !mongosStatus.IsOK() { + return mongosStatus + } + } + + return workflow.OK() +} + +func (r *ShardedClusterReconcileHelper) createOrUpdateMongos(ctx context.Context, s *mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger) workflow.Status { + // we deploy changes to sts to all mongos in all clusters + for _, memberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + mongosOpts := r.getMongosOptions(ctx, *s, opts, log, r.desiredMongosConfiguration, memberCluster) + mongosSts := construct.DatabaseStatefulSet(*s, mongosOpts, log) + if err := create.DatabaseInKubernetes(ctx, memberCluster.Client, *s, mongosSts, mongosOpts, log); err != nil { + return workflow.Failed(xerrors.Errorf("Failed to create Mongos Stateful Set: %w", err)) + } + } + + // we wait for mongos statefulsets here + if workflowStatus := r.getMergedStatefulsetStatus(ctx, s, r.mongosMemberClusters, r.GetMongosStsName); !workflowStatus.IsOK() { + return workflowStatus + } + + log.Infow("Created/updated StatefulSet for mongos servers", "name", s.MongosRsName()) + return workflow.OK() +} + +func (r *ShardedClusterReconcileHelper) createOrUpdateShards(ctx context.Context, s *mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger) workflow.Status { + shardsNames := make([]string, s.Spec.ShardCount) + for shardIdx := 0; shardIdx < s.Spec.ShardCount; shardIdx++ { + // it doesn't matter for which cluster we get scaler as we need it only for ScalingFirstTime which is iterating over all member clusters internally anyway + scalingFirstTime := r.GetShardScaler(shardIdx, r.shardsMemberClustersMap[shardIdx][0]).ScalingFirstTime() + for _, memberCluster := range getHealthyMemberClusters(r.shardsMemberClustersMap[shardIdx]) { + // shardsNames contains shard name, not statefulset name + // in single cluster sts name == shard name + // in multi cluster sts name contains cluster index, but shard name does not (it's a replicaset name) + shardsNames[shardIdx] = s.ShardRsName(shardIdx) + shardOpts := r.getShardOptions(ctx, *s, shardIdx, opts, log, r.desiredShardsConfiguration[shardIdx], memberCluster) + shardSts := construct.DatabaseStatefulSet(*s, shardOpts, log) + + if workflowStatus := r.handlePVCResize(ctx, memberCluster, &shardSts, log); !workflowStatus.IsOK() { + return workflowStatus + } + + if err := create.DatabaseInKubernetes(ctx, memberCluster.Client, *s, shardSts, shardOpts, log); err != nil { + return workflow.Failed(xerrors.Errorf("Failed to create StatefulSet for shard %s: %w", shardSts.Name, err)) + } + + if !scalingFirstTime { + // If we scale for the first time, we deploy all statefulsets across all clusters for the given shard. + // We can do that because when doing the initial deployment there is no automation config, so we can deploy + // everything in parallel and our pods will be spinning up agents only. After everything is ready + // (we have the case in readiness for empty AC to return true) we then publish AC with fully constructed processes + // and all agents are starting to wire things up and configure the replicaset. + // If we don't scale for the first time we need to wait for each individual sts as we need to scale members of the whole replica set one at a time + if workflowStatus := getStatefulSetStatus(ctx, s.Namespace, shardSts.Name, memberCluster.Client); !workflowStatus.IsOK() { + return workflowStatus + } + } + } + // if we scale for the first time we didn't wait for statefulsets to become ready in the loop over member clusters + // we need to wait for all sts here instead after all were deployed/scaled up to desired members + if scalingFirstTime { + getShardStsName := func(memberCluster multicluster.MemberCluster) string { + return r.GetShardStsName(shardIdx, memberCluster) + } + if workflowStatus := r.getMergedStatefulsetStatus(ctx, s, r.shardsMemberClustersMap[shardIdx], getShardStsName); !workflowStatus.IsOK() { + return workflowStatus + } + } + } + + log.Infow("Created/updated Stateful Sets for shards in Kubernetes", "shards", shardsNames) + return workflow.OK() +} + +func (r *ShardedClusterReconcileHelper) createOrUpdateConfigServers(ctx context.Context, s *mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger) workflow.Status { + // it doesn't matter for which cluster we get scaler here as we need it only + // for ScalingFirstTime, which is iterating over all member clusters internally anyway + configSrvScalingFirstTime := r.GetConfigSrvScaler(r.configSrvMemberClusters[0]).ScalingFirstTime() + for _, memberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + configSrvOpts := r.getConfigServerOptions(ctx, *s, opts, log, r.desiredConfigServerConfiguration, memberCluster) + configSrvSts := construct.DatabaseStatefulSet(*s, configSrvOpts, log) + + if workflowStatus := r.handlePVCResize(ctx, memberCluster, &configSrvSts, log); !workflowStatus.IsOK() { + return workflowStatus + } + + if err := create.DatabaseInKubernetes(ctx, memberCluster.Client, *s, configSrvSts, configSrvOpts, log); err != nil { + return workflow.Failed(xerrors.Errorf("Failed to create Config Server Stateful Set: %w", err)) + } + + if !configSrvScalingFirstTime { + if workflowStatus := getStatefulSetStatus(ctx, s.Namespace, r.GetConfigSrvStsName(memberCluster), memberCluster.Client); !workflowStatus.IsOK() { + return workflowStatus + } + } + } + + if configSrvScalingFirstTime { + if workflowStatus := r.getMergedStatefulsetStatus(ctx, s, r.configSrvMemberClusters, r.GetConfigSrvStsName); !workflowStatus.IsOK() { + return workflowStatus + } + } + + log.Infow("Created/updated StatefulSet for config servers", "name", s.ConfigRsName(), "servers count", 0) + return workflow.OK() +} + +func (r *ShardedClusterReconcileHelper) getMergedStatefulsetStatus(ctx context.Context, s *mdbv1.MongoDB, + memberClusters []multicluster.MemberCluster, stsNameProvider func(multicluster.MemberCluster) string, +) workflow.Status { + var mergedStatefulSetStatus workflow.Status = workflow.OK() + for _, memberCluster := range getHealthyMemberClusters(memberClusters) { + statefulSetStatus := getStatefulSetStatus(ctx, s.Namespace, stsNameProvider(memberCluster), memberCluster.Client) + mergedStatefulSetStatus = mergedStatefulSetStatus.Merge(statefulSetStatus) + } + + return mergedStatefulSetStatus +} + +func (r *ShardedClusterReconcileHelper) handlePVCResize(ctx context.Context, memberCluster multicluster.MemberCluster, sts *appsv1.StatefulSet, log *zap.SugaredLogger) workflow.Status { + workflowStatus := create.HandlePVCResize(ctx, memberCluster.Client, sts, log) + if !workflowStatus.IsOK() { + return workflowStatus + } + + if workflow.ContainsPVCOption(workflowStatus.StatusOptions()) { + if _, err := r.updateStatus(ctx, r.sc, workflow.Pending(""), log, workflowStatus.StatusOptions()...); err != nil { + return workflow.Failed(xerrors.Errorf("error updating status: %w", err)) + } + } + return workflow.OK() +} + +func isShardOverridden(shardName string, shardOverrides []mdbv1.ShardOverride) bool { + expandedOverrides := expandShardOverrides(shardOverrides) + foundIdx := slices.IndexFunc(expandedOverrides, func(override mdbv1.ShardOverride) bool { + return len(override.ShardNames) > 0 && override.ShardNames[0] == shardName + }) + return foundIdx != -1 +} + +// calculateSizeStatus computes the current sizes of the sharded cluster deployment and return the structures that are going to be saved to the resource's status and the deployment state. +// It computes the sizes according to the deployment state (previous sizes), the desired state and the sizes returned by the scalers. +// What is important to note it the scalers used here and usage of scale.ReplicasThisReconciliation makes the sizes returned consistent throughout a single reconcile execution and +// with the guarantee that only one node can be added at a time to any replicaset. +// That means we use the scale.ReplicasThisReconciliation function with scalers in other parts of the reconciler logic (e.g. for creating sts and processes for AC, here for status). +func (r *ShardedClusterReconcileHelper) calculateSizeStatus(s *mdbv1.MongoDB) (*mdbstatus.MongodbShardedSizeStatusInClusters, *mdbstatus.MongodbShardedClusterSizeConfig) { + sizeStatusInClusters := r.deploymentState.Status.SizeStatusInClusters.DeepCopy() + sizeStatus := r.deploymentState.Status.MongodbShardedClusterSizeConfig.DeepCopy() + + // We calculate the current member counts for updating the status at the end of the function, after everything is ready according to the current reconcile loop's scalers + // Before making the reconcile loop multi-cluster-first, the counts were saved only when workflow result was OK, so we're keeping the same logic here + + // We iterate over all clusters (not only healthy as it would remove the counts from those) and store counts to deployment state + shardMongodsCountInClusters := map[string]int{} + shardOverridesInClusters := map[string]map[string]int{} + // In all shards, we iterate over all clusters (not only healthy as it would remove the counts from those) and store + // counts to deployment state + for shardIdx := 0; shardIdx < s.Spec.ShardCount; shardIdx++ { + shardName := r.sc.ShardRsName(shardIdx) + isOverridden := isShardOverridden(shardName, r.sc.Spec.ShardOverrides) + + // if all shards are overridden, we have nothing in shardMongodsCountInClusters, followup ticket: https://jira.mongodb.org/browse/CLOUDP-287426 + if isOverridden { + // Initialize the map for this override if needed + if shardOverridesInClusters[shardName] == nil { + shardOverridesInClusters[shardName] = map[string]int{} + } + for _, memberCluster := range r.shardsMemberClustersMap[shardIdx] { + currentReplicas := scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster)) + shardOverridesInClusters[shardName][memberCluster.Name] = currentReplicas + } + } else if len(shardMongodsCountInClusters) == 0 { + // Without override, shardMongodsCountInClusters will be the same for any shard, we need to populate it + // only once, if it's empty + for _, memberCluster := range r.shardsMemberClustersMap[shardIdx] { + currentReplicas := scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster)) + shardMongodsCountInClusters[memberCluster.Name] = currentReplicas + } + } + // If shardMongodsCountInClusters is already populated, no action is needed for non-overridden shards + } + + sizeStatusInClusters.ShardMongodsInClusters = shardMongodsCountInClusters // While we do not address the above to do, this field can be nil in the case where all shards are overridden + sizeStatusInClusters.ShardOverridesInClusters = shardOverridesInClusters + // TODO when we allow changes of the number of nodes in particular shards in shard overrides, then this field might become invalid or will become "MongodsPerShardCount" (for the most shards out there) + sizeStatus.MongodsPerShardCount = sizeStatusInClusters.TotalShardMongodsInClusters() + + // We iterate over all clusters (not only healthy as it would remove the counts from those) and store counts to deployment state + configSrvMongodsTotalCount := map[string]int{} + for _, memberCluster := range r.configSrvMemberClusters { + configSrvMongodsTotalCount[memberCluster.Name] = scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster)) + sizeStatusInClusters.ConfigServerMongodsInClusters[memberCluster.Name] = configSrvMongodsTotalCount[memberCluster.Name] + } + sizeStatus.ConfigServerCount = sizeStatusInClusters.TotalConfigServerMongodsInClusters() + + mongosCountInClusters := map[string]int{} + for _, memberCluster := range r.mongosMemberClusters { + mongosCountInClusters[memberCluster.Name] = scale.ReplicasThisReconciliation(r.GetMongosScaler(memberCluster)) + sizeStatusInClusters.MongosCountInClusters[memberCluster.Name] = mongosCountInClusters[memberCluster.Name] + } + + sizeStatus.MongosCount = sizeStatusInClusters.TotalMongosCountInClusters() + + return sizeStatusInClusters, sizeStatus +} + +func (r *ShardedClusterReconcileHelper) OnDelete(ctx context.Context, obj runtime.Object, log *zap.SugaredLogger) error { + sc := obj.(*mdbv1.MongoDB) + + var errs error + if err := r.cleanOpsManagerState(ctx, sc, log); err != nil { + errs = multierror.Append(errs, err) + } + + for _, item := range getHealthyMemberClusters(r.allMemberClusters) { + c := item.Client + if err := r.deleteClusterResources(ctx, c, sc, log); err != nil { + errs = multierror.Append(errs, xerrors.Errorf("failed deleting dependant resources in cluster %s: %w", item.Name, err)) + } + } + + return errs +} + +func (r *ShardedClusterReconcileHelper) cleanOpsManagerState(ctx context.Context, sc *mdbv1.MongoDB, log *zap.SugaredLogger) error { + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.commonController.client, r.commonController.SecretClient, sc, log) + if err != nil { + return err + } + + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.commonController.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, sc.Namespace, log) + if err != nil { + return err + } + + processNames := make([]string, 0) + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + processNames = d.GetProcessNames(om.ShardedCluster{}, sc.Name) + if e := d.RemoveShardedClusterByName(sc.Name, log); e != nil { + log.Warnf("Failed to remove sharded cluster from automation config: %s", e) + } + return nil + }, + log, + ) + if err != nil { + return err + } + + logDiffOfProcessNames(processNames, r.getHealthyProcessNames(), log.With("ctx", "cleanOpsManagerState")) + if err := om.WaitForReadyState(conn, r.getHealthyProcessNames(), false, log); err != nil { + return err + } + + if sc.Spec.Backup != nil && sc.Spec.Backup.AutoTerminateOnDeletion { + if err := backup.StopBackupIfEnabled(conn, conn, sc.Name, backup.ShardedClusterType, log); err != nil { + return err + } + } + + hostsToRemove := r.getAllHostnames(false) + log.Infow("Stop monitoring removed hosts in Ops Manager", "hostsToBeRemoved", hostsToRemove) + + if err = host.StopMonitoring(conn, hostsToRemove, log); err != nil { + return err + } + + if err := r.commonController.clearProjectAuthenticationSettings(ctx, conn, sc, processNames, log); err != nil { + return err + } + + log.Infow("Clear feature control for group: %s", "groupID", conn.GroupID()) + if result := controlledfeature.ClearFeatureControls(conn, conn.OpsManagerVersion(), log); !result.IsOK() { + result.Log(log) + log.Warnf("Failed to clear feature control from group: %s", conn.GroupID()) + } + + log.Infof("Removed deployment %s from Ops Manager at %s", sc.Name, conn.BaseURL()) + return nil +} + +func logDiffOfProcessNames(acProcesses []string, healthyProcesses []string, log *zap.SugaredLogger) { + sort.Strings(acProcesses) + sort.Strings(healthyProcesses) + if diff := cmp.Diff(acProcesses, healthyProcesses); diff != "" { + log.Debugf("difference of AC processes vs healthy processes: %s\n AC processes: %v, healthy processes: %v", diff, acProcesses, healthyProcesses) + } +} + +func (r *ShardedClusterReconcileHelper) deleteClusterResources(ctx context.Context, c kubernetesClient.Client, sc *mdbv1.MongoDB, log *zap.SugaredLogger) error { + var errs error + + // cleanup resources in the namespace as the MongoDB with the corresponding label. + cleanupOptions := mdbv1.MongodbCleanUpOptions{ + Namespace: sc.Namespace, + Labels: mongoDBLabels(sc.Name), + } + + if err := c.DeleteAllOf(ctx, &corev1.Service{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed Serivces associated with %s/%s", sc.Namespace, sc.Name) + } + + if err := c.DeleteAllOf(ctx, &appsv1.StatefulSet{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed StatefulSets associated with %s/%s", sc.Namespace, sc.Name) + } + + if err := c.DeleteAllOf(ctx, &corev1.ConfigMap{}, &cleanupOptions); err != nil { + errs = multierror.Append(errs, err) + } else { + log.Infof("Removed ConfigMaps associated with %s/%s", sc.Namespace, sc.Name) + } + + r.commonController.resourceWatcher.RemoveDependentWatchedResources(sc.ObjectKey()) + + return errs +} + +func AddShardedClusterController(ctx context.Context, mgr manager.Manager, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, memberClustersMap map[string]cluster.Cluster) error { + // Create a new controller + reconciler := newShardedClusterReconciler(ctx, mgr.GetClient(), imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, multicluster.ClustersMapToClientMap(memberClustersMap), om.NewOpsManagerConnection) + options := controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)} // nolint:forbidigo + c, err := controller.New(util.MongoDbShardedClusterController, mgr, options) + if err != nil { + return err + } + + // watch for changes to sharded cluster MongoDB resources + eventHandler := ResourceEventHandler{deleter: reconciler} + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &mdbv1.MongoDB{}, &eventHandler, watch.PredicatesForMongoDB(mdbv1.ShardedCluster))) + if err != nil { + return err + } + + err = c.Watch(source.Channel[client.Object](OmUpdateChannel, &handler.EnqueueRequestForObject{}, source.WithPredicates(watch.PredicatesForMongoDB(mdbv1.ShardedCluster)))) + if err != nil { + return xerrors.Errorf("not able to setup OmUpdateChannel to listent to update events from OM: %s", err) + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, + &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + // if vault secret backend is enabled watch for Vault secret change and reconcile + if vault.IsVaultSecretBackend() { + eventChannel := make(chan event.GenericEvent) + go vaultwatcher.WatchSecretChangeForMDB(ctx, zap.S(), eventChannel, reconciler.client, reconciler.VaultClient, mdbv1.ShardedCluster) + + err = c.Watch(source.Channel[client.Object](eventChannel, &handler.EnqueueRequestForObject{})) + if err != nil { + zap.S().Errorf("Failed to watch for vault secret changes: %w", err) + } + } + zap.S().Infof("Registered controller %s", util.MongoDbShardedClusterController) + + return nil +} + +func (r *ShardedClusterReconcileHelper) getConfigSrvHostnames(memberCluster multicluster.MemberCluster, replicas int) ([]string, []string) { + externalDomain := r.sc.Spec.ConfigSrvSpec.ClusterSpecList.GetExternalDomainForMemberCluster(memberCluster.Name) + if externalDomain == nil && r.sc.Spec.IsMultiCluster() { + externalDomain = r.sc.Spec.DbCommonSpec.GetExternalDomain() + } + if !memberCluster.Legacy { + return dns.GetMultiClusterProcessHostnamesAndPodNames(r.sc.ConfigRsName(), r.sc.Namespace, memberCluster.Index, replicas, r.sc.Spec.GetClusterDomain(), externalDomain) + } else { + return dns.GetDNSNames(r.GetConfigSrvStsName(memberCluster), r.sc.ConfigSrvServiceName(), r.sc.Namespace, r.sc.Spec.GetClusterDomain(), replicas, externalDomain) + } +} + +func (r *ShardedClusterReconcileHelper) getShardHostnames(shardIdx int, memberCluster multicluster.MemberCluster, replicas int) ([]string, []string) { + externalDomain := r.sc.Spec.ShardSpec.ClusterSpecList.GetExternalDomainForMemberCluster(memberCluster.Name) + if externalDomain == nil && r.sc.Spec.IsMultiCluster() { + externalDomain = r.sc.Spec.DbCommonSpec.GetExternalDomain() + } + if !memberCluster.Legacy { + return dns.GetMultiClusterProcessHostnamesAndPodNames(r.sc.ShardRsName(shardIdx), r.sc.Namespace, memberCluster.Index, replicas, r.sc.Spec.GetClusterDomain(), externalDomain) + } else { + return dns.GetDNSNames(r.GetShardStsName(shardIdx, memberCluster), r.sc.ShardServiceName(), r.sc.Namespace, r.sc.Spec.GetClusterDomain(), replicas, externalDomain) + } +} + +func (r *ShardedClusterReconcileHelper) getMongosHostnames(memberCluster multicluster.MemberCluster, replicas int) ([]string, []string) { + externalDomain := r.sc.Spec.MongosSpec.ClusterSpecList.GetExternalDomainForMemberCluster(memberCluster.Name) + if externalDomain == nil && r.sc.Spec.IsMultiCluster() { + externalDomain = r.sc.Spec.DbCommonSpec.GetExternalDomain() + } + if !memberCluster.Legacy { + return dns.GetMultiClusterProcessHostnamesAndPodNames(r.sc.MongosRsName(), r.sc.Namespace, memberCluster.Index, replicas, r.sc.Spec.GetClusterDomain(), externalDomain) + } else { + // In Single Cluster Mode, only Mongos are exposed to the outside consumption. As such, they need to use proper + // External Domain. + externalDomain = r.sc.Spec.GetExternalDomain() + return dns.GetDNSNames(r.GetMongosStsName(memberCluster), r.sc.ServiceName(), r.sc.Namespace, r.sc.Spec.GetClusterDomain(), replicas, externalDomain) + } +} + +func (r *ShardedClusterReconcileHelper) computeMembersToScaleDown(configSrvMemberClusters []multicluster.MemberCluster, shardsMemberClustersMap map[int][]multicluster.MemberCluster, log *zap.SugaredLogger) map[string][]string { + membersToScaleDown := make(map[string][]string) + for _, memberCluster := range configSrvMemberClusters { + currentReplicas := memberCluster.Replicas + desiredReplicas := scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster)) + _, currentPodNames := r.getConfigSrvHostnames(memberCluster, currentReplicas) + if desiredReplicas < currentReplicas { + log.Debugf("Detected configSrv in cluster %s is scaling down: desiredReplicas=%d, currentReplicas=%d", memberCluster.Name, desiredReplicas, currentReplicas) + configRsName := r.sc.ConfigRsName() + if _, ok := membersToScaleDown[configRsName]; !ok { + membersToScaleDown[configRsName] = []string{} + } + podNamesToScaleDown := currentPodNames[desiredReplicas:currentReplicas] + membersToScaleDown[configRsName] = append(membersToScaleDown[configRsName], podNamesToScaleDown...) + } + } + + // Scaledown size of each shard + for shardIdx, memberClusters := range shardsMemberClustersMap { + for _, memberCluster := range memberClusters { + currentReplicas := memberCluster.Replicas + desiredReplicas := scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster)) + _, currentPodNames := r.getShardHostnames(shardIdx, memberCluster, currentReplicas) + if desiredReplicas < currentReplicas { + log.Debugf("Detected shard idx=%d in cluster %s is scaling down: desiredReplicas=%d, currentReplicas=%d", shardIdx, memberCluster.Name, desiredReplicas, currentReplicas) + shardRsName := r.sc.ShardRsName(shardIdx) + if _, ok := membersToScaleDown[shardRsName]; !ok { + membersToScaleDown[shardRsName] = []string{} + } + podNamesToScaleDown := currentPodNames[desiredReplicas:currentReplicas] + membersToScaleDown[shardRsName] = append(membersToScaleDown[shardRsName], podNamesToScaleDown...) + } + } + } + + return membersToScaleDown +} + +// prepareScaleDownShardedCluster collects all replicasets members to scale down, from configservers and shards, across +// all clusters, and pass them to PrepareScaleDownFromMap, which sets their votes and priorities to 0 +func (r *ShardedClusterReconcileHelper) prepareScaleDownShardedCluster(omClient om.Connection, log *zap.SugaredLogger) error { + membersToScaleDown := r.computeMembersToScaleDown(r.configSrvMemberClusters, r.shardsMemberClustersMap, log) + + if len(membersToScaleDown) > 0 { + healthyProcessesToWaitForReadyState := r.getHealthyProcessNamesToWaitForReadyState(omClient, log) + if err := replicaset.PrepareScaleDownFromMap(omClient, membersToScaleDown, healthyProcessesToWaitForReadyState, log); err != nil { + return err + } + } + return nil +} + +// deploymentOptions contains fields required for creating the OM deployment for the Sharded Cluster. +type deploymentOptions struct { + podEnvVars *env.PodEnvVars + currentAgentAuthMode string + caFilePath string + agentCertSecretName string + certTLSType map[string]bool + finalizing bool + processNames []string + prometheusCertHash string +} + +// updateOmDeploymentShardedCluster performs OM registration operation for the sharded cluster. So the changes will be finally propagated +// to automation agents in containers +// Note that the process may have two phases (if shards number is decreased): +// phase 1: "drain" the shards: remove them from sharded cluster, put replica set names to "draining" array, not remove +// replica sets and processes, wait for agents to reach the goal +// phase 2: remove the "junk" replica sets and their processes, wait for agents to reach the goal. +// The logic is designed to be idempotent: if the reconciliation is retried the controller will never skip the phase 1 +// until the agents have performed draining +func (r *ShardedClusterReconcileHelper) updateOmDeploymentShardedCluster(ctx context.Context, conn om.Connection, sc *mdbv1.MongoDB, opts deploymentOptions, isRecovering bool, log *zap.SugaredLogger) workflow.Status { + err := r.waitForAgentsToRegister(sc, conn, log) + if err != nil { + if !isRecovering { + return workflow.Failed(err) + } + logWarnIgnoredDueToRecovery(log, err) + } + + dep, err := conn.ReadDeployment() + if err != nil { + if !isRecovering { + return workflow.Failed(err) + } + logWarnIgnoredDueToRecovery(log, err) + } + + opts.finalizing = false + opts.processNames = dep.GetProcessNames(om.ShardedCluster{}, sc.Name) + + processNames, shardsRemoving, workflowStatus := r.publishDeployment(ctx, conn, sc, &opts, isRecovering, log) + + if !workflowStatus.IsOK() { + if !isRecovering { + return workflowStatus + } + logWarnIgnoredDueToRecovery(log, workflowStatus) + } + + healthyProcessesToWaitForReadyState := r.getHealthyProcessNamesToWaitForReadyState(conn, log) + logDiffOfProcessNames(processNames, healthyProcessesToWaitForReadyState, log.With("ctx", "updateOmDeploymentShardedCluster")) + if err = om.WaitForReadyState(conn, healthyProcessesToWaitForReadyState, isRecovering, log); err != nil { + if !isRecovering { + if shardsRemoving { + return workflow.Pending("automation agents haven't reached READY state: shards removal in progress: %v", err) + } + return workflow.Failed(err) + } + logWarnIgnoredDueToRecovery(log, err) + } + + if shardsRemoving { + opts.finalizing = true + + log.Infof("Some shards were removed from the sharded cluster, we need to remove them from the deployment completely") + processNames, _, workflowStatus := r.publishDeployment(ctx, conn, sc, &opts, isRecovering, log) + if !workflowStatus.IsOK() { + if !isRecovering { + return workflowStatus + } + logWarnIgnoredDueToRecovery(log, workflowStatus) + } + + healthyProcessesToWaitForReadyState := r.getHealthyProcessNamesToWaitForReadyState(conn, log) + logDiffOfProcessNames(processNames, healthyProcessesToWaitForReadyState, log.With("ctx", "shardsRemoving")) + if err = om.WaitForReadyState(conn, healthyProcessesToWaitForReadyState, isRecovering, log); err != nil { + if !isRecovering { + return workflow.Failed(xerrors.Errorf("automation agents haven't reached READY state while cleaning replica set and processes: %w", err)) + } + logWarnIgnoredDueToRecovery(log, err) + } + } + + currentHosts := r.getAllHostnames(false) + wantedHosts := r.getAllHostnames(true) + + if err = host.CalculateDiffAndStopMonitoring(conn, currentHosts, wantedHosts, log); err != nil { + if !isRecovering { + return workflow.Failed(err) + } + logWarnIgnoredDueToRecovery(log, err) + } + + if workflowStatus := r.commonController.ensureBackupConfigurationAndUpdateStatus(ctx, conn, sc, r.commonController.SecretClient, log); !workflowStatus.IsOK() { + if !isRecovering { + return workflowStatus + } + logWarnIgnoredDueToRecovery(log, err) + } + + log.Info("Updated Ops Manager for sharded cluster") + return workflow.OK() +} + +func (r *ShardedClusterReconcileHelper) publishDeployment(ctx context.Context, conn om.Connection, sc *mdbv1.MongoDB, opts *deploymentOptions, isRecovering bool, log *zap.SugaredLogger) ([]string, bool, workflow.Status) { + // Mongos + var mongosProcesses []om.Process + // We take here the first cluster arbitrarily because the options are used for irrelevant stuff below, same for + // config servers and shards below + mongosMemberCluster := r.mongosMemberClusters[0] + mongosOptionsFunc := r.getMongosOptions(ctx, *sc, *opts, log, r.desiredMongosConfiguration, mongosMemberCluster) + mongosOptions := mongosOptionsFunc(*r.sc) + mongosInternalClusterPath := fmt.Sprintf("%s/%s", util.InternalClusterAuthMountPath, mongosOptions.InternalClusterHash) + mongosMemberCertPath := fmt.Sprintf("%s/%s", util.TLSCertMountPath, mongosOptions.CertificateHash) + if mongosOptions.CertificateHash == "" { + mongosMemberCertPath = util.PEMKeyFilePathInContainer + } + mongosProcesses = append(mongosProcesses, r.createDesiredMongosProcesses(mongosMemberCertPath)...) + + // Config server + configSrvMemberCluster := r.configSrvMemberClusters[0] + configSrvOptionsFunc := r.getConfigServerOptions(ctx, *sc, *opts, log, r.desiredConfigServerConfiguration, configSrvMemberCluster) + configSrvOptions := configSrvOptionsFunc(*r.sc) + + configSrvInternalClusterPath := fmt.Sprintf("%s/%s", util.InternalClusterAuthMountPath, configSrvOptions.InternalClusterHash) + configSrvMemberCertPath := fmt.Sprintf("%s/%s", util.TLSCertMountPath, configSrvOptions.CertificateHash) + if configSrvOptions.CertificateHash == "" { + configSrvMemberCertPath = util.PEMKeyFilePathInContainer + } + + existingDeployment, err := conn.ReadDeployment() + if err != nil { + return nil, false, workflow.Failed(err) + } + + configSrvProcesses, configSrvMemberOptions := r.createDesiredConfigSrvProcessesAndMemberOptions(configSrvMemberCertPath) + configRs, _ := buildReplicaSetFromProcesses(sc.ConfigRsName(), configSrvProcesses, sc, configSrvMemberOptions, existingDeployment) + + // Shards + shards := make([]om.ReplicaSetWithProcesses, sc.Spec.ShardCount) + var shardInternalClusterPaths []string + for shardIdx := 0; shardIdx < r.sc.Spec.ShardCount; shardIdx++ { + shardOptionsFunc := r.getShardOptions(ctx, *sc, shardIdx, *opts, log, r.desiredShardsConfiguration[shardIdx], r.shardsMemberClustersMap[shardIdx][0]) + shardOptions := shardOptionsFunc(*r.sc) + shardInternalClusterPaths = append(shardInternalClusterPaths, fmt.Sprintf("%s/%s", util.InternalClusterAuthMountPath, shardOptions.InternalClusterHash)) + shardMemberCertPath := fmt.Sprintf("%s/%s", util.TLSCertMountPath, shardOptions.CertificateHash) + desiredShardProcesses, desiredShardMemberOptions := r.createDesiredShardProcessesAndMemberOptions(shardIdx, shardMemberCertPath) + shards[shardIdx], _ = buildReplicaSetFromProcesses(r.sc.ShardRsName(shardIdx), desiredShardProcesses, sc, desiredShardMemberOptions, existingDeployment) + } + + // updateOmAuthentication normally takes care of the certfile rotation code, but since sharded-cluster is special pertaining multiple clusterfiles, we code this part here for now. + // We can look into unifying this into updateOmAuthentication at a later stage. + if err := conn.ReadUpdateDeployment(func(d om.Deployment) error { + setInternalAuthClusterFileIfItHasChanged(d, sc.GetSecurity().GetInternalClusterAuthenticationMode(), sc.Name, configSrvInternalClusterPath, mongosInternalClusterPath, shardInternalClusterPaths, isRecovering) + return nil + }, log); err != nil { + return nil, false, workflow.Failed(err) + } + + healthyProcessesToWaitForReadyState := r.getHealthyProcessNamesToWaitForReadyState(conn, log) + + logDiffOfProcessNames(opts.processNames, healthyProcessesToWaitForReadyState, log.With("ctx", "updateOmAuthentication")) + + workflowStatus, additionalReconciliationRequired := r.commonController.updateOmAuthentication(ctx, conn, healthyProcessesToWaitForReadyState, sc, opts.agentCertSecretName, opts.caFilePath, "", isRecovering, log) + if !workflowStatus.IsOK() { + if !isRecovering { + return nil, false, workflowStatus + } + logWarnIgnoredDueToRecovery(log, workflowStatus) + } + + var finalProcesses []string + shardsRemoving := false + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + allProcesses := getAllProcesses(shards, configRs, mongosProcesses) + // it is not possible to disable internal cluster authentication once enabled + if sc.Spec.Security.GetInternalClusterAuthenticationMode() == "" && d.ExistingProcessesHaveInternalClusterAuthentication(allProcesses) { + return xerrors.Errorf("cannot disable x509 internal cluster authentication") + } + numberOfOtherMembers := d.GetNumberOfExcessProcesses(sc.Name) + if numberOfOtherMembers > 0 { + return xerrors.Errorf("cannot have more than 1 MongoDB Cluster per project (see https://docs.mongodb.com/kubernetes-operator/stable/tutorial/migrate-to-single-resource/)") + } + + lastConfigServerConf, err := mdbv1.GetLastAdditionalMongodConfigByType(r.deploymentState.LastAchievedSpec, mdbv1.ConfigServerConfig) + if err != nil { + return err + } + + lastShardServerConf, err := mdbv1.GetLastAdditionalMongodConfigByType(r.deploymentState.LastAchievedSpec, mdbv1.ShardConfig) + if err != nil { + return err + } + + lastMongosServerConf, err := mdbv1.GetLastAdditionalMongodConfigByType(r.deploymentState.LastAchievedSpec, mdbv1.MongosConfig) + if err != nil { + return err + } + + mergeOpts := om.DeploymentShardedClusterMergeOptions{ + Name: sc.Name, + MongosProcesses: mongosProcesses, + ConfigServerRs: configRs, + Shards: shards, + Finalizing: opts.finalizing, + ConfigServerAdditionalOptionsPrev: lastConfigServerConf.ToMap(), + ConfigServerAdditionalOptionsDesired: sc.Spec.ConfigSrvSpec.AdditionalMongodConfig.ToMap(), + ShardAdditionalOptionsPrev: lastShardServerConf.ToMap(), + ShardAdditionalOptionsDesired: sc.Spec.ShardSpec.AdditionalMongodConfig.ToMap(), + MongosAdditionalOptionsPrev: lastMongosServerConf.ToMap(), + MongosAdditionalOptionsDesired: sc.Spec.MongosSpec.AdditionalMongodConfig.ToMap(), + } + + if shardsRemoving, err = d.MergeShardedCluster(mergeOpts); err != nil { + return err + } + + d.AddMonitoringAndBackup(log, sc.Spec.GetSecurity().IsTLSEnabled(), opts.caFilePath) + d.ConfigureTLS(sc.Spec.GetSecurity(), opts.caFilePath) + + setupInternalClusterAuth(d, sc.Name, sc.GetSecurity().GetInternalClusterAuthenticationMode(), + configSrvInternalClusterPath, mongosInternalClusterPath, shardInternalClusterPaths) + + _ = UpdatePrometheus(ctx, &d, conn, sc.GetPrometheus(), r.commonController.SecretClient, sc.GetNamespace(), opts.prometheusCertHash, log) + + finalProcesses = d.GetProcessNames(om.ShardedCluster{}, sc.Name) + + return nil + }, + log, + ) + if err != nil { + return nil, shardsRemoving, workflow.Failed(err) + } + + // Here we only support sc.Spec.Agent on purpose because logRotation for the agents and all processes + // are configured the same way, its unrelated what type of process it is. + if reconcileResult, _ := ReconcileLogRotateSetting(conn, sc.Spec.Agent, log); !reconcileResult.IsOK() { + return nil, shardsRemoving, reconcileResult + } + + healthyProcessesToWaitForReadyState = r.getHealthyProcessNamesToWaitForReadyState(conn, log) + logDiffOfProcessNames(opts.processNames, healthyProcessesToWaitForReadyState, log.With("ctx", "publishDeployment")) + if err := om.WaitForReadyState(conn, healthyProcessesToWaitForReadyState, isRecovering, log); err != nil { + return nil, shardsRemoving, workflow.Failed(err) + } + + if additionalReconciliationRequired { + return nil, shardsRemoving, workflow.Pending("Performing multi stage reconciliation") + } + + return finalProcesses, shardsRemoving, workflow.OK() +} + +func logWarnIgnoredDueToRecovery(log *zap.SugaredLogger, err any) { + log.Warnf("ignoring error due to automatic recovery process: %v", err) +} + +func setInternalAuthClusterFileIfItHasChanged(d om.Deployment, internalAuthMode string, name string, configInternalClusterPath string, mongosInternalClusterPath string, shardsInternalClusterPath []string, isRecovering bool) { + d.SetInternalClusterFilePathOnlyIfItThePathHasChanged(d.GetShardedClusterConfigProcessNames(name), configInternalClusterPath, internalAuthMode, isRecovering) + d.SetInternalClusterFilePathOnlyIfItThePathHasChanged(d.GetShardedClusterMongosProcessNames(name), mongosInternalClusterPath, internalAuthMode, isRecovering) + for i, path := range shardsInternalClusterPath { + d.SetInternalClusterFilePathOnlyIfItThePathHasChanged(d.GetShardedClusterShardProcessNames(name, i), path, internalAuthMode, isRecovering) + } +} + +func setupInternalClusterAuth(d om.Deployment, name string, internalClusterAuthMode string, configInternalClusterPath string, mongosInternalClusterPath string, shardsInternalClusterPath []string) { + d.ConfigureInternalClusterAuthentication(d.GetShardedClusterConfigProcessNames(name), internalClusterAuthMode, configInternalClusterPath) + d.ConfigureInternalClusterAuthentication(d.GetShardedClusterMongosProcessNames(name), internalClusterAuthMode, mongosInternalClusterPath) + + for i, path := range shardsInternalClusterPath { + d.ConfigureInternalClusterAuthentication(d.GetShardedClusterShardProcessNames(name, i), internalClusterAuthMode, path) + } +} + +func getAllProcesses(shards []om.ReplicaSetWithProcesses, configRs om.ReplicaSetWithProcesses, mongosProcesses []om.Process) []om.Process { + allProcesses := make([]om.Process, 0) + for _, shard := range shards { + allProcesses = append(allProcesses, shard.Processes...) + } + allProcesses = append(allProcesses, configRs.Processes...) + allProcesses = append(allProcesses, mongosProcesses...) + return allProcesses +} + +func (r *ShardedClusterReconcileHelper) waitForAgentsToRegister(sc *mdbv1.MongoDB, conn om.Connection, log *zap.SugaredLogger) error { + var mongosHostnames []string + for _, memberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + hostnames, _ := r.getMongosHostnames(memberCluster, scale.ReplicasThisReconciliation(r.GetMongosScaler(memberCluster))) + mongosHostnames = append(mongosHostnames, hostnames...) + } + + if err := agents.WaitForRsAgentsToRegisterSpecifiedHostnames(conn, mongosHostnames, log.With("hostnamesOf", "mongos")); err != nil { + return xerrors.Errorf("Mongos agents didn't register with Ops Manager: %w", err) + } + + var configSrvHostnames []string + for _, memberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + hostnames, _ := r.getConfigSrvHostnames(memberCluster, scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster))) + configSrvHostnames = append(configSrvHostnames, hostnames...) + } + if err := agents.WaitForRsAgentsToRegisterSpecifiedHostnames(conn, configSrvHostnames, log.With("hostnamesOf", "configServer")); err != nil { + return xerrors.Errorf("Config server agents didn't register with Ops Manager: %w", err) + } + + for shardIdx := 0; shardIdx < sc.Spec.ShardCount; shardIdx++ { + var shardHostnames []string + for _, memberCluster := range getHealthyMemberClusters(r.shardsMemberClustersMap[shardIdx]) { + hostnames, _ := r.getShardHostnames(shardIdx, memberCluster, scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster))) + shardHostnames = append(shardHostnames, hostnames...) + } + if err := agents.WaitForRsAgentsToRegisterSpecifiedHostnames(conn, shardHostnames, log.With("hostnamesOf", "shard", "shardIdx", shardIdx)); err != nil { + return xerrors.Errorf("Shards agents didn't register with Ops Manager: %w", err) + } + } + + return nil +} + +func (r *ShardedClusterReconcileHelper) getAllHostnames(desiredReplicas bool) []string { + configSrvHostnames, _ := r.getAllConfigSrvHostnamesAndPodNames(desiredReplicas) + mongosHostnames, _ := r.getAllMongosHostnamesAndPodNames(desiredReplicas) + shardHostnames, _ := r.getAllShardHostnamesAndPodNames(desiredReplicas) + + var hostnames []string + hostnames = append(hostnames, configSrvHostnames...) + hostnames = append(hostnames, mongosHostnames...) + hostnames = append(hostnames, shardHostnames...) + + return hostnames +} + +func (r *ShardedClusterReconcileHelper) getAllConfigSrvHostnamesAndPodNames(desiredReplicas bool) ([]string, []string) { + var configSrvHostnames []string + var configSrvPodNames []string + for _, memberCluster := range r.configSrvMemberClusters { + replicas := memberCluster.Replicas + if desiredReplicas { + replicas = scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster)) + } + hostnames, podNames := r.getConfigSrvHostnames(memberCluster, replicas) + configSrvHostnames = append(configSrvHostnames, hostnames...) + configSrvPodNames = append(configSrvPodNames, podNames...) + } + return configSrvHostnames, configSrvPodNames +} + +func (r *ShardedClusterReconcileHelper) getAllShardHostnamesAndPodNames(desiredReplicas bool) ([]string, []string) { + var shardHostnames []string + var shardPodNames []string + for shardIdx, memberClusterMap := range r.shardsMemberClustersMap { + for _, memberCluster := range memberClusterMap { + replicas := memberCluster.Replicas + if desiredReplicas { + replicas = scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster)) + } + hostnames, podNames := r.getShardHostnames(shardIdx, memberCluster, replicas) + shardHostnames = append(shardHostnames, hostnames...) + shardPodNames = append(shardPodNames, podNames...) + } + } + + return shardHostnames, shardPodNames +} + +func (r *ShardedClusterReconcileHelper) getAllMongosHostnamesAndPodNames(desiredReplicas bool) ([]string, []string) { + var mongosHostnames []string + var mongosPodNames []string + for _, memberCluster := range r.mongosMemberClusters { + replicas := memberCluster.Replicas + if desiredReplicas { + replicas = scale.ReplicasThisReconciliation(r.GetMongosScaler(memberCluster)) + } + hostnames, podNames := r.getMongosHostnames(memberCluster, replicas) + mongosHostnames = append(mongosHostnames, hostnames...) + mongosPodNames = append(mongosPodNames, podNames...) + } + return mongosHostnames, mongosPodNames +} + +func (r *ShardedClusterReconcileHelper) createDesiredMongosProcesses(certificateFilePath string) []om.Process { + var processes []om.Process + for _, memberCluster := range r.mongosMemberClusters { + hostnames, podNames := r.getMongosHostnames(memberCluster, scale.ReplicasThisReconciliation(r.GetMongosScaler(memberCluster))) + for i := range hostnames { + process := om.NewMongosProcess(podNames[i], hostnames[i], r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, r.sc.Spec.MongosSpec.GetAdditionalMongodConfig(), r.sc.GetSpec(), certificateFilePath, r.sc.Annotations, r.sc.CalculateFeatureCompatibilityVersion()) + processes = append(processes, process) + } + } + + return processes +} + +func (r *ShardedClusterReconcileHelper) createDesiredConfigSrvProcessesAndMemberOptions(certificateFilePath string) ([]om.Process, []automationconfig.MemberOptions) { + var processes []om.Process + var memberOptions []automationconfig.MemberOptions + for _, memberCluster := range r.configSrvMemberClusters { + hostnames, podNames := r.getConfigSrvHostnames(memberCluster, scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster))) + for i := range hostnames { + process := om.NewMongodProcess(podNames[i], hostnames[i], r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, r.sc.Spec.ConfigSrvSpec.GetAdditionalMongodConfig(), r.sc.GetSpec(), certificateFilePath, r.sc.Annotations, r.sc.CalculateFeatureCompatibilityVersion()) + processes = append(processes, process) + } + + specMemberConfig := r.desiredConfigServerConfiguration.GetClusterSpecItem(memberCluster.Name).MemberConfig + memberOptions = append(memberOptions, specMemberConfig...) + } + + return processes, memberOptions +} + +func (r *ShardedClusterReconcileHelper) createDesiredShardProcessesAndMemberOptions(shardIdx int, certificateFilePath string) ([]om.Process, []automationconfig.MemberOptions) { + var processes []om.Process + var memberOptions []automationconfig.MemberOptions + for _, memberCluster := range r.shardsMemberClustersMap[shardIdx] { + hostnames, podNames := r.getShardHostnames(shardIdx, memberCluster, scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster))) + for i := range hostnames { + process := om.NewMongodProcess(podNames[i], hostnames[i], r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, r.desiredShardsConfiguration[shardIdx].GetAdditionalMongodConfig(), r.sc.GetSpec(), certificateFilePath, r.sc.Annotations, r.sc.CalculateFeatureCompatibilityVersion()) + processes = append(processes, process) + } + specMemberOptions := r.desiredShardsConfiguration[shardIdx].GetClusterSpecItem(memberCluster.Name).MemberConfig + memberOptions = append(memberOptions, specMemberOptions...) + } + + return processes, memberOptions +} + +func createConfigSrvProcesses(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, mdb *mdbv1.MongoDB, certificateFilePath string) []om.Process { + return createMongodProcessForShardedCluster(mongoDBImage, forceEnterprise, set, mdb.Spec.ConfigSrvSpec.GetAdditionalMongodConfig(), mdb, certificateFilePath) +} + +func createShardProcesses(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, mdb *mdbv1.MongoDB, certificateFilePath string) []om.Process { + return createMongodProcessForShardedCluster(mongoDBImage, forceEnterprise, set, mdb.Spec.ShardSpec.GetAdditionalMongodConfig(), mdb, certificateFilePath) +} + +func createMongodProcessForShardedCluster(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, additionalMongodConfig *mdbv1.AdditionalMongodConfig, mdb *mdbv1.MongoDB, certificateFilePath string) []om.Process { + hostnames, names := dns.GetDnsForStatefulSet(set, mdb.Spec.GetClusterDomain(), nil) + processes := make([]om.Process, len(hostnames)) + + for idx, hostname := range hostnames { + processes[idx] = om.NewMongodProcess(names[idx], hostname, mongoDBImage, forceEnterprise, additionalMongodConfig, &mdb.Spec, certificateFilePath, mdb.Annotations, mdb.CalculateFeatureCompatibilityVersion()) + } + + return processes +} + +// buildReplicaSetFromProcesses creates the 'ReplicaSetWithProcesses' with specified processes. This is of use only +// for sharded cluster (config server, shards) +func buildReplicaSetFromProcesses(name string, members []om.Process, mdb *mdbv1.MongoDB, memberOptions []automationconfig.MemberOptions, deployment om.Deployment) (om.ReplicaSetWithProcesses, error) { + replicaSet := om.NewReplicaSet(name, mdb.Spec.GetMongoDBVersion()) + + existingProcessIds := getReplicaSetProcessIdsFromReplicaSets(replicaSet.Name(), deployment) + var rsWithProcesses om.ReplicaSetWithProcesses + if mdb.Spec.IsMultiCluster() { + // we're passing nil as connectivity argument as in sharded clusters horizons don't make much sense as we don't expose externally individual shards + // we don't support exposing externally individual shards in single cluster as well + // in case of multi-cluster without a service mesh we'll use externalDomains for all shards, so the hostnames will be valid from inside and outside, therefore + // horizons are not needed + rsWithProcesses = om.NewMultiClusterReplicaSetWithProcesses(replicaSet, members, memberOptions, existingProcessIds, nil) + } else { + rsWithProcesses = om.NewReplicaSetWithProcesses(replicaSet, members, memberOptions) + rsWithProcesses.SetHorizons(mdb.Spec.Connectivity.ReplicaSetHorizons) + } + return rsWithProcesses, nil +} + +// getConfigServerOptions returns the Options needed to build the StatefulSet for the config server. +func (r *ShardedClusterReconcileHelper) getConfigServerOptions(ctx context.Context, sc mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger, configSrvSpec *mdbv1.ShardedClusterComponentSpec, memberCluster multicluster.MemberCluster) func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions { + certSecretName := sc.GetSecurity().MemberCertificateSecretName(sc.ConfigRsName()) + internalClusterSecretName := sc.GetSecurity().InternalClusterAuthSecretName(sc.ConfigRsName()) + + var vaultConfig vault.VaultConfiguration + var databaseSecretPath string + if r.commonController.VaultClient != nil { + vaultConfig = r.commonController.VaultClient.VaultConfig + databaseSecretPath = r.commonController.VaultClient.DatabaseSecretPath() + } + + return construct.ConfigServerOptions( + configSrvSpec, + memberCluster.Name, + Replicas(scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster))), + StatefulSetNameOverride(r.GetConfigSrvStsName(memberCluster)), + ServiceName(r.GetConfigSrvServiceName(memberCluster)), + PodEnvVars(opts.podEnvVars), + CurrentAgentAuthMechanism(opts.currentAgentAuthMode), + CertificateHash(enterprisepem.ReadHashFromSecret(ctx, r.commonController.SecretClient, sc.Namespace, certSecretName, databaseSecretPath, log)), + InternalClusterHash(enterprisepem.ReadHashFromSecret(ctx, r.commonController.SecretClient, sc.Namespace, internalClusterSecretName, databaseSecretPath, log)), + PrometheusTLSCertHash(opts.prometheusCertHash), + WithVaultConfig(vaultConfig), + WithAdditionalMongodConfig(configSrvSpec.GetAdditionalMongodConfig()), + WithDefaultConfigSrvStorageSize(), + WithStsLabels(r.statefulsetLabels()), + WithInitDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.InitDatabaseImageUrlEnv, r.initDatabaseNonStaticImageVersion)), + WithDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.NonStaticDatabaseEnterpriseImage, r.databaseNonStaticImageVersion)), + WithAgentImage(images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, r.automationAgentVersion)), + WithMongodbImage(images.GetOfficialImage(r.imageUrls, sc.Spec.Version, sc.GetAnnotations())), + ) +} + +// getMongosOptions returns the Options needed to build the StatefulSet for the mongos. +func (r *ShardedClusterReconcileHelper) getMongosOptions(ctx context.Context, sc mdbv1.MongoDB, opts deploymentOptions, log *zap.SugaredLogger, mongosSpec *mdbv1.ShardedClusterComponentSpec, memberCluster multicluster.MemberCluster) func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions { + certSecretName := sc.GetSecurity().MemberCertificateSecretName(sc.MongosRsName()) + internalClusterSecretName := sc.GetSecurity().InternalClusterAuthSecretName(sc.MongosRsName()) + + var vaultConfig vault.VaultConfiguration + if r.commonController.VaultClient != nil { + vaultConfig = r.commonController.VaultClient.VaultConfig + } + + return construct.MongosOptions( + mongosSpec, + memberCluster.Name, + Replicas(scale.ReplicasThisReconciliation(r.GetMongosScaler(memberCluster))), + StatefulSetNameOverride(r.GetMongosStsName(memberCluster)), + PodEnvVars(opts.podEnvVars), + CurrentAgentAuthMechanism(opts.currentAgentAuthMode), + CertificateHash(enterprisepem.ReadHashFromSecret(ctx, r.commonController.SecretClient, sc.Namespace, certSecretName, vaultConfig.DatabaseSecretPath, log)), + InternalClusterHash(enterprisepem.ReadHashFromSecret(ctx, r.commonController.SecretClient, sc.Namespace, internalClusterSecretName, vaultConfig.DatabaseSecretPath, log)), + PrometheusTLSCertHash(opts.prometheusCertHash), + WithVaultConfig(vaultConfig), + WithAdditionalMongodConfig(sc.Spec.MongosSpec.GetAdditionalMongodConfig()), + WithStsLabels(r.statefulsetLabels()), + WithInitDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.InitDatabaseImageUrlEnv, r.initDatabaseNonStaticImageVersion)), + WithDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.NonStaticDatabaseEnterpriseImage, r.databaseNonStaticImageVersion)), + WithAgentImage(images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, r.automationAgentVersion)), + WithMongodbImage(images.GetOfficialImage(r.imageUrls, sc.Spec.Version, sc.GetAnnotations())), + ) +} + +// getShardOptions returns the Options needed to build the StatefulSet for a given shard. +func (r *ShardedClusterReconcileHelper) getShardOptions(ctx context.Context, sc mdbv1.MongoDB, shardNum int, opts deploymentOptions, log *zap.SugaredLogger, shardSpec *mdbv1.ShardedClusterComponentSpec, memberCluster multicluster.MemberCluster) func(mdb mdbv1.MongoDB) construct.DatabaseStatefulSetOptions { + certSecretName := sc.GetSecurity().MemberCertificateSecretName(sc.ShardRsName(shardNum)) + internalClusterSecretName := sc.GetSecurity().InternalClusterAuthSecretName(sc.ShardRsName(shardNum)) + + var vaultConfig vault.VaultConfiguration + var databaseSecretPath string + if r.commonController.VaultClient != nil { + vaultConfig = r.commonController.VaultClient.VaultConfig + databaseSecretPath = r.commonController.VaultClient.DatabaseSecretPath() + } + + return construct.ShardOptions(shardNum, r.desiredShardsConfiguration[shardNum], memberCluster.Name, + Replicas(scale.ReplicasThisReconciliation(r.GetShardScaler(shardNum, memberCluster))), + StatefulSetNameOverride(r.GetShardStsName(shardNum, memberCluster)), + PodEnvVars(opts.podEnvVars), + CurrentAgentAuthMechanism(opts.currentAgentAuthMode), + CertificateHash(enterprisepem.ReadHashFromSecret(ctx, r.commonController.SecretClient, sc.Namespace, certSecretName, databaseSecretPath, log)), + InternalClusterHash(enterprisepem.ReadHashFromSecret(ctx, r.commonController.SecretClient, sc.Namespace, internalClusterSecretName, databaseSecretPath, log)), + PrometheusTLSCertHash(opts.prometheusCertHash), + WithVaultConfig(vaultConfig), + WithAdditionalMongodConfig(shardSpec.GetAdditionalMongodConfig()), + WithStsLabels(r.statefulsetLabels()), + WithInitDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.InitDatabaseImageUrlEnv, r.initDatabaseNonStaticImageVersion)), + WithDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.NonStaticDatabaseEnterpriseImage, r.databaseNonStaticImageVersion)), + WithAgentImage(images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, r.automationAgentVersion)), + WithMongodbImage(images.GetOfficialImage(r.imageUrls, sc.Spec.Version, sc.GetAnnotations())), + ) +} + +func (r *ShardedClusterReconcileHelper) migrateToNewDeploymentState(sc *mdbv1.MongoDB) error { + // Try to get the last achieved spec from annotations and store it in state + if lastAchievedSpec, err := sc.GetLastSpec(); err != nil { + return err + } else { + r.deploymentState.LastAchievedSpec = lastAchievedSpec + } + r.deploymentState.Status = sc.Status.DeepCopy() + if !sc.Spec.IsMultiCluster() { + r.deploymentState.Status.SizeStatusInClusters = &mdbstatus.MongodbShardedSizeStatusInClusters{ + ShardMongodsInClusters: map[string]int{ + multicluster.LegacyCentralClusterName: r.deploymentState.Status.MongodsPerShardCount, + }, + ShardOverridesInClusters: map[string]map[string]int{}, + MongosCountInClusters: map[string]int{ + multicluster.LegacyCentralClusterName: r.deploymentState.Status.MongosCount, + }, + ConfigServerMongodsInClusters: map[string]int{ + multicluster.LegacyCentralClusterName: r.deploymentState.Status.ConfigServerCount, + }, + } + } else { + r.deploymentState.Status.SizeStatusInClusters = &mdbstatus.MongodbShardedSizeStatusInClusters{ + ShardMongodsInClusters: map[string]int{}, + ShardOverridesInClusters: map[string]map[string]int{}, + MongosCountInClusters: map[string]int{}, + ConfigServerMongodsInClusters: map[string]int{}, + } + } + + return nil +} + +func (r *ShardedClusterReconcileHelper) updateStatus(ctx context.Context, resource *mdbv1.MongoDB, status workflow.Status, log *zap.SugaredLogger, statusOptions ...mdbstatus.Option) (reconcile.Result, error) { + if result, err := r.commonController.updateStatus(ctx, resource, status, log, statusOptions...); err != nil { + return result, err + } else { + // UpdateStatus in the sharded cluster controller should be executed only once per reconcile (always with a return) + // We are saving the status and writing back to the state configmap at this time + r.deploymentState.updateStatusFromResourceStatus(resource.Status) + if err := r.stateStore.WriteState(ctx, r.deploymentState, log); err != nil { + return r.commonController.updateStatus(ctx, resource, workflow.Failed(xerrors.Errorf("Failed to write deployment state after updating status: %w", err)), log, nil) + } + return result, nil + } +} + +func (r *ShardedClusterReconcileHelper) GetShardStsName(shardIdx int, memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.sc.ShardRsName(shardIdx) + } else { + return r.sc.MultiShardRsName(memberCluster.Index, shardIdx) + } +} + +func (r *ShardedClusterReconcileHelper) GetConfigSrvStsName(memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.sc.ConfigRsName() + } else { + return r.sc.MultiConfigRsName(memberCluster.Index) + } +} + +func (r *ShardedClusterReconcileHelper) GetMongosStsName(memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.sc.MongosRsName() + } else { + return r.sc.MultiMongosRsName(memberCluster.Index) + } +} + +func (r *ShardedClusterReconcileHelper) GetConfigSrvScaler(memberCluster multicluster.MemberCluster) interfaces.MultiClusterReplicaSetScaler { + return scalers.NewMultiClusterReplicaSetScaler("configSrv", r.desiredConfigServerConfiguration.ClusterSpecList, memberCluster.Name, memberCluster.Index, r.configSrvMemberClusters) +} + +func (r *ShardedClusterReconcileHelper) GetMongosScaler(memberCluster multicluster.MemberCluster) interfaces.MultiClusterReplicaSetScaler { + return scalers.NewMultiClusterReplicaSetScaler("mongos", r.desiredMongosConfiguration.ClusterSpecList, memberCluster.Name, memberCluster.Index, r.mongosMemberClusters) +} + +func (r *ShardedClusterReconcileHelper) GetShardScaler(shardIdx int, memberCluster multicluster.MemberCluster) interfaces.MultiClusterReplicaSetScaler { + return scalers.NewMultiClusterReplicaSetScaler(fmt.Sprintf("shard idx %d", shardIdx), r.desiredShardsConfiguration[shardIdx].ClusterSpecList, memberCluster.Name, memberCluster.Index, r.shardsMemberClustersMap[shardIdx]) +} + +func (r *ShardedClusterReconcileHelper) getAllScalers() []interfaces.MultiClusterReplicaSetScaler { + var result []interfaces.MultiClusterReplicaSetScaler + for shardIdx := 0; shardIdx < r.sc.Spec.ShardCount; shardIdx++ { + for _, memberCluster := range r.shardsMemberClustersMap[shardIdx] { + scaler := r.GetShardScaler(shardIdx, memberCluster) + result = append(result, scaler) + } + } + + for _, memberCluster := range r.configSrvMemberClusters { + result = append(result, r.GetConfigSrvScaler(memberCluster)) + } + + for _, memberCluster := range r.mongosMemberClusters { + result = append(result, r.GetMongosScaler(memberCluster)) + } + + return result +} + +func (r *ShardedClusterReconcileHelper) GetConfigSrvServiceName(memberCluster multicluster.MemberCluster) string { + if memberCluster.Legacy { + return r.sc.ConfigSrvServiceName() + } else { + return fmt.Sprintf("%s-%d-cs", r.sc.Name, memberCluster.Index) + } +} + +func (r *ShardedClusterReconcileHelper) replicateAgentKeySecret(ctx context.Context, conn om.Connection, agentKey string, log *zap.SugaredLogger) error { + for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) { + var databaseSecretPath string + if memberCluster.SecretClient.VaultClient != nil { + databaseSecretPath = memberCluster.SecretClient.VaultClient.DatabaseSecretPath() + } + if _, err := agents.EnsureAgentKeySecretExists(ctx, memberCluster.SecretClient, conn, r.sc.Namespace, agentKey, conn.GroupID(), databaseSecretPath, log); err != nil { + return xerrors.Errorf("failed to ensure agent key secret in member cluster %s: %w", memberCluster.Name, err) + } + } + return nil +} + +func (r *ShardedClusterReconcileHelper) createHostnameOverrideConfigMap() corev1.ConfigMap { + data := r.createHostnameOverrideConfigMapData() + + cm := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-hostname-override", r.sc.Name), + Namespace: r.sc.Namespace, + Labels: mongoDBLabels(r.sc.Name), + }, + Data: data, + } + return cm +} + +func (r *ShardedClusterReconcileHelper) createHostnameOverrideConfigMapData() map[string]string { + data := make(map[string]string) + + for _, memberCluster := range r.mongosMemberClusters { + mongosScaler := r.GetMongosScaler(memberCluster) + mongosHostnames, mongosPodNames := r.getMongosHostnames(memberCluster, max(mongosScaler.CurrentReplicas(), mongosScaler.DesiredReplicas())) + for i := range mongosPodNames { + data[mongosPodNames[i]] = mongosHostnames[i] + } + } + + for _, memberCluster := range r.configSrvMemberClusters { + configSrvScaler := r.GetConfigSrvScaler(memberCluster) + configSrvHostnames, configSrvPodNames := r.getConfigSrvHostnames(memberCluster, max(configSrvScaler.CurrentReplicas(), configSrvScaler.DesiredReplicas())) + for i := range configSrvPodNames { + data[configSrvPodNames[i]] = configSrvHostnames[i] + } + } + + for shardIdx := 0; shardIdx < max(r.sc.Spec.ShardCount, r.deploymentState.Status.ShardCount); shardIdx++ { + for _, memberCluster := range r.shardsMemberClustersMap[shardIdx] { + shardScaler := r.GetShardScaler(shardIdx, memberCluster) + shardHostnames, shardPodNames := r.getShardHostnames(shardIdx, memberCluster, max(shardScaler.CurrentReplicas(), shardScaler.DesiredReplicas())) + for i := range shardPodNames { + data[shardPodNames[i]] = shardHostnames[i] + } + } + } + return data +} + +func (r *ShardedClusterReconcileHelper) reconcileHostnameOverrideConfigMap(ctx context.Context, log *zap.SugaredLogger) error { + if !r.sc.Spec.IsMultiCluster() { + if r.sc.Spec.DbCommonSpec.GetExternalDomain() == nil { + log.Debugf("Skipping creating hostname override config map in SingleCluster topology (with external domain unspecified)") + return nil + } + } + + cm := r.createHostnameOverrideConfigMap() + for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) { + err := configmap.CreateOrUpdate(ctx, memberCluster.Client, cm) + if err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create configmap: %s/%s in cluster: %s, err: %w", r.sc.Namespace, cm.Name, memberCluster.Name, err) + } + log.Debugf("Successfully ensured configmap: %s/%s in cluster: %s", r.sc.Namespace, cm.Name, memberCluster.Name) + } + + return nil +} + +// reconcileServices creates both internal and external Services. +// +// This method assumes that all overrides have been expanded and are present in the ClusterSpecList. Other fields +// are not taken into consideration. Please ensure you expended and processed them earlier. +func (r *ShardedClusterReconcileHelper) reconcileServices(ctx context.Context, log *zap.SugaredLogger) error { + var allServices []*corev1.Service + for _, memberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + if err := r.reconcileMongosServices(ctx, log, memberCluster, allServices); err != nil { + return err + } + } + + for _, memberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + if err := r.reconcileConfigServerServices(ctx, log, memberCluster, allServices); err != nil { + return err + } + } + + for shardIdx := 0; shardIdx < r.sc.Spec.ShardCount; shardIdx++ { + for _, memberCluster := range getHealthyMemberClusters(r.shardsMemberClustersMap[shardIdx]) { + if err := r.reconcileShardServices(ctx, log, shardIdx, memberCluster, allServices); err != nil { + return err + } + } + } + + if r.sc.Spec.DuplicateServiceObjects != nil && *r.sc.Spec.DuplicateServiceObjects { + for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) { + // the pod services created in their respective clusters will be updated twice here, but this way the code is cleaner + for _, svc := range allServices { + log.Debugf("creating duplicated services for %s in cluster: %s", svc.Name, memberCluster.Name) + err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, *svc) + if err != nil { + return xerrors.Errorf("failed to create (duplicate) pod service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + } + } + } + + return nil +} + +func (r *ShardedClusterReconcileHelper) reconcileConfigServerServices(ctx context.Context, log *zap.SugaredLogger, memberCluster multicluster.MemberCluster, allServices []*corev1.Service) error { + portOrDefault := r.desiredConfigServerConfiguration.AdditionalMongodConfig.GetPortOrDefault() + scaler := r.GetConfigSrvScaler(memberCluster) + + for podNum := 0; podNum < scaler.DesiredReplicas(); podNum++ { + configServerExternalAccess := r.desiredConfigServerConfiguration.ClusterSpecList.GetExternalAccessConfigurationForMemberCluster(memberCluster.Name) + if configServerExternalAccess == nil { + configServerExternalAccess = r.sc.Spec.ExternalAccessConfiguration + } + // Config servers need external services only if an externalDomain is configured + if configServerExternalAccess != nil && configServerExternalAccess.ExternalDomain != nil { + log.Debugf("creating external services for %s in cluster: %s", r.sc.ConfigRsName(), memberCluster.Name) + svc, err := r.getPodExternalService(memberCluster, + r.sc.ConfigRsName(), + configServerExternalAccess, + podNum, + portOrDefault) + if err != nil { + return xerrors.Errorf("failed to create an external service %s in cluster: %s, err: %w", dns.GetMultiExternalServiceName(r.sc.ConfigSrvServiceName(), memberCluster.Index, podNum), memberCluster.Name, err) + } + if err = mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + } + // We don't need internal per-pod services in case we have externalAccess configured AND an external domain + if configServerExternalAccess == nil || configServerExternalAccess.ExternalDomain == nil { + log.Debugf("creating internal services for %s in cluster: %s", r.sc.ConfigRsName(), memberCluster.Name) + svc := r.getPodService(r.sc.ConfigRsName(), memberCluster, podNum, portOrDefault) + if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create pod service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + _ = append(allServices, &svc) + } + } + if err := createHeadlessServiceForStatefulSet(ctx, r.sc.ConfigRsName(), portOrDefault, r.sc.Namespace, memberCluster); err != nil { + return err + } + return nil +} + +func (r *ShardedClusterReconcileHelper) reconcileShardServices(ctx context.Context, log *zap.SugaredLogger, shardIdx int, memberCluster multicluster.MemberCluster, allServices []*corev1.Service) error { + shardsExternalAccess := r.desiredShardsConfiguration[shardIdx].ClusterSpecList.GetExternalAccessConfigurationForMemberCluster(memberCluster.Name) + if shardsExternalAccess == nil { + shardsExternalAccess = r.sc.Spec.ExternalAccessConfiguration + } + portOrDefault := r.desiredShardsConfiguration[shardIdx].AdditionalMongodConfig.GetPortOrDefault() + scaler := r.GetShardScaler(shardIdx, memberCluster) + + for podNum := 0; podNum < scaler.DesiredReplicas(); podNum++ { + // Shards need external services only if an externalDomain is configured + if shardsExternalAccess != nil && shardsExternalAccess.ExternalDomain != nil { + log.Debugf("creating external services for %s in cluster: %s", r.sc.ShardRsName(shardIdx), memberCluster.Name) + svc, err := r.getPodExternalService( + memberCluster, + r.sc.ShardRsName(shardIdx), + shardsExternalAccess, + podNum, + portOrDefault) + if err != nil { + return xerrors.Errorf("failed to create an external service %s in cluster: %s, err: %w", dns.GetMultiExternalServiceName(r.sc.ShardRsName(shardIdx), memberCluster.Index, podNum), memberCluster.Name, err) + } + if err = mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + } + // We don't need internal per-pod services in case we have externalAccess configured AND an external domain + if shardsExternalAccess == nil || shardsExternalAccess.ExternalDomain == nil { + log.Debugf("creating internal services for %s in cluster: %s", r.sc.ShardRsName(shardIdx), memberCluster.Name) + svc := r.getPodService(r.sc.ShardRsName(shardIdx), memberCluster, podNum, portOrDefault) + if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil { + return xerrors.Errorf("failed to create pod service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + + _ = append(allServices, &svc) + } + } + + if err := createHeadlessServiceForStatefulSet(ctx, r.sc.ShardRsName(shardIdx), portOrDefault, r.sc.Namespace, memberCluster); err != nil { + return err + } + return nil +} + +func (r *ShardedClusterReconcileHelper) reconcileMongosServices(ctx context.Context, log *zap.SugaredLogger, memberCluster multicluster.MemberCluster, allServices []*corev1.Service) error { + scaler := r.GetMongosScaler(memberCluster) + portOrDefault := r.desiredMongosConfiguration.AdditionalMongodConfig.GetPortOrDefault() + for podNum := 0; podNum < scaler.DesiredReplicas(); podNum++ { + mongosExternalAccess := r.desiredMongosConfiguration.ClusterSpecList.GetExternalAccessConfigurationForMemberCluster(memberCluster.Name) + if mongosExternalAccess == nil { + mongosExternalAccess = r.sc.Spec.ExternalAccessConfiguration + } + // Mongos always need external services if externalAccess is configured + if mongosExternalAccess != nil { + log.Debugf("creating external services for %s in cluster: %s", r.sc.MongosRsName(), memberCluster.Name) + svc, err := r.getPodExternalService(memberCluster, + r.sc.MongosRsName(), + mongosExternalAccess, + podNum, + portOrDefault) + if err != nil { + return xerrors.Errorf("failed to create an external service %s in cluster: %s, err: %w", dns.GetMultiExternalServiceName(r.sc.MongosRsName(), memberCluster.Index, podNum), memberCluster.Name, err) + } + if err = mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create external service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + } + // We don't need internal per-pod services in case we have externalAccess configured AND an external domain + if mongosExternalAccess == nil || mongosExternalAccess.ExternalDomain == nil { + log.Debugf("creating internal services for %s in cluster: %s", r.sc.MongosRsName(), memberCluster.Name) + svc := r.getPodService(r.sc.MongosRsName(), memberCluster, podNum, portOrDefault) + if err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, svc); err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create pod service %s in cluster: %s, err: %w", svc.Name, memberCluster.Name, err) + } + + _ = append(allServices, &svc) + } + + if err := createHeadlessServiceForStatefulSet(ctx, r.sc.MongosRsName(), portOrDefault, r.sc.Namespace, memberCluster); err != nil { + return err + } + } + return nil +} + +func createHeadlessServiceForStatefulSet(ctx context.Context, stsName string, port int32, namespace string, memberCluster multicluster.MemberCluster) error { + headlessServiceName := dns.GetMultiHeadlessServiceName(stsName, memberCluster.Index) + nameSpacedName := kube.ObjectKey(namespace, headlessServiceName) + headlessService := create.BuildService(nameSpacedName, nil, ptr.To(headlessServiceName), nil, port, omv1.MongoDBOpsManagerServiceDefinition{Type: corev1.ServiceTypeClusterIP}) + err := mekoService.CreateOrUpdateService(ctx, memberCluster.Client, headlessService) + if err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to create pod service %s in cluster: %s, err: %w", headlessService.Name, memberCluster.Name, err) + } + return nil +} + +func (r *ShardedClusterReconcileHelper) getPodExternalService(memberCluster multicluster.MemberCluster, statefulSetName string, externalAccessConfiguration *mdbv1.ExternalAccessConfiguration, podNum int, port int32) (corev1.Service, error) { + svc := r.getPodService(statefulSetName, memberCluster, podNum, port) + svc.Name = dns.GetMultiExternalServiceName(statefulSetName, memberCluster.Index, podNum) + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + + if externalAccessConfiguration.ExternalService.Annotations != nil { + svc.Annotations = merge.StringToStringMap(svc.Annotations, externalAccessConfiguration.ExternalService.Annotations) + } + + placeholderReplacer := create.GetMultiClusterMongoDBPlaceholderReplacer(r.sc.Name, statefulSetName, r.sc.Namespace, memberCluster.Name, memberCluster.Index, externalAccessConfiguration.ExternalDomain, r.sc.Spec.ClusterDomain, podNum) + if processedAnnotations, replacedFlag, err := placeholderReplacer.ProcessMap(svc.Annotations); err != nil { + return corev1.Service{}, xerrors.Errorf("failed to process annotations in external service %s in cluster %s: %w", svc.Name, memberCluster.Name, err) + } else if replacedFlag { + svc.Annotations = processedAnnotations + } + return svc, nil +} + +func (r *ShardedClusterReconcileHelper) replicateTLSCAConfigMap(ctx context.Context, log *zap.SugaredLogger) error { + if !r.sc.Spec.IsMultiCluster() { + return nil + } + caConfigMapName := r.sc.GetSecurity().TLSConfig.CA + if caConfigMapName == "" || !r.sc.Spec.IsMultiCluster() { + return nil + } + + operatorCAConfigMap, err := r.commonController.client.GetConfigMap(ctx, kube.ObjectKey(r.sc.Namespace, caConfigMapName)) + if err != nil { + return xerrors.Errorf("expected CA ConfigMap not found on the operator cluster: %s", caConfigMapName) + } + for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) { + memberCAConfigMap := configmap.Builder(). + SetName(caConfigMapName). + SetNamespace(r.sc.Namespace). + SetData(operatorCAConfigMap.Data). + Build() + err = configmap.CreateOrUpdate(ctx, memberCluster.Client, memberCAConfigMap) + if err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to replicate CA ConfigMap from the operator cluster to cluster %s, err: %w", memberCluster.Name, err) + } + log.Debugf("Successfully ensured configmap: %s/%s in cluster: %s", r.sc.Namespace, caConfigMapName, memberCluster.Name) + } + + return nil +} + +func (r *ShardedClusterReconcileHelper) replicateSSLMMSCAConfigMap(ctx context.Context, projectConfig mdbv1.ProjectConfig, log *zap.SugaredLogger) error { + if !r.sc.Spec.IsMultiCluster() || projectConfig.SSLMMSCAConfigMap == "" { + return nil + } + + cm, err := r.commonController.client.GetConfigMap(ctx, kube.ObjectKey(r.sc.Namespace, projectConfig.SSLMMSCAConfigMap)) + if err != nil { + return xerrors.Errorf("expected SSLMMSCAConfigMap not found on operator cluster: %s", projectConfig.SSLMMSCAConfigMap) + } + + for _, memberCluster := range getHealthyMemberClusters(r.allMemberClusters) { + memberCm := configmap.Builder(). + SetName(projectConfig.SSLMMSCAConfigMap). + SetNamespace(r.sc.Namespace). + SetData(cm.Data). + Build() + err = configmap.CreateOrUpdate(ctx, memberCluster.Client, memberCm) + + if err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed to sync SSLMMSCAConfigMap to cluster: %s, err: %w", memberCluster.Name, err) + } + log.Debugf("Successfully ensured configmap: %s/%s in cluster: %s", r.sc.Namespace, projectConfig.SSLMMSCAConfigMap, memberCluster.Name) + } + + return nil +} + +// isStillScaling checks whether we're in the process of scaling. +// It checks whether any of the components/statefulsets still require scaling by checking +// the actual state from the deployment state vs what's in the spec. +// +// When we're in the last step of the sizing, and the statefulsets were sized to the desired numbers and everything is ready, this function will still +// report that we're in the process of scaling. This is because it gets the previous state from the deployment state, which is updated only after we finished scaling *step* (by one). +// +// This function cannot be used to determine if we're done with the scaling *step* in this reconciliation, so that we can increment +// current sizes in the deployment state. For that case use shouldContinueScalingOneByOne. +func (r *ShardedClusterReconcileHelper) isStillScaling() bool { + for _, s := range r.getAllScalers() { + if s.CurrentReplicas() != s.(*scalers.MultiClusterReplicaSetScaler).TargetReplicas() { + return true + } + } + + return false +} + +// shouldContinueScalingOneByOne iterates over all scalers for each statefulset in the sharded cluster +// and checks whether scale.ReplicasThisReconciliation are equal with the target replicas according to the spec. +// +// If this function returns true, it means that the current sizes reported by ReplicasThisReconciliation are not equal with the desired (spec) sizes. +// So we need to save the current sizes to the deployment state, in order to allow the next reconciliation to calculate next (+1) sizes. +// +// If this function return false, it means we've completely finished scaling process, but it could be that we've just finished the scaling in the current reconciliation. +// The difference vs isStillScaling is subtle. isStillScaling tells us if we're generally in the process of scaling (current sizes != spec). +func (r *ShardedClusterReconcileHelper) shouldContinueScalingOneByOne() bool { + for _, s := range r.getAllScalers() { + if scale.ReplicasThisReconciliation(s) != s.TargetReplicas() { + return true + } + } + + return false +} + +func (r *ShardedClusterReconcileHelper) getPodService(stsName string, memberCluster multicluster.MemberCluster, podNum int, port int32) corev1.Service { + svcLabels := map[string]string{ + "statefulset.kubernetes.io/pod-name": dns.GetMultiPodName(stsName, memberCluster.Index, podNum), + "controller": "mongodb-enterprise-operator", + mdbv1.LabelMongoDBResourceOwner: r.sc.Name, + } + + labelSelectors := map[string]string{ + "statefulset.kubernetes.io/pod-name": dns.GetMultiPodName(stsName, memberCluster.Index, podNum), + "controller": "mongodb-enterprise-operator", + } + + svc := service.Builder(). + SetName(dns.GetMultiServiceName(stsName, memberCluster.Index, podNum)). + SetNamespace(r.sc.Namespace). + SetSelector(labelSelectors). + SetLabels(svcLabels). + SetPublishNotReadyAddresses(true). + AddPort(&corev1.ServicePort{Port: port, Name: "mongodb"}). + // Note: in the agent-launcher.sh We explicitly pass an offset of 1. When port N is exposed + // the agent would use port N+1 for the spinning up of the ephemeral mongod process, which is used for backup + AddPort(&corev1.ServicePort{Port: create.GetNonEphemeralBackupPort(port), Name: "backup", TargetPort: intstr.IntOrString{IntVal: create.GetNonEphemeralBackupPort(port)}}). + Build() + + return svc +} + +func (r *ShardedClusterReconcileHelper) statefulsetLabels() map[string]string { + return merge.StringToStringMap(r.sc.Labels, mongoDBLabels(r.sc.Name)) +} + +func mongoDBLabels(name string) map[string]string { + return map[string]string{ + construct.ControllerLabelName: util.OperatorName, + mdbv1.LabelMongoDBResourceOwner: name, + } +} + +func (r *ShardedClusterReconcileHelper) ShardsMemberClustersMap() map[int][]multicluster.MemberCluster { + return r.shardsMemberClustersMap +} + +func (r *ShardedClusterReconcileHelper) ConfigSrvMemberClusters() []multicluster.MemberCluster { + return r.configSrvMemberClusters +} + +func (r *ShardedClusterReconcileHelper) MongosMemberClusters() []multicluster.MemberCluster { + return r.mongosMemberClusters +} + +func (r *ShardedClusterReconcileHelper) AllShardsMemberClusters() []multicluster.MemberCluster { + return r.allShardsMemberClusters +} + +func (r *ShardedClusterReconcileHelper) AllMemberClusters() []multicluster.MemberCluster { + return r.allMemberClusters +} + +func (r *ShardedClusterReconcileHelper) getHealthyProcessNames() []string { + _, mongosProcessNames := r.getHealthyMongosProcesses() + _, configSrvProcessNames := r.getHealthyConfigSrvProcesses() + _, shardsProcessNames := r.getHealthyShardsProcesses() + + var processNames []string + processNames = append(processNames, mongosProcessNames...) + processNames = append(processNames, configSrvProcessNames...) + processNames = append(processNames, shardsProcessNames...) + + return processNames +} + +func (r *ShardedClusterReconcileHelper) getHealthyProcessNamesToWaitForReadyState(conn om.Connection, log *zap.SugaredLogger) []string { + processList := r.getHealthyProcessNames() + + clusterState, err := agents.GetMongoDBClusterState(conn) + if err != nil { + log.Warnf("Skipping check for mongos deadlock for all the nodes being healthy (deadlock) due to error: %v", err) + return processList + } + + if mongosDeadlock, deadlockedMongos := checkForMongosDeadlock(clusterState, r.sc.MongosRsName(), r.isStillScaling(), log); mongosDeadlock { + deadlockedProcessNames := util.Transform(deadlockedMongos, func(obj agents.ProcessState) string { + return obj.ProcessName + }) + log.Warnf("The following processes are skipped from waiting for the goal state: %+v", deadlockedProcessNames) + processList = slices.DeleteFunc(processList, func(processName string) bool { + for _, processState := range deadlockedMongos { + if processName == processState.ProcessName { + return true + } + } + return false + }) + } + + return processList +} + +func (r *ShardedClusterReconcileHelper) getHealthyMongosProcesses() ([]string, []string) { + var processNames []string + var hostnames []string + for _, memberCluster := range getHealthyMemberClusters(r.mongosMemberClusters) { + clusterHostnames, clusterProcessNames := r.getMongosHostnames(memberCluster, scale.ReplicasThisReconciliation(r.GetMongosScaler(memberCluster))) + hostnames = append(hostnames, clusterHostnames...) + processNames = append(processNames, clusterProcessNames...) + } + return hostnames, processNames +} + +func (r *ShardedClusterReconcileHelper) getHealthyConfigSrvProcesses() ([]string, []string) { + var processNames []string + var hostnames []string + for _, memberCluster := range getHealthyMemberClusters(r.configSrvMemberClusters) { + clusterHostnames, clusterProcessNames := r.getConfigSrvHostnames(memberCluster, scale.ReplicasThisReconciliation(r.GetConfigSrvScaler(memberCluster))) + hostnames = append(hostnames, clusterHostnames...) + processNames = append(processNames, clusterProcessNames...) + } + return hostnames, processNames +} + +func (r *ShardedClusterReconcileHelper) getHealthyShardsProcesses() ([]string, []string) { + var processNames []string + var hostnames []string + for shardIdx := 0; shardIdx < r.sc.Spec.ShardCount; shardIdx++ { + for _, memberCluster := range getHealthyMemberClusters(r.shardsMemberClustersMap[shardIdx]) { + clusterHostnames, clusterProcessNames := r.getShardHostnames(shardIdx, memberCluster, scale.ReplicasThisReconciliation(r.GetShardScaler(shardIdx, memberCluster))) + hostnames = append(hostnames, clusterHostnames...) + processNames = append(processNames, clusterProcessNames...) + } + } + return hostnames, processNames +} + +// checkForMongosDeadlock reports whether the cluster is in a deadlocked state due to mongos waiting on unhealthy +// processes (https://jira.mongodb.org/browse/CLOUDP-288588) +// We are in a deadlock if: +// - We are in the process of scaling. +// - There are healthy mongos process not in goal state (their automation config version is lesser than the goal version). +// - The agent plan of those mongos processes contains 'RollingChangeArgs'. +// - All other healthy processes other than mongos are in goal state. +func checkForMongosDeadlock(clusterState agents.MongoDBClusterStateInOM, mongosReplicaSetName string, isScaling bool, log *zap.SugaredLogger) (isDeadlocked bool, deadlockedMongos []agents.ProcessState) { + if !isScaling { + log.Debugf("Skipping mongos deadlock check as there is no scaling in progress") + return false, nil + } + + staleProcesses := slices.DeleteFunc(clusterState.GetProcesses(), func(processState agents.ProcessState) bool { + return !processState.IsStale() + }) + + if len(staleProcesses) == 0 { + log.Debugf("Mongos deadlock check reported negative. There are no stale processes in the cluster") + return false, nil + } + + allHealthyProcessesNotInGoalState := slices.DeleteFunc(clusterState.GetProcessesNotInGoalState(), func(processState agents.ProcessState) bool { + return processState.IsStale() + }) + + allHealthyMongosNotInGoalState := slices.DeleteFunc(slices.Clone(allHealthyProcessesNotInGoalState), func(processState agents.ProcessState) bool { + return !strings.HasPrefix(processState.ProcessName, mongosReplicaSetName) + }) + + if len(allHealthyMongosNotInGoalState) == 0 { + log.Debugf("Mongos deadlock check reported negative. All healthy mongos processes are in goal state in the cluster") + return false, nil + } + + if len(allHealthyProcessesNotInGoalState) > len(allHealthyMongosNotInGoalState) { + log.Debugf("Mongos deadlock check reported negative. There are other healthy processes not in goal state that are not mongos; allHealthyProcessesNotInGoalState=%+v", allHealthyProcessesNotInGoalState) + return false, nil + } + + allDeadlockedMongos := slices.DeleteFunc(slices.Clone(allHealthyMongosNotInGoalState), func(processState agents.ProcessState) bool { + for _, agentMove := range processState.Plan { + if agentMove == agents.RollingChangeArgs { + return false + } // TODO make a constant + } + return true + }) + + if len(allDeadlockedMongos) > 0 { + staleHostnames := util.Transform(staleProcesses, func(obj agents.ProcessState) string { + return obj.Hostname + }) + log.Warnf("Detected mongos %+v performing RollingChangeArgs operation while there are processes in the cluster that are considered down. "+ + "Skipping waiting for those mongos processes in order to allow the operator to perform scaling. "+ + "Please verify the list of stale (down/unhealthy) processes and change MongoDB resource to remove them from the cluster. "+ + "The operator will not perform removal of those procesess automatically. Hostnames of stale processes: %+v", allDeadlockedMongos, staleHostnames) + + return true, allDeadlockedMongos + } + + return false, nil +} diff --git a/controllers/operator/mongodbshardedcluster_controller_multi_test.go b/controllers/operator/mongodbshardedcluster_controller_multi_test.go new file mode 100644 index 000000000..10604ec3a --- /dev/null +++ b/controllers/operator/mongodbshardedcluster_controller_multi_test.go @@ -0,0 +1,2740 @@ +package operator + +import ( + "context" + "crypto/x509" + "encoding/json" + "fmt" + "os" + "path" + "strconv" + "testing" + "time" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yudai/gojsondiff" + "github.com/yudai/gojsondiff/formatter" + "go.uber.org/zap" + "golang.org/x/exp/constraints" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/test" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func newShardedClusterReconcilerForMultiCluster(ctx context.Context, forceEnterprise bool, sc *mdbv1.MongoDB, globalMemberClustersMap map[string]client.Client, kubeClient kubernetesClient.Client, omConnectionFactory *om.CachedOMConnectionFactory) (*ReconcileMongoDbShardedCluster, *ShardedClusterReconcileHelper, error) { + r := newShardedClusterReconciler(ctx, kubeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, globalMemberClustersMap, omConnectionFactory.GetConnectionFunc) + reconcileHelper, err := NewShardedClusterReconcilerHelper(ctx, r.ReconcileCommonController, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", forceEnterprise, sc, globalMemberClustersMap, omConnectionFactory.GetConnectionFunc, zap.S()) + if err != nil { + return nil, nil, err + } + return r, reconcileHelper, nil +} + +// createMockStateConfigMap creates a configMap with the sizeStatusInClusters populated based on the cluster state +// passed in parameters, to simulate it is the current state of the cluster +func createMockStateConfigMap(kubeClient client.Client, namespace, scName string, state MultiClusterShardedScalingStep) error { + sumMap := func(m map[string]int) int { + sum := 0 + for _, val := range m { + sum += val + } + return sum + } + + configServerSum := sumMap(state.configServerDistribution) + mongosSum := sumMap(state.mongosDistribution) + + sizeStatus := map[string]interface{}{ + "status": map[string]interface{}{ + "shardCount": state.shardCount, + "configServerCount": configServerSum, + "mongosCount": mongosSum, + "sizeStatusInClusters": map[string]interface{}{ + "shardMongodsInClusters": state.shardDistribution, + "mongosCountInClusters": state.mongosDistribution, + "configServerMongodsInClusters": state.configServerDistribution, + "shardOverridesInClusters": ConvertTargetStateToMap(scName, state.shardOverrides), + }, + }, + } + + // Convert state to JSON + stateJSON, err := json.Marshal(sizeStatus) + if err != nil { + return err + } + + // Create ConfigMap definition + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-state", scName), + Namespace: namespace, + }, + Data: map[string]string{ + stateKey: string(stateJSON), + }, + } + + err = kubeClient.Create(context.TODO(), configMap) + return err +} + +// ConvertTargetStateToMap converts a slice of shardOverrides (target state format) into a map format. +func ConvertTargetStateToMap(scName string, shardOverridesDistribution []map[string]int) map[string]map[string]int { + convertedMap := make(map[string]map[string]int) + + for i, distribution := range shardOverridesDistribution { + resourceKey := scName + "-" + strconv.Itoa(i) + convertedMap[resourceKey] = distribution + } + + return convertedMap +} + +type BlockReconcileScalingBothWaysTestCase struct { + name string + initialState MultiClusterShardedScalingStep + targetState MultiClusterShardedScalingStep + expectError bool +} + +// TestBlockReconcileScalingBothWays checks that we block reconciliation when member clusters in a replica set need to +// be scaled both up and down in the same reconciliation +func TestBlockReconcileScalingBothWays(t *testing.T) { + cluster1 := "member-cluster-1" + cluster2 := "member-cluster-2" + cluster3 := "member-cluster-3" + testCases := []BlockReconcileScalingBothWaysTestCase{ + { + name: "No scaling", + initialState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + }, + targetState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + }, + expectError: false, + }, + { + name: "Scaling in the same direction", + initialState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + }, + targetState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 2, cluster3: 1, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 3, + }, + }, + expectError: false, + }, + { + name: "Scaling both directions: cfg server and mongos", + initialState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + }, + targetState: MultiClusterShardedScalingStep{ + shardCount: 3, + // Upscale + configServerDistribution: map[string]int{ + cluster1: 3, cluster2: 1, cluster3: 1, + }, + // Downscale + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 0, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + }, + expectError: true, + }, + { + name: "Scale both ways because of shard override", + initialState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 1, cluster3: 1}, + }, + }, + targetState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 0, cluster3: 2, // cluster 1: 1 -> 3, cluster2 : 1 -> 0, cluster3: 1 -> 2 + }, + shardOverrides: []map[string]int{ + // Downscale shard 0 (was overridden with 5 replicas) + {cluster1: 1, cluster2: 1, cluster3: 1}, + // Upscale shard 1 + {cluster1: 1, cluster2: 3, cluster3: 1}, + }, + }, + expectError: true, + }, + { + // Increasing shardCount creates a new shard, that scales from 0 members. We want to block reconciliation + // in that case + name: "Increase shardcount and downscale override", + initialState: MultiClusterShardedScalingStep{ + shardCount: 2, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 1, cluster3: 1}, + }, + }, + targetState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, // shard distribution stays the same + }, + shardOverrides: []map[string]int{ + // Downscale shard 0 (was overridden with 5 replicas) + {cluster1: 1, cluster2: 1, cluster3: 1}, + }, + }, + expectError: true, + }, + { + // We move replicas from one cluster to another, without changing the total number, and we scale shards up + // Moving replicas necessitate a scale down on one cluster and a scale up on another. + // We need to block reconciliation. + name: "Moving replicas between clusters", + initialState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 2, cluster2: 0, cluster3: 1, + }, + }, + targetState: MultiClusterShardedScalingStep{ + shardCount: 3, + configServerDistribution: map[string]int{ + cluster1: 2, cluster2: 1, cluster3: 0, // No scaling + }, + mongosDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + shardDistribution: map[string]int{ + cluster1: 2, cluster2: 0, cluster3: 1, // No scaling + }, + shardOverrides: []map[string]int{ + {cluster1: 0, cluster2: 2, cluster3: 1}, // Moved two replicas by adding an override, but no scaling + }, + }, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + BlockReconcileScalingBothWaysCase(t, tc) + }) + } +} + +func BlockReconcileScalingBothWaysCase(t *testing.T, tc BlockReconcileScalingBothWaysTestCase) { + ctx := context.Background() + cluster1 := "member-cluster-1" + cluster2 := "member-cluster-2" + cluster3 := "member-cluster-3" + memberClusterNames := []string{ + cluster1, + cluster2, + cluster3, + } + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + _ = omConnectionFactory.GetConnectionFunc(&om.OMContext{GroupName: om.TestGroupName}) + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + + memberClusterMap := getFakeMultiClusterMapWithoutInterceptor(memberClusterNames) + + // The MDB resource applied is defined by the target state. The initial state is the status we store in the + // config map + sc := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(tc.targetState.shardCount). + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + SetShardClusterSpec(test.CreateClusterSpecList(memberClusterNames, tc.targetState.shardDistribution)). + SetConfigSrvClusterSpec(test.CreateClusterSpecList(memberClusterNames, tc.targetState.configServerDistribution)). + SetMongosClusterSpec(test.CreateClusterSpecList(memberClusterNames, tc.targetState.mongosDistribution)). + SetShardOverrides(computeShardOverridesFromDistribution(tc.targetState.shardOverrides)). + Build() + + err := kubeClient.Create(ctx, sc) + require.NoError(t, err) + + err = createMockStateConfigMap(kubeClient, mock.TestNamespace, sc.Name, tc.initialState) + require.NoError(t, err) + + // Checking that we don't scale both ways is done when we initiate the reconciler, not in the reconcile loop. + reconciler, _, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + // The validation happens at the beginning of the reconciliation loop. We expect to fail immediately when scaling is + // invalid, or stay in pending phase otherwise. + if tc.expectError { + checkReconcileFailed(ctx, t, reconciler, sc, true, "Cannot perform scale up and scale down operations at the same time", kubeClient) + } else { + checkReconcilePending(ctx, t, reconciler, sc, "StatefulSet not ready", kubeClient, 3) + } +} + +// TestReconcileCreateMultiClusterShardedClusterWithExternalDomain checks if all components have been exposed using +// their own domains +func TestReconcileCreateMultiClusterShardedClusterWithExternalDomain(t *testing.T) { + memberClusters := test.NewMemberClusters( + test.MemberClusterDetails{ + ClusterName: "member-cluster-1", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 2, + NumberOfMongoses: 2, + }, + test.MemberClusterDetails{ + ClusterName: "member-cluster-2", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 1, + NumberOfMongoses: 1, + }, + ) + + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + WithMultiClusterSetup(memberClusters). + SetExternalAccessDomain(test.ExampleExternalClusterDomains). + SetExternalAccessDomainAnnotations(test.MultiClusterAnnotationsWithPlaceholders). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(sc).WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + memberClusterMap := getFakeMultiClusterMapWithConfiguredInterceptor(memberClusters.ClusterNames, omConnectionFactory, true, true) + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + allHostnames, _ := generateAllHosts(sc, memberClusters.MongosDistribution, clusterMapping, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.ExampleExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + }) + + require.NoError(t, err) + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + expectedHostnameOverrideMap := createExpectedHostnameOverrideMap(sc, clusterMapping, memberClusters.MongosDistribution, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.ExampleExternalClusterDomains) + + for clusterIdx, clusterSpecItem := range sc.Spec.ShardSpec.ClusterSpecList { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, sc.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + configSrvStsName := fmt.Sprintf("%s-config-%d", sc.Name, clusterIdx) + configMembers := memberClusters.ConfigServerDistribution[clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, configSrvStsName, configMembers) + memberClusterChecks.checkInternalServices(ctx, configSrvStsName) + memberClusterChecks.checkPerPodServicesDontExist(ctx, configSrvStsName, configMembers) + memberClusterChecks.checkServiceAnnotations(ctx, configSrvStsName, configMembers, sc, clusterSpecItem.ClusterName, clusterIdx, test.ExampleExternalClusterDomains.ConfigServerExternalDomain) + + mongosStsName := fmt.Sprintf("%s-mongos-%d", sc.Name, clusterIdx) + mongosMembers := memberClusters.MongosDistribution[clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, mongosStsName, mongosMembers) + memberClusterChecks.checkInternalServices(ctx, mongosStsName) + memberClusterChecks.checkPerPodServicesDontExist(ctx, mongosStsName, mongosMembers) + memberClusterChecks.checkServiceAnnotations(ctx, mongosStsName, mongosMembers, sc, clusterSpecItem.ClusterName, clusterIdx, test.ExampleExternalClusterDomains.MongosExternalDomain) + + for shardIdx := 0; shardIdx < memberClusters.ShardCount(); shardIdx++ { + shardStsName := fmt.Sprintf("%s-%d-%d", sc.Name, shardIdx, clusterIdx) + shardMembers := memberClusters.ShardDistribution[shardIdx][clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, shardStsName, shardMembers) + memberClusterChecks.checkInternalServices(ctx, shardStsName) + memberClusterChecks.checkPerPodServicesDontExist(ctx, shardStsName, shardMembers) + memberClusterChecks.checkServiceAnnotations(ctx, shardStsName, shardMembers, sc, clusterSpecItem.ClusterName, clusterIdx, test.ExampleExternalClusterDomains.ShardsExternalDomain) + memberClusterChecks.checkHostnameOverrideConfigMap(ctx, fmt.Sprintf("%s-hostname-override", sc.Name), expectedHostnameOverrideMap) + } + } +} + +// TestReconcileCreateMultiClusterShardedClusterWithExternalAccessAndOnlyTopLevelExternalDomain checks if all components +// have been exposed. +func TestReconcileCreateMultiClusterShardedClusterWithExternalAccessAndOnlyTopLevelExternalDomain(t *testing.T) { + memberClusters := test.NewMemberClusters( + test.MemberClusterDetails{ + ClusterName: "member-cluster-1", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 2, + NumberOfMongoses: 2, + }, + test.MemberClusterDetails{ + ClusterName: "member-cluster-2", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 1, + NumberOfMongoses: 1, + }, + ) + + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + // Specifying it in this order will set only the top-level External Domain (which we're testing here) + SetExternalAccessDomain(test.SingleExternalClusterDomains). + WithMultiClusterSetup(memberClusters). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(sc).WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + memberClusterMap := getFakeMultiClusterMapWithConfiguredInterceptor(memberClusters.ClusterNames, omConnectionFactory, true, true) + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + allHostnames, _ := generateAllHosts(sc, memberClusters.MongosDistribution, clusterMapping, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.SingleExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + }) + + require.NoError(t, err) + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + expectedHostnameOverrideMap := createExpectedHostnameOverrideMap(sc, clusterMapping, memberClusters.MongosDistribution, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.SingleExternalClusterDomains) + + for clusterIdx, clusterSpecItem := range sc.Spec.ShardSpec.ClusterSpecList { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, sc.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + configSrvStsName := fmt.Sprintf("%s-config-%d", sc.Name, clusterIdx) + configMembers := memberClusters.ConfigServerDistribution[clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, configSrvStsName, configMembers) + memberClusterChecks.checkInternalServices(ctx, configSrvStsName) + memberClusterChecks.checkPerPodServicesDontExist(ctx, configSrvStsName, configMembers) + + mongosStsName := fmt.Sprintf("%s-mongos-%d", sc.Name, clusterIdx) + mongosMembers := memberClusters.MongosDistribution[clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, mongosStsName, mongosMembers) + memberClusterChecks.checkInternalServices(ctx, mongosStsName) + memberClusterChecks.checkPerPodServicesDontExist(ctx, mongosStsName, mongosMembers) + + for shardIdx := 0; shardIdx < memberClusters.ShardCount(); shardIdx++ { + shardStsName := fmt.Sprintf("%s-%d-%d", sc.Name, shardIdx, clusterIdx) + shardMembers := memberClusters.ShardDistribution[shardIdx][clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, shardStsName, shardMembers) + memberClusterChecks.checkInternalServices(ctx, shardStsName) + memberClusterChecks.checkPerPodServicesDontExist(ctx, shardStsName, shardMembers) + } + memberClusterChecks.checkHostnameOverrideConfigMap(ctx, fmt.Sprintf("%s-hostname-override", sc.Name), expectedHostnameOverrideMap) + } +} + +// TestReconcileCreateMultiClusterShardedClusterWithExternalAccessAndNoExternalDomain checks if only Mongoses are +// exposed. Other components should be hidden. +func TestReconcileCreateMultiClusterShardedClusterWithExternalAccessAndNoExternalDomain(t *testing.T) { + memberClusters := test.NewMemberClusters( + test.MemberClusterDetails{ + ClusterName: "member-cluster-1", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 2, + NumberOfMongoses: 2, + }, + test.MemberClusterDetails{ + ClusterName: "member-cluster-2", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 1, + NumberOfMongoses: 1, + }, + ) + + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + WithMultiClusterSetup(memberClusters). + SetExternalAccessDomain(test.NoneExternalClusterDomains). + SetExternalAccessDomainAnnotations(test.MultiClusterAnnotationsWithPlaceholders). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(sc).WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + memberClusterMap := getFakeMultiClusterMapWithConfiguredInterceptor(memberClusters.ClusterNames, omConnectionFactory, true, true) + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + allHostnames, _ := generateAllHosts(sc, memberClusters.MongosDistribution, clusterMapping, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + }) + + require.NoError(t, err) + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + expectedHostnameOverrideMap := createExpectedHostnameOverrideMap(sc, clusterMapping, memberClusters.MongosDistribution, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + + for clusterIdx, clusterSpecItem := range sc.Spec.ShardSpec.ClusterSpecList { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, sc.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + configSrvStsName := fmt.Sprintf("%s-config-%d", sc.Name, clusterIdx) + configMembers := memberClusters.ConfigServerDistribution[clusterSpecItem.ClusterName] + memberClusterChecks.checkInternalServices(ctx, configSrvStsName) + memberClusterChecks.checkExternalServicesDontExist(ctx, configSrvStsName, configMembers) + memberClusterChecks.checkPerPodServices(ctx, configSrvStsName, configMembers) + + mongosStsName := fmt.Sprintf("%s-mongos-%d", sc.Name, clusterIdx) + mongosMembers := memberClusters.MongosDistribution[clusterSpecItem.ClusterName] + memberClusterChecks.checkExternalServices(ctx, mongosStsName, mongosMembers) + memberClusterChecks.checkInternalServices(ctx, mongosStsName) + // Without external domain, we need per-pod mongos services + memberClusterChecks.checkPerPodServices(ctx, mongosStsName, mongosMembers) + memberClusterChecks.checkServiceAnnotations(ctx, mongosStsName, mongosMembers, sc, clusterSpecItem.ClusterName, clusterIdx, test.ExampleAccessWithNoExternalDomain.MongosExternalDomain) + + for shardIdx := 0; shardIdx < memberClusters.ShardCount(); shardIdx++ { + shardStsName := fmt.Sprintf("%s-%d-%d", sc.Name, shardIdx, clusterIdx) + shardMembers := memberClusters.ShardDistribution[shardIdx][clusterSpecItem.ClusterName] + memberClusterChecks.checkInternalServices(ctx, shardStsName) + memberClusterChecks.checkExternalServicesDontExist(ctx, shardStsName, shardMembers) + memberClusterChecks.checkPerPodServices(ctx, shardStsName, shardMembers) + } + memberClusterChecks.checkHostnameOverrideConfigMap(ctx, fmt.Sprintf("%s-hostname-override", sc.Name), expectedHostnameOverrideMap) + } +} + +func TestReconcileCreateMultiClusterShardedCluster(t *testing.T) { + // Two Kubernetes clusters, 2 replicaset members of each shard on the first one, 3 on the second one + // This means a MongodPerShardCount of 5 + memberClusters := test.NewMemberClusters( + test.MemberClusterDetails{ + ClusterName: "member-cluster-1", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 2, + NumberOfMongoses: 2, + }, + test.MemberClusterDetails{ + ClusterName: "member-cluster-2", + ShardMap: []int{2, 3}, + NumberOfConfigServers: 1, + NumberOfMongoses: 1, + }, + ) + + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + WithMultiClusterSetup(memberClusters). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(sc).WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + memberClusterMap := getFakeMultiClusterMapWithConfiguredInterceptor(memberClusters.ClusterNames, omConnectionFactory, true, true) + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + allHostnames, _ := generateAllHosts(sc, memberClusters.MongosDistribution, clusterMapping, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + }) + + require.NoError(t, err) + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + checkCorrectShardDistributionInStatus(t, sc) + + expectedHostnameOverrideMap := createExpectedHostnameOverrideMap(sc, clusterMapping, memberClusters.MongosDistribution, memberClusters.ConfigServerDistribution, memberClusters.ShardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + + for clusterIdx, clusterSpecItem := range sc.Spec.ShardSpec.ClusterSpecList { + memberClusterChecks := newClusterChecks(t, clusterSpecItem.ClusterName, clusterIdx, sc.Namespace, memberClusterMap[clusterSpecItem.ClusterName]) + for shardIdx := 0; shardIdx < memberClusters.ShardCount(); shardIdx++ { + shardStsName := fmt.Sprintf("%s-%d-%d", sc.Name, shardIdx, clusterIdx) + memberClusterChecks.checkStatefulSet(ctx, shardStsName, memberClusters.ShardDistribution[shardIdx][clusterSpecItem.ClusterName]) + memberClusterChecks.checkInternalServices(ctx, shardStsName) + memberClusterChecks.checkPerPodServices(ctx, shardStsName, memberClusters.ShardDistribution[shardIdx][clusterSpecItem.ClusterName]) + } + + // Config servers statefulsets should have the names mongoName-config-0, mongoName-config-1 + configSrvStsName := fmt.Sprintf("%s-config-%d", sc.Name, clusterIdx) + memberClusterChecks.checkStatefulSet(ctx, configSrvStsName, memberClusters.ConfigServerDistribution[clusterSpecItem.ClusterName]) + memberClusterChecks.checkInternalServices(ctx, configSrvStsName) + memberClusterChecks.checkPerPodServices(ctx, configSrvStsName, memberClusters.ConfigServerDistribution[clusterSpecItem.ClusterName]) + + // Mongos statefulsets should have the names mongoName-mongos-0, mongoName-mongos-1 + mongosStsName := fmt.Sprintf("%s-mongos-%d", sc.Name, clusterIdx) + memberClusterChecks.checkStatefulSet(ctx, mongosStsName, memberClusters.MongosDistribution[clusterSpecItem.ClusterName]) + memberClusterChecks.checkInternalServices(ctx, mongosStsName) + memberClusterChecks.checkPerPodServices(ctx, mongosStsName, memberClusters.MongosDistribution[clusterSpecItem.ClusterName]) + + memberClusterChecks.checkAgentAPIKeySecret(ctx, om.TestGroupID) + memberClusterChecks.checkHostnameOverrideConfigMap(ctx, fmt.Sprintf("%s-hostname-override", sc.Name), expectedHostnameOverrideMap) + } +} + +func createExpectedHostnameOverrideMap(sc *mdbv1.MongoDB, clusterMapping map[string]int, mongosDistribution map[string]int, configSrvDistribution map[string]int, shardDistribution []map[string]int, clusterDomains test.ClusterDomains, externalClusterDomains test.ClusterDomains) map[string]string { + expectedHostnameOverrideMap := map[string]string{} + allHostnames, allPodNames := generateAllHosts(sc, mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution, clusterDomains, externalClusterDomains) + for i := range allPodNames { + expectedHostnameOverrideMap[allPodNames[i]] = allHostnames[i] + } + return expectedHostnameOverrideMap +} + +type MultiClusterShardedClusterConfigList []MultiClusterShardedClusterConfigItem + +type MultiClusterShardedClusterConfigItem struct { + Name string + ShardsMembersArray []int + MongosMembers int + ConfigSrvMembers int +} + +func (r *MultiClusterShardedClusterConfigList) AddCluster(clusterName string, shards []int, mongosCount int, configSrvCount int) { + clusterSpec := MultiClusterShardedClusterConfigItem{ + Name: clusterName, + ShardsMembersArray: shards, + MongosMembers: mongosCount, + ConfigSrvMembers: configSrvCount, + } + + *r = append(*r, clusterSpec) +} + +func (r *MultiClusterShardedClusterConfigList) GetNames() []string { + clusterNames := make([]string, len(*r)) + for i, clusterSpec := range *r { + clusterNames[i] = clusterSpec.Name + } + + return clusterNames +} + +func (r *MultiClusterShardedClusterConfigList) GenerateAllHosts(sc *mdbv1.MongoDB, clusterMapping map[string]int) ([]string, []string) { + var allHosts []string + var allPodNames []string + for _, clusterSpec := range *r { + memberClusterName := clusterSpec.Name + clusterIdx := clusterMapping[memberClusterName] + + for podIdx := range clusterSpec.MongosMembers { + allHosts = append(allHosts, getMultiClusterFQDN(sc.MongosRsName(), sc.Namespace, clusterIdx, podIdx, "cluster.local", "")) + allPodNames = append(allPodNames, getPodName(sc.MongosRsName(), clusterIdx, podIdx)) + } + + for podIdx := range clusterSpec.ConfigSrvMembers { + allHosts = append(allHosts, getMultiClusterFQDN(sc.ConfigRsName(), sc.Namespace, clusterIdx, podIdx, "cluster.local", "")) + allPodNames = append(allPodNames, getPodName(sc.ConfigRsName(), clusterIdx, podIdx)) + } + + for shardIdx := 0; shardIdx < len(clusterSpec.ShardsMembersArray); shardIdx++ { + for podIdx := 0; podIdx < clusterSpec.ShardsMembersArray[shardIdx]; podIdx++ { + allHosts = append(allHosts, getMultiClusterFQDN(sc.ShardRsName(shardIdx), sc.Namespace, clusterIdx, podIdx, "cluster.local", "")) + allPodNames = append(allPodNames, getPodName(sc.ShardRsName(shardIdx), clusterIdx, podIdx)) + } + } + } + + return allHosts, allPodNames +} + +func TestReconcileMultiClusterShardedClusterCertsAndSecretsReplication(t *testing.T) { + expectedClusterConfigList := make(MultiClusterShardedClusterConfigList, 0) + expectedClusterConfigList.AddCluster("member-cluster-1", []int{2, 2}, 0, 2) + expectedClusterConfigList.AddCluster("member-cluster-2", []int{3, 3}, 1, 1) + expectedClusterConfigList.AddCluster("member-cluster-3", []int{3, 3}, 3, 0) + expectedClusterConfigList.AddCluster("member-cluster-4", []int{0, 0}, 2, 3) + + memberClusterNames := expectedClusterConfigList.GetNames() + + shardCount := 2 + shardDistribution := []map[string]int{ + {expectedClusterConfigList[0].Name: 2, expectedClusterConfigList[1].Name: 3, expectedClusterConfigList[2].Name: 3}, + {expectedClusterConfigList[0].Name: 2, expectedClusterConfigList[1].Name: 3, expectedClusterConfigList[2].Name: 3}, + } + shardClusterSpecList := test.CreateClusterSpecList(memberClusterNames, shardDistribution[0]) + + mongosDistribution := map[string]int{expectedClusterConfigList[1].Name: 1, expectedClusterConfigList[2].Name: 3, expectedClusterConfigList[3].Name: 2} + mongosAndConfigSrvClusterSpecList := test.CreateClusterSpecList(memberClusterNames, mongosDistribution) + + configSrvDistribution := map[string]int{expectedClusterConfigList[0].Name: 2, expectedClusterConfigList[1].Name: 1, expectedClusterConfigList[3].Name: 3} + configSrvDistributionClusterSpecList := test.CreateClusterSpecList(memberClusterNames, configSrvDistribution) + + certificatesSecretsPrefix := "some-prefix" + sc := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(shardCount). + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + SetShardClusterSpec(shardClusterSpecList). + SetConfigSrvClusterSpec(configSrvDistributionClusterSpecList). + SetMongosClusterSpec(mongosAndConfigSrvClusterSpecList). + SetSecurity(mdbv1.Security{ + TLSConfig: &mdbv1.TLSConfig{Enabled: true, CA: "tls-ca-config"}, + CertificatesSecretsPrefix: certificatesSecretsPrefix, + Authentication: &mdbv1.Authentication{ + Enabled: true, + Modes: []mdbv1.AuthMode{"X509"}, + InternalCluster: util.X509, + }, + }). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + projectConfigMap := configmap.Builder(). + SetName(mock.TestProjectConfigMapName). + SetNamespace(mock.TestNamespace). + SetDataField(util.OmBaseUrl, "http://mycompany.example.com:8080"). + SetDataField(util.OmProjectName, om.TestGroupName). + SetDataField(util.SSLMMSCAConfigMap, "mms-ca-config"). + SetDataField(util.OmOrgId, ""). + Build() + + mmsCAConfigMap := configmap.Builder(). + SetName("mms-ca-config"). + SetNamespace(mock.TestNamespace). + SetDataField(util.CaCertMMS, "cert text"). + Build() + + cert, _ := createMockCertAndKeyBytes() + tlsCAConfigMap := configmap.Builder(). + SetName("tls-ca-config"). + SetNamespace(mock.TestNamespace). + SetDataField("ca-pem", string(cert)). + Build() + + // create the secrets for all the shards + shardSecrets := createSecretsForShards(sc.Name, sc.Spec.ShardCount, certificatesSecretsPrefix) + + // create secrets for mongos + mongosSecrets := createMongosSecrets(sc.Name, certificatesSecretsPrefix) + + // create secrets for config server + configSrvSecrets := createConfigSrvSecrets(sc.Name, certificatesSecretsPrefix) + + // create `agent-certs` secret + agentCertsSecret := createAgentCertsSecret(sc.Name, certificatesSecretsPrefix) + + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(sc). + WithObjects(&projectConfigMap, &mmsCAConfigMap, &tlsCAConfigMap). + WithObjects(shardSecrets...). + WithObjects(mongosSecrets...). + WithObjects(configSrvSecrets...). + WithObjects(agentCertsSecret). + WithObjects(mock.GetCredentialsSecret(om.TestUser, om.TestApiKey)). + Build() + + kubeClient := kubernetesClient.NewClient(fakeClient) + memberClusterMap := getFakeMultiClusterMapWithConfiguredInterceptor(memberClusterNames, omConnectionFactory, true, false) + + ctx := context.Background() + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + allHostnames, _ := generateAllHosts(sc, mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + }) + + require.NoError(t, err) + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + for clusterIdx, clusterDef := range expectedClusterConfigList { + memberClusterChecks := newClusterChecks(t, clusterDef.Name, clusterIdx, sc.Namespace, memberClusterMap[clusterDef.Name]) + + memberClusterChecks.checkMMSCAConfigMap(ctx, "mms-ca-config") + memberClusterChecks.checkTLSCAConfigMap(ctx, "tls-ca-config") + memberClusterChecks.checkAgentAPIKeySecret(ctx, om.TestGroupID) + memberClusterChecks.checkAgentCertsSecret(ctx, certificatesSecretsPrefix, sc.Name) + + memberClusterChecks.checkMongosCertsSecret(ctx, certificatesSecretsPrefix, sc.Name, clusterDef.MongosMembers > 0) + memberClusterChecks.checkConfigSrvCertsSecret(ctx, certificatesSecretsPrefix, sc.Name, clusterDef.ConfigSrvMembers > 0) + + for shardIdx, shardMembers := range clusterDef.ShardsMembersArray { + memberClusterChecks.checkInternalClusterCertSecret(ctx, certificatesSecretsPrefix, sc.Name, shardIdx, shardMembers > 0) + memberClusterChecks.checkMemberCertSecret(ctx, certificatesSecretsPrefix, sc.Name, shardIdx, shardMembers > 0) + } + } +} + +func createSecretsForShards(resourceName string, shardCount int, certificatesSecretsPrefix string) []client.Object { + var shardSecrets []client.Object + for i := 0; i < shardCount; i++ { + shardData := make(map[string][]byte) + shardData["tls.crt"], shardData["tls.key"] = createMockCertAndKeyBytes() + + shardSecrets = append(shardSecrets, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s-%d-cert", certificatesSecretsPrefix, resourceName, i), Namespace: mock.TestNamespace}, + Data: shardData, + Type: corev1.SecretTypeTLS, + }) + + shardSecrets = append(shardSecrets, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s-%d-%s", certificatesSecretsPrefix, resourceName, i, util.ClusterFileName), Namespace: mock.TestNamespace}, + Data: shardData, + Type: corev1.SecretTypeTLS, + }) + } + return shardSecrets +} + +func createMongosSecrets(resourceName string, certificatesSecretsPrefix string) []client.Object { + mongosData := make(map[string][]byte) + mongosData["tls.crt"], mongosData["tls.key"] = createMockCertAndKeyBytes() + + // create the mongos secret + mongosSecretName := fmt.Sprintf("%s-%s-mongos-cert", certificatesSecretsPrefix, resourceName) + mongosSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: mongosSecretName, Namespace: mock.TestNamespace}, + Data: mongosData, + Type: corev1.SecretTypeTLS, + } + + mongosClusterFileSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s-mongos-%s", certificatesSecretsPrefix, resourceName, util.ClusterFileName), Namespace: mock.TestNamespace}, + Data: mongosData, + Type: corev1.SecretTypeTLS, + } + + return []client.Object{mongosSecret, mongosClusterFileSecret} +} + +func createConfigSrvSecrets(resourceName string, certificatesSecretsPrefix string) []client.Object { + configData := make(map[string][]byte) + configData["tls.crt"], configData["tls.key"] = createMockCertAndKeyBytes() + + configSrvSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s-config-cert", certificatesSecretsPrefix, resourceName), Namespace: mock.TestNamespace}, + Data: configData, + Type: corev1.SecretTypeTLS, + } + + configSrvClusterSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s-config-%s", certificatesSecretsPrefix, resourceName, util.ClusterFileName), Namespace: mock.TestNamespace}, + Data: configData, + Type: corev1.SecretTypeTLS, + } + + return []client.Object{configSrvSecret, configSrvClusterSecret} +} + +func createAgentCertsSecret(resourceName string, certificatesSecretsPrefix string) *corev1.Secret { + subjectModifier := func(cert *x509.Certificate) { + cert.Subject.OrganizationalUnit = []string{"cloud"} + cert.Subject.Locality = []string{"New York"} + cert.Subject.Province = []string{"New York"} + cert.Subject.Country = []string{"US"} + } + cert, key := createMockCertAndKeyBytes(subjectModifier, func(cert *x509.Certificate) { cert.Subject.CommonName = util.AutomationAgentName }) + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%s", certificatesSecretsPrefix, resourceName, util.AgentSecretName), + Namespace: mock.TestNamespace, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + "tls.crt": cert, + "tls.key": key, + }, + } +} + +func TestReconcileForComplexMultiClusterYaml(t *testing.T) { + ctx := context.Background() + sc, err := loadMongoDBResource("testdata/mdb-sharded-multi-cluster-complex.yaml") + require.NoError(t, err) + + cluster0 := "cluster-0" + cluster1 := "cluster-1" + cluster2 := "cluster-2" + clusterAnalytics := "cluster-analytics" + clusterAnalytics2 := "cluster-analytics-2" + memberClusterNames := []string{ + cluster0, + cluster1, + cluster2, + clusterAnalytics, + clusterAnalytics2, + } + + // expected distributions of shards are copied from testdata/mdb-sharded-multi-cluster-complex-expected-shardmap.yaml + shardDistribution := []map[string]int{ + { + cluster0: 2, + cluster1: 2, + cluster2: 1, + }, // shard 0 + { + cluster0: 1, + cluster1: 2, + cluster2: 3, + }, // shard 1 + { + cluster0: 2, + cluster1: 3, + cluster2: 0, + clusterAnalytics: 1, + clusterAnalytics2: 2, + }, // shard 2 + { + cluster0: 2, + cluster1: 2, + cluster2: 1, + }, // shard 3 + } + + // expected distributions of mongos and config srv are copied from testdata/mdb-sharded-multi-cluster-complex.yaml + mongosDistribution := map[string]int{ + cluster0: 1, + cluster1: 1, + cluster2: 0, + } + + configSrvDistribution := map[string]int{ + cluster0: 2, + cluster1: 2, + cluster2: 1, + } + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(sc) + memberClusterMap := getFakeMultiClusterMapWithClusters(memberClusterNames, omConnectionFactory) + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + require.NoError(t, err) + + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + hosts, _ := generateAllHosts(sc, mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(hosts) + }) + + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + expectedReplicaSets, err := loadExpectedReplicaSets("testdata/mdb-sharded-multi-cluster-complex-expected-replicasets.yaml") + require.NoError(t, err) + normalizedExpectedReplicaSets, err := normalizeObjectToInterfaceMap(expectedReplicaSets) + require.NoError(t, err) + automationConfig, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + require.NoError(t, err) + normalizedActualReplicaSets, err := normalizeObjectToInterfaceMap(map[string]any{"replicaSets": automationConfig.Deployment.ReplicaSets()}) + require.NoError(t, err) + if !assert.Equal(t, normalizedExpectedReplicaSets, normalizedActualReplicaSets) { + visualDiff, err := getVisualJsonDiff(normalizedExpectedReplicaSets, normalizedActualReplicaSets) + require.NoError(t, err) + fmt.Printf("\n%s\n", visualDiff) + } + + for shardIdx := 0; shardIdx < sc.Spec.ShardCount; shardIdx++ { + for clusterName, expectedMembersCount := range shardDistribution[shardIdx] { + memberClusterChecks := newClusterChecks(t, clusterName, clusterMapping[clusterName], sc.Namespace, memberClusterMap[clusterName]) + if expectedMembersCount > 0 { + memberClusterChecks.checkStatefulSet(ctx, sc.MultiShardRsName(clusterMapping[clusterName], shardIdx), expectedMembersCount) + } else { + memberClusterChecks.checkStatefulSetDoesNotExist(ctx, sc.MultiShardRsName(clusterMapping[clusterName], shardIdx)) + } + } + } + + for clusterName, expectedMembersCount := range mongosDistribution { + memberClusterChecks := newClusterChecks(t, clusterName, clusterMapping[clusterName], sc.Namespace, memberClusterMap[clusterName]) + if expectedMembersCount > 0 { + memberClusterChecks.checkStatefulSet(ctx, sc.MultiMongosRsName(clusterMapping[clusterName]), expectedMembersCount) + } else { + memberClusterChecks.checkStatefulSetDoesNotExist(ctx, sc.MultiMongosRsName(clusterMapping[clusterName])) + } + } + + for clusterName, expectedMembersCount := range configSrvDistribution { + memberClusterChecks := newClusterChecks(t, clusterName, clusterMapping[clusterName], sc.Namespace, memberClusterMap[clusterName]) + memberClusterChecks.checkStatefulSet(ctx, sc.MultiConfigRsName(clusterMapping[clusterName]), expectedMembersCount) + } + + expectedHostnameOverrideMap := createExpectedHostnameOverrideMap(sc, clusterMapping, mongosDistribution, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + for _, clusterName := range memberClusterNames { + memberClusterChecks := newClusterChecks(t, clusterName, clusterMapping[clusterName], sc.Namespace, memberClusterMap[clusterName]) + memberClusterChecks.checkHostnameOverrideConfigMap(ctx, fmt.Sprintf("%s-hostname-override", sc.Name), expectedHostnameOverrideMap) + } +} + +func generateAllHosts(sc *mdbv1.MongoDB, mongosDistribution map[string]int, clusterMapping map[string]int, configSrvDistribution map[string]int, shardDistribution []map[string]int, clusterDomain test.ClusterDomains, externalClusterDomain test.ClusterDomains) ([]string, []string) { + var allHosts []string + var allPodNames []string + podNames, hosts := generateHostsWithDistribution(sc.MongosRsName(), sc.Namespace, mongosDistribution, clusterMapping, clusterDomain.MongosExternalDomain, externalClusterDomain.MongosExternalDomain) + allHosts = append(allHosts, hosts...) + allPodNames = append(allPodNames, podNames...) + + podNames, hosts = generateHostsWithDistribution(sc.ConfigRsName(), sc.Namespace, configSrvDistribution, clusterMapping, clusterDomain.ConfigServerExternalDomain, externalClusterDomain.ConfigServerExternalDomain) + allHosts = append(allHosts, hosts...) + allPodNames = append(allPodNames, podNames...) + + for shardIdx := 0; shardIdx < sc.Spec.ShardCount; shardIdx++ { + podNames, hosts = generateHostsWithDistribution(sc.ShardRsName(shardIdx), sc.Namespace, shardDistribution[shardIdx], clusterMapping, clusterDomain.ShardsExternalDomain, externalClusterDomain.ShardsExternalDomain) + allHosts = append(allHosts, hosts...) + allPodNames = append(allPodNames, podNames...) + } + return allHosts, allPodNames +} + +func TestMigrateToNewDeploymentState(t *testing.T) { + ctx := context.Background() + + // These annotations should be preserved, but not appear in the migrated config map + initialAnnotations := map[string]string{ + "key1": "value1", + "key2": "value2", + } + sc := test.DefaultClusterBuilder(). + SetAnnotations(initialAnnotations). + Build() + + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(sc) + memberClusterMap := getFakeMultiClusterMapWithClusters([]string{multicluster.LegacyCentralClusterName}, omConnectionFactory) + + reconciler, _, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + + // Migration is performed at reconciliation, when needed + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + // Ensure that reconciliation generated the correct deployment state + configMapName := fmt.Sprintf("%s-state", sc.Name) + stateConfigMap := &corev1.ConfigMap{} + err = kubeClient.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: sc.Namespace}, stateConfigMap) + require.NoError(t, err) + + expectedDeploymentState := generateExpectedDeploymentState(t, sc) + require.Contains(t, stateConfigMap.Data, stateKey) + require.JSONEq(t, expectedDeploymentState, stateConfigMap.Data[stateKey]) + + // Original annotations must be preserved + updatedSc := &mdbv1.MongoDB{} + err = kubeClient.Get(ctx, types.NamespacedName{Name: sc.Name, Namespace: sc.Namespace}, updatedSc) + require.NoError(t, err) + for key, value := range initialAnnotations { + require.Equal(t, value, updatedSc.Annotations[key], "Annotation %s should be preserved", key) + } + + // Verify that we also store the state in the annotations (everything below) + // This way, downgrading the operator is possible without breaking the state + require.Contains(t, updatedSc.Annotations, util.LastAchievedSpec) + actualLastAchievedSpec := updatedSc.Annotations[util.LastAchievedSpec] + + var configMapData, actualLastAchievedSpecData map[string]interface{} + // Deserialize the JSON data from the annotation + err = json.Unmarshal([]byte(actualLastAchievedSpec), &actualLastAchievedSpecData) + require.NoError(t, err) + + // Extract lastAchievedSpec from the state Config Map + err = json.Unmarshal([]byte(stateConfigMap.Data[stateKey]), &configMapData) + require.NoError(t, err) + expectedLastAchievedSpec, ok := configMapData["lastAchievedSpec"].(map[string]interface{}) + require.True(t, ok, "Expected lastAchievedSpec field is missing or invalid") + + require.Equal(t, expectedLastAchievedSpec, actualLastAchievedSpecData) +} + +// Without genericity and type hinting, when unmarshalling the file in a struct, fields that should be omitted when empty +// are not, and the actual/expected configurations are not compared correctly +func testDesiredConfigurationFromYAML[T *mdbv1.ShardedClusterComponentSpec | map[int]*mdbv1.ShardedClusterComponentSpec](t *testing.T, mongoDBResourceFile string, expectedConfigurationFile string, shardedComponentType string) { + ctx := context.Background() + sc, err := loadMongoDBResource(mongoDBResourceFile) + require.NoError(t, err) + + memberClusterNames := []string{"cluster-0", "cluster-1", "cluster-2", "cluster-analytics", "cluster-analytics-2"} + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(sc) + memberClusterMap := getFakeMultiClusterMapWithClusters(memberClusterNames, omConnectionFactory) + + _, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + + var actual interface{} + // no reconcile here, we just test prepareDesiredConfiguration + switch shardedComponentType { + case "shard": + actual = reconcilerHelper.prepareDesiredShardsConfiguration() + case "config": + actual = reconcilerHelper.prepareDesiredConfigServerConfiguration() + case "mongos": + actual = reconcilerHelper.prepareDesiredMongosConfiguration() + } + + expected, err := unmarshalYamlFileInStruct[T](expectedConfigurationFile) + require.NoError(t, err) + + normalizedActual, err := normalizeObjectToInterfaceMap(actual) + require.NoError(t, err) + normalizedExpected, err := normalizeObjectToInterfaceMap(expected) + require.NoError(t, err) + + assert.Equal(t, normalizedExpected, normalizedActual) + visualDiff, err := getVisualJsonDiff(normalizedExpected, normalizedActual) + require.NoError(t, err) + + if !assert.Empty(t, visualDiff) { + // it is extremely difficult to diagnose problems in IDE's console as the diff dump is very large >400 lines, + // therefore we're saving visual diffs in ops-manager-kubernetes/tmp dir to a temp file + tmpFile, err := os.CreateTemp(path.Join(os.Getenv("PROJECT_DIR"), "tmp"), "jsondiff") // nolint:forbidigo + if err != nil { + // ignore the error, it's not part of the actual test + fmt.Printf("error saving diff to tmp file: %v", err) + } else { + if tmpFile != nil { + defer func() { _ = tmpFile.Close() }() + } + _, _ = tmpFile.WriteString(visualDiff) + if tmpFile != nil { + fmt.Printf("Diff written to %s\n", tmpFile.Name()) + } + } + } +} + +// Multi-Cluster +func TestShardMapForComplexMultiClusterYaml(t *testing.T) { + testDesiredConfigurationFromYAML[map[int]*mdbv1.ShardedClusterComponentSpec](t, "testdata/mdb-sharded-multi-cluster-complex.yaml", "testdata/mdb-sharded-multi-cluster-complex-expected-shardmap.yaml", "shard") +} + +// Config servers and Mongos share a lot of logic, and have the same settings in CRDs, the two below tests use the same files, and are almost identical +func TestConfigServerdExpectedConfigFromMultiClusterYaml(t *testing.T) { + testDesiredConfigurationFromYAML[*mdbv1.ShardedClusterComponentSpec](t, "testdata/mdb-sharded-multi-cluster-configsrv-mongos.yaml", "testdata/mdb-sharded-multi-cluster-configsrv-mongos-expected-config.yaml", "config") +} + +func TestMongosExpectedConfigFromMultiClusterYaml(t *testing.T) { + testDesiredConfigurationFromYAML[*mdbv1.ShardedClusterComponentSpec](t, "testdata/mdb-sharded-multi-cluster-configsrv-mongos.yaml", "testdata/mdb-sharded-multi-cluster-configsrv-mongos-expected-config.yaml", "mongos") +} + +// Single-Cluster +func TestShardMapForSingleClusterWithOverridesYaml(t *testing.T) { + testDesiredConfigurationFromYAML[map[int]*mdbv1.ShardedClusterComponentSpec](t, "testdata/mdb-sharded-single-with-overrides.yaml", "testdata/mdb-sharded-single-with-overrides-expected-shardmap.yaml", "shard") +} + +// Config servers and Mongos share a lot of logic, and have the same settings in CRDs, the two below tests use the same files, and are almost identical +func TestConfigServerdExpectedConfigFromSingleClusterYaml(t *testing.T) { + testDesiredConfigurationFromYAML[*mdbv1.ShardedClusterComponentSpec](t, "testdata/mdb-sharded-single-cluster-configsrv-mongos.yaml", "testdata/mdb-sharded-single-cluster-configsrv-mongos-expected-config.yaml", "config") +} + +func TestMongosExpectedConfigFromSingleClusterYaml(t *testing.T) { + testDesiredConfigurationFromYAML[*mdbv1.ShardedClusterComponentSpec](t, "testdata/mdb-sharded-single-cluster-configsrv-mongos.yaml", "testdata/mdb-sharded-single-cluster-configsrv-mongos-expected-config.yaml", "mongos") +} + +func TestMultiClusterShardedSetRace(t *testing.T) { + cluster1 := "cluster-member-1" + cluster2 := "cluster-member-2" + + memberClusterNames := []string{ + cluster1, + cluster2, + } + + shardCount := 2 + // Two Kubernetes clusters, 2 replicaset members of each shard on the first one, 3 on the second one + // This means a MongodPerShardCount of 5 + shardDistribution := []map[string]int{ + {cluster1: 2, cluster2: 3}, + {cluster1: 2, cluster2: 3}, + } + shardClusterSpecList := test.CreateClusterSpecList(memberClusterNames, shardDistribution[0]) + + // For Mongos and Config servers, 2 replicaset members on the first one, 1 on the second one + mongosDistribution := map[string]int{cluster1: 2, cluster2: 1} + mongosAndConfigSrvClusterSpecList := test.CreateClusterSpecList(memberClusterNames, mongosDistribution) + + configSrvDistribution := map[string]int{cluster1: 2, cluster2: 1} + configSrvDistributionClusterSpecList := test.CreateClusterSpecList(memberClusterNames, configSrvDistribution) + + sc, cfgMap, projectName := buildShardedClusterWithCustomProjectName("mc-sharded", shardCount, shardClusterSpecList, mongosAndConfigSrvClusterSpecList, configSrvDistributionClusterSpecList) + sc1, cfgMap1, projectName1 := buildShardedClusterWithCustomProjectName("mc-sharded-1", shardCount, shardClusterSpecList, mongosAndConfigSrvClusterSpecList, configSrvDistributionClusterSpecList) + sc2, cfgMap2, projectName2 := buildShardedClusterWithCustomProjectName("mc-sharded-2", shardCount, shardClusterSpecList, mongosAndConfigSrvClusterSpecList, configSrvDistributionClusterSpecList) + + resourceToProjectMapping := map[string]string{ + "mc-sharded": projectName, + "mc-sharded-1": projectName1, + "mc-sharded-2": projectName2, + } + + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(sc, sc1, sc2). + WithObjects(cfgMap, cfgMap1, cfgMap2). + WithObjects(mock.GetCredentialsSecret(om.TestUser, om.TestApiKey)). + Build() + + kubeClient := kubernetesClient.NewClient(fakeClient) + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory().WithResourceToProjectMapping(resourceToProjectMapping) + globalMemberClustersMap := getFakeMultiClusterMapWithConfiguredInterceptor(memberClusterNames, omConnectionFactory, true, false) + + ctx := context.Background() + reconciler := newShardedClusterReconciler(ctx, kubeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, globalMemberClustersMap, omConnectionFactory.GetConnectionFunc) + + allHostnames := generateHostsForCluster(ctx, reconciler, false, sc, mongosDistribution, configSrvDistribution, shardDistribution) + allHostnames1 := generateHostsForCluster(ctx, reconciler, false, sc1, mongosDistribution, configSrvDistribution, shardDistribution) + allHostnames2 := generateHostsForCluster(ctx, reconciler, false, sc2, mongosDistribution, configSrvDistribution, shardDistribution) + + projectHostMapping := map[string][]string{ + projectName: allHostnames, + projectName1: allHostnames1, + projectName2: allHostnames2, + } + + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + hostnames := projectHostMapping[connection.GroupName()] + connection.(*om.MockedOmConnection).AddHosts(hostnames) + }) + + testConcurrentReconciles(ctx, t, fakeClient, reconciler, sc, sc1, sc2) +} + +// TODO extract this, please don't review +func TestMultiClusterShardedMongosDeadlock(t *testing.T) { + t.Skip("The test is not finished and will fail") + /* + + + waitForReadyState: we wait on all the process, even if they are down + BUT even if we fix that + mongos will report ready only if all down processes are removed from the project + + + + We need to mock automationStatus (wait for goal/ready state) and agentStatus (wait for agents to be registered) + + === AutomationStatus: list of processes + + JSON + "hostname": "sh-disaster-recovery-mongos-2-0-svc.mongodb-test.svc.cluster.local", + "lastGoalVersionAchieved": 8, + "name": "sh-disaster-recovery-mongos-2-0", + + GO struct + // AutomationStatus represents the status of automation agents registered with Ops Manager + type AutomationStatus struct { + GoalVersion int `json:"goalVersion"` + Processes []ProcessStatus `json:"processes"` + } + + // ProcessStatus status of the process and what's the last version achieved + type ProcessStatus struct { + Hostname string `json:"hostname"` + Name string `json:"name"` + LastGoalVersionAchieved int `json:"lastGoalVersionAchieved"` + Plan []string `json:"plan"` + } + + ReadAutomationStatus is built from deployment lists + + + + WaitForReadyState fetches the automation status -> checkAutomationStatusIsGoal -> + if p.LastGoalVersionAchieved == as.GoalVersion { + goalsAchievedMap[p.Name] = p.LastGoalVersionAchieved + + + + === AutomationAgentStatus: List of + + type AgentStatus struct { + ConfCount int `json:"confCount"` + Hostname string `json:"hostname"` + LastConf string `json:"lastConf"` + StateName string `json:"stateName"` + TypeName string `json:"typeName"` + } + + automationStatus.json + + "results": [ + { + "confCount": 24683, + "hostname": "sh-disaster-recovery-0-0-0-svc.mongodb-test.svc.cluster.local", + "lastConf": "2025-01-24T09:48:58Z", + "stateName": "ACTIVE", + "typeName": "AUTOMATION" + }, + + ReadAutomationAgents returns automation agent status based on what is in + AgentStatus{Hostname: r.Hostname, LastConf: time.Now().Add(time.Second * -1).Format(time.RFC3339)}) + + Results []Host + type Host struct { + Username string `json:"username"` + Hostname string `json:"hostname"` + [...] + } + + What is in results is not necessarily relevant, but we need to extend what the method puts in AgentStatus + + */ + + ctx := context.Background() + cluster1 := "member-cluster-1" + cluster2 := "member-cluster-2" + cluster3 := "member-cluster-3" + memberClusterNames := []string{ + cluster1, + cluster2, + cluster3, + } + + mongosDistribution := map[string]int{cluster1: 1, cluster2: 0, cluster3: 2} + shardDistribution := map[string]int{cluster1: 2, cluster2: 1, cluster3: 2} + shardFullDistribution := []map[string]int{ + {cluster1: 2, cluster2: 1, cluster3: 2}, + {cluster1: 2, cluster2: 1, cluster3: 2}, + } + configServerDistribution := shardDistribution + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + omConnection := omConnectionFactory.GetConnectionFunc(&om.OMContext{GroupName: om.TestGroupName}) + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + + // TODO: remove cluster 2 from config map + memberClusterMap := getFakeMultiClusterMapWithoutInterceptor(memberClusterNames) + var memberClusterClients []client.Client + for _, c := range memberClusterMap { + memberClusterClients = append(memberClusterClients, c) + } + + sc := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(2). + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + SetShardClusterSpec(test.CreateClusterSpecList(memberClusterNames, shardDistribution)). + SetConfigSrvClusterSpec(test.CreateClusterSpecList(memberClusterNames, configServerDistribution)). + SetMongosClusterSpec(test.CreateClusterSpecList(memberClusterNames, mongosDistribution)). + Build() + + sc.Name = "sh-disaster-recovery" + + err := kubeClient.Create(ctx, sc) + require.NoError(t, err) + + addAllHostsWithDistribution := func(connection om.Connection, mongosDistribution map[string]int, clusterMapping map[string]int, configSrvDistribution map[string]int, shardDistribution []map[string]int) { + allHostnames, _ := generateAllHosts(sc, mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + } + + //We need to mock automation status: + //- Config servers 2 1 2 + //- Mongos 1 0 2 + //- (2) Shards 2 1 2 + // + //Cluster with index 2 is down + + deadlockedGoalVersion := 13 + deadlockedProcesses := []om.ProcessStatus{ + // Mongos + { + Hostname: "sh-disaster-recovery-mongos-2-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-mongos-2-0", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-mongos-2-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-mongos-2-1", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-mongos-0-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-mongos-0-0", + LastGoalVersionAchieved: deadlockedGoalVersion - 1, // Cluster up but mongos deadlock, cannot reach goal version + Plan: []string{agents.RollingChangeArgs}, + }, + + // Config server + // Cluster 0 + { + Hostname: "sh-disaster-recovery-config-0-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-config-0-0", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-config-0-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-config-0-1", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + // Cluster 1 + { + Hostname: "sh-disaster-recovery-config-1-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-config-1-0", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + // Cluster 2 + { + Hostname: "sh-disaster-recovery-config-2-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-config-2-0", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, // Cluster 2 down, not ready + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-config-2-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-config-2-1", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, // Cluster 2 down, not ready + Plan: []string{}, + }, + + // Shards + // Shard 0 + { + Hostname: "sh-disaster-recovery-0-0-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-0-0-0", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-0-1-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-0-1-0", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-0-1-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-0-1-1", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-0-2-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-0-2-0", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, // Cluster 2 down, not ready + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-0-2-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-0-2-1", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, // Cluster 2 down, not ready + Plan: []string{}, + }, + + // Shard 1 + { + Hostname: "sh-disaster-recovery-1-0-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-1-0-0", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-1-1-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-1-1-0", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-1-1-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-1-1-1", + LastGoalVersionAchieved: deadlockedGoalVersion, + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-1-2-0-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-1-2-0", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, // Cluster 2 down, not ready + Plan: []string{}, + }, + { + Hostname: "sh-disaster-recovery-1-2-1-svc.my-namespace.svc.cluster.local", + Name: "sh-disaster-recovery-1-2-1", + LastGoalVersionAchieved: deadlockedGoalVersion - 5, // Cluster 2 down, not ready + Plan: []string{}, + }, + } + + deadlockedAutomationStatus := om.AutomationStatus{ + GoalVersion: deadlockedGoalVersion, + Processes: deadlockedProcesses, + } + + readyTimestamp := time.Now().Add(-20 * time.Second).Format(time.RFC3339) + notReadyTimestamp := time.Now().Add(-100 * time.Second).Format(time.RFC3339) + + // An agent is considered registered if its last ping was <1min ago + deadLockedAgentStatus := []om.AgentStatus{ + // Mongos + { + Hostname: "sh-disaster-recovery-mongos-0-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-mongos-2-0-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-mongos-2-1-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + + // Config server + // Cluster 0 + { + Hostname: "sh-disaster-recovery-config-0-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-config-0-1-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + // Cluster 1 + { + Hostname: "sh-disaster-recovery-config-1-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + // Cluster 2 + { + Hostname: "sh-disaster-recovery-config-2-0-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-config-2-1-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + + // Shards + // Shard 0 + { + Hostname: "sh-disaster-recovery-0-0-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-0-1-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-0-1-1-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-0-2-0-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-0-2-1-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + + // Shard 1 + { + Hostname: "sh-disaster-recovery-1-0-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-1-1-0-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-1-1-1-svc.my-namespace.svc.cluster.local", + LastConf: readyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-1-2-0-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + { + Hostname: "sh-disaster-recovery-1-2-1-svc.my-namespace.svc.cluster.local", + LastConf: notReadyTimestamp, + }, + } + + omConnection.(*om.MockedOmConnection).ReadAutomationStatusFunc = func() (*om.AutomationStatus, error) { + return &deadlockedAutomationStatus, nil + } + + omConnection.(*om.MockedOmConnection).ReadAutomationAgentsFunc = func(int) (om.Paginated, error) { + response := om.AutomationAgentStatusResponse{ + OMPaginated: om.OMPaginated{ + TotalCount: 1, + Links: nil, + }, + AutomationAgents: deadLockedAgentStatus, + } + return response, nil + } + + // TODO: statuses in OM mock + // TODO: OM mock: set agent ready depending on a clusterDown parameter ? + set mongos not ready if anything is not ready + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configServerDistribution, shardFullDistribution) + + err = kubeClient.Get(ctx, mock.ObjectKeyFromApiObject(sc), sc) + require.NoError(t, err) + reconcileUntilSuccessful(ctx, t, reconciler, sc, kubeClient, memberClusterClients, nil, false) + + // End of reconciliation, verify state is as expected +} + +func TestCheckForMongosDeadlock(t *testing.T) { + type CheckForMongosDeadlockTestCase struct { + name string + clusterState agents.MongoDBClusterStateInOM + mongosReplicaSetName string + isScaling bool + expectedDeadlock bool + expectedProcessStatesSize int + } + + goalVersion := 3 + mongosReplicaSetName := "mongos-" + // Processes are considered stale if the last agent ping is >2 min + healthyPingTime := time.Now().Add(-20 * time.Second) + unHealthyPingTime := time.Now().Add(-200 * time.Second) + + testCases := []CheckForMongosDeadlockTestCase{ + { + name: "Mongos Deadlock", + clusterState: agents.MongoDBClusterStateInOM{ + GoalVersion: goalVersion, + ProcessStateMap: map[string]agents.ProcessState{ + "1": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion - 1, + Plan: []string{agents.RollingChangeArgs}, + ProcessName: mongosReplicaSetName, + }, + "2": { + Hostname: "", + LastAgentPing: unHealthyPingTime, // We need at least one stale process + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + "3": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + }, + }, + mongosReplicaSetName: mongosReplicaSetName, + isScaling: true, + expectedDeadlock: true, + expectedProcessStatesSize: 1, + }, + { + name: "Unhealthy mongos", + clusterState: agents.MongoDBClusterStateInOM{ + GoalVersion: goalVersion, + ProcessStateMap: map[string]agents.ProcessState{ + "1": { + Hostname: "", + LastAgentPing: unHealthyPingTime, + GoalVersionAchieved: goalVersion - 1, + Plan: []string{agents.RollingChangeArgs}, + ProcessName: mongosReplicaSetName, + }, + "2": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + "3": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + }, + }, + mongosReplicaSetName: mongosReplicaSetName, + isScaling: true, + expectedDeadlock: false, + expectedProcessStatesSize: 0, + }, + { + name: "Other process not in goal state", + clusterState: agents.MongoDBClusterStateInOM{ + GoalVersion: goalVersion, + ProcessStateMap: map[string]agents.ProcessState{ + "1": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion - 1, + Plan: []string{agents.RollingChangeArgs}, + ProcessName: mongosReplicaSetName, + }, + "2": { + Hostname: "", + LastAgentPing: unHealthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + "3": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion - 1, + Plan: nil, + ProcessName: "shard", + }, + }, + }, + mongosReplicaSetName: mongosReplicaSetName, + isScaling: true, + expectedDeadlock: false, + expectedProcessStatesSize: 0, + }, + { + name: "Not scaling", + clusterState: agents.MongoDBClusterStateInOM{ + GoalVersion: goalVersion, + ProcessStateMap: map[string]agents.ProcessState{ + "1": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion - 1, + Plan: []string{agents.RollingChangeArgs}, + ProcessName: mongosReplicaSetName, + }, + "2": { + Hostname: "", + LastAgentPing: unHealthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + "3": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + }, + }, + mongosReplicaSetName: mongosReplicaSetName, + isScaling: false, + expectedDeadlock: false, + expectedProcessStatesSize: 0, + }, + { + name: "All healthy mongos in goal state", + clusterState: agents.MongoDBClusterStateInOM{ + GoalVersion: goalVersion, + ProcessStateMap: map[string]agents.ProcessState{ + "1": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: []string{agents.RollingChangeArgs}, + ProcessName: mongosReplicaSetName, + }, + "2": { + Hostname: "", + LastAgentPing: unHealthyPingTime, + GoalVersionAchieved: goalVersion - 1, + Plan: nil, + ProcessName: mongosReplicaSetName, + }, + "3": { + Hostname: "", + LastAgentPing: unHealthyPingTime, // We need at least one stale process + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + "4": { + Hostname: "", + LastAgentPing: healthyPingTime, + GoalVersionAchieved: goalVersion, + Plan: nil, + ProcessName: "shard", + }, + }, + }, + mongosReplicaSetName: mongosReplicaSetName, + isScaling: true, + expectedDeadlock: false, + expectedProcessStatesSize: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + isDeadLocked, processStates := checkForMongosDeadlock(tc.clusterState, tc.mongosReplicaSetName, tc.isScaling, zap.S()) + assert.Equal(t, tc.expectedDeadlock, isDeadLocked) + assert.Equal(t, tc.expectedProcessStatesSize, len(processStates)) + }) + } +} + +func computeShardOverridesFromDistribution(shardOverridesDistribution []map[string]int) []mdbv1.ShardOverride { + var shardOverrides []mdbv1.ShardOverride + + // This will create shard overrides for shards 0...len(shardOverridesDistribution-1), shardCount can be greater + for i, distribution := range shardOverridesDistribution { + // Cluster builder has slaney as default name + shardName := test.SCBuilderDefaultName + "-" + strconv.Itoa(i) + + // Build the ClusterSpecList for the current shard + var clusterSpecList []mdbv1.ClusterSpecItemOverride + for clusterName, members := range distribution { + clusterSpecList = append(clusterSpecList, mdbv1.ClusterSpecItemOverride{ + ClusterName: clusterName, + Members: ptr.To(members), + }) + } + + // Construct the ShardOverride for the current shard + shardOverride := mdbv1.ShardOverride{ + ShardNames: []string{shardName}, + ShardedClusterComponentOverrideSpec: mdbv1.ShardedClusterComponentOverrideSpec{ + ClusterSpecList: clusterSpecList, + }, + } + + // Append the constructed ShardOverride to the shardOverrides slice + shardOverrides = append(shardOverrides, shardOverride) + } + return shardOverrides +} + +type MultiClusterShardedScalingTestCase struct { + name string + scalingSteps []MultiClusterShardedScalingStep +} + +type MultiClusterShardedScalingStep struct { + name string + shardCount int + shardDistribution map[string]int + configServerDistribution map[string]int + mongosDistribution map[string]int + shardOverrides []map[string]int + expectedShardDistribution []map[string]int +} + +func MultiClusterShardedScalingWithOverridesTestCase(t *testing.T, tc MultiClusterShardedScalingTestCase) { + ctx := context.Background() + cluster1 := "member-cluster-1" + cluster2 := "member-cluster-2" + cluster3 := "member-cluster-3" + memberClusterNames := []string{ + cluster1, + cluster2, + cluster3, + } + + mongosDistribution := map[string]int{cluster2: 1} + configSrvDistribution := map[string]int{cluster3: 1} + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + _ = omConnectionFactory.GetConnectionFunc(&om.OMContext{GroupName: om.TestGroupName}) + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + + memberClusterMap := getFakeMultiClusterMapWithoutInterceptor(memberClusterNames) + var memberClusterClients []client.Client + for _, c := range memberClusterMap { + memberClusterClients = append(memberClusterClients, c) + } + + sc := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(tc.scalingSteps[0].shardCount). + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + SetShardClusterSpec(test.CreateClusterSpecList(memberClusterNames, tc.scalingSteps[0].shardDistribution)). + SetConfigSrvClusterSpec(test.CreateClusterSpecList(memberClusterNames, configSrvDistribution)). + SetMongosClusterSpec(test.CreateClusterSpecList(memberClusterNames, mongosDistribution)). + SetShardOverrides(computeShardOverridesFromDistribution(tc.scalingSteps[0].shardOverrides)). + Build() + + err := kubeClient.Create(ctx, sc) + require.NoError(t, err) + + addAllHostsWithDistribution := func(connection om.Connection, mongosDistribution map[string]int, clusterMapping map[string]int, configSrvDistribution map[string]int, shardDistribution []map[string]int) { + allHostnames, _ := generateAllHosts(sc, mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + } + + for _, scalingStep := range tc.scalingSteps { + t.Run(scalingStep.name, func(t *testing.T) { + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + + err = kubeClient.Get(ctx, mock.ObjectKeyFromApiObject(sc), sc) + require.NoError(t, err) + sc.Spec.ShardCount = scalingStep.shardCount + sc.Spec.ShardSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, scalingStep.shardDistribution) + sc.Spec.ShardOverrides = computeShardOverridesFromDistribution(scalingStep.shardOverrides) + + // Hosts must be added after updating the spec because the function below depends on spec.ShardCount + // to generate the shard distribution. + // We pass the *expected* distribution as a parameter, ensuring that all hosts expected to be registered + // by the end of the full reconciliation process are added to OM. + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configSrvDistribution, scalingStep.expectedShardDistribution) + + err = kubeClient.Update(ctx, sc) + require.NoError(t, err) + reconcileUntilSuccessful(ctx, t, reconciler, sc, kubeClient, memberClusterClients, nil, false) + + // Verify scaled deployment + checkCorrectShardDistributionInStatefulSets(t, ctx, sc, clusterMapping, memberClusterMap, scalingStep.expectedShardDistribution) + }) + } +} + +func TestMultiClusterShardedScalingWithOverrides(t *testing.T) { + cluster1 := "member-cluster-1" + cluster2 := "member-cluster-2" + cluster3 := "member-cluster-3" + testCases := []MultiClusterShardedScalingTestCase{ + { + name: "Scale down shard in cluster1", + scalingSteps: []MultiClusterShardedScalingStep{ + { + name: "Initial scaling without overrides", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + }, + }, + { + name: "Scale down a shard, add an override", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 0, cluster2: 1, cluster3: 1, // cluster1: 1-0 + }, + shardOverrides: []map[string]int{ + {cluster1: 0, cluster2: 1, cluster3: 1}, // no changes in scaling + {cluster1: 1, cluster2: 1, cluster3: 0}, // cluster3: 1->3 + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 0, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 0}, + {cluster1: 0, cluster2: 1, cluster3: 1}, + }, + }, + }, + }, + { + name: "Scale up from zero members", + scalingSteps: []MultiClusterShardedScalingStep{ + { + name: "Initial scaling without overrides", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 0, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 0}, + {cluster1: 1, cluster2: 1, cluster3: 0}, + {cluster1: 1, cluster2: 1, cluster3: 0}, + }, + }, + { + name: "Scale up from zero members", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, // cluster3: 0->1 + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + }, + }, + }, + }, + { + name: "Scale up from zero members using shard overrides", + scalingSteps: []MultiClusterShardedScalingStep{ + { + name: "Initial scaling with overrides", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + shardOverrides: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 0}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 0}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + }, + }, + { + name: "Scale up from zero members using shard override", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + shardOverrides: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 1}, // cluster3: 0->1 + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + {cluster1: 1, cluster2: 1, cluster3: 1}, + }, + }, + }, + }, + { + name: "All shards contain overrides", + scalingSteps: []MultiClusterShardedScalingStep{ + { + name: "Deploy with overrides on all shards", + shardCount: 2, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + {cluster1: 1, cluster2: 2, cluster3: 3}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + {cluster1: 1, cluster2: 2, cluster3: 3}, + }, + }, + { + name: "Scale shards", + shardCount: 2, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, // we don't change the default distribution + }, + shardOverrides: []map[string]int{ + {cluster1: 0, cluster2: 2, cluster3: 1}, // cluster1: 3->0 + {cluster1: 1, cluster2: 2, cluster3: 0}, // cluster3: 3->0 + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 0, cluster2: 2, cluster3: 1}, + {cluster1: 1, cluster2: 2, cluster3: 0}, + }, + }, + { + name: "Scale zero to one in one shard override", + shardCount: 2, + shardDistribution: map[string]int{ + cluster1: 1, cluster2: 1, cluster3: 1, // we don't change the default distribution + }, + shardOverrides: []map[string]int{ + {cluster1: 0, cluster2: 3, cluster3: 1}, + {cluster1: 3, cluster2: 2, cluster3: 1}, // {cluster3: 0->1}; + }, + /* + slaney-1-0-1-svc.my-namespace.svc.cluster.local + slaney-1-0-2-svc.my-namespace.svc.cluster.local + + slaney-1-1-0-svc.my-namespace.svc.cluster.local + slaney-1-1-1-svc.my-namespace.svc.cluster.local + + slaney-1-2-0-svc.my-namespace.svc.cluster.local + slaney-1-2-1-svc.my-namespace.svc.cluster.local + slaney-1-2-2-svc.my-namespace.svc.cluster.local + + */ + expectedShardDistribution: []map[string]int{ + {cluster1: 0, cluster2: 3, cluster3: 1}, + {cluster1: 3, cluster2: 2, cluster3: 1}, + }, + }, + // This scaling step test an edge case: when all shards contain overrides, the mongod distribution is + // empty in the status (sizeStatusInClusters) + { + name: "Add a shard", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 3, cluster3: 3, // We set default distribution to 3 + }, + shardOverrides: []map[string]int{ + {cluster1: 0, cluster2: 3, cluster3: 1}, + {cluster1: 3, cluster2: 2, cluster3: 1}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 0, cluster2: 3, cluster3: 1}, + {cluster1: 3, cluster2: 2, cluster3: 1}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + }, + }, + { + name: "Remove one override", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 3, cluster3: 3, + }, + shardOverrides: []map[string]int{ + {cluster1: 0, cluster2: 3, cluster3: 1}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 0, cluster2: 3, cluster3: 1}, + {cluster1: 3, cluster2: 3, cluster3: 3}, // This shard should scale to the default distribution + {cluster1: 3, cluster2: 3, cluster3: 3}, + }, + }, + }, + }, + { + // In this test, we try adding shards after the initial deployment. We expect the sizeStatusInClusters + // field to be incorrect, as it will be set to 3 3 3, but is shared between all shards. + // When adding a new shard, operator will think it is scaled to this distribution, while in reality it + // has no replicas + // We use an override, but this edge case doesn't need one to happen + name: "Add shards after deployment", + scalingSteps: []MultiClusterShardedScalingStep{ + { + name: "Initial deployment", + shardCount: 2, + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 3, cluster3: 3, + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + }, + }, + { + name: "Add two shards", + shardCount: 4, // We only change shardCount + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 3, cluster3: 3, + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + }, + }, + { + name: "Remove one shard", + shardCount: 3, // We only change shardCount + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 3, cluster3: 3, + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 3, cluster2: 2, cluster3: 1}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + {cluster1: 3, cluster2: 3, cluster3: 3}, + }, + }, + { + name: "Re-scale", + shardCount: 3, + shardDistribution: map[string]int{ + cluster1: 3, cluster2: 1, cluster3: 1, // Scale down base shards + }, + shardOverrides: []map[string]int{ + {cluster1: 3, cluster2: 1, cluster3: 1}, // Scale down override on cluter 2 + }, + expectedShardDistribution: []map[string]int{ + {cluster1: 3, cluster2: 1, cluster3: 1}, + {cluster1: 3, cluster2: 1, cluster3: 1}, + {cluster1: 3, cluster2: 1, cluster3: 1}, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + MultiClusterShardedScalingWithOverridesTestCase(t, tc) + }) + } +} + +func TestMultiClusterShardedScaling(t *testing.T) { + cluster1 := "member-cluster-1" + cluster2 := "member-cluster-2" + cluster3 := "member-cluster-3" + memberClusterNames := []string{ + cluster1, + cluster2, + cluster3, + } + + shardCount := 2 + shardDistribution := []map[string]int{ + {cluster1: 1, cluster2: 2}, + {cluster1: 1, cluster2: 2}, + } + mongosDistribution := map[string]int{cluster2: 1} + configSrvDistribution := map[string]int{cluster3: 1} + + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(shardCount). + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + SetShardClusterSpec(test.CreateClusterSpecList(memberClusterNames, shardDistribution[0])). + SetConfigSrvClusterSpec(test.CreateClusterSpecList(memberClusterNames, configSrvDistribution)). + SetMongosClusterSpec(test.CreateClusterSpecList(memberClusterNames, mongosDistribution)). + Build() + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + fakeClient := mock.NewEmptyFakeClientBuilder().WithObjects(sc).WithObjects(mock.GetDefaultResources()...).Build() + kubeClient := kubernetesClient.NewClient(fakeClient) + + memberClusterMap := getFakeMultiClusterMapWithoutInterceptor(memberClusterNames) + var memberClusterClients []client.Client + for _, c := range memberClusterMap { + memberClusterClients = append(memberClusterClients, c) + } + + reconciler, reconcilerHelper, err := newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + addAllHostsWithDistribution := func(connection om.Connection, mongosDistribution map[string]int, clusterMapping map[string]int, configSrvDistribution map[string]int, shardDistribution []map[string]int) { + allHostnames, _ := generateAllHosts(sc, mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + } + + // first reconciler run is with failure, we didn't yet add hosts to OM + // we do this just to initialize omConnectionFactory to contain a mock connection + _, err = reconciler.Reconcile(ctx, requestFromObject(sc)) + require.NoError(t, err) + require.NoError(t, mock.MarkAllStatefulSetsAsReady(ctx, sc.Namespace, memberClusterClients...)) + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution) + reconcileUntilSuccessful(ctx, t, reconciler, sc, kubeClient, memberClusterClients, nil, false) + + // Ensure that reconciliation generated the correct deployment state + checkCorrectShardDistributionInStatus(t, sc) + + // 1 successful reconcile finished, we have initial scaling done and Phase=Running + + // shardDistribution := []map[string]int{ + // {cluster1: 1, cluster2: 2}, + // {cluster1: 1, cluster2: 2}, + // } + // add two members to each shard + shardDistribution = []map[string]int{ + {cluster3: 2, cluster1: 1, cluster2: 2}, + {cluster3: 2, cluster1: 1, cluster2: 2}, + } + // add two mongos + // mongosDistribution := map[string]int{cluster2: 1} + mongosDistribution = map[string]int{cluster1: 0, cluster2: 1, cluster3: 2} + // add two config servers + // configSrvDistribution := map[string]int{cluster3: 1} + configSrvDistribution = map[string]int{cluster1: 2, cluster3: 1} + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution) + + err = kubeClient.Get(ctx, mock.ObjectKeyFromApiObject(sc), sc) + require.NoError(t, err) + sc.Spec.ConfigSrvSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, configSrvDistribution) + sc.Spec.ShardSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, shardDistribution[0]) + sc.Spec.MongosSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, mongosDistribution) + err = kubeClient.Update(ctx, sc) + require.NoError(t, err) + + reconciler, reconcilerHelper, err = newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + clusterMapping = reconcilerHelper.deploymentState.ClusterMapping + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution) + + require.NoError(t, err) + reconcileUntilSuccessful(ctx, t, reconciler, sc, kubeClient, memberClusterClients, nil, false) + // Ensure that reconciliation generated the correct deployment state + checkCorrectShardDistributionInStatus(t, sc) + + // remove members from cluster 1 + shardDistribution = []map[string]int{ + {cluster3: 2, cluster1: 0, cluster2: 2}, + {cluster3: 2, cluster1: 0, cluster2: 2}, + } + + mongosDistribution = map[string]int{cluster1: 0, cluster2: 1, cluster3: 1} + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution) + + err = kubeClient.Get(ctx, mock.ObjectKeyFromApiObject(sc), sc) + require.NoError(t, err) + sc.Spec.ConfigSrvSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, configSrvDistribution) + sc.Spec.ShardSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, shardDistribution[0]) + sc.Spec.MongosSpec.ClusterSpecList = test.CreateClusterSpecList(memberClusterNames, mongosDistribution) + err = kubeClient.Update(ctx, sc) + require.NoError(t, err) + + reconciler, reconcilerHelper, err = newShardedClusterReconcilerForMultiCluster(ctx, false, sc, memberClusterMap, kubeClient, omConnectionFactory) + require.NoError(t, err) + clusterMapping = reconcilerHelper.deploymentState.ClusterMapping + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosDistribution, clusterMapping, configSrvDistribution, shardDistribution) + + require.NoError(t, err) + reconcileUntilSuccessful(ctx, t, reconciler, sc, kubeClient, memberClusterClients, nil, false) + checkCorrectShardDistributionInStatus(t, sc) +} + +func reconcileUntilSuccessful(ctx context.Context, t *testing.T, reconciler reconcile.Reconciler, object *mdbv1.MongoDB, operatorClient client.Client, memberClusterClients []client.Client, expectedReconciles *int, ignoreFailures bool) { + maxReconcileCount := 20 + actualReconciles := 0 + + for { + result, err := reconciler.Reconcile(ctx, requestFromObject(object)) + require.NoError(t, err) + require.NoError(t, mock.MarkAllStatefulSetsAsReady(ctx, object.Namespace, memberClusterClients...)) + + actualReconciles++ + if actualReconciles >= maxReconcileCount { + require.FailNow(t, "Reconcile not successful after maximum (%d) attempts", maxReconcileCount) + return + } + require.NoError(t, operatorClient.Get(ctx, mock.ObjectKeyFromApiObject(object), object)) + + if object.Status.Phase == status.PhaseRunning { + assert.Equal(t, reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS}, result) + if expectedReconciles != nil { + assert.Equal(t, *expectedReconciles, actualReconciles) + } + zap.S().Debugf("Reconcile successful on %d try", actualReconciles) + return + } else if object.Status.Phase == status.PhaseFailed { + if !ignoreFailures { + require.FailNow(t, "", "Reconcile failed on %d try", actualReconciles) + } + } + } +} + +func generateHostsForCluster(ctx context.Context, reconciler *ReconcileMongoDbShardedCluster, forceEnterprise bool, sc *mdbv1.MongoDB, mongosDistribution map[string]int, configSrvDistribution map[string]int, shardDistribution []map[string]int) []string { + reconcileHelper, _ := NewShardedClusterReconcilerHelper(ctx, reconciler.ReconcileCommonController, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", forceEnterprise, sc, reconciler.memberClustersMap, reconciler.omConnectionFactory, zap.S()) + allHostnames, _ := generateAllHosts(sc, mongosDistribution, reconcileHelper.deploymentState.ClusterMapping, configSrvDistribution, shardDistribution, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + return allHostnames +} + +func buildShardedClusterWithCustomProjectName(mcShardedClusterName string, shardCount int, shardClusterSpecList mdbv1.ClusterSpecList, mongosAndConfigSrvClusterSpecList mdbv1.ClusterSpecList, configSrvDistributionClusterSpecList mdbv1.ClusterSpecList) (*mdbv1.MongoDB, *corev1.ConfigMap, string) { + configMapName := mock.TestProjectConfigMapName + "-" + mcShardedClusterName + projectName := om.TestGroupName + "-" + mcShardedClusterName + + return test.DefaultClusterBuilder(). + SetName(mcShardedClusterName). + SetOpsManagerConfigMapName(configMapName). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(shardCount). + // The below parameters should be ignored when a clusterSpecList is configured/for multiClusterTopology + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + // Same pods repartition for + SetShardClusterSpec(shardClusterSpecList). + SetConfigSrvClusterSpec(configSrvDistributionClusterSpecList). + SetMongosClusterSpec(mongosAndConfigSrvClusterSpecList). + Build(), mock.GetProjectConfigMap(configMapName, projectName, ""), projectName +} + +func checkCorrectShardDistributionInStatefulSets(t *testing.T, ctx context.Context, sc *mdbv1.MongoDB, clusterMapping map[string]int, + memberClusterMap map[string]client.Client, expectedShardsDistributions []map[string]int, +) { + for shardIdx, shardExpectedDistributions := range expectedShardsDistributions { + for memberClusterName, expectedMemberCount := range shardExpectedDistributions { + c := memberClusterMap[memberClusterName] + sts := appsv1.StatefulSet{} + var stsName string + if memberClusterName == multicluster.LegacyCentralClusterName { + stsName = fmt.Sprintf("%s-%d", sc.Name, shardIdx) + } else { + stsName = fmt.Sprintf("%s-%d-%d", sc.Name, shardIdx, clusterMapping[memberClusterName]) + } + err := c.Get(ctx, types.NamespacedName{Namespace: sc.Namespace, Name: stsName}, &sts) + stsMessage := fmt.Sprintf("shardIdx: %d, clusterName: %s, stsName: %s", shardIdx, memberClusterName, stsName) + require.NoError(t, err) + assert.Equal(t, int32(expectedMemberCount), sts.Status.ReadyReplicas, stsMessage) + assert.Equal(t, int32(expectedMemberCount), sts.Status.Replicas, stsMessage) + } + } +} + +func checkCorrectShardDistributionInStatus(t *testing.T, sc *mdbv1.MongoDB) { + clusterSpecItemToClusterNameMembers := func(clusterSpecItem mdbv1.ClusterSpecItem, _ int) (string, int) { + return clusterSpecItem.ClusterName, clusterSpecItem.Members + } + clusterSpecItemOverrideToClusterNameMembers := func(clusterSpecItem mdbv1.ClusterSpecItemOverride, _ int) (string, int) { + return clusterSpecItem.ClusterName, *clusterSpecItem.Members + } + expectedShardSizeStatusInClusters := util.TransformToMap(sc.Spec.ShardSpec.ClusterSpecList, clusterSpecItemToClusterNameMembers) + var expectedShardOverridesInClusters map[string]map[string]int + for _, shardOverride := range sc.Spec.ShardOverrides { + for _, shardName := range shardOverride.ShardNames { + if expectedShardOverridesInClusters == nil { + // we need to initialize it only when there are any shard overrides because we receive nil from status) + expectedShardOverridesInClusters = map[string]map[string]int{} + } + expectedShardOverridesInClusters[shardName] = util.TransformToMap(shardOverride.ClusterSpecList, clusterSpecItemOverrideToClusterNameMembers) + } + } + + expectedMongosSizeStatusInClusters := util.TransformToMap(sc.Spec.MongosSpec.ClusterSpecList, clusterSpecItemToClusterNameMembers) + expectedConfigSrvSizeStatusInClusters := util.TransformToMap(sc.Spec.ConfigSrvSpec.ClusterSpecList, clusterSpecItemToClusterNameMembers) + + assert.Equal(t, expectedMongosSizeStatusInClusters, sc.Status.SizeStatusInClusters.MongosCountInClusters) + assert.Equal(t, expectedShardSizeStatusInClusters, sc.Status.SizeStatusInClusters.ShardMongodsInClusters) + assert.Equal(t, expectedShardOverridesInClusters, sc.Status.SizeStatusInClusters.ShardOverridesInClusters) + assert.Equal(t, expectedConfigSrvSizeStatusInClusters, sc.Status.SizeStatusInClusters.ConfigServerMongodsInClusters) + + clusterSpecItemToMembers := func(item mdbv1.ClusterSpecItem) int { + return item.Members + } + assert.Equal(t, sumSlice(util.Transform(sc.Spec.MongosSpec.ClusterSpecList, clusterSpecItemToMembers)), sc.Status.MongosCount) + assert.Equal(t, sumSlice(util.Transform(sc.Spec.ConfigSrvSpec.ClusterSpecList, clusterSpecItemToMembers)), sc.Status.ConfigServerCount) + assert.Equal(t, sumSlice(util.Transform(sc.Spec.ShardSpec.ClusterSpecList, clusterSpecItemToMembers)), sc.Status.MongodsPerShardCount) +} + +func TestComputeMembersToScaleDown(t *testing.T) { + ctx := context.Background() + memberCluster1 := "cluster1" + memberCluster2 := "cluster2" + memberClusterNames := []string{memberCluster1, memberCluster2} + + type testCase struct { + name string + shardCount int + cfgServerCurrentClusters []multicluster.MemberCluster + shardsCurrentClusters map[int][]multicluster.MemberCluster + targetCfgServerDistribution map[string]int + targetShardDistribution map[string]int + expected map[string][]string + } + + testCases := []testCase{ + { + name: "Case 1: Downscale config server and shard", + shardCount: 1, + cfgServerCurrentClusters: []multicluster.MemberCluster{ + {Name: memberCluster1, Index: 0, Replicas: 5}, + {Name: memberCluster2, Index: 1, Replicas: 0}, + }, + shardsCurrentClusters: map[int][]multicluster.MemberCluster{ + 0: { + {Name: memberCluster1, Index: 0, Replicas: 3}, + {Name: memberCluster2, Index: 1, Replicas: 2}, + }, + }, + targetCfgServerDistribution: map[string]int{ + memberCluster1: 2, + memberCluster2: 1, + }, + targetShardDistribution: map[string]int{ + memberCluster1: 1, + memberCluster2: 2, + }, + expected: map[string][]string{ + // For the config replica set: downscale from 5 to 2 means remove members with indices 2, 3, 4 + test.SCBuilderDefaultName + "-config": { + test.SCBuilderDefaultName + "-config-0-2", + test.SCBuilderDefaultName + "-config-0-3", + test.SCBuilderDefaultName + "-config-0-4", + }, + // For the shard replica set (shard 0): downscale from 3 to 1, so remove two members + test.SCBuilderDefaultName + "-0": { + test.SCBuilderDefaultName + "-0-0-1", + test.SCBuilderDefaultName + "-0-0-2", + }, + }, + }, + { + name: "Case 2: Scale down and move replicas among clusters", + shardCount: 2, + cfgServerCurrentClusters: []multicluster.MemberCluster{ + {Name: memberCluster1, Index: 0, Replicas: 2}, + {Name: memberCluster2, Index: 1, Replicas: 1}, + }, + shardsCurrentClusters: map[int][]multicluster.MemberCluster{ + 0: { + {Name: memberCluster1, Index: 0, Replicas: 3}, + {Name: memberCluster2, Index: 1, Replicas: 2}, + }, + 1: { + {Name: memberCluster1, Index: 0, Replicas: 3}, + {Name: memberCluster2, Index: 1, Replicas: 2}, + }, + }, + targetCfgServerDistribution: map[string]int{ + memberCluster1: 1, + memberCluster2: 2, + }, + targetShardDistribution: map[string]int{ + memberCluster1: 3, + memberCluster2: 0, + }, + expected: map[string][]string{ + test.SCBuilderDefaultName + "-config": { + test.SCBuilderDefaultName + "-config" + "-0" + "-1", + }, + // For each shard replica set, we remove two members from cluster with index 1 + test.SCBuilderDefaultName + "-0": { + test.SCBuilderDefaultName + "-0-1-0", + test.SCBuilderDefaultName + "-0-1-1", + }, + test.SCBuilderDefaultName + "-1": { + test.SCBuilderDefaultName + "-1-1-0", + test.SCBuilderDefaultName + "-1-1-1", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + targetSpec := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologyMultiCluster). + SetShardCountSpec(tc.shardCount). + SetMongodsPerShardCountSpec(0). + SetConfigServerCountSpec(0). + SetMongosCountSpec(0). + SetConfigSrvClusterSpec(test.CreateClusterSpecList(memberClusterNames, tc.targetCfgServerDistribution)). + SetShardClusterSpec(test.CreateClusterSpecList(memberClusterNames, tc.targetShardDistribution)). + Build() + + _, omConnectionFactory := mock.NewDefaultFakeClient(targetSpec) + memberClusterMap := getFakeMultiClusterMapWithClusters(memberClusterNames, omConnectionFactory) + + _, reconcileHelper, _, _, err := defaultClusterReconciler(ctx, nil, "", "", targetSpec, memberClusterMap) + assert.NoError(t, err) + + membersToScaleDown := reconcileHelper.computeMembersToScaleDown(tc.cfgServerCurrentClusters, tc.shardsCurrentClusters, zap.S()) + + assert.Equal(t, tc.expected, membersToScaleDown) + }) + } +} + +func sumSlice[T constraints.Integer](s []T) int { + result := 0 + for i := range s { + result += int(s[i]) + } + return result +} + +func generateHostsWithDistribution(stsName string, namespace string, distribution map[string]int, clusterIndexMapping map[string]int, clusterDomain string, externalClusterDomain string) ([]string, []string) { + var hosts []string + var podNames []string + for memberClusterName, memberCount := range distribution { + for podIdx := range memberCount { + hosts = append(hosts, getMultiClusterFQDN(stsName, namespace, clusterIndexMapping[memberClusterName], podIdx, clusterDomain, externalClusterDomain)) + podNames = append(podNames, getPodName(stsName, clusterIndexMapping[memberClusterName], podIdx)) + } + } + + return podNames, hosts +} + +func getPodName(stsName string, clusterIdx int, podIdx int) string { + return fmt.Sprintf("%s-%d-%d", stsName, clusterIdx, podIdx) +} + +func getMultiClusterFQDN(stsName string, namespace string, clusterIdx int, podIdx int, clusterDomain string, externalClusterDomain string) string { + if len(externalClusterDomain) != 0 { + return fmt.Sprintf("%s.%s", getPodName(stsName, clusterIdx, podIdx), externalClusterDomain) + } + return fmt.Sprintf("%s-svc.%s.svc.%s", getPodName(stsName, clusterIdx, podIdx), namespace, clusterDomain) +} + +func generateExpectedDeploymentState(t *testing.T, sc *mdbv1.MongoDB) string { + lastSpec, _ := sc.GetLastSpec() + expectedState := ShardedClusterDeploymentState{ + CommonDeploymentState: CommonDeploymentState{ + ClusterMapping: map[string]int{}, + }, + LastAchievedSpec: lastSpec, + Status: &sc.Status, + } + lastSpecBytes, err := json.Marshal(expectedState) + require.NoError(t, err) + return string(lastSpecBytes) +} + +func loadMongoDBResource(resourceYamlPath string) (*mdbv1.MongoDB, error) { + mdbBytes, err := os.ReadFile(resourceYamlPath) + if err != nil { + return nil, err + } + + mdb := mdbv1.MongoDB{} + if err := yaml.Unmarshal(mdbBytes, &mdb); err != nil { + return nil, err + } + return &mdb, nil +} + +func unmarshalYamlFileInStruct[T *mdbv1.ShardedClusterComponentSpec | map[int]*mdbv1.ShardedClusterComponentSpec](path string) (*T, error) { + bytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + componentSpecStruct := new(T) + if err := yaml.Unmarshal(bytes, &componentSpecStruct); err != nil { + return nil, err + } + return componentSpecStruct, nil +} + +func loadExpectedReplicaSets(path string) (map[string]any, error) { + bytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var ac map[string]any + if err := yaml.Unmarshal(bytes, &ac); err != nil { + return nil, err + } + return ac, nil +} + +func normalizeObjectToInterfaceMap(obj any) (map[string]interface{}, error) { + objJson, err := json.Marshal(obj) + if err != nil { + return nil, err + } + result := map[string]interface{}{} + err = json.Unmarshal(objJson, &result) + if err != nil { + return nil, err + } + return result, nil +} + +func visualJsonDiffOfAnyObjects(t *testing.T, expectedObj any, actualObj any) string { + normalizedExpectedObj, err := normalizeObjectToInterfaceMap(expectedObj) + require.NoError(t, err) + normalizedActualObj, err := normalizeObjectToInterfaceMap(actualObj) + require.NoError(t, err) + + visualDiff, err := getVisualJsonDiff(normalizedExpectedObj, normalizedActualObj) + require.NoError(t, err) + + return visualDiff +} + +func getVisualJsonDiff(expectedMap map[string]interface{}, actualMap map[string]interface{}) (string, error) { + differ := gojsondiff.New() + diff := differ.CompareObjects(expectedMap, actualMap) + if !diff.Modified() { + fmt.Println("No diffs found") + return "", nil + } + jsonFormatter := formatter.NewAsciiFormatter(expectedMap, formatter.AsciiFormatterConfig{ + ShowArrayIndex: false, + Coloring: true, + }) + + diffString, err := jsonFormatter.Format(diff) + if err != nil { + return "", err + } + + return diffString, nil +} diff --git a/controllers/operator/mongodbshardedcluster_controller_test.go b/controllers/operator/mongodbshardedcluster_controller_test.go new file mode 100644 index 000000000..4663f981c --- /dev/null +++ b/controllers/operator/mongodbshardedcluster_controller_test.go @@ -0,0 +1,1917 @@ +package operator + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/status/pvc" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/backup" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/test" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func TestChangingFCVShardedCluster(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().Build() + reconciler, _, cl, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + // Helper function to update and verify FCV + verifyFCV := func(version, expectedFCV string, fcvOverride *string, t *testing.T) { + if fcvOverride != nil { + sc.Spec.FeatureCompatibilityVersion = fcvOverride + } + + sc.Spec.Version = version + _ = cl.Update(ctx, sc) + checkReconcileSuccessful(ctx, t, reconciler, sc, cl) + assert.Equal(t, expectedFCV, sc.Status.FeatureCompatibilityVersion) + } + + testFCVsCases(t, verifyFCV) +} + +func TestReconcileCreateShardedCluster(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().Build() + + reconciler, _, kubeClient, omConnectionFactory, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + c := kubeClient + require.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, c) + assert.Len(t, mock.GetMapForObject(c, &corev1.Secret{}), 2) + assert.Len(t, mock.GetMapForObject(c, &corev1.Service{}), 3) + assert.Len(t, mock.GetMapForObject(c, &appsv1.StatefulSet{}), 4) + assert.Equal(t, getStsReplicas(ctx, c, kube.ObjectKey(sc.Namespace, sc.ConfigRsName()), t), int32(sc.Spec.ConfigServerCount)) + assert.Equal(t, getStsReplicas(ctx, c, kube.ObjectKey(sc.Namespace, sc.MongosRsName()), t), int32(sc.Spec.MongosCount)) + assert.Equal(t, getStsReplicas(ctx, c, kube.ObjectKey(sc.Namespace, sc.ShardRsName(0)), t), int32(sc.Spec.MongodsPerShardCount)) + assert.Equal(t, getStsReplicas(ctx, c, kube.ObjectKey(sc.Namespace, sc.ShardRsName(1)), t), int32(sc.Spec.MongodsPerShardCount)) + + mockedConn := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + expectedDeployment := createDeploymentFromShardedCluster(t, sc) + if !mockedConn.CheckDeployment(t, expectedDeployment, "auth", "tls") { + // this is to diagnose problems using visual diff as the automation config is large + // it is very difficult to spot what's wrong using assert's Equal dump + // NOTE: this sometimes get mangled in IntelliJ's console. If it's not showing correctly, put a time.Sleep here. + fmt.Printf("deployment diff:\n%s", visualJsonDiffOfAnyObjects(t, expectedDeployment, mockedConn.GetDeployment())) + } + mockedConn.CheckNumberOfUpdateRequests(t, 2) + // we don't remove hosts from monitoring if there is no scale down + mockedConn.CheckOperationsDidntHappen(t, reflect.ValueOf(mockedConn.GetHosts), reflect.ValueOf(mockedConn.RemoveHost)) +} + +// TestReconcileCreateSingleClusterShardedClusterWithNoServiceMeshSimplest assumes only Services for Mongos +// will be created. +func TestReconcileCreateSingleClusterShardedClusterWithExternalDomainSimplest(t *testing.T) { + // given + ctx := context.Background() + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + + sc := test.DefaultClusterBuilder(). + SetExternalAccessDomain(test.ExampleExternalClusterDomains). + SetExternalAccessDomainAnnotations(test.SingleClusterAnnotationsWithPlaceholders). + Build() + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(sc). + WithObjects(mock.GetDefaultResources()...). + WithInterceptorFuncs(interceptor.Funcs{ + Get: mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, true, true), + }). + Build() + + kubeClient := kubernetesClient.NewClient(fakeClient) + reconciler, _, _ := newShardedClusterReconcilerFromResource(ctx, nil, "", "", sc, nil, kubeClient, omConnectionFactory) + + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + var allHostnames []string + // Note that only Mongos uses external domains. The other components use domains internal to the cluster. + mongosHostNames, _ := dns.GetDNSNames(sc.MongosRsName(), sc.ServiceName(), sc.Namespace, "", sc.Spec.MongosCount, &test.ExampleExternalClusterDomains.SingleClusterDomain) + allHostnames = append(allHostnames, mongosHostNames...) + configServersHostNames, _ := dns.GetDNSNames(sc.ConfigRsName(), sc.ConfigSrvServiceName(), sc.Namespace, "", sc.Spec.ConfigServerCount, &test.NoneExternalClusterDomains.ConfigServerExternalDomain) + allHostnames = append(allHostnames, configServersHostNames...) + for shardIdx := range sc.Spec.ShardCount { + shardHostNames, _ := dns.GetDNSNames(sc.ShardRsName(shardIdx), sc.ShardServiceName(), sc.Namespace, "", sc.Spec.MongodsPerShardCount, &test.NoneExternalClusterDomains.ShardsExternalDomain) + allHostnames = append(allHostnames, shardHostNames...) + } + + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + }) + + // when + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + // then + memberClusterChecks := newClusterChecks(t, multicluster.LegacyCentralClusterName, 0, sc.Namespace, kubeClient) + + mongosStatefulSetName := fmt.Sprintf("%s-mongos", sc.Name) + memberClusterChecks.checkExternalServices(ctx, mongosStatefulSetName, sc.Spec.MongosCount) + memberClusterChecks.checkPerPodServicesDontExist(ctx, mongosStatefulSetName, sc.Spec.MongosCount) + memberClusterChecks.checkServiceAnnotations(ctx, mongosStatefulSetName, sc.Spec.MongosCount, sc, multicluster.LegacyCentralClusterName, 0, test.ExampleExternalClusterDomains.SingleClusterDomain) + + configServerStatefulSetName := fmt.Sprintf("%s-config", sc.Name) + memberClusterChecks.checkExternalServicesDontExist(ctx, configServerStatefulSetName, sc.Spec.ConfigServerCount) + memberClusterChecks.checkPerPodServicesDontExist(ctx, configServerStatefulSetName, sc.Spec.ConfigServerCount) + // This is something to be unified - why MC and SC Services are called differently? + configServerInternalServiceName := fmt.Sprintf("%s-cs", sc.Name) + memberClusterChecks.checkServiceExists(ctx, configServerInternalServiceName) + + memberClusterChecks.checkExternalServicesDontExist(ctx, fmt.Sprintf("%s-config", sc.Name), sc.Spec.ConfigServerCount) + for shardIdx := 0; shardIdx < sc.Spec.ShardCount; shardIdx++ { + shardStatefulSetName := fmt.Sprintf("%s-%d", sc.Name, shardIdx) + memberClusterChecks.checkExternalServicesDontExist(ctx, shardStatefulSetName, sc.Spec.ShardCount) + memberClusterChecks.checkPerPodServicesDontExist(ctx, shardStatefulSetName, sc.Spec.ShardCount) + // This is something to be unified - why MC and SC Services are called differently? + shardInternalServiceName := fmt.Sprintf("%s-sh", sc.Name) + memberClusterChecks.checkServiceExists(ctx, shardInternalServiceName) + } +} + +func getStsReplicas(ctx context.Context, client kubernetesClient.Client, key client.ObjectKey, t *testing.T) int32 { + sts, err := client.GetStatefulSet(ctx, key) + require.NoError(t, err) + + return *sts.Spec.Replicas +} + +func TestShardedClusterRace(t *testing.T) { + ctx := context.Background() + sc1, cfgMap1, projectName1 := buildShardedClusterWithCustomProject("my-sh1") + sc2, cfgMap2, projectName2 := buildShardedClusterWithCustomProject("my-sh2") + sc3, cfgMap3, projectName3 := buildShardedClusterWithCustomProject("my-sh3") + + resourceToProjectMapping := map[string]string{ + "my-sh1": projectName1, + "my-sh2": projectName2, + "my-sh3": projectName3, + } + + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory().WithResourceToProjectMapping(resourceToProjectMapping) + fakeClient := mock.NewEmptyFakeClientBuilder(). + WithObjects(sc1, sc2, sc3). + WithObjects(cfgMap1, cfgMap2, cfgMap3). + WithInterceptorFuncs(interceptor.Funcs{ + Get: mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, true, true), + }). + WithObjects(mock.GetDefaultResources()...). + Build() + + reconciler := newShardedClusterReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, nil, omConnectionFactory.GetConnectionFunc) + + testConcurrentReconciles(ctx, t, fakeClient, reconciler, sc1, sc2, sc3) +} + +func buildShardedClusterWithCustomProject(scName string) (*mdbv1.MongoDB, *corev1.ConfigMap, string) { + configMapName := mock.TestProjectConfigMapName + "-" + scName + projectName := om.TestGroupName + "-" + scName + + return test.DefaultClusterBuilder(). + SetName(scName). + SetOpsManagerConfigMapName(configMapName). + SetShardCountSpec(4). + SetShardCountStatus(4). + Build(), mock.GetProjectConfigMap(configMapName, projectName, ""), projectName +} + +// TODO this is to be removed as it's testing whether we scale down entire shards one by one, but it's actually testing only scale by one; and we actually don't scale one by one but prune all the shards to be removed immediately" +func TestReconcileCreateShardedCluster_ScaleDown(t *testing.T) { + t.Skip("this test should probably be deleted") + ctx := context.Background() + // First creation + sc := test.DefaultClusterBuilder().SetShardCountSpec(4).SetShardCountStatus(4).Build() + reconciler, _, clusterClient, omConnectionFactory, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + mockedConn := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + mockedConn.CleanHistory() + + // Scale down then + sc.Spec.ShardCount = 3 + + _ = clusterClient.Update(ctx, sc) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + // Two deployment modifications are expected + mockedConn.CheckOrderOfOperations(t, reflect.ValueOf(mockedConn.ReadUpdateDeployment), reflect.ValueOf(mockedConn.ReadUpdateDeployment)) + + // todo ideally we need to check the "transitive" deployment that was created on first step, but let's check the + // final version at least + + // the updated deployment should reflect that of a ShardedCluster with one fewer member + scWith3Members := test.DefaultClusterBuilder().SetShardCountStatus(3).SetShardCountSpec(3).Build() + mockedConn.CheckDeployment(t, createDeploymentFromShardedCluster(t, scWith3Members), "auth", "tls") + + // No matter how many members we scale down by, we will only have one fewer each reconciliation + assert.Len(t, mock.GetMapForObject(clusterClient, &appsv1.StatefulSet{}), 5) +} + +func TestShardedClusterReconcileContainerImages(t *testing.T) { + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_1_0_0", util.NonStaticDatabaseEnterpriseImage) + initDatabaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_2_0_0", util.InitDatabaseImageUrlEnv) + + imageUrlsMock := images.ImageUrls{ + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", + initDatabaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", + } + + ctx := context.Background() + sc := test.DefaultClusterBuilder().SetVersion("8.0.0").SetShardCountSpec(1).Build() + + reconciler, _, kubeClient, _, err := defaultClusterReconciler(ctx, imageUrlsMock, "2.0.0", "1.0.0", sc, nil) + require.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + for stsAlias, stsName := range map[string]string{ + "config": sc.ConfigRsName(), + "mongos": sc.MongosRsName(), + "shard-0": sc.ShardRsName(0), + } { + t.Run(stsAlias, func(t *testing.T) { + sts := &appsv1.StatefulSet{} + err = kubeClient.Get(ctx, kube.ObjectKey(sc.Namespace, stsName), sts) + assert.NoError(t, err) + + require.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", sts.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[0].Image) + }) + } +} + +func TestShardedClusterReconcileContainerImagesWithStaticArchitecture(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0_ubi9", mcoConstruct.MongodbImageEnv) + + ctx := context.Background() + sc := test.DefaultClusterBuilder().SetVersion("8.0.0").SetShardCountSpec(1).Build() + + imageUrlsMock := images.ImageUrls{ + architectures.MdbAgentImageRepo: "quay.io/mongodb/mongodb-agent-ubi", + mcoConstruct.MongodbImageEnv: "quay.io/mongodb/mongodb-enterprise-server", + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", + } + + reconciler, _, kubeClient, omConnectionFactory, err := defaultClusterReconciler(ctx, imageUrlsMock, "", "", sc, nil) + require.NoError(t, err) + + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + connection.(*om.MockedOmConnection).SetAgentVersion("12.0.30.7791-1", "") + }) + + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + for stsAlias, stsName := range map[string]string{ + "config": sc.ConfigRsName(), + "mongos": sc.MongosRsName(), + "shard-0": sc.ShardRsName(0), + } { + t.Run(stsAlias, func(t *testing.T) { + sts := &appsv1.StatefulSet{} + err = kubeClient.Get(ctx, kube.ObjectKey(sc.Namespace, stsName), sts) + assert.NoError(t, err) + + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 0) + require.Len(t, sts.Spec.Template.Spec.Containers, 2) + + // Version from OM + operator version + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:12.0.30.7791-1_9.9.9-test", sts.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[1].Image) + }) + } +} + +func TestReconcilePVCResizeShardedCluster(t *testing.T) { + ctx := context.Background() + // First creation + sc := test.DefaultClusterBuilder().SetShardCountSpec(2).SetShardCountStatus(2).Build() + persistence := mdbv1.Persistence{ + SingleConfig: &mdbv1.PersistenceConfig{ + Storage: "1Gi", + }, + } + sc.Spec.Persistent = util.BooleanRef(true) + sc.Spec.ConfigSrvPodSpec.Persistence = &persistence + sc.Spec.ShardPodSpec.Persistence = &persistence + reconciler, _, c, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + assert.NoError(t, err) + + // first, we create the shardedCluster with sts and pvc, + // no resize happening, even after running reconcile multiple times + checkReconcileSuccessful(ctx, t, reconciler, sc, c) + testNoResize(t, c, ctx, sc) + + checkReconcileSuccessful(ctx, t, reconciler, sc, c) + testNoResize(t, c, ctx, sc) + + createdConfigPVCs, createdSharded0PVCs, createdSharded1PVCs := getPVCs(t, c, ctx, sc) + + newSize := "2Gi" + // increasing the storage now and start a new reconciliation + persistence.SingleConfig.Storage = newSize + + sc.Spec.ConfigSrvPodSpec.Persistence = &persistence + sc.Spec.ShardPodSpec.Persistence = &persistence + err = c.Update(ctx, sc) + assert.NoError(t, err) + + _, e := reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, e) + + // its only one sts in the pvc status, since we haven't started the next one yet + testMDBStatus(t, c, ctx, sc, status.PhasePending, status.PVCS{{Phase: pvc.PhasePVCResize, StatefulsetName: test.SCBuilderDefaultName + "-config"}}) + + testPVCSizeHasIncreased(t, c, ctx, newSize, test.SCBuilderDefaultName+"-config") + + // Running the same resize makes no difference, we are still resizing + _, e = reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, e) + + testMDBStatus(t, c, ctx, sc, status.PhasePending, status.PVCS{{Phase: pvc.PhasePVCResize, StatefulsetName: test.SCBuilderDefaultName + "-config"}}) + + for _, claim := range createdConfigPVCs { + setPVCWithUpdatedResource(ctx, t, c, &claim) + } + + // Running reconcile again should go into orphan + _, e = reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, e) + + // the second pvc is now getting resized + testMDBStatus(t, c, ctx, sc, status.PhasePending, status.PVCS{ + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: sc.Name + "-config"}, + {Phase: pvc.PhasePVCResize, StatefulsetName: sc.Name + "-0"}, + }) + testPVCSizeHasIncreased(t, c, ctx, newSize, sc.Name+"-0") + testStatefulsetHasAnnotationAndCorrectSize(t, c, ctx, sc.Namespace, sc.Name+"-config") + + for _, claim := range createdSharded0PVCs { + setPVCWithUpdatedResource(ctx, t, c, &claim) + } + + // Running reconcile again second pvcState should go into orphan, third one should start + _, e = reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, e) + + testMDBStatus(t, c, ctx, sc, status.PhasePending, status.PVCS{ + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: sc.Name + "-config"}, + {Phase: pvc.PhaseSTSOrphaned, StatefulsetName: sc.Name + "-0"}, + {Phase: pvc.PhasePVCResize, StatefulsetName: sc.Name + "-1"}, + }) + testPVCSizeHasIncreased(t, c, ctx, newSize, sc.Name+"-1") + testStatefulsetHasAnnotationAndCorrectSize(t, c, ctx, sc.Namespace, sc.Name+"-0") + + for _, claim := range createdSharded1PVCs { + setPVCWithUpdatedResource(ctx, t, c, &claim) + } + + // We move from resize → orphaned and in the final call in the reconciling to running and + // remove the PVCs. + _, err = reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, err) + + // We are now in the running phase, since all statefulsets have finished resizing; therefore, + // no pvc phase is shown anymore + testMDBStatus(t, c, ctx, sc, status.PhaseRunning, nil) + testStatefulsetHasAnnotationAndCorrectSize(t, c, ctx, sc.Namespace, sc.Name+"-1") +} + +func testStatefulsetHasAnnotationAndCorrectSize(t *testing.T, c client.Client, ctx context.Context, namespace, stsName string) { + // verify config-sts has been re-created with new annotation + sts := &appsv1.StatefulSet{} + err := c.Get(ctx, kube.ObjectKey(namespace, stsName), sts) + assert.NoError(t, err) + assert.Equal(t, "[{\"Name\":\"data\",\"Size\":\"2Gi\"}]", sts.Spec.Template.Annotations["mongodb.com/storageSize"]) + assert.Equal(t, "2Gi", sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().String()) + assert.Len(t, sts.Spec.VolumeClaimTemplates, 1) +} + +func testMDBStatus(t *testing.T, c kubernetesClient.Client, ctx context.Context, sc *mdbv1.MongoDB, expectedMDBPhase status.Phase, expectedPVCS status.PVCS) { + mdb := mdbv1.MongoDB{} + err := c.Get(ctx, kube.ObjectKey(sc.Namespace, sc.Name), &mdb) + assert.NoError(t, err) + require.Equal(t, expectedMDBPhase, mdb.Status.Phase) + require.Equal(t, expectedPVCS, mdb.Status.PVCs) +} + +func getPVCs(t *testing.T, c kubernetesClient.Client, ctx context.Context, sc *mdbv1.MongoDB) ([]corev1.PersistentVolumeClaim, []corev1.PersistentVolumeClaim, []corev1.PersistentVolumeClaim) { + sts, err := c.GetStatefulSet(ctx, kube.ObjectKey(sc.Namespace, sc.Name+"-config")) + assert.NoError(t, err) + createdConfigPVCs := createPVCs(t, sts, c) + + sts, err = c.GetStatefulSet(ctx, kube.ObjectKey(sc.Namespace, sc.Name+"-0")) + assert.NoError(t, err) + createdSharded0PVCs := createPVCs(t, sts, c) + + sts, err = c.GetStatefulSet(ctx, kube.ObjectKey(sc.Namespace, sc.Name+"-1")) + assert.NoError(t, err) + createdSharded1PVCs := createPVCs(t, sts, c) + return createdConfigPVCs, createdSharded0PVCs, createdSharded1PVCs +} + +func testNoResize(t *testing.T, c kubernetesClient.Client, ctx context.Context, sc *mdbv1.MongoDB) { + mdb := mdbv1.MongoDB{} + err := c.Get(ctx, kube.ObjectKey(sc.Namespace, sc.Name), &mdb) + assert.NoError(t, err) + assert.Nil(t, mdb.Status.PVCs) +} + +func testPVCSizeHasIncreased(t *testing.T, c client.Client, ctx context.Context, newSize string, pvcName string) { + list := corev1.PersistentVolumeClaimList{} + err := c.List(ctx, &list) + require.NoError(t, err) + for _, item := range list.Items { + if strings.Contains(item.Name, pvcName) { + assert.Equal(t, item.Spec.Resources.Requests.Storage().String(), newSize) + } + } + require.NoError(t, err) +} + +func createPVCs(t *testing.T, sts appsv1.StatefulSet, c client.Writer) []corev1.PersistentVolumeClaim { + var createdPVCs []corev1.PersistentVolumeClaim + // Manually create the PVCs that would be generated by the StatefulSet controller + for i := 0; i < int(*sts.Spec.Replicas); i++ { + pvcName := fmt.Sprintf("%s-%s-%d", sts.Spec.VolumeClaimTemplates[0].Name, sts.Name, i) + p := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: sts.Namespace, + Labels: sts.Spec.Template.Labels, + }, + Spec: sts.Spec.VolumeClaimTemplates[0].Spec, + } + err := c.Create(context.TODO(), p) + require.NoError(t, err) + createdPVCs = append(createdPVCs, *p) + } + return createdPVCs +} + +// TestAddDeleteShardedCluster checks that no state is left in OpsManager on removal of the sharded cluster +func TestAddDeleteShardedCluster(t *testing.T) { + ctx := context.Background() + // First we need to create a sharded cluster + sc := test.DefaultClusterBuilder().Build() + + reconciler, _, clusterClient, omConnectionFactory, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + connection.(*om.MockedOmConnection).AgentsDelayCount = 1 + }) + require.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + // Now delete it + assert.NoError(t, reconciler.OnDelete(ctx, sc, zap.S())) + + // Operator doesn't mutate K8s state, so we don't check its changes, only OM + mockedOmConnection := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + mockedOmConnection.CheckResourcesDeleted(t) + + mockedOmConnection.CheckOrderOfOperations(t, + reflect.ValueOf(mockedOmConnection.ReadUpdateDeployment), reflect.ValueOf(mockedOmConnection.ReadAutomationStatus), + reflect.ValueOf(mockedOmConnection.GetHosts), reflect.ValueOf(mockedOmConnection.RemoveHost)) +} + +func getEmptyDeploymentOptions() deploymentOptions { + return deploymentOptions{ + podEnvVars: &env.PodEnvVars{}, + certTLSType: map[string]bool{}, + prometheusCertHash: "", + } +} + +// TestPrepareScaleDownShardedCluster tests the scale down operation for config servers and mongods per shard. It checks +// that all members that will be removed are marked as unvoted +func TestPrepareScaleDownShardedCluster_ConfigMongodsUp(t *testing.T) { + ctx := context.Background() + + initialState := MultiClusterShardedScalingStep{ + shardCount: 2, + configServerDistribution: map[string]int{ + multicluster.LegacyCentralClusterName: 3, + }, + shardDistribution: map[string]int{ + multicluster.LegacyCentralClusterName: 4, + }, + } + + scBeforeScale := test.DefaultClusterBuilder(). + SetConfigServerCountSpec(3). + SetMongodsPerShardCountSpec(4). + Build() + + scAfterScale := test.DefaultClusterBuilder(). + SetConfigServerCountSpec(2). + SetMongodsPerShardCountSpec(3). + Build() + + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(createDeploymentFromShardedCluster(t, scBeforeScale))) + kubeClient, _ := mock.NewDefaultFakeClient(scAfterScale) + // Store the initial scaling status in state configmap + assert.NoError(t, createMockStateConfigMap(kubeClient, mock.TestNamespace, scBeforeScale.Name, initialState)) + _, reconcileHelper, err := newShardedClusterReconcilerFromResource(ctx, nil, "", "", scAfterScale, nil, kubeClient, omConnectionFactory) + assert.NoError(t, err) + assert.NoError(t, reconcileHelper.prepareScaleDownShardedCluster(omConnectionFactory.GetConnection(), zap.S())) + + // create the expected deployment from the sharded cluster that has not yet scaled + // expected change of state: rs members are marked unvoted + expectedDeployment := createDeploymentFromShardedCluster(t, scBeforeScale) + firstConfig := scAfterScale.ConfigRsName() + "-2" + firstShard := scAfterScale.ShardRsName(0) + "-3" + secondShard := scAfterScale.ShardRsName(1) + "-3" + + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scAfterScale.ConfigRsName(), []string{firstConfig})) + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scAfterScale.ShardRsName(0), []string{firstShard})) + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scAfterScale.ShardRsName(1), []string{secondShard})) + + mockedOmConnection := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + mockedOmConnection.CheckNumberOfUpdateRequests(t, 1) + mockedOmConnection.CheckDeployment(t, expectedDeployment) + // we don't remove hosts from monitoring at this stage + mockedOmConnection.CheckMonitoredHostsRemoved(t, []string{}) +} + +// TestPrepareScaleDownShardedCluster_ShardsUpMongodsDown checks the situation when shards count increases and mongods +// count per shard is decreased - scale down operation is expected to be called only for existing shards +func TestPrepareScaleDownShardedCluster_ShardsUpMongodsDown(t *testing.T) { + ctx := context.Background() + + initialState := MultiClusterShardedScalingStep{ + shardCount: 4, + shardDistribution: map[string]int{ + multicluster.LegacyCentralClusterName: 4, + }, + } + + scBeforeScale := test.DefaultClusterBuilder(). + SetShardCountSpec(4). + SetMongodsPerShardCountSpec(4). + Build() + + scAfterScale := test.DefaultClusterBuilder(). + SetShardCountSpec(2). + SetMongodsPerShardCountSpec(3). + Build() + + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(createDeploymentFromShardedCluster(t, scBeforeScale))) + kubeClient, _ := mock.NewDefaultFakeClient(scAfterScale) + assert.NoError(t, createMockStateConfigMap(kubeClient, mock.TestNamespace, scBeforeScale.Name, initialState)) + _, reconcileHelper, err := newShardedClusterReconcilerFromResource(ctx, nil, "", "", scAfterScale, nil, kubeClient, omConnectionFactory) + assert.NoError(t, err) + + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + deployment := createDeploymentFromShardedCluster(t, scBeforeScale) + if _, err := connection.UpdateDeployment(deployment); err != nil { + panic(err) + } + connection.(*om.MockedOmConnection).AddHosts(deployment.GetAllHostnames()) + connection.(*om.MockedOmConnection).CleanHistory() + }) + + assert.NoError(t, reconcileHelper.prepareScaleDownShardedCluster(omConnectionFactory.GetConnection(), zap.S())) + + // expected change of state: rs members are marked unvoted only for two shards (old state) + expectedDeployment := createDeploymentFromShardedCluster(t, scBeforeScale) + firstShard := scBeforeScale.ShardRsName(0) + "-3" + secondShard := scBeforeScale.ShardRsName(1) + "-3" + thirdShard := scBeforeScale.ShardRsName(2) + "-3" + fourthShard := scBeforeScale.ShardRsName(3) + "-3" + + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scBeforeScale.ShardRsName(0), []string{firstShard})) + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scBeforeScale.ShardRsName(1), []string{secondShard})) + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scBeforeScale.ShardRsName(2), []string{thirdShard})) + assert.NoError(t, expectedDeployment.MarkRsMembersUnvoted(scBeforeScale.ShardRsName(3), []string{fourthShard})) + + mockedOmConnection := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + mockedOmConnection.CheckNumberOfUpdateRequests(t, 1) + mockedOmConnection.CheckDeployment(t, expectedDeployment) + // we don't remove hosts from monitoring at this stage + mockedOmConnection.CheckOperationsDidntHappen(t, reflect.ValueOf(mockedOmConnection.RemoveHost)) +} + +func TestConstructConfigSrv(t *testing.T) { + sc := test.DefaultClusterBuilder().Build() + configSrvSpec := createConfigSrvSpec(sc) + assert.NotPanics(t, func() { + construct.DatabaseStatefulSet(*sc, construct.ConfigServerOptions(configSrvSpec, multicluster.LegacyCentralClusterName, construct.GetPodEnvOptions()), zap.S()) + }) +} + +// TestPrepareScaleDownShardedCluster_OnlyMongos checks that if only mongos processes are scaled down - then no preliminary +// actions are done +func TestPrepareScaleDownShardedCluster_OnlyMongos(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().SetMongosCountStatus(4).SetMongosCountSpec(2).Build() + _, reconcileHelper, _, omConnectionFactory, _ := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + oldDeployment := createDeploymentFromShardedCluster(t, sc) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + if _, err := connection.UpdateDeployment(oldDeployment); err != nil { + panic(err) + } + connection.(*om.MockedOmConnection).CleanHistory() + }) + + // necessary otherwise next omConnectionFactory.GetConnection() will return nil as the connectionFactoryFunc hasn't been called yet + initializeOMConnection(t, ctx, reconcileHelper, sc, zap.S(), omConnectionFactory) + + assert.NoError(t, reconcileHelper.prepareScaleDownShardedCluster(omConnectionFactory.GetConnection(), zap.S())) + mockedOmConnection := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + mockedOmConnection.CheckNumberOfUpdateRequests(t, 0) + mockedOmConnection.CheckDeployment(t, createDeploymentFromShardedCluster(t, sc)) + mockedOmConnection.CheckOperationsDidntHappen(t, reflect.ValueOf(mockedOmConnection.RemoveHost)) +} + +// initializeOMConnection reads project config maps and initializes connection to OM. +// It's useful for cases when the full Reconcile is not caller or the reconcile is not calling omConnectionFactoryFunc to get (create and cache) actual connection. +// Without it subsequent calls to omConnectionFactory.GetConnection() will return nil. +func initializeOMConnection(t *testing.T, ctx context.Context, reconcileHelper *ShardedClusterReconcileHelper, sc *mdbv1.MongoDB, log *zap.SugaredLogger, omConnectionFactory *om.CachedOMConnectionFactory) { + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, reconcileHelper.commonController.client, reconcileHelper.commonController.SecretClient, sc, log) + require.NoError(t, err) + _, _, err = connection.PrepareOpsManagerConnection(ctx, reconcileHelper.commonController.SecretClient, projectConfig, credsConfig, omConnectionFactory.GetConnectionFunc, sc.Namespace, log) + require.NoError(t, err) +} + +// TestUpdateOmDeploymentShardedCluster_HostsRemovedFromMonitoring verifies that if scale down operation was performed - +// hosts are removed +func TestUpdateOmDeploymentShardedCluster_HostsRemovedFromMonitoring(t *testing.T) { + ctx := context.Background() + + initialState := MultiClusterShardedScalingStep{ + mongosDistribution: map[string]int{ + multicluster.LegacyCentralClusterName: 2, + }, + configServerDistribution: map[string]int{ + multicluster.LegacyCentralClusterName: 4, + }, + } + + sc := test.DefaultClusterBuilder(). + SetMongosCountSpec(2). + SetConfigServerCountSpec(4). + Build() + + // we need to create a different sharded cluster that is currently in the process of scaling down + scScaledDown := test.DefaultClusterBuilder(). + SetMongosCountSpec(1). + SetConfigServerCountSpec(3). + Build() + + omConnectionFactory := om.NewCachedOMConnectionFactoryWithInitializedConnection(om.NewMockedOmConnection(createDeploymentFromShardedCluster(t, sc))) + kubeClient, _ := mock.NewDefaultFakeClient(sc) + assert.NoError(t, createMockStateConfigMap(kubeClient, mock.TestNamespace, sc.Name, initialState)) + _, reconcileHelper, err := newShardedClusterReconcilerFromResource(ctx, nil, "", "", scScaledDown, nil, kubeClient, omConnectionFactory) + assert.NoError(t, err) + + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + // the initial deployment we create should have all processes + deployment := createDeploymentFromShardedCluster(t, sc) + if _, err := connection.UpdateDeployment(deployment); err != nil { + panic(err) + } + connection.(*om.MockedOmConnection).AddHosts(deployment.GetAllHostnames()) + connection.(*om.MockedOmConnection).CleanHistory() + }) + + // updateOmDeploymentShardedCluster checks an element from ac.Auth.DeploymentAuthMechanisms + // so we need to ensure it has a non-nil value. An empty list implies no authentication + _ = omConnectionFactory.GetConnection().ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.DeploymentAuthMechanisms = []string{} + return nil + }, nil) + + mockOm := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + assert.Equal(t, workflow.OK(), reconcileHelper.updateOmDeploymentShardedCluster(ctx, mockOm, scScaledDown, deploymentOptions{podEnvVars: &env.PodEnvVars{ProjectID: "abcd"}}, false, zap.S())) + + mockOm.CheckOrderOfOperations(t, reflect.ValueOf(mockOm.ReadUpdateDeployment), reflect.ValueOf(mockOm.RemoveHost)) + + // expected change of state: no unvoting - just monitoring deleted + firstConfig := sc.ConfigRsName() + "-3" + firstMongos := sc.MongosRsName() + "-1" + + mockOm.CheckMonitoredHostsRemoved(t, []string{ + firstConfig + fmt.Sprintf(".%s-cs.mongodb.svc.cluster.local", scScaledDown.Name), + firstMongos + fmt.Sprintf(".%s-svc.mongodb.svc.cluster.local", scScaledDown.Name), + }) +} + +// CLOUDP-32765: checks that pod anti affinity rule spreads mongods inside one shard, not inside all shards +func TestPodAntiaffinity_MongodsInsideShardAreSpread(t *testing.T) { + sc := test.DefaultClusterBuilder().Build() + + kubeClient, _ := mock.NewDefaultFakeClient(sc) + shardSpec, memberCluster := createShardSpecAndDefaultCluster(kubeClient, sc) + firstShardSet := construct.DatabaseStatefulSet(*sc, construct.ShardOptions(0, shardSpec, memberCluster.Name, construct.GetPodEnvOptions()), zap.S()) + secondShardSet := construct.DatabaseStatefulSet(*sc, construct.ShardOptions(1, shardSpec, memberCluster.Name, construct.GetPodEnvOptions()), zap.S()) + + assert.Equal(t, sc.ShardRsName(0), firstShardSet.Spec.Selector.MatchLabels[construct.PodAntiAffinityLabelKey]) + assert.Equal(t, sc.ShardRsName(1), secondShardSet.Spec.Selector.MatchLabels[construct.PodAntiAffinityLabelKey]) + + firstShartPodAffinityTerm := firstShardSet.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm + assert.Equal(t, firstShartPodAffinityTerm.LabelSelector.MatchLabels[construct.PodAntiAffinityLabelKey], sc.ShardRsName(0)) + + secondShartPodAffinityTerm := secondShardSet.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm + assert.Equal(t, secondShartPodAffinityTerm.LabelSelector.MatchLabels[construct.PodAntiAffinityLabelKey], sc.ShardRsName(1)) +} + +func createShardSpecAndDefaultCluster(client kubernetesClient.Client, sc *mdbv1.MongoDB) (*mdbv1.ShardedClusterComponentSpec, multicluster.MemberCluster) { + shardSpec := sc.Spec.ShardSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec, multicluster.GetLegacyCentralMemberCluster(sc.Spec.MongodsPerShardCount, 0, client, secrets.SecretClient{KubeClient: client}) +} + +func createConfigSrvSpec(sc *mdbv1.MongoDB) *mdbv1.ShardedClusterComponentSpec { + shardSpec := sc.Spec.ConfigSrvSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec +} + +func createMongosSpec(sc *mdbv1.MongoDB) *mdbv1.ShardedClusterComponentSpec { + shardSpec := sc.Spec.ConfigSrvSpec.DeepCopy() + shardSpec.ClusterSpecList = mdbv1.ClusterSpecList{ + { + ClusterName: multicluster.LegacyCentralClusterName, + Members: sc.Spec.MongodsPerShardCount, + }, + } + + return shardSpec +} + +func TestShardedCluster_WithTLSEnabled_AndX509Enabled_Succeeds(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + EnableTLS(). + EnableX509(). + SetTLSCA("custom-ca"). + Build() + + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + addKubernetesTlsResources(ctx, clusterClient, sc) + + actualResult, err := reconciler.Reconcile(ctx, requestFromObject(sc)) + expectedResult := reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS} + + assert.Equal(t, expectedResult, actualResult) + assert.Nil(t, err) +} + +func TestShardedCluster_NeedToPublishState(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + EnableTLS(). + SetTLSCA("custom-ca"). + Build() + + // perform successful reconciliation to populate all the stateful sets in the mocked client + reconciler, reconcilerHelper, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + addKubernetesTlsResources(ctx, clusterClient, sc) + actualResult, err := reconciler.Reconcile(ctx, requestFromObject(sc)) + expectedResult := reconcile.Result{RequeueAfter: util.TWENTY_FOUR_HOURS} + + assert.Equal(t, expectedResult, actualResult) + assert.Nil(t, err) + + allConfigs := reconcilerHelper.getAllConfigs(ctx, *sc, getEmptyDeploymentOptions(), zap.S()) + + assert.False(t, anyStatefulSetNeedsToPublishStateToOM(ctx, *sc, clusterClient, reconcilerHelper.deploymentState.LastAchievedSpec, allConfigs, zap.S())) + + // attempting to set tls to false + require.NoError(t, clusterClient.Get(ctx, kube.ObjectKeyFromApiObject(sc), sc)) + + sc.Spec.Security.TLSConfig.Enabled = false + + err = clusterClient.Update(ctx, sc) + assert.NoError(t, err) + + // Ops Manager state needs to be published first as we want to reach goal state before unmounting certificates + allConfigs = reconcilerHelper.getAllConfigs(ctx, *sc, getEmptyDeploymentOptions(), zap.S()) + assert.True(t, anyStatefulSetNeedsToPublishStateToOM(ctx, *sc, clusterClient, reconcilerHelper.deploymentState.LastAchievedSpec, allConfigs, zap.S())) +} + +func TestShardedCustomPodSpecTemplate(t *testing.T) { + ctx := context.Background() + shardPodSpec := corev1.PodSpec{ + NodeName: "some-node-name", + Hostname: "some-host-name", + Containers: []corev1.Container{{ + Name: "my-custom-container-sc", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + } + + mongosPodSpec := corev1.PodSpec{ + NodeName: "some-node-name-mongos", + Hostname: "some-host-name-mongos", + Containers: []corev1.Container{{ + Name: "my-custom-container-mongos", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyNever, + } + + configSrvPodSpec := corev1.PodSpec{ + NodeName: "some-node-name-config", + Hostname: "some-host-name-config", + Containers: []corev1.Container{{ + Name: "my-custom-container-config", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyOnFailure, + } + + sc := test.DefaultClusterBuilder().SetName("pod-spec-sc").EnableTLS().SetTLSCA("custom-ca"). + SetShardPodSpec(corev1.PodTemplateSpec{ + Spec: shardPodSpec, + }).SetMongosPodSpecTemplate(corev1.PodTemplateSpec{ + Spec: mongosPodSpec, + }).SetPodConfigSvrSpecTemplate(corev1.PodTemplateSpec{ + Spec: configSrvPodSpec, + }).Build() + + reconciler, _, kubeClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + addKubernetesTlsResources(ctx, kubeClient, sc) + + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + // read the stateful sets that were created by the operator + statefulSetSc0, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-0")) + assert.NoError(t, err) + statefulSetSc1, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-1")) + assert.NoError(t, err) + statefulSetScConfig, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-config")) + assert.NoError(t, err) + statefulSetMongoS, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-mongos")) + assert.NoError(t, err) + + // assert Pod Spec for Sharded cluster + assertPodSpecSts(t, &statefulSetSc0, shardPodSpec.NodeName, shardPodSpec.Hostname, shardPodSpec.RestartPolicy) + assertPodSpecSts(t, &statefulSetSc1, shardPodSpec.NodeName, shardPodSpec.Hostname, shardPodSpec.RestartPolicy) + + // assert Pod Spec for Mongos + assertPodSpecSts(t, &statefulSetMongoS, mongosPodSpec.NodeName, mongosPodSpec.Hostname, mongosPodSpec.RestartPolicy) + + // assert Pod Spec for ConfigServer + assertPodSpecSts(t, &statefulSetScConfig, configSrvPodSpec.NodeName, configSrvPodSpec.Hostname, configSrvPodSpec.RestartPolicy) + + podSpecTemplateSc0 := statefulSetSc0.Spec.Template.Spec + assert.Len(t, podSpecTemplateSc0.Containers, 2, "Should have 2 containers now") + assert.Equal(t, util.DatabaseContainerName, podSpecTemplateSc0.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-sc", podSpecTemplateSc0.Containers[1].Name, "Custom container should be second") + + podSpecTemplateSc1 := statefulSetSc1.Spec.Template.Spec + assert.Len(t, podSpecTemplateSc1.Containers, 2, "Should have 2 containers now") + assert.Equal(t, util.DatabaseContainerName, podSpecTemplateSc1.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-sc", podSpecTemplateSc1.Containers[1].Name, "Custom container should be second") + + podSpecTemplateMongoS := statefulSetMongoS.Spec.Template.Spec + assert.Len(t, podSpecTemplateMongoS.Containers, 2, "Should have 2 containers now") + assert.Equal(t, util.DatabaseContainerName, podSpecTemplateMongoS.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-mongos", podSpecTemplateMongoS.Containers[1].Name, "Custom container should be second") + + podSpecTemplateScConfig := statefulSetScConfig.Spec.Template.Spec + assert.Len(t, podSpecTemplateScConfig.Containers, 2, "Should have 2 containers now") + assert.Equal(t, util.DatabaseContainerName, podSpecTemplateScConfig.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-config", podSpecTemplateScConfig.Containers[1].Name, "Custom container should be second") +} + +func TestShardedCustomPodStaticSpecTemplate(t *testing.T) { + ctx := context.Background() + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + shardPodSpec := corev1.PodSpec{ + NodeName: "some-node-name", + Hostname: "some-host-name", + Containers: []corev1.Container{{ + Name: "my-custom-container-sc", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + } + + mongosPodSpec := corev1.PodSpec{ + NodeName: "some-node-name-mongos", + Hostname: "some-host-name-mongos", + Containers: []corev1.Container{{ + Name: "my-custom-container-mongos", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyNever, + } + + configSrvPodSpec := corev1.PodSpec{ + NodeName: "some-node-name-config", + Hostname: "some-host-name-config", + Containers: []corev1.Container{{ + Name: "my-custom-container-config", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyOnFailure, + } + + sc := test.DefaultClusterBuilder().SetName("pod-spec-sc").EnableTLS().SetTLSCA("custom-ca"). + SetShardPodSpec(corev1.PodTemplateSpec{ + Spec: shardPodSpec, + }).SetMongosPodSpecTemplate(corev1.PodTemplateSpec{ + Spec: mongosPodSpec, + }).SetPodConfigSvrSpecTemplate(corev1.PodTemplateSpec{ + Spec: configSrvPodSpec, + }).Build() + + reconciler, _, kubeClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + addKubernetesTlsResources(ctx, kubeClient, sc) + + checkReconcileSuccessful(ctx, t, reconciler, sc, kubeClient) + + // read the stateful sets that were created by the operator + statefulSetSc0, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-0")) + assert.NoError(t, err) + statefulSetSc1, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-1")) + assert.NoError(t, err) + statefulSetScConfig, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-config")) + assert.NoError(t, err) + statefulSetMongoS, err := kubeClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "pod-spec-sc-mongos")) + assert.NoError(t, err) + + // assert Pod Spec for Sharded cluster + assertPodSpecSts(t, &statefulSetSc0, shardPodSpec.NodeName, shardPodSpec.Hostname, shardPodSpec.RestartPolicy) + assertPodSpecSts(t, &statefulSetSc1, shardPodSpec.NodeName, shardPodSpec.Hostname, shardPodSpec.RestartPolicy) + + // assert Pod Spec for Mongos + assertPodSpecSts(t, &statefulSetMongoS, mongosPodSpec.NodeName, mongosPodSpec.Hostname, mongosPodSpec.RestartPolicy) + + // assert Pod Spec for ConfigServer + assertPodSpecSts(t, &statefulSetScConfig, configSrvPodSpec.NodeName, configSrvPodSpec.Hostname, configSrvPodSpec.RestartPolicy) + + podSpecTemplateSc0 := statefulSetSc0.Spec.Template.Spec + assert.Len(t, podSpecTemplateSc0.Containers, 3, "Should have 2 containers now") + assert.Equal(t, util.AgentContainerName, podSpecTemplateSc0.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-sc", podSpecTemplateSc0.Containers[2].Name, "Custom container should be second") + + podSpecTemplateSc1 := statefulSetSc1.Spec.Template.Spec + assert.Len(t, podSpecTemplateSc1.Containers, 3, "Should have 2 containers now") + assert.Equal(t, util.AgentContainerName, podSpecTemplateSc1.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-sc", podSpecTemplateSc1.Containers[2].Name, "Custom container should be second") + + podSpecTemplateMongoS := statefulSetMongoS.Spec.Template.Spec + assert.Len(t, podSpecTemplateMongoS.Containers, 3, "Should have 2 containers now") + assert.Equal(t, util.AgentContainerName, podSpecTemplateMongoS.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-mongos", podSpecTemplateMongoS.Containers[2].Name, "Custom container should be second") + + podSpecTemplateScConfig := statefulSetScConfig.Spec.Template.Spec + assert.Len(t, podSpecTemplateScConfig.Containers, 3, "Should have 2 containers now") + assert.Equal(t, util.AgentContainerName, podSpecTemplateScConfig.Containers[0].Name, "Database container should always be first") + assert.Equal(t, "my-custom-container-config", podSpecTemplateScConfig.Containers[2].Name, "Custom container should be second") +} + +func TestFeatureControlsNoAuth(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().RemoveAuth().Build() + omConnectionFactory := om.NewCachedOMConnectionFactory(omConnectionFactoryFuncSettingVersion()) + fakeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, sc) + reconciler := newShardedClusterReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, nil, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, sc, fakeClient) + + cf, _ := omConnectionFactory.GetConnection().GetControlledFeature() + + assert.Len(t, cf.Policies, 2) + + assert.Equal(t, cf.ManagementSystem.Version, util.OperatorVersion) + assert.Equal(t, cf.ManagementSystem.Name, util.OperatorName) + assert.Equal(t, cf.Policies[0].PolicyType, controlledfeature.ExternallyManaged) + assert.Equal(t, cf.Policies[1].PolicyType, controlledfeature.DisableMongodVersion) + assert.Len(t, cf.Policies[0].DisabledParams, 0) +} + +func TestScalingShardedCluster_ScalesOneMemberAtATime_WhenScalingUp(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + SetMongodsPerShardCountSpec(3). + SetMongodsPerShardCountStatus(0). + SetConfigServerCountSpec(1). + SetConfigServerCountStatus(0). + SetMongosCountSpec(1). + SetMongosCountStatus(0). + SetShardCountSpec(1). + SetShardCountStatus(0). + Build() + + clusterClient, omConnectionFactory := mock.NewDefaultFakeClient(sc) + reconciler, _, err := newShardedClusterReconcilerFromResource(ctx, nil, "", "", sc, nil, clusterClient, omConnectionFactory) + require.NoError(t, err) + + // perform initial reconciliation, so we are not creating a new resource + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + getShard := func(i int) appsv1.StatefulSet { + sts := appsv1.StatefulSet{} + err := clusterClient.Get(ctx, types.NamespacedName{Name: sc.ShardRsName(i), Namespace: sc.Namespace}, &sts) + assert.NoError(t, err) + return sts + } + + assert.Equal(t, 1, sc.Status.MongosCount) + assert.Equal(t, 1, sc.Status.ConfigServerCount) + require.Equal(t, 1, sc.Status.ShardCount) + assert.Equal(t, int32(3), *getShard(0).Spec.Replicas) + + // Scale up the Sharded Cluster + sc.Spec.MongodsPerShardCount = 6 + sc.Spec.MongosCount = 3 + sc.Spec.ShardCount = 2 + sc.Spec.ConfigServerCount = 2 + + err = clusterClient.Update(ctx, sc) + assert.NoError(t, err) + + var deployment om.Deployment + performReconciliation := func(shouldRetry bool) { + res, err := reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, err) + if shouldRetry { + assert.Equal(t, time.Duration(10000000000), res.RequeueAfter) + } else { + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) + } + err = clusterClient.Get(ctx, sc.ObjectKey(), sc) + assert.NoError(t, err) + + deployment, err = omConnectionFactory.GetConnection().ReadDeployment() + assert.NoError(t, err) + } + + t.Run("1st reconciliation", func(t *testing.T) { + performReconciliation(true) + + assert.Equal(t, 2, sc.Status.MongosCount) + assert.Equal(t, 2, sc.Status.ConfigServerCount) + // Shard 0 is scaling one replica at a time + assert.Equal(t, int32(4), *getShard(0).Spec.Replicas) + // Shard 1 is scaling for the first time (new shard), we create it directly with the target number of replicas + assert.Equal(t, int32(6), *getShard(1).Spec.Replicas) + assert.Len(t, deployment.GetAllProcessNames(), 14) + }) + + t.Run("2nd reconciliation", func(t *testing.T) { + performReconciliation(true) + assert.Equal(t, 3, sc.Status.MongosCount) + assert.Equal(t, 2, sc.Status.ConfigServerCount) + // Shard 0 is scaling one replica at a time + assert.Equal(t, int32(5), *getShard(0).Spec.Replicas) + assert.Equal(t, int32(6), *getShard(1).Spec.Replicas) + assert.Len(t, deployment.GetAllProcessNames(), 16) + }) + + t.Run("3rd reconciliation", func(t *testing.T) { + performReconciliation(false) + assert.Equal(t, 3, sc.Status.MongosCount) + assert.Equal(t, 2, sc.Status.ConfigServerCount) + assert.Equal(t, int32(6), *getShard(0).Spec.Replicas) + assert.Equal(t, int32(6), *getShard(1).Spec.Replicas) + assert.Len(t, deployment.GetAllProcessNames(), 17) + }) +} + +func TestScalingShardedCluster_ScalesOneMemberAtATime_WhenScalingDown(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + SetMongodsPerShardCountSpec(6). + SetMongodsPerShardCountStatus(6). + SetConfigServerCountSpec(3). + SetConfigServerCountStatus(3). + SetMongosCountSpec(3). + SetMongosCountStatus(3). + SetShardCountSpec(3). + SetShardCountStatus(3). + Build() + + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + // perform initial reconciliation so we are not creating a new resource + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + err = clusterClient.Get(ctx, sc.ObjectKey(), sc) + assert.NoError(t, err) + + assert.Equal(t, 3, sc.Status.ShardCount) + assert.Equal(t, 3, sc.Status.ConfigServerCount) + assert.Equal(t, 3, sc.Status.MongosCount) + assert.Equal(t, 6, sc.Status.MongodsPerShardCount) + + // Scale up the Sharded Cluster + sc.Spec.MongodsPerShardCount = 3 // from 6 + sc.Spec.MongosCount = 1 // from 3 + sc.Spec.ShardCount = 1 // from 2 + sc.Spec.ConfigServerCount = 1 // from 3 + + err = clusterClient.Update(ctx, sc) + assert.NoError(t, err) + + performReconciliation := func(shouldRetry bool) { + res, err := reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.NoError(t, err) + if shouldRetry { + assert.Equal(t, time.Duration(10000000000), res.RequeueAfter) + } else { + ok, _ := workflow.OK().ReconcileResult() + assert.Equal(t, ok, res) + } + err = clusterClient.Get(ctx, sc.ObjectKey(), sc) + assert.NoError(t, err) + } + + getShard := func(i int) *appsv1.StatefulSet { + sts := appsv1.StatefulSet{} + err := clusterClient.Get(ctx, types.NamespacedName{Name: sc.ShardRsName(i), Namespace: sc.Namespace}, &sts) + if errors.IsNotFound(err) { + return nil + } + return &sts + } + + t.Run("1st reconciliation", func(t *testing.T) { + performReconciliation(true) + assert.Equal(t, 3, sc.Status.ShardCount) + assert.Equal(t, 2, sc.Status.MongosCount) + assert.Equal(t, 2, sc.Status.ConfigServerCount) + assert.Equal(t, int32(5), *getShard(0).Spec.Replicas) + // shards to be deleted are not updated anymore + assert.Equal(t, int32(6), *getShard(1).Spec.Replicas) + assert.Equal(t, int32(6), *getShard(2).Spec.Replicas) + assert.NotNil(t, getShard(1), "Shard 1 should not be removed until the scaling operation is complete") + assert.NotNil(t, getShard(2), "Shard 2 should not be removed until the scaling operation is complete") + }) + t.Run("2nd reconciliation", func(t *testing.T) { + performReconciliation(true) + assert.Equal(t, 3, sc.Status.ShardCount) + assert.Equal(t, 1, sc.Status.MongosCount) + assert.Equal(t, 1, sc.Status.ConfigServerCount) + assert.Equal(t, int32(4), *getShard(0).Spec.Replicas) + assert.Equal(t, int32(6), *getShard(1).Spec.Replicas) + assert.Equal(t, int32(6), *getShard(2).Spec.Replicas) + assert.NotNil(t, getShard(1), "Shard 1 should not be removed until the scaling operation is complete") + assert.NotNil(t, getShard(2), "Shard 2 should not be removed until the scaling operation is complete") + }) + t.Run("Final reconciliation", func(t *testing.T) { + performReconciliation(false) + assert.Equal(t, 1, sc.Status.ShardCount, "Upon finishing reconciliation, the original shard count should be set to the current value") + assert.Equal(t, 1, sc.Status.MongosCount) + assert.Equal(t, 1, sc.Status.ConfigServerCount) + assert.Equal(t, int32(3), *getShard(0).Spec.Replicas) + assert.Nil(t, getShard(1), "Shard 1 should be removed as we have reached have finished scaling") + assert.Nil(t, getShard(2), "Shard 2 should be removed as we have reached have finished scaling") + }) +} + +func TestFeatureControlsAuthEnabled(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().Build() + omConnectionFactory := om.NewCachedOMConnectionFactory(omConnectionFactoryFuncSettingVersion()) + fakeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, sc) + reconciler := newShardedClusterReconciler(ctx, fakeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, nil, omConnectionFactory.GetConnectionFunc) + + checkReconcileSuccessful(ctx, t, reconciler, sc, fakeClient) + + cf, _ := omConnectionFactory.GetConnection().GetControlledFeature() + + assert.Len(t, cf.Policies, 3) + + assert.Equal(t, cf.ManagementSystem.Version, util.OperatorVersion) + assert.Equal(t, cf.ManagementSystem.Name, util.OperatorName) + + var policies []controlledfeature.PolicyType + for _, p := range cf.Policies { + policies = append(policies, p.PolicyType) + } + + assert.Contains(t, policies, controlledfeature.ExternallyManaged) + assert.Contains(t, policies, controlledfeature.DisableAuthenticationMechanisms) + assert.Contains(t, policies, controlledfeature.DisableMongodVersion) +} + +func TestShardedClusterPortsAreConfigurable_WithAdditionalMongoConfig(t *testing.T) { + ctx := context.Background() + configSrvConfig := mdbv1.NewAdditionalMongodConfig("net.port", 30000) + mongosConfig := mdbv1.NewAdditionalMongodConfig("net.port", 30001) + shardConfig := mdbv1.NewAdditionalMongodConfig("net.port", 30002) + + sc := mdbv1.NewClusterBuilder(). + SetNamespace(mock.TestNamespace). + SetConnectionSpec(testConnectionSpec()). + SetConfigSrvAdditionalConfig(configSrvConfig). + SetMongosAdditionalConfig(mongosConfig). + SetShardAdditionalConfig(shardConfig). + Build() + + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + t.Run("Config Server Port is configured", func(t *testing.T) { + configSrvSvc, err := clusterClient.GetService(ctx, kube.ObjectKey(sc.Namespace, sc.ConfigSrvServiceName())) + assert.NoError(t, err) + assert.Equal(t, int32(30000), configSrvSvc.Spec.Ports[0].Port) + }) + + t.Run("Mongos Port is configured", func(t *testing.T) { + mongosSvc, err := clusterClient.GetService(ctx, kube.ObjectKey(sc.Namespace, sc.ServiceName())) + assert.NoError(t, err) + assert.Equal(t, int32(30001), mongosSvc.Spec.Ports[0].Port) + }) + + t.Run("Shard Port is configured", func(t *testing.T) { + shardSvc, err := clusterClient.GetService(ctx, kube.ObjectKey(sc.Namespace, sc.ShardServiceName())) + assert.NoError(t, err) + assert.Equal(t, int32(30002), shardSvc.Spec.Ports[0].Port) + }) +} + +// TestShardedCluster_ConfigMapAndSecretWatched verifies that config map and secret are added to the internal +// map that allows to watch them for changes +func TestShardedCluster_ConfigMapAndSecretWatched(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().Build() + + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, sc.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, sc.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, sc.Name)}, + } + + assert.Equal(t, reconciler.resourceWatcher.GetWatchedResources(), expected) +} + +// TestShardedClusterTLSResourcesWatched verifies that TLS config map and secret are added to the internal +// map that allows to watch them for changes +func TestShardedClusterTLSAndInternalAuthResourcesWatched(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder().SetShardCountSpec(1).EnableTLS().SetTLSCA("custom-ca").Build() + sc.Spec.Security.Authentication.InternalCluster = "x509" + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + addKubernetesTlsResources(ctx, clusterClient, sc) + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + expectedWatchedResources := []watch.Object{ + getWatch(sc.Namespace, sc.Name+"-config-cert", watch.Secret), + getWatch(sc.Namespace, sc.Name+"-config-clusterfile", watch.Secret), + getWatch(sc.Namespace, sc.Name+"-mongos-cert", watch.Secret), + getWatch(sc.Namespace, sc.Name+"-mongos-clusterfile", watch.Secret), + getWatch(sc.Namespace, sc.Name+"-0-cert", watch.Secret), + getWatch(sc.Namespace, sc.Name+"-0-clusterfile", watch.Secret), + getWatch(sc.Namespace, "custom-ca", watch.ConfigMap), + getWatch(sc.Namespace, "my-credentials", watch.Secret), + getWatch(sc.Namespace, "my-project", watch.ConfigMap), + } + + var actual []watch.Object + for obj := range reconciler.resourceWatcher.GetWatchedResources() { + actual = append(actual, obj) + } + + assert.ElementsMatch(t, expectedWatchedResources, actual) + + // ReconcileMongoDbShardedCluster.publishDeployment - once internal cluster authentication is enabled, + // it is impossible to turn it off. + sc.Spec.Security.TLSConfig.Enabled = false + sc.Spec.Security.Authentication.InternalCluster = "" + err = clusterClient.Update(ctx, sc) + assert.NoError(t, err) + + res, err := reconciler.Reconcile(ctx, requestFromObject(sc)) + assert.Equal(t, reconcile.Result{RequeueAfter: 10 * time.Second}, res) + assert.NoError(t, err) + assert.Len(t, reconciler.resourceWatcher.GetWatchedResources(), 2) +} + +func TestBackupConfiguration_ShardedCluster(t *testing.T) { + ctx := context.Background() + sc := mdbv1.NewClusterBuilder(). + SetNamespace(mock.TestNamespace). + SetConnectionSpec(testConnectionSpec()). + SetBackup(mdbv1.Backup{ + Mode: "enabled", + }). + Build() + + reconciler, _, clusterClient, omConnectionFactory, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + omConnectionFactory.SetPostCreateHook(func(c om.Connection) { + // 4 because config server + num shards + 1 for entity to represent the sharded cluster itself + clusterIds := []string{"1", "2", "3", "4"} + typeNames := []string{"SHARDED_REPLICA_SET", "REPLICA_SET", "REPLICA_SET", "CONFIG_SERVER_REPLICA_SET"} + for i, clusterId := range clusterIds { + _, err := c.UpdateBackupConfig(&backup.Config{ + ClusterId: clusterId, + Status: backup.Inactive, + }) + require.NoError(t, err) + + c.(*om.MockedOmConnection).BackupHostClusters[clusterId] = &backup.HostCluster{ + ClusterName: sc.Name, + ShardName: "ShardedCluster", + TypeName: typeNames[i], + } + c.(*om.MockedOmConnection).CleanHistory() + } + }) + + assertAllOtherBackupConfigsRemainUntouched := func(t *testing.T) { + for _, configId := range []string{"2", "3", "4"} { + config, err := omConnectionFactory.GetConnection().ReadBackupConfig(configId) + assert.NoError(t, err) + // backup status should remain INACTIVE for all non "SHARDED_REPLICA_SET" configs. + assert.Equal(t, backup.Inactive, config.Status) + } + } + + t.Run("Backup can be started", func(t *testing.T) { + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + config, err := omConnectionFactory.GetConnection().ReadBackupConfig("1") + assert.NoError(t, err) + assert.Equal(t, backup.Started, config.Status) + assert.Equal(t, "1", config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + assertAllOtherBackupConfigsRemainUntouched(t) + }) + + t.Run("Backup snapshot schedule tests", backupSnapshotScheduleTests(sc, clusterClient, reconciler, omConnectionFactory, "1")) + + t.Run("Backup can be stopped", func(t *testing.T) { + sc.Spec.Backup.Mode = "disabled" + err := clusterClient.Update(ctx, sc) + assert.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + config, err := omConnectionFactory.GetConnection().ReadBackupConfig("1") + assert.NoError(t, err) + assert.Equal(t, backup.Stopped, config.Status) + assert.Equal(t, "1", config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + assertAllOtherBackupConfigsRemainUntouched(t) + }) + + t.Run("Backup can be terminated", func(t *testing.T) { + sc.Spec.Backup.Mode = "terminated" + err := clusterClient.Update(ctx, sc) + assert.NoError(t, err) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + config, err := omConnectionFactory.GetConnection().ReadBackupConfig("1") + assert.NoError(t, err) + assert.Equal(t, backup.Terminating, config.Status) + assert.Equal(t, "1", config.ClusterId) + assert.Equal(t, "PRIMARY", config.SyncSource) + assertAllOtherBackupConfigsRemainUntouched(t) + }) +} + +// createShardedClusterTLSSecretsFromCustomCerts creates and populates all the required +// secrets required to enabled TLS with custom certs for all sharded cluster components. +func createShardedClusterTLSSecretsFromCustomCerts(ctx context.Context, sc *mdbv1.MongoDB, prefix string, client kubernetesClient.Client) { + mongosSecret := secret.Builder(). + SetName(fmt.Sprintf("%s-%s-cert", prefix, sc.MongosRsName())). + SetNamespace(sc.Namespace).SetDataType(corev1.SecretTypeTLS). + Build() + + mongosSecret.Data["tls.crt"], mongosSecret.Data["tls.key"] = createMockCertAndKeyBytes() + + err := client.CreateSecret(ctx, mongosSecret) + if err != nil { + panic(err) + } + + configSrvSecret := secret.Builder(). + SetName(fmt.Sprintf("%s-%s-cert", prefix, sc.ConfigRsName())). + SetNamespace(sc.Namespace).SetDataType(corev1.SecretTypeTLS). + Build() + + configSrvSecret.Data["tls.crt"], configSrvSecret.Data["tls.key"] = createMockCertAndKeyBytes() + + err = client.CreateSecret(ctx, configSrvSecret) + if err != nil { + panic(err) + } + + for i := 0; i < sc.Spec.ShardCount; i++ { + shardSecret := secret.Builder(). + SetName(fmt.Sprintf("%s-%s-cert", prefix, sc.ShardRsName(i))). + SetNamespace(sc.Namespace).SetDataType(corev1.SecretTypeTLS). + Build() + + shardSecret.Data["tls.crt"], shardSecret.Data["tls.key"] = createMockCertAndKeyBytes() + + err = client.CreateSecret(ctx, shardSecret) + if err != nil { + panic(err) + } + } +} + +func TestTlsConfigPrefix_ForShardedCluster(t *testing.T) { + ctx := context.Background() + sc := test.DefaultClusterBuilder(). + SetTLSConfig(mdbv1.TLSConfig{ + Enabled: false, + }). + Build() + + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + + createShardedClusterTLSSecretsFromCustomCerts(ctx, sc, "my-prefix", clusterClient) + + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) +} + +func TestShardSpecificPodSpec(t *testing.T) { + ctx := context.Background() + shardPodSpec := corev1.PodSpec{ + NodeName: "some-node-name", + Hostname: "some-host-name", + Containers: []corev1.Container{{ + Name: "my-custom-container-sc", + Image: "my-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "my-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + } + + shard0PodSpec := corev1.PodSpec{ + NodeName: "shard0-node-name", + Containers: []corev1.Container{{ + Name: "shard0-container", + Image: "shard0-custom-image", + VolumeMounts: []corev1.VolumeMount{{ + Name: "shard0-volume-mount", + }}, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + } + + sc := test.DefaultClusterBuilder().SetName("shard-specific-pod-spec").EnableTLS().SetTLSCA("custom-ca"). + SetShardPodSpec(corev1.PodTemplateSpec{ + Spec: shardPodSpec, + }).SetShardSpecificPodSpecTemplate([]corev1.PodTemplateSpec{ + { + Spec: shard0PodSpec, + }, + }).Build() + + reconciler, _, clusterClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + addKubernetesTlsResources(ctx, clusterClient, sc) + checkReconcileSuccessful(ctx, t, reconciler, sc, clusterClient) + + // read the statefulsets from the cluster + statefulSetSc0, err := clusterClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "shard-specific-pod-spec-0")) + assert.NoError(t, err) + statefulSetSc1, err := clusterClient.GetStatefulSet(ctx, kube.ObjectKey(mock.TestNamespace, "shard-specific-pod-spec-1")) + assert.NoError(t, err) + + // shard0 should have the override + assertPodSpecSts(t, &statefulSetSc0, shard0PodSpec.NodeName, shard0PodSpec.Hostname, shard0PodSpec.RestartPolicy) + + // shard1 should have the common one + assertPodSpecSts(t, &statefulSetSc1, shardPodSpec.NodeName, shardPodSpec.Hostname, shardPodSpec.RestartPolicy) +} + +func TestShardedClusterAgentVersionMapping(t *testing.T) { + ctx := context.Background() + defaultResource := test.DefaultClusterBuilder().Build() + reconcilerFactory := func(sc *mdbv1.MongoDB) (reconcile.Reconciler, kubernetesClient.Client) { + // Go couldn't infer correctly that *ReconcileMongoDbShardedCluster implemented *reconciler.Reconciler interface + // without this anonymous function + reconciler, _, mockClient, _, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + require.NoError(t, err) + return reconciler, mockClient + } + + defaultResources := testReconciliationResources{ + Resource: defaultResource, + ReconcilerFactory: reconcilerFactory, + } + + containers := []corev1.Container{{Name: util.AgentContainerName, Image: "foo"}} + podTemplate := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: containers, + }, + } + + // Override each sts of the sharded cluster + overridenResource := test.DefaultClusterBuilder().SetMongosPodSpecTemplate(podTemplate).SetPodConfigSvrSpecTemplate(podTemplate).SetShardPodSpec(podTemplate).Build() + overridenResources := testReconciliationResources{ + Resource: overridenResource, + ReconcilerFactory: reconcilerFactory, + } + + agentVersionMappingTest(ctx, t, defaultResources, overridenResources) +} + +func assertPodSpecSts(t *testing.T, sts *appsv1.StatefulSet, nodeName, hostName string, restartPolicy corev1.RestartPolicy) { + podSpecTemplate := sts.Spec.Template.Spec + // ensure values were passed to the stateful set + assert.Equal(t, nodeName, podSpecTemplate.NodeName) + assert.Equal(t, hostName, podSpecTemplate.Hostname) + assert.Equal(t, restartPolicy, podSpecTemplate.RestartPolicy) + + if architectures.IsRunningStaticArchitecture(nil) { + assert.Equal(t, util.AgentContainerName, podSpecTemplate.Containers[0].Name, "Database container should always be first") + } else { + assert.Equal(t, util.DatabaseContainerName, podSpecTemplate.Containers[0].Name, "Database container should always be first") + assert.True(t, statefulset.VolumeMountWithNameExists(podSpecTemplate.Containers[0].VolumeMounts, construct.PvcNameDatabaseScripts)) + } +} + +func createMongosProcesses(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, mdb *mdbv1.MongoDB, certificateFilePath string) []om.Process { + hostnames, names := dns.GetDnsForStatefulSet(set, mdb.Spec.GetClusterDomain(), nil) + processes := make([]om.Process, len(hostnames)) + + for idx, hostname := range hostnames { + processes[idx] = om.NewMongosProcess(names[idx], hostname, mongoDBImage, forceEnterprise, mdb.Spec.MongosSpec.GetAdditionalMongodConfig(), mdb.GetSpec(), certificateFilePath, mdb.Annotations, mdb.CalculateFeatureCompatibilityVersion()) + } + + return processes +} + +func createDeploymentFromShardedCluster(t *testing.T, updatable v1.CustomResourceReadWriter) om.Deployment { + sh := updatable.(*mdbv1.MongoDB) + + shards := make([]om.ReplicaSetWithProcesses, sh.Spec.ShardCount) + kubeClient, _ := mock.NewDefaultFakeClient(sh) + shardSpec, memberCluster := createShardSpecAndDefaultCluster(kubeClient, sh) + + for i := 0; i < sh.Spec.ShardCount; i++ { + shardOptions := construct.ShardOptions(i, shardSpec, memberCluster.Name, + Replicas(sh.Spec.MongodsPerShardCount), + construct.GetPodEnvOptions(), + ) + shardSts := construct.DatabaseStatefulSet(*sh, shardOptions, zap.S()) + shards[i], _ = buildReplicaSetFromProcesses(shardSts.Name, createShardProcesses("fake-mongoDBImage", false, shardSts, sh, ""), sh, sh.Spec.GetMemberOptions(), om.NewDeployment()) + } + + desiredMongosConfig := createMongosSpec(sh) + mongosOptions := construct.MongosOptions( + desiredMongosConfig, + memberCluster.Name, + Replicas(sh.Spec.MongosCount), + construct.GetPodEnvOptions(), + ) + mongosSts := construct.DatabaseStatefulSet(*sh, mongosOptions, zap.S()) + mongosProcesses := createMongosProcesses("fake-mongoDBImage", false, mongosSts, sh, util.PEMKeyFilePathInContainer) + + desiredConfigSrvConfig := createConfigSrvSpec(sh) + configServerOptions := construct.ConfigServerOptions( + desiredConfigSrvConfig, + memberCluster.Name, + Replicas(sh.Spec.ConfigServerCount), + construct.GetPodEnvOptions(), + ) + configSvrSts := construct.DatabaseStatefulSet(*sh, configServerOptions, zap.S()) + configRs, _ := buildReplicaSetFromProcesses(configSvrSts.Name, createConfigSrvProcesses("fake-mongoDBImage", false, configSvrSts, sh, ""), sh, sh.Spec.GetMemberOptions(), om.NewDeployment()) + + d := om.NewDeployment() + _, err := d.MergeShardedCluster(om.DeploymentShardedClusterMergeOptions{ + Name: sh.Name, + MongosProcesses: mongosProcesses, + ConfigServerRs: configRs, + Shards: shards, + Finalizing: false, + }) + assert.NoError(t, err) + d.AddMonitoringAndBackup(zap.S(), sh.Spec.GetSecurity().IsTLSEnabled(), util.CAFilePathInContainer) + return d +} + +func defaultClusterReconciler(ctx context.Context, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, sc *mdbv1.MongoDB, globalMemberClustersMap map[string]client.Client) (*ReconcileMongoDbShardedCluster, *ShardedClusterReconcileHelper, kubernetesClient.Client, *om.CachedOMConnectionFactory, error) { + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(sc) + r, reconcileHelper, err := newShardedClusterReconcilerFromResource(ctx, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, sc, globalMemberClustersMap, kubeClient, omConnectionFactory) + if err != nil { + return nil, nil, nil, nil, err + } + return r, reconcileHelper, kubeClient, omConnectionFactory, nil +} + +func newShardedClusterReconcilerFromResource(ctx context.Context, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, sc *mdbv1.MongoDB, globalMemberClustersMap map[string]client.Client, kubeClient kubernetesClient.Client, omConnectionFactory *om.CachedOMConnectionFactory) (*ReconcileMongoDbShardedCluster, *ShardedClusterReconcileHelper, error) { + r := newShardedClusterReconciler(ctx, kubeClient, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, false, globalMemberClustersMap, omConnectionFactory.GetConnectionFunc) + reconcileHelper, err := NewShardedClusterReconcilerHelper(ctx, r.ReconcileCommonController, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, false, sc, globalMemberClustersMap, omConnectionFactory.GetConnectionFunc, zap.S()) + if err != nil { + return nil, nil, err + } + if err := kubeClient.Get(ctx, kube.ObjectKeyFromApiObject(sc), sc); err != nil { + return nil, nil, err + } + return r, reconcileHelper, nil +} + +func computeSingleClusterShardOverridesFromDistribution(shardOverridesDistribution map[string]int) []mdbv1.ShardOverride { + var shardOverrides []mdbv1.ShardOverride + + // This will create shard overrides for shards references by shardNames, shardCount can be greater than the length + // of the map + for shardName, memberCount := range shardOverridesDistribution { + // Construct the ShardOverride for that shard + shardOverride := mdbv1.ShardOverride{ + ShardNames: []string{shardName}, + Members: ptr.To(memberCount), + } + + // Append the constructed ShardOverride to the shardOverrides slice + shardOverrides = append(shardOverrides, shardOverride) + } + return shardOverrides +} + +type SingleClusterShardedScalingTestCase struct { + name string + scalingSteps []SingleClusterShardedScalingStep +} + +type SingleClusterShardedScalingStep struct { + name string + shardCount int + mongodsPerShardCount int + shardOverrides map[string]int + expectedShardDistribution []int +} + +// This scaling test simulates multiple reconciliation loops until the reconciliation succeeds. +// We add hostnames to the OM mock so that the operator sees them as ready, and we mark the sts ready in the kube client +// as well. Because we mark all hostnames in OM ready at the beginning of the test, there are edge cases where we don't +// catch errors. +func SingleClusterShardedScalingWithOverridesTestCase(t *testing.T, tc SingleClusterShardedScalingTestCase) { + ctx := context.Background() + + mongosCount := 1 + configSrvCount := 1 + + sc := test.DefaultClusterBuilder(). + SetTopology(mdbv1.ClusterTopologySingleCluster). + SetShardCountSpec(tc.scalingSteps[0].shardCount). + SetMongodsPerShardCountSpec(tc.scalingSteps[0].mongodsPerShardCount). + SetConfigServerCountSpec(configSrvCount). + SetMongosCountSpec(mongosCount). + SetShardOverrides(computeSingleClusterShardOverridesFromDistribution(tc.scalingSteps[0].shardOverrides)). + Build() + + sc.Status = mdbv1.MongoDbStatus{} // The default builder fill scaling status with incorrect values by default, TODO change the builder + + // We add these hosts so that they are available in the mocked OM connection + addAllHostsWithDistribution := func(connection om.Connection, mongosCount int, configSrvCount int, shardDistribution []map[string]int) { + var shardMemberCounts []int + for _, distribution := range shardDistribution { + memberCount := distribution[multicluster.LegacyCentralClusterName] + shardMemberCounts = append(shardMemberCounts, memberCount) + } + allHostnames, _ := generateAllHostsSingleCluster(sc, mongosCount, configSrvCount, shardMemberCounts, test.ClusterLocalDomains, test.NoneExternalClusterDomains) + connection.(*om.MockedOmConnection).AddHosts(allHostnames) + } + + for _, scalingStep := range tc.scalingSteps { + t.Run(scalingStep.name, func(t *testing.T) { + reconciler, reconcilerHelper, kubeClient, omConnectionFactory, err := defaultClusterReconciler(ctx, nil, "", "", sc, nil) + _ = omConnectionFactory.GetConnectionFunc(&om.OMContext{GroupName: om.TestGroupName}) + require.NoError(t, err) + clusterMapping := reconcilerHelper.deploymentState.ClusterMapping + + var expectedShardDistribution []map[string]int + for _, memberCount := range scalingStep.expectedShardDistribution { + expectedShardDistribution = append(expectedShardDistribution, map[string]int{multicluster.LegacyCentralClusterName: memberCount}) + } + + addAllHostsWithDistribution(omConnectionFactory.GetConnection(), mongosCount, configSrvCount, expectedShardDistribution) + + err = kubeClient.Get(ctx, mock.ObjectKeyFromApiObject(sc), sc) + require.NoError(t, err) + sc.Spec.MongodsPerShardCount = scalingStep.mongodsPerShardCount + sc.Spec.ShardCount = scalingStep.shardCount + sc.Spec.ShardOverrides = computeSingleClusterShardOverridesFromDistribution(scalingStep.shardOverrides) + err = kubeClient.Update(ctx, sc) + require.NoError(t, err) + reconcileUntilSuccessful(ctx, t, reconciler, sc, kubeClient, []client.Client{kubeClient}, nil, false) + + // Verify scaled deployment + checkCorrectShardDistributionInStatefulSets(t, ctx, sc, clusterMapping, map[string]client.Client{multicluster.LegacyCentralClusterName: kubeClient}, expectedShardDistribution) + }) + } +} + +func TestSingleClusterShardedScalingWithOverrides(t *testing.T) { + scDefaultName := test.SCBuilderDefaultName + "-" + testCases := []SingleClusterShardedScalingTestCase{ + { + name: "Basic sample test", + scalingSteps: []SingleClusterShardedScalingStep{ + { + name: "Initial scaling", + shardCount: 3, + mongodsPerShardCount: 3, + shardOverrides: map[string]int{ + scDefaultName + "0": 5, + }, + expectedShardDistribution: []int{ + 5, + 3, + 3, + }, + }, + { + name: "Scale up mongodsPerShard", + shardCount: 3, + mongodsPerShardCount: 5, + shardOverrides: map[string]int{ + scDefaultName + "0": 5, + }, + expectedShardDistribution: []int{ + 5, + 5, + 5, + }, + }, + }, + }, + { + // This operation works in unit test only + // In e2e tests, the operator is waiting for uncreated hostnames to be ready + name: "Scale overrides up and down", + scalingSteps: []SingleClusterShardedScalingStep{ + { + name: "Initial deployment", + shardCount: 4, + mongodsPerShardCount: 2, + shardOverrides: map[string]int{ + scDefaultName + "0": 3, + scDefaultName + "1": 3, + scDefaultName + "3": 2, + }, + expectedShardDistribution: []int{ + 3, + 3, + 2, // Not overridden + 2, + }, + }, + { + name: "Scale overrides", + shardCount: 4, + mongodsPerShardCount: 2, + shardOverrides: map[string]int{ + scDefaultName + "0": 2, // Scaled down + scDefaultName + "1": 2, // Scaled down + scDefaultName + "3": 3, // Scaled up + }, + expectedShardDistribution: []int{ + 2, // Scaled down + 2, // Scaled down + 2, // Not overridden + 3, // Scaled up + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + SingleClusterShardedScalingWithOverridesTestCase(t, tc) + }) + } +} + +func generateHostsWithDistributionSingleCluster(stsName string, namespace string, memberCount int, clusterDomain string, externalClusterDomain string) ([]string, []string) { + var hosts []string + var podNames []string + for podIdx := range memberCount { + hosts = append(hosts, getSingleClusterFQDN(stsName, namespace, podIdx, clusterDomain, externalClusterDomain)) + podNames = append(podNames, getPodNameSingleCluster(stsName, podIdx)) + } + + return podNames, hosts +} + +func getPodNameSingleCluster(stsName string, podIdx int) string { + return fmt.Sprintf("%s-%d", stsName, podIdx) +} + +func getSingleClusterFQDN(stsName string, namespace string, podIdx int, clusterDomain string, externalClusterDomain string) string { + if len(externalClusterDomain) != 0 { + return fmt.Sprintf("%s.%s", getPodNameSingleCluster(stsName, podIdx), externalClusterDomain) + } + return fmt.Sprintf("%s-svc.%s.svc.%s", getPodNameSingleCluster(stsName, podIdx), namespace, clusterDomain) +} + +func generateAllHostsSingleCluster(sc *mdbv1.MongoDB, mongosCount int, configSrvCount int, shardsMemberCounts []int, clusterDomain test.ClusterDomains, externalClusterDomain test.ClusterDomains) ([]string, []string) { + var allHosts []string + var allPodNames []string + podNames, hosts := generateHostsWithDistributionSingleCluster(sc.MongosRsName(), sc.Namespace, mongosCount, clusterDomain.MongosExternalDomain, externalClusterDomain.MongosExternalDomain) + allHosts = append(allHosts, hosts...) + allPodNames = append(allPodNames, podNames...) + + podNames, hosts = generateHostsWithDistributionSingleCluster(sc.ConfigRsName(), sc.Namespace, configSrvCount, clusterDomain.ConfigServerExternalDomain, externalClusterDomain.ConfigServerExternalDomain) + allHosts = append(allHosts, hosts...) + allPodNames = append(allPodNames, podNames...) + + for shardIdx := 0; shardIdx < sc.Spec.ShardCount; shardIdx++ { + podNames, hosts = generateHostsWithDistributionSingleCluster(sc.ShardRsName(shardIdx), sc.Namespace, shardsMemberCounts[shardIdx], clusterDomain.ShardsExternalDomain, externalClusterDomain.ShardsExternalDomain) + allHosts = append(allHosts, hosts...) + allPodNames = append(allPodNames, podNames...) + } + return allHosts, allPodNames +} diff --git a/controllers/operator/mongodbstandalone_controller.go b/controllers/operator/mongodbstandalone_controller.go new file mode 100644 index 000000000..a5291d560 --- /dev/null +++ b/controllers/operator/mongodbstandalone_controller.go @@ -0,0 +1,414 @@ +package operator + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbstatus "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/deployment" + "github.com/10gen/ops-manager-kubernetes/controllers/om/host" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/agents" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" + "github.com/10gen/ops-manager-kubernetes/pkg/vault/vaultwatcher" +) + +// AddStandaloneController creates a new MongoDbStandalone Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func AddStandaloneController(ctx context.Context, mgr manager.Manager, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool) error { + // Create a new controller + reconciler := newStandaloneReconciler(ctx, mgr.GetClient(), imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, om.NewOpsManagerConnection) + c, err := controller.New(util.MongoDbStandaloneController, mgr, controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}) // nolint:forbidigo + if err != nil { + return err + } + + // watch for changes to standalone MongoDB resources + eventHandler := ResourceEventHandler{deleter: reconciler} + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &mdbv1.MongoDB{}, &eventHandler, watch.PredicatesForMongoDB(mdbv1.Standalone))) + if err != nil { + return err + } + + err = c.Watch( + source.Channel[client.Object](OmUpdateChannel, + &handler.EnqueueRequestForObject{}, + source.WithPredicates(watch.PredicatesForMongoDB(mdbv1.Standalone)), + )) + if err != nil { + return xerrors.Errorf("not able to setup OmUpdateChannel to listent to update events from OM: %s", err) + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, + &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + // if vault secret backend is enabled watch for Vault secret change and trigger reconcile + if vault.IsVaultSecretBackend() { + eventChannel := make(chan event.GenericEvent) + go vaultwatcher.WatchSecretChangeForMDB(ctx, zap.S(), eventChannel, reconciler.client, reconciler.VaultClient, mdbv1.Standalone) + + err = c.Watch( + source.Channel[client.Object](eventChannel, + &handler.EnqueueRequestForObject{}, + )) + if err != nil { + zap.S().Errorf("Failed to watch for vault secret changes: %w", err) + } + } + zap.S().Infof("Registered controller %s", util.MongoDbStandaloneController) + + return nil +} + +func newStandaloneReconciler(ctx context.Context, kubeClient client.Client, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, omFunc om.ConnectionFactory) *ReconcileMongoDbStandalone { + return &ReconcileMongoDbStandalone{ + ReconcileCommonController: NewReconcileCommonController(ctx, kubeClient), + omConnectionFactory: omFunc, + imageUrls: imageUrls, + forceEnterprise: forceEnterprise, + + initDatabaseNonStaticImageVersion: initDatabaseNonStaticImageVersion, + databaseNonStaticImageVersion: databaseNonStaticImageVersion, + } +} + +// ReconcileMongoDbStandalone reconciles a MongoDbStandalone object +type ReconcileMongoDbStandalone struct { + *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + imageUrls images.ImageUrls + forceEnterprise bool + + initDatabaseNonStaticImageVersion string + databaseNonStaticImageVersion string +} + +func (r *ReconcileMongoDbStandalone) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, e error) { + log := zap.S().With("Standalone", request.NamespacedName) + s := &mdbv1.MongoDB{} + + if reconcileResult, err := r.prepareResourceForReconciliation(ctx, request, s, log); err != nil { + if errors.IsNotFound(err) { + return workflow.Invalid("Object for reconciliation not found").ReconcileResult() + } + return reconcileResult, err + } + + if !architectures.IsRunningStaticArchitecture(s.Annotations) { + agents.UpgradeAllIfNeeded(ctx, agents.ClientSecret{Client: r.client, SecretClient: r.SecretClient}, r.omConnectionFactory, GetWatchedNamespace(), false) + } + + if err := s.ProcessValidationsOnReconcile(nil); err != nil { + return r.updateStatus(ctx, s, workflow.Invalid("%s", err.Error()), log) + } + + log.Info("-> Standalone.Reconcile") + log.Infow("Standalone.Spec", "spec", s.Spec) + log.Infow("Standalone.Status", "status", s.Status) + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, s, log) + if err != nil { + return r.updateStatus(ctx, s, workflow.Failed(err), log) + } + + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, s.Namespace, log) + if err != nil { + return r.updateStatus(ctx, s, workflow.Failed(xerrors.Errorf("Failed to prepare Ops Manager connection: %w", err)), log) + } + + if status := ensureSupportedOpsManagerVersion(conn); status.Phase() != mdbstatus.PhaseRunning { + return r.updateStatus(ctx, s, status, log) + } + + r.SetupCommonWatchers(s, nil, nil, s.Name) + + reconcileResult := checkIfHasExcessProcesses(conn, s.Name, log) + if !reconcileResult.IsOK() { + return r.updateStatus(ctx, s, reconcileResult, log) + } + + if status := controlledfeature.EnsureFeatureControls(*s, conn, conn.OpsManagerVersion(), log); !status.IsOK() { + return r.updateStatus(ctx, s, status, log) + } + + // cannot have a non-tls deployment in an x509 environment + // TODO move to webhook validations + security := s.Spec.Security + if security.Authentication != nil && security.Authentication.Enabled && security.Authentication.IsX509Enabled() && !s.Spec.GetSecurity().IsTLSEnabled() { + return r.updateStatus(ctx, s, workflow.Invalid("cannot have a non-tls deployment when x509 authentication is enabled"), log) + } + + currentAgentAuthMode, err := conn.GetAgentAuthMode() + if err != nil { + return r.updateStatus(ctx, s, workflow.Failed(err), log) + } + + podVars := newPodVars(conn, projectConfig, s.Spec.LogLevel) + + if status := validateMongoDBResource(s, conn); !status.IsOK() { + return r.updateStatus(ctx, s, status, log) + } + + if status := certs.EnsureSSLCertsForStatefulSet(ctx, r.SecretClient, r.SecretClient, *s.Spec.Security, certs.StandaloneConfig(*s), log); !status.IsOK() { + return r.updateStatus(ctx, s, status, log) + } + + // TODO separate PR + certConfigurator := certs.StandaloneX509CertConfigurator{MongoDB: s, SecretClient: r.SecretClient} + if status := r.ensureX509SecretAndCheckTLSType(ctx, certConfigurator, currentAgentAuthMode, log); !status.IsOK() { + return r.updateStatus(ctx, s, status, log) + } + + if status := ensureRoles(s.Spec.GetSecurity().Roles, conn, log); !status.IsOK() { + return r.updateStatus(ctx, s, status, log) + } + + var vaultConfig vault.VaultConfiguration + if r.VaultClient != nil { + vaultConfig = r.VaultClient.VaultConfig + } + standaloneCertSecretName := certs.StandaloneConfig(*s).CertSecretName + + var databaseSecretPath string + if r.VaultClient != nil { + databaseSecretPath = r.VaultClient.DatabaseSecretPath() + } + + var automationAgentVersion string + if architectures.IsRunningStaticArchitecture(s.Annotations) { + // In case the Agent *is* overridden, its version will be merged into the StatefulSet. The merging process + // happens after creating the StatefulSet definition. + if !s.IsAgentImageOverridden() { + automationAgentVersion, err = r.getAgentVersion(conn, conn.OpsManagerVersion().VersionString, false, log) + if err != nil { + log.Errorf("Impossible to get agent version, please override the agent image by providing a pod template") + status := workflow.Failed(xerrors.Errorf("Failed to get agent version: %w", err)) + return r.updateStatus(ctx, s, status, log) + } + } + } + + standaloneOpts := construct.StandaloneOptions( + CertificateHash(pem.ReadHashFromSecret(ctx, r.SecretClient, s.Namespace, standaloneCertSecretName, databaseSecretPath, log)), + CurrentAgentAuthMechanism(currentAgentAuthMode), + PodEnvVars(podVars), + WithVaultConfig(vaultConfig), + WithAdditionalMongodConfig(s.Spec.GetAdditionalMongodConfig()), + WithInitDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.InitDatabaseImageUrlEnv, r.initDatabaseNonStaticImageVersion)), + WithDatabaseNonStaticImage(images.ContainerImage(r.imageUrls, util.NonStaticDatabaseEnterpriseImage, r.databaseNonStaticImageVersion)), + WithAgentImage(images.ContainerImage(r.imageUrls, architectures.MdbAgentImageRepo, automationAgentVersion)), + WithMongodbImage(images.GetOfficialImage(r.imageUrls, s.Spec.Version, s.GetAnnotations())), + ) + + sts := construct.DatabaseStatefulSet(*s, standaloneOpts, log) + + workflowStatus := create.HandlePVCResize(ctx, r.client, &sts, log) + if !workflowStatus.IsOK() { + return r.updateStatus(ctx, s, workflowStatus, log) + } + + lastSpec, err := s.GetLastSpec() + if err != nil { + lastSpec = &mdbv1.MongoDbSpec{} + } + + status := workflow.RunInGivenOrder(publishAutomationConfigFirst(ctx, r.client, *s, lastSpec, standaloneOpts, log), + func() workflow.Status { + return r.updateOmDeployment(ctx, conn, s, sts, false, log).OnErrorPrepend("Failed to create/update (Ops Manager reconciliation phase):") + }, + func() workflow.Status { + if err = create.DatabaseInKubernetes(ctx, r.client, *s, sts, standaloneOpts, log); err != nil { + return workflow.Failed(xerrors.Errorf("Failed to create/update (Kubernetes reconciliation phase): %w", err)) + } + + if status := getStatefulSetStatus(ctx, sts.Namespace, sts.Name, r.client); !status.IsOK() { + return status + } + + log.Info("Updated StatefulSet for standalone") + return workflow.OK() + }) + + if !status.IsOK() { + return r.updateStatus(ctx, s, status, log) + } + + annotationsToAdd, err := getAnnotationsForResource(s) + if err != nil { + return r.updateStatus(ctx, s, workflow.Failed(err), log) + } + + if vault.IsVaultSecretBackend() { + secrets := s.GetSecretsMountedIntoDBPod() + vaultMap := make(map[string]string) + for _, secret := range secrets { + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.DatabaseSecretMetadataPath(), s.Namespace, secret) + vaultMap = merge.StringToStringMap(vaultMap, r.VaultClient.GetSecretAnnotation(path)) + } + path := fmt.Sprintf("%s/%s/%s", r.VaultClient.OperatorScretMetadataPath(), s.Namespace, s.Spec.Credentials) + vaultMap = merge.StringToStringMap(vaultMap, r.VaultClient.GetSecretAnnotation(path)) + for k, val := range vaultMap { + annotationsToAdd[k] = val + } + } + if err := annotations.SetAnnotations(ctx, s, annotationsToAdd, r.client); err != nil { + return r.updateStatus(ctx, s, workflow.Failed(err), log) + } + + log.Infof("Finished reconciliation for MongoDbStandalone! %s", completionMessage(conn.BaseURL(), conn.GroupID())) + return r.updateStatus(ctx, s, status, log, mdbstatus.NewBaseUrlOption(deployment.Link(conn.BaseURL(), conn.GroupID()))) +} + +func (r *ReconcileMongoDbStandalone) updateOmDeployment(ctx context.Context, conn om.Connection, s *mdbv1.MongoDB, set appsv1.StatefulSet, isRecovering bool, log *zap.SugaredLogger) workflow.Status { + if err := agents.WaitForRsAgentsToRegister(set, 0, s.Spec.GetClusterDomain(), conn, log, s); err != nil { + return workflow.Failed(err) + } + + // TODO standalone PR + status, additionalReconciliationRequired := r.updateOmAuthentication(ctx, conn, []string{set.Name}, s, "", "", "", isRecovering, log) + if !status.IsOK() { + return status + } + + standaloneOmObject := createProcess(r.imageUrls[mcoConstruct.MongodbImageEnv], r.forceEnterprise, set, util.DatabaseContainerName, s) + err := conn.ReadUpdateDeployment( + func(d om.Deployment) error { + excessProcesses := d.GetNumberOfExcessProcesses(s.Name) + if excessProcesses > 0 { + return xerrors.Errorf("cannot have more than 1 MongoDB Cluster per project (see https://docs.mongodb.com/kubernetes-operator/stable/tutorial/migrate-to-single-resource/)") + } + + lastStandaloneConfig, err := s.GetLastAdditionalMongodConfigByType(mdbv1.StandaloneConfig) + if err != nil { + return err + } + + d.MergeStandalone(standaloneOmObject, s.Spec.AdditionalMongodConfig.ToMap(), lastStandaloneConfig.ToMap(), nil) + // TODO change last argument in separate PR + d.AddMonitoringAndBackup(log, s.Spec.GetSecurity().IsTLSEnabled(), util.CAFilePathInContainer) + d.ConfigureTLS(s.Spec.GetSecurity(), util.CAFilePathInContainer) + return nil + }, + log, + ) + if err != nil { + return workflow.Failed(err) + } + + if err := om.WaitForReadyState(conn, []string{set.Name}, isRecovering, log); err != nil { + return workflow.Failed(err) + } + + if additionalReconciliationRequired { + return workflow.Pending("Performing multi stage reconciliation") + } + + log.Info("Updated Ops Manager for standalone") + return workflow.OK() +} + +func (r *ReconcileMongoDbStandalone) OnDelete(ctx context.Context, obj runtime.Object, log *zap.SugaredLogger) error { + s := obj.(*mdbv1.MongoDB) + + log.Infow("Removing standalone from Ops Manager", "config", s.Spec) + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, s, log) + if err != nil { + return err + } + + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, s.Namespace, log) + if err != nil { + return err + } + + processNames := make([]string, 0) + err = conn.ReadUpdateDeployment( + func(d om.Deployment) error { + processNames = d.GetProcessNames(om.Standalone{}, s.Name) + // error means that process is not in the deployment - it's ok and we can proceed (could happen if + // deletion cleanup happened twice and the first one cleaned OM state already) + if e := d.RemoveProcessByName(s.Name, log); e != nil { + log.Warnf("Failed to remove standalone from automation config: %s", e) + } + return nil + }, + log, + ) + if err != nil { + return xerrors.Errorf("failed to update Ops Manager automation config: %w", err) + } + + if err := om.WaitForReadyState(conn, processNames, false, log); err != nil { + return err + } + + hostsToRemove, _ := dns.GetDNSNames(s.Name, s.ServiceName(), s.Namespace, s.Spec.GetClusterDomain(), 1, nil) + log.Infow("Stop monitoring removed hosts", "removedHosts", hostsToRemove) + if err = host.StopMonitoring(conn, hostsToRemove, log); err != nil { + return err + } + if err := r.clearProjectAuthenticationSettings(ctx, conn, s, processNames, log); err != nil { + return err + } + + r.resourceWatcher.RemoveDependentWatchedResources(s.ObjectKey()) + + log.Infow("Clear feature control for group: %s", "groupID", conn.GroupID()) + if result := controlledfeature.ClearFeatureControls(conn, conn.OpsManagerVersion(), log); !result.IsOK() { + result.Log(log) + log.Warnf("Failed to clear feature control from group: %s", conn.GroupID()) + } + + log.Info("Removed standalone from Ops Manager!") + return nil +} + +func createProcess(mongoDBImage string, forceEnterprise bool, set appsv1.StatefulSet, containerName string, s *mdbv1.MongoDB) om.Process { + hostnames, _ := dns.GetDnsForStatefulSet(set, s.Spec.GetClusterDomain(), nil) + process := om.NewMongodProcess(s.Name, hostnames[0], mongoDBImage, forceEnterprise, s.Spec.GetAdditionalMongodConfig(), s.GetSpec(), "", s.Annotations, s.CalculateFeatureCompatibilityVersion()) + return process +} diff --git a/controllers/operator/mongodbstandalone_controller_test.go b/controllers/operator/mongodbstandalone_controller_test.go new file mode 100644 index 000000000..3586a739d --- /dev/null +++ b/controllers/operator/mongodbstandalone_controller_test.go @@ -0,0 +1,417 @@ +package operator + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/controlledfeature" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/pkg/dns" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +func TestCreateOmProcess(t *testing.T) { + const mongodbImage = "quay.io/mongodb/mongodb-enterprise-server" + sts := construct.DatabaseStatefulSet(*DefaultReplicaSetBuilder().SetName("dublin").Build(), construct.StandaloneOptions(construct.GetPodEnvOptions()), zap.S()) + process := createProcess(mongodbImage, false, sts, util.AgentContainerName, DefaultStandaloneBuilder().Build()) + // Note, that for standalone the name of process is the name of statefulset - not the pod inside it. + assert.Equal(t, "dublin", process.Name()) + assert.Equal(t, "dublin-0.dublin-svc.my-namespace.svc.cluster.local", process.HostName()) + assert.Equal(t, "4.0.0", process.Version()) +} + +func TestCreateOmProcesStatic(t *testing.T) { + const mongodbImage = "quay.io/mongodb/mongodb-enterprise-server" + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + sts := construct.DatabaseStatefulSet(*DefaultReplicaSetBuilder().SetName("dublin").Build(), construct.StandaloneOptions(construct.GetPodEnvOptions()), zap.S()) + process := createProcess(mongodbImage, false, sts, util.AgentContainerName, DefaultStandaloneBuilder().Build()) + // Note, that for standalone the name of process is the name of statefulset - not the pod inside it. + assert.Equal(t, "dublin", process.Name()) + assert.Equal(t, "dublin-0.dublin-svc.my-namespace.svc.cluster.local", process.HostName()) + assert.Equal(t, "4.0.0-ent", process.Version()) +} + +func TestOnAddStandalone(t *testing.T) { + ctx := context.Background() + st := DefaultStandaloneBuilder().SetVersion("4.1.0").SetService("mysvc").Build() + st.Status.FeatureCompatibilityVersion = "4.1" + + reconciler, kubeClient, omConnectionFactory := defaultStandaloneReconciler(ctx, nil, "", "", om.NewEmptyMockedOmConnection, st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + omConn := omConnectionFactory.GetConnection() + + // seems we don't need very deep checks here as there should be smaller tests specially for those methods + assert.Len(t, mock.GetMapForObject(kubeClient, &corev1.Service{}), 1) + assert.Len(t, mock.GetMapForObject(kubeClient, &appsv1.StatefulSet{}), 1) + assert.Equal(t, *mock.GetMapForObject(kubeClient, &appsv1.StatefulSet{})[st.ObjectKey()].(*appsv1.StatefulSet).Spec.Replicas, int32(1)) + assert.Len(t, mock.GetMapForObject(kubeClient, &corev1.Secret{}), 2) + + omConn.(*om.MockedOmConnection).CheckDeployment(t, createDeploymentFromStandalone(st), "auth", "tls") + omConn.(*om.MockedOmConnection).CheckNumberOfUpdateRequests(t, 1) +} + +func TestStandaloneClusterReconcileContainerImages(t *testing.T) { + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_1_0_0", util.NonStaticDatabaseEnterpriseImage) + initDatabaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_2_0_0", util.InitDatabaseImageUrlEnv) + + imageUrlsMock := images.ImageUrls{ + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", + initDatabaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", + } + + ctx := context.Background() + st := DefaultStandaloneBuilder().SetVersion("8.0.0").Build() + reconciler, kubeClient, _ := defaultReplicaSetReconciler(ctx, imageUrlsMock, "2.0.0", "1.0.0", st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + sts := &appsv1.StatefulSet{} + err := kubeClient.Get(ctx, kube.ObjectKey(st.Namespace, st.Name), sts) + assert.NoError(t, err) + + require.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + require.Len(t, sts.Spec.Template.Spec.Containers, 1) + + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database:@sha256:MONGODB_INIT_DATABASE", sts.Spec.Template.Spec.InitContainers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-database:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[0].Image) +} + +func TestStandaloneClusterReconcileContainerImagesWithStaticArchitecture(t *testing.T) { + t.Setenv(architectures.DefaultEnvArchitecture, string(architectures.Static)) + + databaseRelatedImageEnv := fmt.Sprintf("RELATED_IMAGE_%s_8_0_0_ubi9", mcoConstruct.MongodbImageEnv) + + imageUrlsMock := images.ImageUrls{ + architectures.MdbAgentImageRepo: "quay.io/mongodb/mongodb-agent-ubi", + mcoConstruct.MongodbImageEnv: "quay.io/mongodb/mongodb-enterprise-server", + databaseRelatedImageEnv: "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", + } + + ctx := context.Background() + st := DefaultStandaloneBuilder().SetVersion("8.0.0").Build() + reconciler, kubeClient, omConnectionFactory := defaultReplicaSetReconciler(ctx, imageUrlsMock, "", "", st) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + connection.(*om.MockedOmConnection).SetAgentVersion("12.0.30.7791-1", "") + }) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + sts := &appsv1.StatefulSet{} + err := kubeClient.Get(ctx, kube.ObjectKey(st.Namespace, st.Name), sts) + assert.NoError(t, err) + + assert.Len(t, sts.Spec.Template.Spec.InitContainers, 0) + require.Len(t, sts.Spec.Template.Spec.Containers, 2) + + // Version from OM + operator version + assert.Equal(t, "quay.io/mongodb/mongodb-agent-ubi:12.0.30.7791-1_9.9.9-test", sts.Spec.Template.Spec.Containers[0].Image) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-server:@sha256:MONGODB_DATABASE", sts.Spec.Template.Spec.Containers[1].Image) +} + +// TestOnAddStandaloneWithDelay checks the reconciliation on standalone creation with some "delay" in getting +// StatefulSet ready. The first reconciliation gets to Pending while the second reconciliation suceeds +func TestOnAddStandaloneWithDelay(t *testing.T) { + ctx := context.Background() + st := DefaultStandaloneBuilder().SetVersion("4.1.0").SetService("mysvc").Build() + + markStsAsReady := false + omConnectionFactory := om.NewDefaultCachedOMConnectionFactory() + kubeClient := mock.NewEmptyFakeClientBuilder().WithObjects(st).WithObjects(mock.GetDefaultResources()...).Build() + kubeClient = interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + // we reference markAsStsReady from outside scope, which is modified to false below + return mock.GetFakeClientInterceptorGetFunc(omConnectionFactory, markStsAsReady, markStsAsReady)(ctx, client, key, obj, opts...) + }, + }) + + reconciler := newStandaloneReconciler(ctx, kubeClient, nil, "fake-initDatabaseNonStaticImageVersion", "fake-databaseNonStaticImageVersion", false, omConnectionFactory.GetConnectionFunc) + + checkReconcilePending(ctx, t, reconciler, st, "StatefulSet not ready", kubeClient, 3) + // this affects Get interceptor func, blocking automatically marking sts as ready + markStsAsReady = true + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) +} + +// TestAddDeleteStandalone checks that no state is left in OpsManager on removal of the standalone +func TestAddDeleteStandalone(t *testing.T) { + ctx := context.Background() + // First we need to create a standalone + st := DefaultStandaloneBuilder().SetVersion("4.0.0").Build() + + reconciler, kubeClient, omConnectionFactory := defaultStandaloneReconciler(ctx, nil, "", "", om.NewEmptyMockedOmConnection, st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + // Now delete it + assert.NoError(t, reconciler.OnDelete(ctx, st, zap.S())) + + mockedConn := omConnectionFactory.GetConnection().(*om.MockedOmConnection) + // Operator doesn't mutate K8s state, so we don't check its changes, only OM + mockedConn.CheckResourcesDeleted(t) + + // Note, that 'omConn.ReadAutomationStatus' happened twice - because the connection emulates agents delay in reaching goal state + mockedConn.CheckOrderOfOperations(t, + reflect.ValueOf(mockedConn.ReadUpdateDeployment), reflect.ValueOf(mockedConn.ReadAutomationStatus), + reflect.ValueOf(mockedConn.ReadAutomationStatus), reflect.ValueOf(mockedConn.GetHosts), reflect.ValueOf(mockedConn.RemoveHost)) +} + +func TestStandaloneAuthenticationOwnedByOpsManager(t *testing.T) { + ctx := context.Background() + stBuilder := DefaultStandaloneBuilder() + stBuilder.Spec.Security = nil + st := stBuilder.Build() + + reconciler, kubeClient, omConnectionFactory := defaultStandaloneReconciler(ctx, nil, "", "", omConnectionFactoryFuncSettingVersion(), st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + cf, _ := omConnectionFactory.GetConnection().GetControlledFeature() + + assert.Len(t, cf.Policies, 2) + assert.Equal(t, cf.ManagementSystem.Version, util.OperatorVersion) + assert.Equal(t, cf.ManagementSystem.Name, util.OperatorName) + assert.Equal(t, cf.Policies[0].PolicyType, controlledfeature.ExternallyManaged) + assert.Len(t, cf.Policies[0].DisabledParams, 0) +} + +func omConnectionFactoryFuncSettingVersion() func(context *om.OMContext) om.Connection { + return func(context *om.OMContext) om.Connection { + context.Version = versionutil.OpsManagerVersion{ + VersionString: "5.0.0", + } + conn := om.NewEmptyMockedOmConnection(context) + return conn + } +} + +func TestStandaloneAuthenticationOwnedByOperator(t *testing.T) { + ctx := context.Background() + st := DefaultStandaloneBuilder().Build() + + reconciler, kubeClient, omConnectionFactory := defaultStandaloneReconciler(ctx, nil, "", "", omConnectionFactoryFuncSettingVersion(), st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + mockedConn := omConnectionFactory.GetConnection() + cf, _ := mockedConn.GetControlledFeature() + + assert.Len(t, cf.Policies, 3) + assert.Equal(t, cf.ManagementSystem.Version, util.OperatorVersion) + assert.Equal(t, cf.ManagementSystem.Name, util.OperatorName) + + var policies []controlledfeature.PolicyType + for _, p := range cf.Policies { + policies = append(policies, p.PolicyType) + } + + assert.Contains(t, policies, controlledfeature.ExternallyManaged) + assert.Contains(t, policies, controlledfeature.DisableAuthenticationMechanisms) +} + +func TestStandalonePortIsConfigurable_WithAdditionalMongoConfig(t *testing.T) { + ctx := context.Background() + config := mdbv1.NewAdditionalMongodConfig("net.port", 30000) + st := mdbv1.NewStandaloneBuilder(). + SetNamespace(mock.TestNamespace). + SetAdditionalConfig(config). + SetConnectionSpec(testConnectionSpec()). + Build() + + reconciler, kubeClient, _ := defaultStandaloneReconciler(ctx, nil, "", "", om.NewEmptyMockedOmConnection, st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + svc, err := kubeClient.GetService(ctx, kube.ObjectKey(st.Namespace, st.ServiceName())) + assert.NoError(t, err) + assert.Equal(t, int32(30000), svc.Spec.Ports[0].Port) +} + +func TestStandaloneCustomPodSpecTemplate(t *testing.T) { + ctx := context.Background() + st := DefaultStandaloneBuilder().SetPodSpecTemplate(corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"first": "val"}}, + }).Build() + + reconciler, kubeClient, _ := defaultStandaloneReconciler(ctx, nil, "", "", om.NewEmptyMockedOmConnection, st) + + checkReconcileSuccessful(ctx, t, reconciler, st, kubeClient) + + statefulSet, err := kubeClient.GetStatefulSet(ctx, mock.ObjectKeyFromApiObject(st)) + assert.NoError(t, err) + + expectedLabels := map[string]string{ + "app": "dublin-svc", "controller": "mongodb-enterprise-operator", + "first": "val", "pod-anti-affinity": "dublin", + } + assert.Equal(t, expectedLabels, statefulSet.Spec.Template.Labels) +} + +// TestStandalone_ConfigMapAndSecretWatched +func TestStandalone_ConfigMapAndSecretWatched(t *testing.T) { + ctx := context.Background() + s := DefaultStandaloneBuilder().Build() + + reconciler, kubeClient, _ := defaultStandaloneReconciler(ctx, nil, "", "", om.NewEmptyMockedOmConnection, s) + + checkReconcileSuccessful(ctx, t, reconciler, s, kubeClient) + + expected := map[watch.Object][]types.NamespacedName{ + {ResourceType: watch.ConfigMap, Resource: kube.ObjectKey(mock.TestNamespace, mock.TestProjectConfigMapName)}: {kube.ObjectKey(mock.TestNamespace, s.Name)}, + {ResourceType: watch.Secret, Resource: kube.ObjectKey(mock.TestNamespace, s.Spec.Credentials)}: {kube.ObjectKey(mock.TestNamespace, s.Name)}, + } + + actual := reconciler.resourceWatcher.GetWatchedResources() + assert.Equal(t, expected, actual) +} + +func TestStandaloneAgentVersionMapping(t *testing.T) { + ctx := context.Background() + defaultResource := DefaultStandaloneBuilder().Build() + // Go couldn't infer correctly that *ReconcileMongoDbReplicaset implemented *reconciler.Reconciler interface + // without this anonymous function + reconcilerFactory := func(s *mdbv1.MongoDB) (reconcile.Reconciler, kubernetesClient.Client) { + // Call the original defaultReplicaSetReconciler, which returns a *ReconcileMongoDbReplicaSet that implements reconcile.Reconciler + reconciler, mockClient, _ := defaultStandaloneReconciler(ctx, nil, "", "", om.NewEmptyMockedOmConnection, s) + // Return the reconciler as is, because it implements the reconcile.Reconciler interface + return reconciler, mockClient + } + defaultResources := testReconciliationResources{ + Resource: defaultResource, + ReconcilerFactory: reconcilerFactory, + } + + containers := []corev1.Container{{Name: util.AgentContainerName, Image: "foo"}} + podTemplate := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: containers, + }, + } + + overriddenResource := DefaultStandaloneBuilder().SetPodSpecTemplate(podTemplate).Build() + overriddenResources := testReconciliationResources{ + Resource: overriddenResource, + ReconcilerFactory: reconcilerFactory, + } + + agentVersionMappingTest(ctx, t, defaultResources, overriddenResources) +} + +// defaultStandaloneReconciler is the standalone reconciler used in unit test. It "adds" necessary +// additional K8s objects (st, connection config map and secrets) necessary for reconciliation, +// so it's possible to call 'reconcileAppDB()' on it right away +func defaultStandaloneReconciler(ctx context.Context, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, omConnectionFactoryFunc om.ConnectionFactory, rs *mdbv1.MongoDB) (*ReconcileMongoDbStandalone, kubernetesClient.Client, *om.CachedOMConnectionFactory) { + omConnectionFactory := om.NewCachedOMConnectionFactory(omConnectionFactoryFunc) + kubeClient := mock.NewDefaultFakeClientWithOMConnectionFactory(omConnectionFactory, rs) + return newStandaloneReconciler(ctx, kubeClient, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, false, omConnectionFactory.GetConnectionFunc), kubeClient, omConnectionFactory +} + +// TODO remove in favor of '/api/mongodbbuilder.go' +type StandaloneBuilder struct { + *mdbv1.MongoDB +} + +func DefaultStandaloneBuilder() *StandaloneBuilder { + spec := mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + Version: "4.0.0", + Persistent: util.BooleanRef(true), + ConnectionSpec: mdbv1.ConnectionSpec{ + SharedConnectionSpec: mdbv1.SharedConnectionSpec{ + OpsManagerConfig: &mdbv1.PrivateCloudConfig{ + ConfigMapRef: mdbv1.ConfigMapRef{ + Name: mock.TestProjectConfigMapName, + }, + }, + }, Credentials: mock.TestCredentialsSecretName, + }, + Security: &mdbv1.Security{ + Authentication: &mdbv1.Authentication{ + Modes: []mdbv1.AuthMode{}, + }, + TLSConfig: &mdbv1.TLSConfig{}, + }, + ResourceType: mdbv1.Standalone, + }, + Members: 1, + } + resource := &mdbv1.MongoDB{ObjectMeta: metav1.ObjectMeta{Name: "dublin", Namespace: mock.TestNamespace}, Spec: spec} + return &StandaloneBuilder{resource} +} + +func (b *StandaloneBuilder) SetName(name string) *StandaloneBuilder { + b.Name = name + return b +} + +func (b *StandaloneBuilder) SetVersion(version string) *StandaloneBuilder { + b.Spec.Version = version + return b +} + +func (b *StandaloneBuilder) SetPersistent(p *bool) *StandaloneBuilder { + b.Spec.Persistent = p + return b +} + +func (b *StandaloneBuilder) SetService(s string) *StandaloneBuilder { + b.Spec.Service = s + return b +} + +func (b *StandaloneBuilder) SetPodSpecTemplate(spec corev1.PodTemplateSpec) *StandaloneBuilder { + if b.Spec.PodSpec == nil { + b.Spec.PodSpec = &mdbv1.MongoDbPodSpec{} + } + b.Spec.PodSpec.PodTemplateWrapper.PodTemplate = &spec + return b +} + +func (b *StandaloneBuilder) Build() *mdbv1.MongoDB { + b.Spec.ResourceType = mdbv1.Standalone + b.InitDefaults() + return b.DeepCopy() +} + +func createDeploymentFromStandalone(st *mdbv1.MongoDB) om.Deployment { + d := om.NewDeployment() + sts := construct.DatabaseStatefulSet(*st, construct.StandaloneOptions(construct.GetPodEnvOptions()), zap.S()) + hostnames, _ := dns.GetDnsForStatefulSet(sts, st.Spec.GetClusterDomain(), nil) + process := om.NewMongodProcess(st.Name, hostnames[0], "fake-mongoDBImage", false, st.Spec.AdditionalMongodConfig, st.GetSpec(), "", nil, st.Status.FeatureCompatibilityVersion) + + lastConfig, err := st.GetLastAdditionalMongodConfigByType(mdbv1.StandaloneConfig) + if err != nil { + panic(err) + } + + d.MergeStandalone(process, st.Spec.AdditionalMongodConfig.ToMap(), lastConfig.ToMap(), nil) + d.AddMonitoringAndBackup(zap.S(), st.Spec.GetSecurity().IsTLSEnabled(), util.CAFilePathInContainer) + return d +} diff --git a/controllers/operator/mongodbuser_controller.go b/controllers/operator/mongodbuser_controller.go new file mode 100644 index 000000000..91d24a05c --- /dev/null +++ b/controllers/operator/mongodbuser_controller.go @@ -0,0 +1,498 @@ +package operator + +import ( + "context" + "encoding/json" + "time" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connection" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/connectionstring" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/project" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +type MongoDBUserReconciler struct { + *ReconcileCommonController + omConnectionFactory om.ConnectionFactory + memberClusterClientsMap map[string]kubernetesClient.Client + memberClusterSecretClientsMap map[string]secrets.SecretClient +} + +func newMongoDBUserReconciler(ctx context.Context, kubeClient client.Client, omFunc om.ConnectionFactory, memberClustersMap map[string]client.Client) *MongoDBUserReconciler { + clientsMap := make(map[string]kubernetesClient.Client) + secretClientsMap := make(map[string]secrets.SecretClient) + + for k, v := range memberClustersMap { + clientsMap[k] = kubernetesClient.NewClient(v) + secretClientsMap[k] = secrets.SecretClient{ + VaultClient: nil, + KubeClient: clientsMap[k], + } + } + return &MongoDBUserReconciler{ + ReconcileCommonController: NewReconcileCommonController(ctx, kubeClient), + omConnectionFactory: omFunc, + memberClusterClientsMap: clientsMap, + memberClusterSecretClientsMap: secretClientsMap, + } +} + +func (r *MongoDBUserReconciler) getUser(ctx context.Context, request reconcile.Request, log *zap.SugaredLogger) (*userv1.MongoDBUser, error) { + user := &userv1.MongoDBUser{} + if _, err := r.getResource(ctx, request, user, log); err != nil { + return nil, err + } + + // if database isn't specified default to the admin database, the recommended + // place for creating non-$external users + if user.Spec.Database == "" { + user.Spec.Database = "admin" + } + + return user, nil +} + +// Use MongoDBResourceRef namespace if specified, otherwise default to user's namespace. +func getMongoDBObjectKey(user userv1.MongoDBUser) client.ObjectKey { + mongoDBResourceNamespace := user.Namespace + if user.Spec.MongoDBResourceRef.Namespace != "" { + mongoDBResourceNamespace = user.Spec.MongoDBResourceRef.Namespace + } + return kube.ObjectKey(mongoDBResourceNamespace, user.Spec.MongoDBResourceRef.Name) +} + +// getMongoDB return a MongoDB deployment of type Single or Multi cluster based on the clusterType passed +func (r *MongoDBUserReconciler) getMongoDB(ctx context.Context, user userv1.MongoDBUser) (project.Reader, error) { + name := getMongoDBObjectKey(user) + + // Try the single cluster resource + mdb := &mdbv1.MongoDB{} + if err := r.client.Get(ctx, name, mdb); err == nil { + return mdb, nil + } + + // Try the multi-cluster next + mdbm := &mdbmulti.MongoDBMultiCluster{} + err := r.client.Get(ctx, name, mdbm) + return mdbm, err +} + +// getMongoDBConnectionBuilder returns an object that can construct a MongoDB Connection String on itself. +func (r *MongoDBUserReconciler) getMongoDBConnectionBuilder(ctx context.Context, user userv1.MongoDBUser) (connectionstring.ConnectionStringBuilder, error) { + name := getMongoDBObjectKey(user) + + // Try single cluster resource + mdb := &mdbv1.MongoDB{} + if err := r.client.Get(ctx, name, mdb); err == nil { + return mdb, nil + } + + // Try the multi-cluster next + mdbm := &mdbmulti.MongoDBMultiCluster{} + err := r.client.Get(ctx, name, mdbm) + return mdbm, err +} + +// +kubebuilder:rbac:groups=mongodb.com,resources={mongodbusers,mongodbusers/status,mongodbusers/finalizers},verbs=*,namespace=placeholder + +// Reconciles a mongodbusers.mongodb.com Custom resource. +func (r *MongoDBUserReconciler) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, e error) { + log := zap.S().With("MongoDBUser", request.NamespacedName) + log.Info("-> MongoDBUser.Reconcile") + + user, err := r.getUser(ctx, request, log) + if err != nil { + log.Warnf("error getting user %s", err) + return reconcile.Result{RequeueAfter: time.Second * util.RetryTimeSec}, nil + } + + log.Infow("MongoDBUser.Spec", "spec", user.Spec) + var mdb project.Reader + + if user.Spec.MongoDBResourceRef.Name != "" { + if mdb, err = r.getMongoDB(ctx, *user); err != nil { + log.Warnf("Couldn't fetch MongoDB Single/Multi Cluster Resource with name: %s, namespace: %s, err: %s", + user.Spec.MongoDBResourceRef.Name, user.Spec.MongoDBResourceRef.Namespace, err) + + if controllerutil.ContainsFinalizer(user, util.Finalizer) { + controllerutil.RemoveFinalizer(user, util.Finalizer) + if err := r.client.Update(ctx, user); err != nil { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("Failed to update the user with the removed finalizer: %w", err)), log) + } + return r.updateStatus(ctx, user, workflow.Pending("Finalizer will be removed. MongoDB resource not found"), log) + } + + return r.updateStatus(ctx, user, workflow.Pending("%s", err.Error()), log) + } + } else { + log.Warn("MongoDB reference not specified. Using deprecated project field.") + } + + // this can happen when a user has registered a configmap as watched resource + // but the user gets deleted. Reconciliation happens to this user even though it is deleted. + // TODO: unregister config map upon MongoDBUser deletion + if user.Namespace == "" && user.Name == "" { + // stop reconciliation + return workflow.Invalid("User or namespace is empty or nil").ReconcileResult() + } + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, mdb, log) + if err != nil { + return r.updateStatus(ctx, user, workflow.Failed(err), log) + } + + conn, _, err := connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, user.Namespace, log) + if err != nil { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("Failed to prepare Ops Manager connection: %w", err)), log) + } + + if err = r.updateConnectionStringSecret(ctx, *user, log); err != nil { + return r.updateStatus(ctx, user, workflow.Failed(err), log) + } + + if !user.DeletionTimestamp.IsZero() { + log.Info("MongoDBUser is being deleted") + + if controllerutil.ContainsFinalizer(user, util.Finalizer) { + return r.preDeletionCleanup(ctx, user, conn, log) + } + } + + if err := r.ensureFinalizer(ctx, user, log); err != nil { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("Failed to add finalizer: %w", err)), log) + } + + if user.Spec.Database == authentication.ExternalDB { + return r.handleExternalAuthUser(ctx, user, conn, log) + } else { + return r.handleScramShaUser(ctx, user, conn, log) + } +} + +func (r *MongoDBUserReconciler) delete(ctx context.Context, obj interface{}, log *zap.SugaredLogger) error { + user := obj.(*userv1.MongoDBUser) + + mdb, err := r.getMongoDB(ctx, *user) + if err != nil { + return err + } + + projectConfig, credsConfig, err := project.ReadConfigAndCredentials(ctx, r.client, r.SecretClient, mdb, log) + if err != nil { + return err + } + + _, _, err = connection.PrepareOpsManagerConnection(ctx, r.SecretClient, projectConfig, credsConfig, r.omConnectionFactory, user.Namespace, log) + if err != nil { + log.Errorf("Failed to prepare Ops Manager connection: %s", err) + return err + } + + r.resourceWatcher.RemoveAllDependentWatchedResources(user.Namespace, kube.ObjectKeyFromApiObject(user)) + + return nil +} + +func (r *MongoDBUserReconciler) updateConnectionStringSecret(ctx context.Context, user userv1.MongoDBUser, log *zap.SugaredLogger) error { + var err error + var password string + + if user.Spec.Database != authentication.ExternalDB { + password, err = user.GetPassword(ctx, r.SecretClient) + if err != nil { + log.Debug("User does not have a configured password.") + } + } + + connectionBuilder, err := r.getMongoDBConnectionBuilder(ctx, user) + if err != nil { + return err + } + + secretName := user.GetConnectionStringSecretName() + existingSecret, err := r.client.GetSecret(ctx, types.NamespacedName{Name: secretName, Namespace: user.Namespace}) + if err != nil && !apiErrors.IsNotFound(err) { + return err + } + if err == nil && !secret.HasOwnerReferences(existingSecret, user.GetOwnerReferences()) { + return xerrors.Errorf("connection string secret %s already exists and is not managed by the operator", secretName) + } + + mongoAuthUserURI := connectionBuilder.BuildConnectionString(user.Spec.Username, password, connectionstring.SchemeMongoDB, map[string]string{}) + mongoAuthUserSRVURI := connectionBuilder.BuildConnectionString(user.Spec.Username, password, connectionstring.SchemeMongoDBSRV, map[string]string{}) + + connectionStringSecret := secret.Builder(). + SetName(secretName). + SetNamespace(user.Namespace). + SetField("connectionString.standard", mongoAuthUserURI). + SetField("connectionString.standardSrv", mongoAuthUserSRVURI). + SetField("username", user.Spec.Username). + SetField("password", password). + SetOwnerReferences(user.GetOwnerReferences()). + Build() + + for _, c := range r.memberClusterSecretClientsMap { + err = secret.CreateOrUpdate(ctx, c, connectionStringSecret) + if err != nil { + return err + } + } + return secret.CreateOrUpdate(ctx, r.SecretClient, connectionStringSecret) +} + +func AddMongoDBUserController(ctx context.Context, mgr manager.Manager, memberClustersMap map[string]cluster.Cluster) error { + reconciler := newMongoDBUserReconciler(ctx, mgr.GetClient(), om.NewOpsManagerConnection, multicluster.ClustersMapToClientMap(memberClustersMap)) + c, err := controller.New(util.MongoDbUserController, mgr, controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: env.ReadIntOrDefault(util.MaxConcurrentReconcilesEnv, 1)}) // nolint:forbidigo + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.ConfigMap{}, + &watch.ResourcesHandler{ResourceType: watch.ConfigMap, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &corev1.Secret{}, + &watch.ResourcesHandler{ResourceType: watch.Secret, ResourceWatcher: reconciler.resourceWatcher})) + if err != nil { + return err + } + + // watch for changes to MongoDBUser resources + eventHandler := MongoDBUserEventHandler{reconciler: reconciler} + err = c.Watch(source.Kind[client.Object](mgr.GetCache(), &userv1.MongoDBUser{}, &eventHandler, watch.PredicatesForUser())) + if err != nil { + return err + } + + zap.S().Infof("Registered controller %s", util.MongoDbUserController) + return nil +} + +// toOmUser converts a MongoDBUser specification and optional password into an +// automation config MongoDB user. If the user has no password then a blank +// password should be provided. +func toOmUser(spec userv1.MongoDBUserSpec, password string) (om.MongoDBUser, error) { + user := om.MongoDBUser{ + Database: spec.Database, + Username: spec.Username, + Roles: []*om.Role{}, + AuthenticationRestrictions: []string{}, + Mechanisms: []string{}, + } + + // only specify password if we're dealing with non-x509 users + if spec.Database != authentication.ExternalDB { + if err := authentication.ConfigureScramCredentials(&user, password); err != nil { + return om.MongoDBUser{}, xerrors.Errorf("error generating SCRAM credentials: %w", err) + } + } + + for _, r := range spec.Roles { + user.AddRole(&om.Role{Role: r.RoleName, Database: r.Database}) + } + return user, nil +} + +func (r *MongoDBUserReconciler) handleScramShaUser(ctx context.Context, user *userv1.MongoDBUser, conn om.Connection, log *zap.SugaredLogger) (res reconcile.Result, e error) { + // watch the password secret in order to trigger reconciliation if the + // password is updated + if user.Spec.PasswordSecretKeyRef.Name != "" { + r.resourceWatcher.AddWatchedResourceIfNotAdded( + user.Spec.PasswordSecretKeyRef.Name, + user.Namespace, + watch.Secret, + kube.ObjectKeyFromApiObject(user), + ) + } + + shouldRetry := false + err := conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + if ac.Auth.Disabled || + (!stringutil.ContainsAny(ac.Auth.DeploymentAuthMechanisms, util.AutomationConfigScramSha256Option, util.AutomationConfigScramSha1Option)) { + shouldRetry = true + return xerrors.Errorf("scram Sha has not yet been configured") + } + + password, err := user.GetPassword(ctx, r.SecretClient) + if err != nil { + return err + } + + auth := ac.Auth + if user.ChangedIdentifier() { // we've changed username or database, we need to remove the old user before adding new + auth.RemoveUser(user.Status.Username, user.Status.Database) + } + + desiredUser, err := toOmUser(user.Spec, password) + if err != nil { + return err + } + + auth.EnsureUser(desiredUser) + return nil + }, log) + if err != nil { + if shouldRetry { + return r.updateStatus(ctx, user, workflow.Pending("%s", err.Error()).WithRetry(10), log) + } + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("error updating user %w", err)), log) + } + + // Before we update the MongoDBUser's status to Updated, + // we need to wait for the cluster to be in a ready state + // to ensure that the user has been created successfully and is usable. + if err := waitForReadyState(conn, log); err != nil { + return r.updateStatus(ctx, user, workflow.Pending("error waiting for ready state: %s", err.Error()).WithRetry(10), log) + } + + annotationsToAdd, err := getAnnotationsForUserResource(user) + if err != nil { + return r.updateStatus(ctx, user, workflow.Failed(err), log) + } + + if err := annotations.SetAnnotations(ctx, user, annotationsToAdd, r.client); err != nil { + return r.updateStatus(ctx, user, workflow.Failed(err), log) + } + + log.Infof("Finished reconciliation for MongoDBUser!") + return r.updateStatus(ctx, user, workflow.OK(), log) +} + +func (r *MongoDBUserReconciler) handleExternalAuthUser(ctx context.Context, user *userv1.MongoDBUser, conn om.Connection, log *zap.SugaredLogger) (reconcile.Result, error) { + desiredUser, err := toOmUser(user.Spec, "") + if err != nil { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("error updating user %w", err)), log) + } + + shouldRetry := false + updateFunction := func(ac *om.AutomationConfig) error { + if !externalAuthMechanismsAvailable(ac.Auth.DeploymentAuthMechanisms) { + shouldRetry = true + return xerrors.Errorf("no external authentication mechanisms (LDAP or x509) have been configured") + } + + auth := ac.Auth + if user.ChangedIdentifier() { + auth.RemoveUser(user.Status.Username, user.Status.Database) + } + + auth.EnsureUser(desiredUser) + return nil + } + + err = conn.ReadUpdateAutomationConfig(updateFunction, log) + if err != nil { + if shouldRetry { + return r.updateStatus(ctx, user, workflow.Pending("%s", err.Error()).WithRetry(10), log) + } + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("error updating user %w", err)), log) + } + + // Before we update the MongoDBUser's status to Updated, + // we need to wait for the cluster to be in a ready state + // to ensure that the user has been created successfully and is usable. + if err := waitForReadyState(conn, log); err != nil { + return r.updateStatus(ctx, user, workflow.Pending("error waiting for ready state: %s", err.Error()).WithRetry(10), log) + } + + annotationsToAdd, err := getAnnotationsForUserResource(user) + if err != nil { + return r.updateStatus(ctx, user, workflow.Failed(err), log) + } + + if err := annotations.SetAnnotations(ctx, user, annotationsToAdd, r.client); err != nil { + return r.updateStatus(ctx, user, workflow.Failed(err), log) + } + + log.Infow("Finished reconciliation for MongoDBUser!") + return r.updateStatus(ctx, user, workflow.OK(), log) +} + +func waitForReadyState(conn om.Connection, log *zap.SugaredLogger) error { + automationConfig, err := conn.ReadAutomationConfig() + if err != nil { + return err + } + + processes := automationConfig.Deployment.GetAllProcessNames() + return om.WaitForReadyState(conn, processes, false, log) +} + +func externalAuthMechanismsAvailable(mechanisms []string) bool { + return stringutil.ContainsAny(mechanisms, util.AutomationConfigLDAPOption, util.AutomationConfigX509Option) +} + +func getAnnotationsForUserResource(user *userv1.MongoDBUser) (map[string]string, error) { + finalAnnotations := make(map[string]string) + specBytes, err := json.Marshal(user.Spec) + if err != nil { + return nil, err + } + finalAnnotations[util.LastAchievedSpec] = string(specBytes) + return finalAnnotations, nil +} + +func (r *MongoDBUserReconciler) preDeletionCleanup(ctx context.Context, user *userv1.MongoDBUser, conn om.Connection, log *zap.SugaredLogger) (reconcile.Result, error) { + log.Info("Performing pre deletion cleanup before deleting MongoDBUser") + + err := conn.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.EnsureUserRemoved(user.Spec.Username, user.Spec.Database) + return nil + }, log) + if err != nil { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("Failed to perform AutomationConfig cleanup: %w", err)), log) + } + + if finalizerRemoved := controllerutil.RemoveFinalizer(user, util.Finalizer); !finalizerRemoved { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("Failed to remove finalizer: %w", err)), log) + } + + if err := r.client.Update(ctx, user); err != nil { + return r.updateStatus(ctx, user, workflow.Failed(xerrors.Errorf("Failed to update the user with the removed finalizer: %w", err)), log) + } + return r.updateStatus(ctx, user, workflow.OK(), log) +} + +func (r *MongoDBUserReconciler) ensureFinalizer(ctx context.Context, user *userv1.MongoDBUser, log *zap.SugaredLogger) error { + log.Info("Adding finalizer to the MongoDBUser resource") + + if finalizerAdded := controllerutil.AddFinalizer(user, util.Finalizer); finalizerAdded { + if err := r.client.Update(ctx, user); err != nil { + return err + } + } + + return nil +} diff --git a/controllers/operator/mongodbuser_controller_test.go b/controllers/operator/mongodbuser_controller_test.go new file mode 100644 index 000000000..da30bcbe6 --- /dev/null +++ b/controllers/operator/mongodbuser_controller_test.go @@ -0,0 +1,633 @@ +package operator + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/authentication" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/watch" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/workflow" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +func TestSettingUserStatus_ToPending_IsFilteredOut(t *testing.T) { + userInUpdatedPhase := &userv1.MongoDBUser{ObjectMeta: metav1.ObjectMeta{Name: "mms-user", Namespace: mock.TestNamespace}, Status: userv1.MongoDBUserStatus{Common: status.Common{Phase: status.PhaseUpdated}}} + userInPendingPhase := &userv1.MongoDBUser{ObjectMeta: metav1.ObjectMeta{Name: "mms-user", Namespace: mock.TestNamespace}, Status: userv1.MongoDBUserStatus{Common: status.Common{Phase: status.PhasePending}}} + + predicates := watch.PredicatesForUser() + updateEvent := event.UpdateEvent{ + ObjectOld: userInUpdatedPhase, + ObjectNew: userInPendingPhase, + } + assert.False(t, predicates.UpdateFunc(updateEvent), "changing phase from updated to pending should be filtered out") +} + +func TestUserIsAdded_ToAutomationConfig_OnSuccessfulReconciliation(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().EnableAuth().AgentAuthMode("SCRAM"). + SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + // the automation config should have been updated during reconciliation + assert.Len(t, ac.Auth.Users, 1, "the MongoDBUser should have been added to the AutomationConfig") + + _, createdUser := ac.Auth.GetUser("my-user", "admin") + assert.Equal(t, user.Spec.Username, createdUser.Username) + assert.Equal(t, user.Spec.Database, createdUser.Database) + assert.Equal(t, len(user.Spec.Roles), len(createdUser.Roles)) +} + +func TestReconciliationSucceed_OnAddingUser_FromADifferentNamespace(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + user.Spec.MongoDBResourceRef.Namespace = user.Namespace + // the operator should correctly reconcile if the user resource is in a different namespace than the mongodb resource + otherNamespace := "userNamespace" + user.Namespace = otherNamespace + + reconciler, client, _ := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + // we enable auth because we need to explicitly put the password secret in same namespace + _ = client.Create(ctx, DefaultReplicaSetBuilder().EnableAuth().AgentAuthMode("SCRAM"). + SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + // the secret must be in the same namespace as the user, we do not support cross-referencing + createPasswordSecretInNamespace(ctx, client, user.Spec.PasswordSecretKeyRef, "password", otherNamespace) + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if MongoDBUser and MongoDB resources are in different namespaces") +} + +func TestReconciliationSucceed_OnAddingUser_WithNoMongoDBNamespaceSpecified(t *testing.T) { + ctx := context.Background() + // DefaultMongoDBUserBuilder doesn't provide a namespace to the MongoDBResourceRef by default + // The reconciliation should succeed + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, _ := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().EnableAuth().AgentAuthMode("SCRAM"). + SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if MongoDBResourceRef is not provided") +} + +func TestUserIsUpdated_IfNonIdentifierFieldIsUpdated_OnSuccessfulReconciliation(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().SetName("my-rs").EnableAuth().AgentAuthMode("SCRAM"). + Build()) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + // remove roles from the same user + updateUser(ctx, user, client, func(user *userv1.MongoDBUser) { + user.Spec.Roles = []userv1.Role{} + }) + + actual, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + assert.Len(t, ac.Auth.Users, 1, "we should still have a single MongoDBUser, no users should have been deleted") + _, updatedUser := ac.Auth.GetUser("my-user", "admin") + assert.Len(t, updatedUser.Roles, 0) +} + +func TestUserIsReplaced_IfIdentifierFieldsAreChanged_OnSuccessfulReconciliation(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().SetName("my-rs").EnableAuth().AgentAuthMode("SCRAM").Build()) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + // change the username and database (these are the values used to identify a user) + updateUser(ctx, user, client, func(user *userv1.MongoDBUser) { + user.Spec.Username = "changed-name" + user.Spec.Database = "changed-db" + }) + + actual, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + assert.Len(t, ac.Auth.Users, 2, "we should have a new user with the updated fields and a nil value for the deleted user") + assert.False(t, ac.Auth.HasUser("my-user", "admin"), "the deleted user should no longer be present") + assert.True(t, containsNil(ac.Auth.Users), "the deleted user should have been assigned a nil value") + _, updatedUser := ac.Auth.GetUser("changed-name", "changed-db") + assert.Equal(t, "changed-name", updatedUser.Username, "new user name should be reflected") + assert.Equal(t, "changed-db", updatedUser.Database, "new database should be reflected") +} + +// updateUser applies and updates the changes to the user after getting the most recent version +// from the mocked client +func updateUser(ctx context.Context, user *userv1.MongoDBUser, client client.Client, updateFunc func(*userv1.MongoDBUser)) { + _ = client.Get(ctx, kube.ObjectKey(user.Namespace, user.Name), user) + updateFunc(user) + _ = client.Update(ctx, user) +} + +func TestRetriesReconciliation_IfNoPasswordSecretExists(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := defaultUserReconciler(ctx, user) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + + // No password has been created + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected := reconcile.Result{RequeueAfter: time.Second * 10} + assert.Nil(t, err, "should be no error on retry") + assert.Equal(t, expected, actual, "the reconciliation should be retried as there is no password") + + connection := omConnectionFactory.GetConnection() + ac, _ := connection.ReadAutomationConfig() + + assert.Len(t, ac.Auth.Users, 0, "the MongoDBUser should not have been added to the AutomationConfig") +} + +func TestRetriesReconciliation_IfPasswordSecretExists_ButHasNoPassword(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := defaultUserReconciler(ctx, user) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + + // use the wrong key to store the password + createPasswordSecret(ctx, client, userv1.SecretKeyRef{Name: user.Spec.PasswordSecretKeyRef.Name, Key: "non-existent-key"}, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected := reconcile.Result{RequeueAfter: time.Second * 10} + assert.Nil(t, err, "should be no error on retry") + assert.Equal(t, expected, actual, "the reconciliation should be retried as there is a secret, but the key contains no password") + + connection := omConnectionFactory.GetConnection() + ac, _ := connection.ReadAutomationConfig() + + assert.Len(t, ac.Auth.Users, 0, "the MongoDBUser should not have been added to the AutomationConfig") +} + +func TestX509User_DoesntRequirePassword(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetDatabase(authentication.ExternalDB).Build() + reconciler, client, _ := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigX509Option) + + // initialize resources required for x590 tests + createMongoDBForUserWithAuth(ctx, client, *user, util.X509) + + createUserControllerConfigMap(ctx, client) + + // No password has been created + + // in order for x509 to be configurable, "util.AutomationConfigX509Option" needs to be enabled on the automation config + // pre-configure the connection + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "the reconciliation should be successful as x509 does not require a password") +} + +func AssertAuthModeTest(ctx context.Context, t *testing.T, mode mdbv1.AuthMode) { + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").SetDatabase(authentication.ExternalDB).Build() + + reconciler, client, _ := defaultUserReconciler(ctx, user) + err := client.Create(ctx, DefaultReplicaSetBuilder().EnableAuth().SetAuthModes([]mdbv1.AuthMode{mode}).SetName("my-rs0").Build()) + assert.NoError(t, err) + createUserControllerConfigMap(ctx, client) + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + // reconciles if a $external user creation is attempted with no configured backends. + expected := reconcile.Result{Requeue: false, RequeueAfter: 10 * time.Second} + + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} + +func TestExternalAuthUserReconciliation_RequiresExternalAuthConfigured(t *testing.T) { + ctx := context.Background() + AssertAuthModeTest(ctx, t, "LDAP") + AssertAuthModeTest(ctx, t, "X509") +} + +func TestScramShaUserReconciliation_CreatesAgentUsers(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + err := client.Create(ctx, DefaultReplicaSetBuilder().AgentAuthMode("SCRAM").EnableAuth().SetName("my-rs").Build()) + assert.NoError(t, err) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + expected, _ := workflow.OK().ReconcileResult() + + assert.NoError(t, err) + assert.Equal(t, expected, actual) + + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + + assert.Len(t, ac.Auth.Users, 1, "users list should contain 1 user just added") +} + +func TestMultipleAuthMethod_CreateAgentUsers(t *testing.T) { + ctx := context.Background() + t.Run("When SCRAM and X509 auth modes are enabled, and agent mode is SCRAM, 3 users are created", func(t *testing.T) { + ac := BuildAuthenticationEnabledReplicaSet(ctx, t, util.AutomationConfigX509Option, 0, "SCRAM", []mdbv1.AuthMode{"SCRAM", "X509"}) + + assert.Equal(t, ac.Auth.AutoUser, "mms-automation-agent") + assert.Len(t, ac.Auth.Users, 1, "users list should contain a created user") + + expectedUsernames := []string{"mms-backup-agent", "mms-monitoring-agent", "my-user"} + for _, user := range ac.Auth.Users { + assert.True(t, stringutil.Contains(expectedUsernames, user.Username)) + } + }) + + t.Run("When X509 and SCRAM auth modes are enabled, and agent mode is X509, 1 user is created", func(t *testing.T) { + ac := BuildAuthenticationEnabledReplicaSet(ctx, t, util.AutomationConfigX509Option, 1, "X509", []mdbv1.AuthMode{"X509", "SCRAM"}) + assert.Equal(t, util.AutomationAgentName, ac.Auth.AutoUser) + assert.Len(t, ac.Auth.Users, 1, "users list should contain only 1 user") + assert.Equal(t, "$external", ac.Auth.Users[0].Database) + assert.Equal(t, "my-user", ac.Auth.Users[0].Username) + }) + + t.Run("When X509 and SCRAM auth modes are enabled, SCRAM is AgentAuthMode, 3 users are created", func(t *testing.T) { + ac := BuildAuthenticationEnabledReplicaSet(ctx, t, util.AutomationConfigX509Option, 0, "SCRAM", []mdbv1.AuthMode{"X509", "SCRAM"}) + + assert.Equal(t, ac.Auth.AutoUser, "mms-automation-agent") + assert.Len(t, ac.Auth.Users, 1, "users list should contain only one actual user") + assert.Equal(t, "my-user", ac.Auth.Users[0].Username) + }) + + t.Run("When X509 auth mode is enabled, 1 user will be created", func(t *testing.T) { + ac := BuildAuthenticationEnabledReplicaSet(ctx, t, util.AutomationConfigX509Option, 3, "X509", []mdbv1.AuthMode{"X509"}) + assert.Equal(t, util.AutomationAgentName, ac.Auth.AutoUser) + assert.Len(t, ac.Auth.Users, 1, "users list should contain only an actual user") + + expectedUsernames := []string{ + "my-user", + } + for _, user := range ac.Auth.Users { + assert.True(t, stringutil.Contains(expectedUsernames, user.Username)) + } + }) + + t.Run("When LDAP and X509 are enabled, 1 X509 user will be created", func(t *testing.T) { + ac := BuildAuthenticationEnabledReplicaSet(ctx, t, util.AutomationConfigLDAPOption, 3, "X509", []mdbv1.AuthMode{"LDAP", "X509"}) + assert.Equal(t, util.AutomationAgentName, ac.Auth.AutoUser) + assert.Len(t, ac.Auth.Users, 1, "users list should contain only an actual user") + + expectedUsernames := []string{ + "my-user", + } + for _, user := range ac.Auth.Users { + assert.True(t, stringutil.Contains(expectedUsernames, user.Username)) + } + }) + + t.Run("When LDAP is enabled, 1 SCRAM agent will be created", func(t *testing.T) { + ac := BuildAuthenticationEnabledReplicaSet(ctx, t, util.AutomationConfigLDAPOption, 0, "LDAP", []mdbv1.AuthMode{"LDAP"}) + assert.Equal(t, "mms-automation-agent", ac.Auth.AutoUser) + + assert.Len(t, ac.Auth.Users, 1, "users list should contain only 1 user") + assert.Equal(t, "my-user", ac.Auth.Users[0].Username) + assert.Equal(t, "$external", ac.Auth.Users[0].Database) + }) +} + +func TestFinalizerIsAdded_WhenUserIsCreated(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().EnableAuth().AgentAuthMode("SCRAM"). + SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + // the automation config should have been updated during reconciliation + assert.Len(t, ac.Auth.Users, 1, "the MongoDBUser should have been added to the AutomationConfig") + + _ = client.Get(ctx, kube.ObjectKey(user.Namespace, user.Name), user) + + assert.Contains(t, user.GetFinalizers(), util.Finalizer) +} + +func TestFinalizerIsRemoved_WhenUserIsDeleted(t *testing.T) { + ctx := context.Background() + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").Build() + reconciler, client, omConnectionFactory := userReconcilerWithAuthMode(ctx, user, util.AutomationConfigScramSha256Option) + + // initialize resources required for the tests + _ = client.Create(ctx, DefaultReplicaSetBuilder().EnableAuth().AgentAuthMode("SCRAM"). + SetName("my-rs").Build()) + createUserControllerConfigMap(ctx, client) + createPasswordSecret(ctx, client, user.Spec.PasswordSecretKeyRef, "password") + + actual, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + expected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, expected, actual, "there should be a successful reconciliation if the password is a valid reference") + + ac, _ := omConnectionFactory.GetConnection().ReadAutomationConfig() + + // the automation config should have been updated during reconciliation + assert.Len(t, ac.Auth.Users, 1, "the MongoDBUser should have been added to the AutomationConfig") + + _ = client.Delete(ctx, user) + + newResult, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + + newExpected, _ := workflow.OK().ReconcileResult() + + assert.Nil(t, err, "there should be no error on successful reconciliation") + assert.Equal(t, newExpected, newResult, "there should be a successful reconciliation if the password is a valid reference") + + assert.Empty(t, user.GetFinalizers()) +} + +// BuildAuthenticationEnabledReplicaSet returns a AutomationConfig after creating a Replica Set with a set of +// different Authentication values. It should be used to test different combination of authentication modes enabled +// and agent authentication modes. +func BuildAuthenticationEnabledReplicaSet(ctx context.Context, t *testing.T, automationConfigOption string, numAgents int, agentAuthMode string, authModes []mdbv1.AuthMode) *om.AutomationConfig { + user := DefaultMongoDBUserBuilder().SetMongoDBResourceName("my-rs").SetDatabase(authentication.ExternalDB).Build() + + reconciler, client, omConnectionFactory := defaultUserReconciler(ctx, user) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + _ = connection.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.DeploymentAuthMechanisms = append(ac.Auth.DeploymentAuthMechanisms, automationConfigOption) + return nil + }, nil) + }) + + builder := DefaultReplicaSetBuilder().EnableAuth().SetAuthModes(authModes).SetName("my-rs") + if agentAuthMode != "" { + builder.AgentAuthMode(agentAuthMode) + } + + err := client.Create(ctx, builder.Build()) + assert.NoError(t, err) + createUserControllerConfigMap(ctx, client) + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: kube.ObjectKey(user.Namespace, user.Name)}) + assert.NoError(t, err) + + ac, err := omConnectionFactory.GetConnection().ReadAutomationConfig() + assert.NoError(t, err) + + return ac +} + +// createUserControllerConfigMap creates a configmap with credentials present +func createUserControllerConfigMap(ctx context.Context, client client.Client) { + _ = client.Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: om.TestGroupName, Namespace: mock.TestNamespace}, + Data: map[string]string{ + util.OmBaseUrl: om.TestURL, + util.OmOrgId: om.TestOrgID, + util.OmProjectName: om.TestGroupName, + util.OmCredentials: mock.TestCredentialsSecretName, + }, + }) +} + +func containsNil(users []*om.MongoDBUser) bool { + for _, user := range users { + if user == nil { + return true + } + } + return false +} + +func createPasswordSecret(ctx context.Context, client client.Client, secretRef userv1.SecretKeyRef, password string) { + createPasswordSecretInNamespace(ctx, client, secretRef, password, mock.TestNamespace) +} + +func createPasswordSecretInNamespace(ctx context.Context, client client.Client, secretRef userv1.SecretKeyRef, password string, namespace string) { + _ = client.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretRef.Name, Namespace: namespace}, + Data: map[string][]byte{ + secretRef.Key: []byte(password), + }, + }) +} + +func createMongoDBForUserWithAuth(ctx context.Context, client client.Client, user userv1.MongoDBUser, authModes ...mdbv1.AuthMode) { + mdbBuilder := DefaultReplicaSetBuilder().SetName(user.Spec.MongoDBResourceRef.Name) + + _ = client.Create(ctx, mdbBuilder.SetAuthModes(authModes).Build()) +} + +// defaultUserReconciler is the user reconciler used in unit test. It "adds" necessary +// additional K8s objects (st, connection config map and secrets) necessary for reconciliation +func defaultUserReconciler(ctx context.Context, user *userv1.MongoDBUser) (*MongoDBUserReconciler, client.Client, *om.CachedOMConnectionFactory) { + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(user) + memberClusterMap := getFakeMultiClusterMap(omConnectionFactory) + return newMongoDBUserReconciler(ctx, kubeClient, omConnectionFactory.GetConnectionFunc, memberClusterMap), kubeClient, omConnectionFactory +} + +func userReconcilerWithAuthMode(ctx context.Context, user *userv1.MongoDBUser, authMode string) (*MongoDBUserReconciler, client.Client, *om.CachedOMConnectionFactory) { + kubeClient, omConnectionFactory := mock.NewDefaultFakeClient(user) + memberClusterMap := getFakeMultiClusterMap(omConnectionFactory) + reconciler := newMongoDBUserReconciler(ctx, kubeClient, omConnectionFactory.GetConnectionFunc, memberClusterMap) + omConnectionFactory.SetPostCreateHook(func(connection om.Connection) { + _ = connection.ReadUpdateAutomationConfig(func(ac *om.AutomationConfig) error { + ac.Auth.DeploymentAuthMechanisms = append(ac.Auth.DeploymentAuthMechanisms, authMode) + // Enabling auth as it's required to be enabled for the user controller to proceed + ac.Auth.Disabled = false + return nil + }, nil) + }) + return reconciler, kubeClient, omConnectionFactory +} + +type MongoDBUserBuilder struct { + project string + passwordRef userv1.SecretKeyRef + roles []userv1.Role + username string + database string + resourceName string + mongodbResourceName string + namespace string +} + +func (b *MongoDBUserBuilder) SetPasswordRef(secretName, key string) *MongoDBUserBuilder { + b.passwordRef = userv1.SecretKeyRef{Name: secretName, Key: key} + return b +} + +func (b *MongoDBUserBuilder) SetMongoDBResourceName(name string) *MongoDBUserBuilder { + b.mongodbResourceName = name + return b +} + +func (b *MongoDBUserBuilder) SetUsername(username string) *MongoDBUserBuilder { + b.username = username + return b +} + +func (b *MongoDBUserBuilder) SetNamespace(namespace string) *MongoDBUserBuilder { + b.namespace = namespace + return b +} + +func (b *MongoDBUserBuilder) SetDatabase(db string) *MongoDBUserBuilder { + b.database = db + return b +} + +func (b *MongoDBUserBuilder) SetProject(project string) *MongoDBUserBuilder { + b.project = project + return b +} + +func (b *MongoDBUserBuilder) SetResourceName(resourceName string) *MongoDBUserBuilder { + b.resourceName = resourceName + return b +} + +func (b *MongoDBUserBuilder) SetRoles(roles []userv1.Role) *MongoDBUserBuilder { + b.roles = roles + return b +} + +func DefaultMongoDBUserBuilder() *MongoDBUserBuilder { + return &MongoDBUserBuilder{ + roles: []userv1.Role{{ + RoleName: "role-1", + Database: "admin", + }, { + RoleName: "role-2", + Database: "admin", + }, { + RoleName: "role-3", + Database: "admin", + }}, + project: mock.TestProjectConfigMapName, + passwordRef: userv1.SecretKeyRef{ + Name: "password-secret", + Key: "password", + }, + username: "my-user", + database: "admin", + mongodbResourceName: mock.TestMongoDBName, + namespace: mock.TestNamespace, + } +} + +func (b *MongoDBUserBuilder) Build() *userv1.MongoDBUser { + if b.roles == nil { + b.roles = make([]userv1.Role, 0) + } + if b.resourceName == "" { + b.resourceName = b.username + } + + return &userv1.MongoDBUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.resourceName, + Namespace: b.namespace, + }, + Spec: userv1.MongoDBUserSpec{ + Roles: b.roles, + PasswordSecretKeyRef: b.passwordRef, + Username: b.username, + Database: b.database, + MongoDBResourceRef: userv1.MongoDBResourceRef{ + Name: b.mongodbResourceName, + }, + }, + } +} diff --git a/controllers/operator/mongodbuser_eventhandler.go b/controllers/operator/mongodbuser_eventhandler.go new file mode 100644 index 000000000..f678d0907 --- /dev/null +++ b/controllers/operator/mongodbuser_eventhandler.go @@ -0,0 +1,29 @@ +package operator + +import ( + "context" + + "go.uber.org/zap" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + + "github.com/10gen/ops-manager-kubernetes/pkg/kube" +) + +type MongoDBUserEventHandler struct { + *handler.EnqueueRequestForObject + reconciler interface { + delete(ctx context.Context, obj interface{}, log *zap.SugaredLogger) error + } +} + +func (eh *MongoDBUserEventHandler) Delete(ctx context.Context, e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + zap.S().Infow("Cleaning up MongoDBUser resource", "resource", e.Object) + logger := zap.S().With("resource", kube.ObjectKey(e.Object.GetNamespace(), e.Object.GetName())) + if err := eh.reconciler.delete(ctx, e.Object, logger); err != nil { + logger.Errorf("MongoDBUser resource removed from Kubernetes, but failed to clean some state in Ops Manager: %s", err) + return + } + logger.Info("Removed MongoDBUser resource from Kubernetes and Ops Manager") +} diff --git a/controllers/operator/namespace_watched.go b/controllers/operator/namespace_watched.go new file mode 100644 index 000000000..59ab605e6 --- /dev/null +++ b/controllers/operator/namespace_watched.go @@ -0,0 +1,55 @@ +package operator + +import ( + "os" + "strings" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// GetWatchedNamespace returns a namespace or namespaces to watch. +// +// If `WATCH_NAMESPACE` has not been set, is an empty string or is a star ("*"), it watches all namespaces. +// If `WATCH_NAMESPACE` is set, it watches over that namespace, unless there are commas in there, in which +// the namespaces to watch will be a comma-separated list. +// +// If `WATCH_NAMESPACE` is '*' it will return []string{""}, which means all namespaces will be watched. +func GetWatchedNamespace() []string { + watchNamespace, nsSpecified := os.LookupEnv(util.WatchNamespace) // nolint:forbidigo + + // If WatchNamespace is not specified - we assume the Operator is watching all namespaces. + // In contrast to the common way to configure cluster-wide operators we additionally support '*' + // see: https://sdk.operatorframework.io/docs/building-operators/golang/operator-scope/#configuring-watch-namespaces-dynamically + if !nsSpecified || len(watchNamespace) == 0 || strings.TrimSpace(watchNamespace) == "" || strings.TrimSpace(watchNamespace) == "*" { + return []string{""} + } + + if strings.Contains(watchNamespace, ",") { + namespaceSplit := strings.Split(watchNamespace, ",") + namespaceList := []string{} + for i := range namespaceSplit { + namespace := strings.TrimSpace(namespaceSplit[i]) + if namespace != "" { + namespaceList = append(namespaceList, namespace) + } + } + + if stringutil.Contains(namespaceList, "*") { + // If `WATCH_NAMESPACE` contains a single *, then we return a list + // of "" as a defensive measure, to avoid cases where + // WATCH_NAMESPACE could be "*,another-namespace" which could at + // some point make the Operator traverse "another-namespace" twice. + return []string{""} + } + + if len(namespaceList) > 0 { + return namespaceList + } + + return []string{env.ReadOrDefault(util.CurrentNamespace, "")} // nolint:forbidigo + } + + return []string{watchNamespace} +} diff --git a/controllers/operator/namespace_watched_test.go b/controllers/operator/namespace_watched_test.go new file mode 100644 index 000000000..69338f9a1 --- /dev/null +++ b/controllers/operator/namespace_watched_test.go @@ -0,0 +1,32 @@ +package operator + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestGetWatchedNamespace(t *testing.T) { + t.Setenv(util.WatchNamespace, "one-namespace") + assert.Equal(t, []string{"one-namespace"}, GetWatchedNamespace()) + + t.Setenv(util.WatchNamespace, "one-namespace, two-namespace,three-namespace") + assert.Equal(t, []string{"one-namespace", "two-namespace", "three-namespace"}, GetWatchedNamespace()) + + t.Setenv(util.WatchNamespace, "") + assert.Equal(t, []string{""}, GetWatchedNamespace()) + + t.Setenv(util.WatchNamespace, ",") + assert.Equal(t, []string{OperatorNamespace}, GetWatchedNamespace()) + + t.Setenv(util.WatchNamespace, ",one-namespace") + assert.Equal(t, []string{"one-namespace"}, GetWatchedNamespace()) + + t.Setenv(util.WatchNamespace, "*") + assert.Equal(t, []string{""}, GetWatchedNamespace()) + + t.Setenv(util.WatchNamespace, "*,hi") + assert.Equal(t, []string{""}, GetWatchedNamespace()) +} diff --git a/controllers/operator/operator_configuration.go b/controllers/operator/operator_configuration.go new file mode 100644 index 000000000..dc5c2bf82 --- /dev/null +++ b/controllers/operator/operator_configuration.go @@ -0,0 +1,11 @@ +package operator + +import ( + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +// operatorNamespace returns the current namespace where the Operator is deployed +func operatorNamespace() string { + return env.ReadOrPanic(util.CurrentNamespace) // nolint:forbidigo +} diff --git a/controllers/operator/pem/pem_collection.go b/controllers/operator/pem/pem_collection.go new file mode 100644 index 000000000..1c6bb4f19 --- /dev/null +++ b/controllers/operator/pem/pem_collection.go @@ -0,0 +1,182 @@ +package pem + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/base32" + "encoding/json" + "encoding/pem" + "strings" + + "golang.org/x/xerrors" +) + +type Collection struct { + PemFiles map[string]File +} + +// GetHash returns a cryptographically hashed representation of the collection +// of PEM files. +func (p Collection) GetHash() (string, error) { + // this relies on the implementation detail that json.Marshal sorts the keys + // in a map when performing the serialisation, thus resulting in a + // deterministic representation of the struct + jsonBytes, err := json.Marshal(p.PemFiles) + if err != nil { + // this should never happen + return "", xerrors.Errorf("could not marshal PEM files to JSON: %w", err) + } + hashBytes := sha256.Sum256(jsonBytes) + + // base32 encoding without padding (i.e. no '=' character) is used as this + // guarantees a strictly alphanumeric output. Since the result is a hash, and + // thus needs not be reversed, removing the padding is not an issue. + return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hashBytes[:]), nil +} + +// NewCollection creates a new Pem Collection with an initialized empty map. +func NewCollection() *Collection { + return &Collection{ + PemFiles: make(map[string]File), + } +} + +// AddPrivateKey ensures a Pem File exists for the given hostname and key. +func (p Collection) AddPrivateKey(hostname, key string) { + if key == "" { + return + } + pem, ok := p.PemFiles[hostname] + if !ok { + pem = File{PrivateKey: key} + } else { + pem.PrivateKey = key + } + p.PemFiles[hostname] = pem +} + +// AddCertificate ensures a Pem File is added for the given hostname and cert. +func (p Collection) AddCertificate(hostname, cert string) { + if cert == "" { + return + } + pem, ok := p.PemFiles[hostname] + if !ok { + pem = File{Certificate: cert} + } else { + pem.Certificate = cert + } + p.PemFiles[hostname] = pem +} + +// MergeEntry merges a given PEM file into the collection of PEM files. If a +// file with the same hostname exists in the collection, then existing +// components will not be overridden. +func (p Collection) MergeEntry(hostname string, pem File) { + existingPem := p.PemFiles[hostname] + if existingPem.PrivateKey == "" { + p.AddPrivateKey(hostname, pem.PrivateKey) + } + if existingPem.Certificate == "" { + p.AddCertificate(hostname, pem.Certificate) + } +} + +// Merge combines all Pem Files into a map[string]string. +func (p *Collection) Merge() map[string]string { + result := make(map[string]string) + + for k, v := range p.PemFiles { + result[k+"-pem"] = v.String() + } + + return result +} + +// MergeWith merges the provided entry into this Collection. +func (p *Collection) MergeWith(data map[string][]byte) map[string]string { + for k, v := range data { + hostname := strings.TrimSuffix(k, "-pem") + p.MergeEntry(hostname, NewFileFrom(string(v))) + } + + return p.Merge() +} + +func (p File) ParseCertificate() ([]*x509.Certificate, error) { + var certs []*x509.Certificate + for block, rest := pem.Decode([]byte(p.Certificate)); block != nil; block, rest = pem.Decode(rest) { + if block == nil { + return []*x509.Certificate{}, xerrors.Errorf("failed to parse certificate PEM, please ensure validity of the file") + } + switch block.Type { + case "CERTIFICATE": + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return []*x509.Certificate{}, err + } + certs = append(certs, cert) + default: + return []*x509.Certificate{}, xerrors.Errorf("failed to parse certificate PEM, please ensure validity of the file") + } + + } + return certs, nil +} + +type File struct { + PrivateKey string `json:"privateKey"` + Certificate string `json:"certificate"` +} + +func NewFileFrom(data string) File { + parts := separatePemFile(data) + privateKey := "" + certificate := "" + + for _, el := range parts { + if strings.Contains(el, "BEGIN CERTIFICATE") { + certificate += el + } else if strings.Contains(el, "PRIVATE KEY") { + privateKey = el + } + } + + return File{ + PrivateKey: privateKey, + Certificate: certificate, + } +} + +func NewFileFromData(data []byte) File { + return NewFileFrom(string(data)) +} + +func (p *File) IsValid() bool { + return p.PrivateKey != "" +} + +func (p *File) IsComplete() bool { + return p.IsValid() && p.Certificate != "" +} + +func (p *File) String() string { + return p.Certificate + p.PrivateKey +} + +func separatePemFile(data string) []string { + parts := strings.Split(data, "\n") + certificates := make([]string, 0) + certificatePart := "" + + for _, el := range parts { + if strings.HasPrefix(el, "-----END") { + certificates = append(certificates, certificatePart+el+"\n") + certificatePart = "" + continue + } + certificatePart += el + "\n" + } + + return certificates +} diff --git a/controllers/operator/pem/pem_collection_test.go b/controllers/operator/pem/pem_collection_test.go new file mode 100644 index 000000000..57ebb00c8 --- /dev/null +++ b/controllers/operator/pem/pem_collection_test.go @@ -0,0 +1,95 @@ +package pem + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseCertificate(t *testing.T) { + tests := []struct { + inp File + err error + }{ + { + inp: File{ + PrivateKey: "mykey", + Certificate: "mycert", + }, + err: errors.New("failed to parse certificate PEM, please ensure validity of the file"), + }, + { + inp: File{ + PrivateKey: "mykey", + Certificate: `-----BEGIN CERTIFICATE----- +MIIB/jCCAWICCQDscdUxw16XFDAJBgcqhkjOPQQBMEUxCzAJBgNVBAYTAkFVMRMw +EQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0 +eSBMdGQwHhcNMTIxMTE0MTI0MDQ4WhcNMTUxMTE0MTI0MDQ4WjBFMQswCQYDVQQG +EwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lk +Z2l0cyBQdHkgTHRkMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBY9+my9OoeSUR +lDQdV/x8LsOuLilthhiS1Tz4aGDHIPwC1mlvnf7fg5lecYpMCrLLhauAc1UJXcgl +01xoLuzgtAEAgv2P/jgytzRSpUYvgLBt1UA0leLYBy6mQQbrNEuqT3INapKIcUv8 +XxYP0xMEUksLPq6Ca+CRSqTtrd/23uTnapkwCQYHKoZIzj0EAQOBigAwgYYCQXJo +A7Sl2nLVf+4Iu/tAX/IF4MavARKC4PPHK3zfuGfPR3oCCcsAoz3kAzOeijvd0iXb +H5jBImIxPL4WxQNiBTexAkF8D1EtpYuWdlVQ80/h/f4pBcGiXPqX5h2PQSQY7hP1 ++jwM1FGS4fREIOvlBYr/SzzQRtwrvrzGYxDEDbsC0ZGRnA== +-----END CERTIFICATE----- +`, + }, + err: nil, + }, + { + inp: File{ + PrivateKey: "mykey", + Certificate: `-----BEGIN CERTIFICATE----- +MIIB/jCCAWICCQDscdUxw16XFDAJBgcqhkjOPQQBMEUxCzAJBgNVBAYTAkFVMRMw +EQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0 +eSBMdGQwHhcNMTIxMTE0MTI0MDQ4WhcNMTUxMTE0MTI0MDQ4WjBFMQswCQYDVQQG +EwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lk +Z2l0cyBQdHkgTHRkMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBY9+my9OoeSUR +lDQdV/x8LsOuLilthhiS1Tz4aGDHIPwC1mlvnf7fg5lecYpMCrLLhauAc1UJXcgl +01xoLuzgtAEAgv2P/jgytzRSpUYvgLBt1UA0leLYBy6mQQbrNEuqT3INapKIcUv8 +XxYP0xMEUksLPq6Ca+CRSqTtrd/23uTnapkwCQYHKoZIzj0EAQOBigAwgYYCQXJo +A7Sl2nLVf+4Iu/tAX/IF4MavARKC4PPHK3zfuGfPR3oCCcsAoz3kAzOeijvd0iXb +H5jBImIxPL4WxQNiBTexAkF8D1EtpYuWdlVQ80/h/f4pBcGiXPqX5h2PQSQY7hP1 ++jwM1FGS4fREIOvlBYr/SzzQRtwrvrzGYxDEDbsC0ZGRnA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIB/jCCAWICCQDscdUxw16XFDAJBgcqhkjOPQQBMEUxCzAJBgNVBAYTAkFVMRMw +EQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0 +eSBMdGQwHhcNMTIxMTE0MTI0MDQ4WhcNMTUxMTE0MTI0MDQ4WjBFMQswCQYDVQQG +EwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lk +Z2l0cyBQdHkgTHRkMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBY9+my9OoeSUR +lDQdV/x8LsOuLilthhiS1Tz4aGDHIPwC1mlvnf7fg5lecYpMCrLLhauAc1UJXcgl +01xoLuzgtAEAgv2P/jgytzRSpUYvgLBt1UA0leLYBy6mQQbrNEuqT3INapKIcUv8 +XxYP0xMEUksLPq6Ca+CRSqTtrd/23uTnapkwCQYHKoZIzj0EAQOBigAwgYYCQXJo +A7Sl2nLVf+4Iu/tAX/IF4MavARKC4PPHK3zfuGfPR3oCCcsAoz3kAzOeijvd0iXb +H5jBImIxPL4WxQNiBTexAkF8D1EtpYuWdlVQ80/h/f4pBcGiXPqX5h2PQSQY7hP1 ++jwM1FGS4fREIOvlBYr/SzzQRtwrvrzGYxDEDbsC0ZGRnA== +-----END CERTIFICATE----- +`, + }, + err: nil, + }, + } + + for _, tt := range tests { + _, err := tt.inp.ParseCertificate() + errors.Is(err, tt.err) + } +} + +func TestFile(t *testing.T) { + data := []byte(`{ + "H7NQYI7GBGJBC3RZ76DQEAEOX6L6HALZLPIVZI77RHJRDMGPQEYA": "-----BEGIN CERTIFICATE----- +mycert +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +mykey== +-----END RSA PRIVATE KEY----- +" +}`) + f := NewFileFromData(data) + assert.True(t, f.IsValid()) +} diff --git a/controllers/operator/pem/secret.go b/controllers/operator/pem/secret.go new file mode 100644 index 000000000..e2a130b5b --- /dev/null +++ b/controllers/operator/pem/secret.go @@ -0,0 +1,56 @@ +package pem + +import ( + "context" + "fmt" + + "go.uber.org/zap" + + corev1 "k8s.io/api/core/v1" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +// ReadHashFromSecret reads the existing Pem from +// the secret that stores this StatefulSet's Pem collection. +func ReadHashFromSecret(ctx context.Context, secretClient secrets.SecretClient, namespace, name, basePath string, log *zap.SugaredLogger) string { + var secretData map[string]string + var err error + if vault.IsVaultSecretBackend() { + path := fmt.Sprintf("%s/%s/%s", basePath, namespace, name) + secretData, err = secretClient.VaultClient.ReadSecretString(path) + if err != nil { + log.Debugf("tls secret %s doesn't exist yet, unable to compute hash of pem", name) + return "" + } + } else { + s, err := secretClient.KubeClient.GetSecret(ctx, kube.ObjectKey(namespace, name)) + if err != nil { + log.Debugf("tls secret %s doesn't exist yet, unable to compute hash of pem", name) + return "" + } + + if s.Type != corev1.SecretTypeTLS { + log.Debugf("tls secret %s is not of type corev1.SecretTypeTLS; we will not use hash as key name", name) + return "" + } + + secretData = secrets.DataToStringData(s.Data) + } + return ReadHashFromData(secretData, log) +} + +func ReadHashFromData(secretData map[string]string, log *zap.SugaredLogger) string { + pemCollection := NewCollection() + for k, v := range secretData { + pemCollection.MergeEntry(k, NewFileFrom(v)) + } + pemHash, err := pemCollection.GetHash() + if err != nil { + log.Errorf("error computing pem hash: %s", err) + return "" + } + return pemHash +} diff --git a/controllers/operator/pem_test.go b/controllers/operator/pem_test.go new file mode 100644 index 000000000..fce7ee0e1 --- /dev/null +++ b/controllers/operator/pem_test.go @@ -0,0 +1,179 @@ +package operator + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/pem" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" +) + +func TestGetPEMHashIsDeterministic(t *testing.T) { + pemCollection := pem.Collection{ + PemFiles: map[string]pem.File{ + "myhostname1": { + PrivateKey: "mykey", + Certificate: "mycert", + }, + "myhostname2": { + PrivateKey: "mykey", + Certificate: "mycert", + }, + }, + } + firstHash, err := pemCollection.GetHash() + assert.NoError(t, err) + + // modify the PEM collection and check the hash is different + pemCollection.PemFiles["myhostname3"] = pem.File{ + PrivateKey: "thirdey", + Certificate: "thirdcert", + } + secondHash, err := pemCollection.GetHash() + assert.NoError(t, err) + assert.NotEqual(t, firstHash, secondHash) + + // revert the changes to the PEM collection and check the hash is the same + delete(pemCollection.PemFiles, "myhostname3") + thirdHash, err := pemCollection.GetHash() + assert.NoError(t, err) + assert.Equal(t, firstHash, thirdHash) +} + +func TestMergeEntryOverwritesOldSecret(t *testing.T) { + p := pem.Collection{ + PemFiles: map[string]pem.File{ + "myhostname": { + PrivateKey: "mykey", + Certificate: "mycert", + }, + }, + } + + secretData := pem.File{ + PrivateKey: "oldkey", + Certificate: "oldcert", + } + + p.MergeEntry("myhostname", secretData) + assert.Equal(t, "mykey", p.PemFiles["myhostname"].PrivateKey) + assert.Equal(t, "mycert", p.PemFiles["myhostname"].Certificate) +} + +func TestMergeEntryOnlyCertificate(t *testing.T) { + p := pem.Collection{ + PemFiles: map[string]pem.File{ + "myhostname": { + PrivateKey: "mykey", + }, + }, + } + + secretData := pem.File{ + PrivateKey: "oldkey", + Certificate: "oldcert", + } + + p.MergeEntry("myhostname", secretData) + assert.Equal(t, "mykey", p.PemFiles["myhostname"].PrivateKey) + assert.Equal(t, "oldcert", p.PemFiles["myhostname"].Certificate) +} + +func TestMergeEntryPreservesOldSecret(t *testing.T) { + p := pem.Collection{ + PemFiles: map[string]pem.File{ + "myexistinghostname": { + PrivateKey: "mykey", + Certificate: "mycert", + }, + }, + } + + secretData := pem.File{ + PrivateKey: "oldkey", + Certificate: "oldcert", + } + + p.MergeEntry("myhostname", secretData) + assert.Equal(t, "oldkey", p.PemFiles["myhostname"].PrivateKey) + assert.Equal(t, "oldcert", p.PemFiles["myhostname"].Certificate) + assert.Equal(t, "mykey", p.PemFiles["myexistinghostname"].PrivateKey) + assert.Equal(t, "mycert", p.PemFiles["myexistinghostname"].Certificate) +} + +type mockSecretGetter struct { + secret *corev1.Secret +} + +func (m mockSecretGetter) GetSecret(_ context.Context, _ client.ObjectKey) (corev1.Secret, error) { + if m.secret == nil { + return corev1.Secret{}, xerrors.Errorf("not found") + } + return *m.secret, nil +} + +func (m mockSecretGetter) CreateSecret(_ context.Context, _ corev1.Secret) error { + return nil +} + +func (m mockSecretGetter) UpdateSecret(_ context.Context, _ corev1.Secret) error { + return nil +} + +func (m mockSecretGetter) DeleteSecret(_ context.Context, _ types.NamespacedName) error { + return nil +} + +func TestReadPemHashFromSecret(t *testing.T) { + ctx := context.Background() + name := "res-name" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name + "-cert", Namespace: mock.TestNamespace}, + Data: map[string][]byte{"hello": []byte("world")}, + Type: corev1.SecretTypeTLS, + } + + assert.Empty(t, pem.ReadHashFromSecret(ctx, secrets.SecretClient{ + VaultClient: nil, + KubeClient: mockSecretGetter{}, + }, mock.TestNamespace, name, "", zap.S()), "secret does not exist so pem hash should be empty") + + hash := pem.ReadHashFromSecret(ctx, secrets.SecretClient{ + VaultClient: nil, + KubeClient: mockSecretGetter{secret: secret}, + }, mock.TestNamespace, name, "", zap.S()) + + hash2 := pem.ReadHashFromSecret(ctx, secrets.SecretClient{ + VaultClient: nil, + KubeClient: mockSecretGetter{secret: secret}, + }, mock.TestNamespace, name, "", zap.S()) + + assert.NotEmpty(t, hash, "pem hash should be read from the secret") + assert.Equal(t, hash, hash2, "hash creation should be idempotent") +} + +func TestReadPemHashFromSecretOpaqueType(t *testing.T) { + ctx := context.Background() + + name := "res-name" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name + "-cert", Namespace: mock.TestNamespace}, + Data: map[string][]byte{"hello": []byte("world")}, + Type: corev1.SecretTypeOpaque, + } + + assert.Empty(t, pem.ReadHashFromSecret(ctx, secrets.SecretClient{ + VaultClient: nil, + KubeClient: mockSecretGetter{secret: secret}, + }, mock.TestNamespace, name, "", zap.S()), "if secret type is not TLS the empty string should be returned") +} diff --git a/controllers/operator/project/credentials.go b/controllers/operator/project/credentials.go new file mode 100644 index 000000000..ecd423615 --- /dev/null +++ b/controllers/operator/project/credentials.go @@ -0,0 +1,58 @@ +package project + +import ( + "context" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "sigs.k8s.io/controller-runtime/pkg/client" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +// ReadCredentials reads the Secret containing the credentials to authenticate in Ops Manager and creates a matching 'Credentials' object +func ReadCredentials(ctx context.Context, secretClient secrets.SecretClient, credentialsSecret client.ObjectKey, log *zap.SugaredLogger) (mdbv1.Credentials, error) { + var operatorSecretPath string + if vault.IsVaultSecretBackend() { + operatorSecretPath = secretClient.VaultClient.OperatorSecretPath() + } + secret, err := secretClient.ReadSecret(ctx, credentialsSecret, operatorSecretPath) + if err != nil { + return mdbv1.Credentials{}, err + } + oldSecretEntries, user, publicAPIKey := secretContainsPairOfKeys(secret, util.OldOmUser, util.OldOmPublicApiKey) + + newSecretEntries, publicKey, privateKey := secretContainsPairOfKeys(secret, util.OmPublicApiKey, util.OmPrivateKey) + + if !oldSecretEntries && !newSecretEntries { + return mdbv1.Credentials{}, xerrors.Errorf("secret %s does not contain the required entries. It should contain either %s and %s, or %s and %s", credentialsSecret, util.OldOmUser, util.OldOmPublicApiKey, util.OmPublicApiKey, util.OmPrivateKey) + } + + if oldSecretEntries { + log.Infof("Usage of old entries for the credentials secret (\"%s\" and \"%s\") is deprecated, prefer using \"%s\" and \"%s\"", util.OldOmUser, util.OldOmPublicApiKey, util.OmPublicApiKey, util.OmPrivateKey) + return mdbv1.Credentials{ + PublicAPIKey: user, + PrivateAPIKey: publicAPIKey, + }, nil + } + + return mdbv1.Credentials{ + PublicAPIKey: publicKey, + PrivateAPIKey: privateKey, + }, nil +} + +func secretContainsPairOfKeys(secret map[string]string, key1 string, key2 string) (bool, string, string) { + val1, ok := secret[key1] + if !ok { + return false, "", "" + } + val2, ok := secret[key2] + if !ok { + return false, "", "" + } + return true, val1, val2 +} diff --git a/controllers/operator/project/project.go b/controllers/operator/project/project.go new file mode 100644 index 000000000..b12bcc6c5 --- /dev/null +++ b/controllers/operator/project/project.go @@ -0,0 +1,245 @@ +package project + +import ( + "context" + "fmt" + "strings" + + "go.uber.org/zap" + "golang.org/x/xerrors" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om/apierror" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// Reader returns the name of a ConfigMap which contains Ops Manager project details. +// and the name of a secret containing project credentials. +type Reader interface { + metav1.Object + GetProjectConfigMapName() string + GetProjectConfigMapNamespace() string + GetCredentialsSecretName() string + GetCredentialsSecretNamespace() string +} + +// ReadConfigAndCredentials returns the ProjectConfig and Credentials for a given resource which are +// used to communicate with Ops Manager. +func ReadConfigAndCredentials(ctx context.Context, cmGetter configmap.Getter, secretGetter secrets.SecretClient, reader Reader, log *zap.SugaredLogger) (mdbv1.ProjectConfig, mdbv1.Credentials, error) { + projectConfig, err := ReadProjectConfig(ctx, cmGetter, kube.ObjectKey(reader.GetProjectConfigMapNamespace(), reader.GetProjectConfigMapName()), reader.GetName()) + if err != nil { + return mdbv1.ProjectConfig{}, mdbv1.Credentials{}, xerrors.Errorf("error reading project %w", err) + } + credsConfig, err := ReadCredentials(ctx, secretGetter, kube.ObjectKey(reader.GetCredentialsSecretNamespace(), reader.GetCredentialsSecretName()), log) + if err != nil { + return mdbv1.ProjectConfig{}, mdbv1.Credentials{}, xerrors.Errorf("error reading Credentials secret: %w", err) + } + return projectConfig, credsConfig, nil +} + +/* +ReadOrCreateProject +Communication with groups is tricky. +In connection ConfigMap user must provide the project name and the id of organization. +The only way to find out if the project exists already is to check if its +organization name is the same as the projects one and that's what Operator is doing. So if ConfigMap specifies the +project with name "A" and no org id and there is already project in Ops manager named "A" in organization with name"B" +then Operator won't find it as the names don't match. + +Note, that the method is performed holding the "groupName+orgId" mutex which allows to avoid race conditions and avoid +duplicated groups/organizations creation. So if for example the standalone and the replica set which reference the same +configMap are created in parallel - this function will be invoked sequentially and the second caller will see the group +created on the first call +*/ +func ReadOrCreateProject(config mdbv1.ProjectConfig, credentials mdbv1.Credentials, connectionFactory om.ConnectionFactory, log *zap.SugaredLogger) (*om.Project, om.Connection, error) { + projectName := config.ProjectName + mutex := om.GetMutex(projectName, config.OrgID) + mutex.Lock() + defer mutex.Unlock() + + log = log.With("project", projectName) + + // we need to create a temporary connection object without group id + omContext := om.OMContext{ + GroupID: "", + GroupName: projectName, + OrgID: config.OrgID, + BaseURL: config.BaseURL, + PublicKey: credentials.PublicAPIKey, + PrivateKey: credentials.PrivateAPIKey, + + // The OM Client expects the inverse of "Require valid cert" because in Go + // The "zero" value of bool is "False", hence this default. + AllowInvalidSSLCertificate: !config.SSLRequireValidMMSServerCertificates, + + // The CA certificate passed to the OM client needs to be an actual certificate, + // and not a location in disk, because each "project" will have its own CA cert. + CACertificate: config.SSLMMSCAConfigMapContents, + } + + conn := connectionFactory(&omContext) + + org, err := findOrganization(config.OrgID, projectName, conn, log) + if err != nil { + return nil, nil, err + } + + var project *om.Project + if org != nil { + project, err = findProject(projectName, org, conn, log) + if err != nil { + return nil, nil, err + } + } + + if project == nil { + project, err = tryCreateProject(org, projectName, config.OrgID, conn, log) + if err != nil { + return nil, nil, err + } + } + + conn.ConfigureProject(project) + + return project, conn, nil +} + +func findOrganization(orgID string, projectName string, conn om.Connection, log *zap.SugaredLogger) (*om.Organization, error) { + if orgID == "" { + // Note: this org_id = "" has to be explicitly set by the customer. + // If org id is not specified - then the contract is that the organization for the project must have the same + // name as project has (as it was created automatically for the project), so we need to find relevant organization + log.Debugf("Organization id is not specified - trying to find the organization with name \"%s\"", projectName) + var err error + if orgID, err = findOrganizationByName(conn, projectName, log); err != nil { + return nil, err + } + if orgID == "" { + log.Debugf("Organization \"%s\" not found", projectName) + return nil, nil + } + } + + organization, err := conn.ReadOrganization(orgID) + if err != nil { + return nil, xerrors.Errorf("organization with id %s not found: %w", orgID, err) + } + return organization, nil +} + +// findProject tries to find if the group already exists. +func findProject(projectName string, organization *om.Organization, conn om.Connection, log *zap.SugaredLogger) (*om.Project, error) { + project, err := findProjectInsideOrganization(conn, projectName, organization, log) + if err != nil { + return nil, xerrors.Errorf("error finding project %s in organization with id %s: %w", projectName, organization, err) + } + if project != nil { + return project, nil + } + log.Debugf("Project \"%s\" not found in organization %s (\"%s\")", projectName, organization.ID, organization.Name) + return nil, nil +} + +// findProjectInsideOrganization looks up a project by name inside an organization and returns the project if it was the only one found by that name. +// If no project was found, the function returns a nil project to indicate that no such project exists. +// In all other cases, a non nil error is returned. +func findProjectInsideOrganization(conn om.Connection, projectName string, organization *om.Organization, log *zap.SugaredLogger) (*om.Project, error) { + projects, err := conn.ReadProjectsInOrganizationByName(organization.ID, projectName) + if err != nil { + if v, ok := err.(*apierror.Error); ok { + // If the project was not found, return an empty project and no error. + if v.ErrorCode == apierror.ProjectNotFound { + // ProjectNotFound is an expected condition. + + return nil, nil + } + } + // Return an empty project and the OM api error in case there is a different API error. + return nil, xerrors.Errorf("error looking up project %s in organization %s: %w", projectName, organization.ID, err) + } + + // There is no API error. We check if the project found has the exact name. + // The API endpoint returns a list of projects and in case of no exact match, it would return the first item that matches the search term as a prefix. + var projectsFound []*om.Project + for _, project := range projects { + if project.Name == projectName { + projectsFound = append(projectsFound, project) + } + } + + if len(projectsFound) == 1 { + // If there is just one project returned, and it matches the name, return it. + return projectsFound[0], nil + } else if len(projectsFound) > 0 { + projectsList := util.Transform(projectsFound, func(project *om.Project) string { + return fmt.Sprintf("%s (%s)", project.Name, project.ID) + }) + // This should not happen, but older versions of OM supported the same name for a project in an org. We cannot proceed here so we return an error. + return nil, xerrors.Errorf("found more than one project with name %s in organization %s (%s): %v", projectName, organization.ID, organization.Name, strings.Join(projectsList, ", ")) + } + + // If there is no error from the API and no match in the response, return an empty project and no error. + return nil, nil +} + +func findOrganizationByName(conn om.Connection, name string, log *zap.SugaredLogger) (string, error) { + // 1. We try to find the organization using 'name' filter parameter first + organizations, err := conn.ReadOrganizationsByName(name) + if err != nil { + // This code is Ops Manager < 7 variant. Once 6 is EOL, it can be removed. + if v, ok := err.(*apierror.Error); ok { + if v.ErrorCode == apierror.OrganizationNotFound { + // the "name" API is supported and the organization not found - returning nil + return "", nil + } + } + return "", xerrors.Errorf("could not find organization %s: %w", name, err) + } + // There's no error, so now we're checking organizations and matching names. + // This code is needed as the Ops Manager selects organizations that "start by" the query. So potentially, + // it can return manu results. + // For example: + // When looking for name "development", it may return ["development", "development-oplog"]. + for _, organization := range organizations { + if organization.Name == name { + return organization.ID, nil + } + } + // This is the fallback case - there is no org and in subsequent steps we'll create it. + return "", nil +} + +func tryCreateProject(organization *om.Organization, projectName, orgId string, conn om.Connection, log *zap.SugaredLogger) (*om.Project, error) { + // We can face the following scenario: for the project "foo" with 'orgId=""' the organization "foo" already exists + // - so we need to reuse its orgId instead of creating the new Organization with the same name (OM API is quite + // poor here - it may create duplicates) + if organization != nil { + orgId = organization.ID + } + // Creating the group as it doesn't exist + log.Infow("Creating the project as it doesn't exist", "orgId", orgId) + if orgId == "" { + log.Infof("Note that as the orgId is not specified the organization with name \"%s\" will be created "+ + "automatically by Ops Manager", projectName) + } + group := &om.Project{ + Name: projectName, + OrgID: orgId, + Tags: []string{}, // Project creation no longer applies the EXTERNALLY_MANAGED tag, this is added afterwards + } + ans, err := conn.CreateProject(group) + if err != nil { + return nil, xerrors.Errorf("Error creating project \"%s\" in Ops Manager: %w", group, err) + } + + log.Infow("Project successfully created", "id", ans.ID) + + return ans, nil +} diff --git a/controllers/operator/project/projectconfig.go b/controllers/operator/project/projectconfig.go new file mode 100644 index 000000000..66ce5cbb5 --- /dev/null +++ b/controllers/operator/project/projectconfig.go @@ -0,0 +1,109 @@ +package project + +import ( + "context" + + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +func validateProjectConfig(ctx context.Context, cmGetter configmap.Getter, projectConfigMap client.ObjectKey) (map[string]string, error) { + data, err := configmap.ReadData(ctx, cmGetter, projectConfigMap) + if err != nil { + return nil, err + } + + requiredFields := []string{util.OmBaseUrl, util.OmOrgId} + + for _, requiredField := range requiredFields { + if _, ok := data[requiredField]; !ok { + return nil, xerrors.Errorf(`property "%s" is not specified in ConfigMap %s`, requiredField, projectConfigMap) + } + } + return data, nil +} + +// ReadProjectConfig returns a "Project" config build from a ConfigMap with a series of attributes +// like `projectName`, `baseUrl` and a series of attributes related to SSL. +// If configMap doesn't have a projectName defined - the name of MongoDB resource is used as a name of project +func ReadProjectConfig(ctx context.Context, cmGetter configmap.Getter, projectConfigMap client.ObjectKey, mdbName string) (mdbv1.ProjectConfig, error) { + data, err := validateProjectConfig(ctx, cmGetter, projectConfigMap) + if err != nil { + return mdbv1.ProjectConfig{}, err + } + + baseURL := data[util.OmBaseUrl] + orgID := data[util.OmOrgId] + + projectName := data[util.OmProjectName] + if projectName == "" { + projectName = mdbName + } + + sslRequireValid := true + sslRequireValidData, ok := data[util.SSLRequireValidMMSServerCertificates] + const falseCustomCASetting = "false" + if ok { + sslRequireValid = sslRequireValidData != falseCustomCASetting + } + + sslCaConfigMap, ok := data[util.SSLMMSCAConfigMap] + caFile := "" + if ok { + sslCaConfigMapKey := types.NamespacedName{Name: sslCaConfigMap, Namespace: projectConfigMap.Namespace} + + cacrt, err := configmap.ReadData(ctx, cmGetter, sslCaConfigMapKey) + if err != nil { + return mdbv1.ProjectConfig{}, xerrors.Errorf("failed to read the specified ConfigMap %s (%w)", sslCaConfigMapKey, err) + } + for k, v := range cacrt { + if k == util.CaCertMMS { + caFile = v + break + } + } + } + + var useCustomCA bool + useCustomCAData, ok := data[util.UseCustomCAConfigMap] + if ok { + useCustomCA = useCustomCAData != falseCustomCASetting + } + + return mdbv1.ProjectConfig{ + BaseURL: baseURL, + ProjectName: projectName, + OrgID: orgID, + + // Options related with SSL on OM side. + SSLProjectConfig: env.SSLProjectConfig{ + // Relevant to + // + operator (via golang http configuration) + // + curl (via command line argument [--insecure]) + // + automation-agent (via env variable configuration [SSL_REQUIRE_VALID_MMS_CERTIFICATES]) + // + EnvVarSSLRequireValidMMSCertificates and automation agent option + // + -sslRequireValidMMSServerCertificates + SSLRequireValidMMSServerCertificates: sslRequireValid, + + // SSLMMSCAConfigMap is name of the configmap with the CA. This CM + // will be mounted in the database Pods. + SSLMMSCAConfigMap: sslCaConfigMap, + + // This needs to be passed for the operator itself to be able to + // recognize the CA -- as it can't be mounted on an already running + // Pod. + SSLMMSCAConfigMapContents: caFile, + }, + + Credentials: data[util.OmCredentials], + + UseCustomCA: useCustomCA, + }, nil +} diff --git a/controllers/operator/project/projectconfig_test.go b/controllers/operator/project/projectconfig_test.go new file mode 100644 index 000000000..517791a09 --- /dev/null +++ b/controllers/operator/project/projectconfig_test.go @@ -0,0 +1,177 @@ +package project + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + corev1 "k8s.io/api/core/v1" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestSSLOptionsArePassedCorrectly_SSLRequireValidMMSServerCertificates(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + cm := defaultConfigMap("cm1") + cm.Data[util.SSLRequireValidMMSServerCertificates] = "true" + err := client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err := ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm1"), "") + + assert.NoError(t, err) + assert.True(t, projectConfig.SSLRequireValidMMSServerCertificates) + + assert.Equal(t, projectConfig.SSLMMSCAConfigMap, "") + assert.Equal(t, projectConfig.SSLMMSCAConfigMapContents, "") + + cm = defaultConfigMap("cm2") + cm.Data[util.SSLRequireValidMMSServerCertificates] = "1" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm2"), "") + + assert.NoError(t, err) + assert.True(t, projectConfig.SSLRequireValidMMSServerCertificates) + + assert.Equal(t, projectConfig.SSLMMSCAConfigMap, "") + assert.Equal(t, projectConfig.SSLMMSCAConfigMapContents, "") + + cm = defaultConfigMap("cm3") + // Setting this attribute to "false" will make it false, any other + // value will result in this attribute being set to true. + cm.Data[util.SSLRequireValidMMSServerCertificates] = "false" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm3"), "") + + assert.NoError(t, err) + assert.False(t, projectConfig.SSLRequireValidMMSServerCertificates) + + assert.Equal(t, projectConfig.SSLMMSCAConfigMap, "") + assert.Equal(t, projectConfig.SSLMMSCAConfigMapContents, "") +} + +func TestSSLOptionsArePassedCorrectly_SSLMMSCAConfigMap(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + // This represents the ConfigMap holding the CustomCA + cm := defaultConfigMap("configmap-with-ca-entry") + cm.Data["mms-ca.crt"] = "---- some cert ----" + cm.Data["this-field-is-not-required"] = "bla bla" + err := client.Create(ctx, &cm) + assert.NoError(t, err) + + // The second CM (the "Project" one) refers to the previous one, where + // the certificate entry is stored. + cm = defaultConfigMap("cm") + cm.Data[util.SSLMMSCAConfigMap] = "configmap-with-ca-entry" + cm.Data[util.SSLRequireValidMMSServerCertificates] = "false" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err := ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm"), "") + + assert.NoError(t, err) + assert.False(t, projectConfig.SSLRequireValidMMSServerCertificates) + + assert.Equal(t, projectConfig.SSLMMSCAConfigMap, "configmap-with-ca-entry") + assert.Equal(t, projectConfig.SSLMMSCAConfigMapContents, "---- some cert ----") +} + +func TestSSLOptionsArePassedCorrectly_UseCustomCAConfigMap(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + // Passing "false" results in false to UseCustomCA + cm := defaultConfigMap("cm") + cm.Data[util.UseCustomCAConfigMap] = "false" + err := client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err := ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm"), "") + + assert.NoError(t, err) + assert.False(t, projectConfig.UseCustomCA) + + // Passing "true" results in true to UseCustomCA + cm = defaultConfigMap("cm2") + cm.Data[util.UseCustomCAConfigMap] = "true" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm2"), "") + + assert.NoError(t, err) + assert.True(t, projectConfig.UseCustomCA) + + // Passing any value different from "false" results in true. + cm = defaultConfigMap("cm3") + cm.Data[util.UseCustomCAConfigMap] = "" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm3"), "") + assert.NoError(t, err) + assert.True(t, projectConfig.UseCustomCA) + + // "1" also results in a true value + cm = defaultConfigMap("cm4") + cm.Data[util.UseCustomCAConfigMap] = "1" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm4"), "") + assert.NoError(t, err) + assert.True(t, projectConfig.UseCustomCA) + + // This last section only tests that the unit test is working fine + // and having multiple ConfigMaps in the mocked client will not + // result in contaminated checks. + cm = defaultConfigMap("cm5") + cm.Data[util.UseCustomCAConfigMap] = "false" + err = client.Create(ctx, &cm) + assert.NoError(t, err) + + projectConfig, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm5"), "") + assert.NoError(t, err) + assert.False(t, projectConfig.UseCustomCA) +} + +func TestMissingRequiredFieldsFromCM(t *testing.T) { + ctx := context.Background() + client, _ := mock.NewDefaultFakeClient() + t.Run("missing url", func(t *testing.T) { + cm := defaultConfigMap("cm1") + delete(cm.Data, util.OmBaseUrl) + err := client.Create(ctx, &cm) + assert.NoError(t, err) + _, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm1"), "") + assert.Error(t, err) + }) + t.Run("missing orgID", func(t *testing.T) { + cm := defaultConfigMap("cm1") + delete(cm.Data, util.OmOrgId) + err := client.Create(ctx, &cm) + assert.NotNil(t, err) + _, err = ReadProjectConfig(ctx, client, kube.ObjectKey(mock.TestNamespace, "cm1"), "") + assert.Error(t, err) + }) +} + +func defaultConfigMap(name string) corev1.ConfigMap { + return configmap.Builder(). + SetName(name). + SetNamespace(mock.TestNamespace). + SetDataField(util.OmBaseUrl, "http://mycompany.example.com:8080"). + SetDataField(util.OmOrgId, "123abc"). + SetDataField(util.OmProjectName, "my-name"). + Build() +} diff --git a/controllers/operator/recovery/recovery.go b/controllers/operator/recovery/recovery.go new file mode 100644 index 000000000..c08a3f963 --- /dev/null +++ b/controllers/operator/recovery/recovery.go @@ -0,0 +1,38 @@ +package recovery + +import ( + "time" + + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +const ( + EnableRecoveryEnvVar = "MDB_AUTOMATIC_RECOVERY_ENABLE" + RecoveryBackoffTimeEnvVar = "MDB_AUTOMATIC_RECOVERY_BACKOFF_TIME_S" + DefaultAutomaticRecoveryBackoffTimeSeconds = 20 * 60 +) + +func isAutomaticRecoveryTurnedOn() bool { + return env.ReadBoolOrDefault(EnableRecoveryEnvVar, true) // nolint:forbidigo +} + +func automaticRecoveryBackoffSeconds() int { + return env.ReadIntOrDefault(RecoveryBackoffTimeEnvVar, DefaultAutomaticRecoveryBackoffTimeSeconds) // nolint:forbidigo +} + +func ShouldTriggerRecovery(isResourceFailing bool, lastTransitionTime string) bool { + if isAutomaticRecoveryTurnedOn() && isResourceFailing { + parsedTime, err := time.Parse(time.RFC3339, lastTransitionTime) + if err != nil { + // We silently ignore all the errors and just prevent the recovery from happening + return false + } + zap.S().Debugf("The configured delay before recovery is %d seconds", automaticRecoveryBackoffSeconds()) + if parsedTime.Add(time.Duration(automaticRecoveryBackoffSeconds()) * time.Second).Before(time.Now()) { + return true + } + } + return false +} diff --git a/controllers/operator/secrets/secrets.go b/controllers/operator/secrets/secrets.go new file mode 100644 index 000000000..2ec6407a7 --- /dev/null +++ b/controllers/operator/secrets/secrets.go @@ -0,0 +1,203 @@ +package secrets + +import ( + "context" + "encoding/base64" + "fmt" + "reflect" + "strings" + + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/types" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +type SecretClientInterface interface { + ReadSecret(ctx context.Context, secretName types.NamespacedName, basePath string) (map[string]string, error) +} + +var _ SecretClientInterface = (*SecretClient)(nil) + +type SecretClient struct { + VaultClient *vault.VaultClient + KubeClient kubernetesClient.KubernetesSecretClient +} + +func namespacedNameToVaultPath(nsName types.NamespacedName, basePath string) string { + return fmt.Sprintf("%s/%s/%s", basePath, nsName.Namespace, nsName.Name) +} + +func secretNamespacedName(s corev1.Secret) types.NamespacedName { + return types.NamespacedName{ + Namespace: s.Namespace, + Name: s.Name, + } +} + +func (r SecretClient) ReadSecretKey(ctx context.Context, secretName types.NamespacedName, basePath string, key string) (string, error) { + secret, err := r.ReadSecret(ctx, secretName, basePath) + if err != nil { + return "", xerrors.Errorf("can't read secret %s: %w", secretName, err) + } + val, ok := secret[key] + if !ok { + return "", xerrors.Errorf("secret %s does not contain key %s", secretName, key) + } + return val, nil +} + +func (r SecretClient) ReadSecret(ctx context.Context, secretName types.NamespacedName, basePath string) (map[string]string, error) { + secrets := make(map[string]string) + if vault.IsVaultSecretBackend() { + var err error + secretPath := namespacedNameToVaultPath(secretName, basePath) + secrets, err = r.VaultClient.ReadSecretString(secretPath) + if err != nil { + return nil, err + } + } else { + stringData, err := secret.ReadStringData(ctx, r.KubeClient, secretName) + if err != nil { + return nil, err + } + for k, v := range stringData { + secrets[k] = strings.TrimSuffix(v[:], "\n") + } + } + return secrets, nil +} + +func (r SecretClient) ReadBinarySecret(ctx context.Context, secretName types.NamespacedName, basePath string) (map[string][]byte, error) { + var secrets map[string][]byte + var err error + if vault.IsVaultSecretBackend() { + secretPath := namespacedNameToVaultPath(secretName, basePath) + secrets, err = r.VaultClient.ReadSecretBytes(secretPath) + if err != nil { + return nil, err + } + } else { + secrets, err = secret.ReadByteData(ctx, r.KubeClient, secretName) + if err != nil { + return nil, err + } + } + return secrets, nil +} + +// PutSecret copies secret.Data into vault. Note: we don't rely on secret.StringData since our builder does not use the field. +func (r SecretClient) PutSecret(ctx context.Context, s corev1.Secret, basePath string) error { + if vault.IsVaultSecretBackend() { + secretPath := namespacedNameToVaultPath(secretNamespacedName(s), basePath) + secretData := map[string]interface{}{} + for k, v := range s.Data { + secretData[k] = string(v) + } + data := map[string]interface{}{ + "data": secretData, + } + return r.VaultClient.PutSecret(secretPath, data) + } + + return secret.CreateOrUpdate(ctx, r.KubeClient, s) +} + +// PutBinarySecret copies secret.Data as base64 into vault. +func (r SecretClient) PutBinarySecret(ctx context.Context, s corev1.Secret, basePath string) error { + if vault.IsVaultSecretBackend() { + secretPath := namespacedNameToVaultPath(secretNamespacedName(s), basePath) + secretData := map[string]interface{}{} + for k, v := range s.Data { + secretData[k] = base64.StdEncoding.EncodeToString(v) + } + data := map[string]interface{}{ + "data": secretData, + } + return r.VaultClient.PutSecret(secretPath, data) + } + + return secret.CreateOrUpdate(ctx, r.KubeClient, s) +} + +// PutSecretIfChanged updates a Secret only if it has changed. Equality is based on s.Data. +// `basePath` is only used when Secrets backend is `Vault`. +func (r SecretClient) PutSecretIfChanged(ctx context.Context, s corev1.Secret, basePath string) error { + if vault.IsVaultSecretBackend() { + secret, err := r.ReadSecret(ctx, secretNamespacedName(s), basePath) + if err != nil && !strings.Contains(err.Error(), "not found") { + return err + } + if err != nil || !reflect.DeepEqual(secret, DataToStringData(s.Data)) { + return r.PutSecret(ctx, s, basePath) + } + } + + return secret.CreateOrUpdateIfNeeded(ctx, r.KubeClient, s) +} + +func SecretNotExist(err error) bool { + if err == nil { + return false + } + return apiErrors.IsNotFound(err) || strings.Contains(err.Error(), "secret not found") +} + +// These methods implement the secretGetterUpdateCreateDeleter interface from community. +// We hardcode here the AppDB sub-path for Vault since community is used only to deploy +// AppDB pods. This allows us to minimize the changes to Community. +// TODO this method is very fishy as it has hardcoded AppDBSecretPath, but is used not only for AppDB +// We should probably use ReadSecret instead -> https://jira.mongodb.org/browse/CLOUDP-277863 +func (r SecretClient) GetSecret(ctx context.Context, secretName types.NamespacedName) (corev1.Secret, error) { + if vault.IsVaultSecretBackend() { + s := corev1.Secret{} + + data, err := r.ReadSecret(ctx, secretName, r.VaultClient.AppDBSecretPath()) + if err != nil { + return s, err + } + s.Data = make(map[string][]byte) + for k, v := range data { + s.Data[k] = []byte(v) + } + return s, nil + } + return r.KubeClient.GetSecret(ctx, secretName) +} + +func (r SecretClient) CreateSecret(ctx context.Context, s corev1.Secret) error { + var appdbSecretPath string + if r.VaultClient != nil { + appdbSecretPath = r.VaultClient.AppDBSecretPath() + } + return r.PutSecret(ctx, s, appdbSecretPath) +} + +func (r SecretClient) UpdateSecret(ctx context.Context, s corev1.Secret) error { + if vault.IsVaultSecretBackend() { + return r.CreateSecret(ctx, s) + } + return r.KubeClient.UpdateSecret(ctx, s) +} + +func (r SecretClient) DeleteSecret(ctx context.Context, secretName types.NamespacedName) error { + if vault.IsVaultSecretBackend() { + // TODO deletion logic + return nil + } + return r.KubeClient.DeleteSecret(ctx, secretName) +} + +func DataToStringData(data map[string][]byte) map[string]string { + stringData := make(map[string]string) + for k, v := range data { + stringData[k] = string(v) + } + return stringData +} diff --git a/controllers/operator/state_store.go b/controllers/operator/state_store.go new file mode 100644 index 000000000..7464ac418 --- /dev/null +++ b/controllers/operator/state_store.go @@ -0,0 +1,124 @@ +package operator + +import ( + "context" + "encoding/json" + "fmt" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + corev1 "k8s.io/api/core/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +const stateKey = "state" + +// StateStore is a wrapper for a custom, per-resource deployment state required for the operator to reconciler the resource correctly. +// It handles serialization/deserialization of any deployment state structure of type S. +// The deployment state is saved to a config map -state in the resource's namespace in the operator's cluster. +type StateStore[S any] struct { + namespace string + resourceName string + client kubernetesClient.Client + + data map[string]string +} + +// NewStateStore constructs a new instance of the StateStore. +// It is intended to be instantiated with each execution of the Reconcile method and therefore it is not +// designed to be thread safe. +func NewStateStore[S any](namespace string, resourceName string, client kubernetesClient.Client) *StateStore[S] { + return &StateStore[S]{ + namespace: namespace, + resourceName: resourceName, + client: client, + data: map[string]string{}, + } +} + +func (s *StateStore[S]) read(ctx context.Context) error { + cm := corev1.ConfigMap{} + if err := s.client.Get(ctx, kube.ObjectKey(s.namespace, s.getStateConfigMapName()), &cm); err != nil { + return err + } + + s.data = cm.Data + return nil +} + +func (s *StateStore[S]) write(ctx context.Context, log *zap.SugaredLogger) error { + dataCM := configmap.Builder(). + SetName(s.getStateConfigMapName()). + SetLabels(map[string]string{ + construct.ControllerLabelName: util.OperatorName, + mdbv1.LabelMongoDBResourceOwner: s.resourceName, + }). + SetNamespace(s.namespace). + SetData(s.data). + Build() + + log.Debugf("Saving deployment state to %s config map: %s", s.getStateConfigMapName(), s.data) + return configmap.CreateOrUpdate(ctx, s.client, dataCM) +} + +func (s *StateStore[S]) getStateConfigMapName() string { + return fmt.Sprintf("%s-state", s.resourceName) +} + +func (s *StateStore[S]) WriteState(ctx context.Context, state *S, log *zap.SugaredLogger) error { + if err := s.setDataValue(stateKey, state); err != nil { + return err + } + return s.write(ctx, log) +} + +func (s *StateStore[S]) ReadState(ctx context.Context) (*S, error) { + state := new(S) + + // If we don't find the state ConfigMap, return an error + if err := s.read(ctx); err != nil { + return nil, err + } + + // Deserialize the state + if ok, err := s.getDataValue(stateKey, state); err != nil { + return nil, err + } else if !ok { + // if we don't have state key it's like we don't have state at all + return nil, errors.NewNotFound(schema.GroupResource{}, s.getStateConfigMapName()) + } else { + return state, nil + } +} + +func (s *StateStore[S]) getDataValue(key string, obj any) (bool, error) { + if jsonStr, ok := s.data[key]; !ok { + return false, nil + } else { + if err := json.Unmarshal([]byte(jsonStr), obj); err != nil { + return true, xerrors.Errorf("cannot unmarshal deployment state %s/%s key %s from the value: %s: %w", s.namespace, s.getStateConfigMapName(), key, jsonStr, err) + } + } + + return true, nil +} + +func (s *StateStore[S]) setDataValue(key string, value any) error { + if jsonBytes, err := json.Marshal(value); err != nil { + return xerrors.Errorf("cannot marshal deployment state %s/%s key %s from the value: %v: %w", s.namespace, s.getStateConfigMapName(), key, value, err) + } else { + s.data[key] = string(jsonBytes) + } + + return nil +} diff --git a/controllers/operator/testdata/certificates/cert_auto b/controllers/operator/testdata/certificates/cert_auto new file mode 100644 index 000000000..b25135597 --- /dev/null +++ b/controllers/operator/testdata/certificates/cert_auto @@ -0,0 +1,137 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 3 (0x3) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=New York, L=New York, O=MongoDB, OU=mongodb, CN=www.mongodb.com + Validity + Not Before: Jul 16 09:51:56 2020 GMT + Not After : Apr 12 09:51:56 2023 GMT + Subject: C=US, ST=New York, L=New York, O=MongoDB, OU=cloud, CN=mms-automation-agent + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a8:9c:e4:7b:36:ca:65:61:89:41:69:81:40:c8: + b3:5d:fa:0b:4f:48:4d:29:b2:20:36:1d:42:f5:f9: + 80:c1:b8:42:45:46:c9:b3:38:8c:00:c8:d3:e0:f2: + 7d:62:ca:15:38:52:43:0b:56:ec:14:94:81:87:5d: + 58:a6:23:d2:a9:29:81:9e:5b:b4:d2:9e:79:b3:5b: + 36:bf:9f:1f:e5:a4:06:73:85:08:98:8c:7b:66:50: + 8b:fd:5d:45:14:61:97:1e:f7:93:f1:de:7d:7e:8f: + 6d:b4:ff:f6:e7:22:38:df:e5:6e:7b:00:5b:1f:30: + 9d:e0:4f:9f:b0:86:a2:60:69:1c:ee:e1:b2:f9:1f: + 92:fb:c0:bc:56:8b:04:a4:e9:a3:33:04:68:7a:c5: + a9:92:45:82:52:4e:62:19:36:31:b1:e1:ec:83:34: + f8:2d:47:b7:b9:e7:71:4c:b6:2f:1c:f4:a2:87:a3: + 10:41:62:e0:1a:c6:36:ee:5d:23:fd:7d:ef:b0:28: + fe:66:c7:51:6d:16:34:28:67:6c:fc:48:fa:bc:01: + 0c:31:ef:08:0a:bf:38:9d:6b:1f:ab:5f:5c:8b:9a: + 81:82:14:07:2a:20:ea:96:d1:ff:b3:ad:cd:80:23: + dc:f5:e1:6f:8e:38:f9:0e:1d:69:ff:ef:d4:48:98: + 34:ab + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 70:73:AF:1D:81:B8:F5:5B:72:DE:04:03:65:C9:76:38:9D:0E:6C:BB + X509v3 Authority Key Identifier: + keyid:13:DA:D9:CD:E0:CC:51:84:5D:2C:7C:A5:46:33:4E:02:E4:F2:D9:6C + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication + X509v3 Subject Alternative Name: + DNS:ec2-3-82-109-30.compute-1.amazonaws.com + Netscape Comment: + OpenSSL Generated Client Certificate + Signature Algorithm: sha256WithRSAEncryption + 33:b1:fb:61:13:38:5e:9e:e5:c0:99:95:4f:7d:7a:5c:b7:2e: + ca:3e:3f:f3:03:7c:a6:34:fc:e9:87:22:4a:cd:00:9c:15:c2: + aa:31:6b:31:2f:e3:72:2b:6c:45:eb:c7:ce:39:5d:df:ab:3e: + eb:80:db:ce:2d:f3:2b:79:06:7d:af:06:66:d9:1b:8c:f7:ff: + 26:d4:7b:69:a2:d8:b9:09:e5:f9:00:59:9b:05:31:27:c5:74: + 6b:49:1d:4b:cf:f1:0d:b5:21:d0:3c:ca:92:91:1c:2c:97:77: + c8:4c:20:5a:3d:43:f4:7b:9a:0d:6d:3d:88:5a:4a:74:1b:52: + d1:32:72:1d:cb:78:a0:89:c4:98:4d:1e:5a:26:d8:a7:8a:aa: + 86:70:21:72:7e:20:a2:11:39:61:08:51:22:94:1c:2b:d6:8a: + 84:16:6f:6b:4c:7c:6f:9b:b2:ec:a8:4d:7a:5b:e3:cc:73:d0: + fe:7b:30:b7:6d:1d:7d:41:da:a5:a5:19:47:9b:ce:5d:0c:78: + 13:4c:bd:cd:71:72:da:de:b5:ca:f4:0e:7e:8c:a3:30:55:ff: + 94:0f:6d:2c:54:40:4f:6d:92:8c:0b:d7:33:3b:cd:9d:4a:b8: + 00:9e:c8:1b:d6:e7:7c:0c:cf:fe:a8:3f:97:0a:13:bd:ca:6a: + 60:9e:ea:ee:e7:9e:42:c0:4d:c1:ec:a7:93:68:bd:16:84:f3: + 11:3d:51:61:d8:a5:1a:63:c1:59:e3:d4:ef:26:17:47:87:99: + f8:0b:63:09:6b:84:2c:ab:e4:56:9c:15:af:df:a7:ac:9f:8f: + 73:e2:3b:57:bc:98:91:f7:e0:5e:ff:e2:65:40:a5:25:c9:aa: + f1:8f:1d:a5:17:26:ee:9d:4c:6b:28:57:50:5f:c1:48:d0:f2: + c9:ea:6b:28:c4:d2:c5:b8:e9:79:f2:86:92:34:20:3c:52:2f: + c2:00:9d:b6:51:f7:61:d5:69:14:d6:75:66:89:f8:77:98:06: + a8:c0:0a:0d:82:ac:3a:40:9b:02:7e:ba:5c:e7:58:37:e9:6a: + ad:c6:93:35:67:ca:6b:ae:eb:84:5c:bf:0a:e1:94:67:37:84: + 05:3e:4a:00:39:ff:dc:30:04:c2:4a:19:63:2f:0a:fe:76:5d: + 3a:8e:d7:dd:22:b8:84:1d:fa:5e:65:c5:7b:44:6d:bd:0f:15: + 15:bb:cf:cf:fb:29:4f:61:4c:81:51:9d:86:0a:be:e7:02:ec: + d6:7f:9a:36:b8:73:94:af:aa:a7:6b:c8:33:a2:fb:ab:53:30: + 4a:1f:ec:2f:9d:ec:df:26:18:16:5b:c0:bc:25:8e:1b:13:68: + b6:18:28:0d:bd:09:19:e7 +-----BEGIN CERTIFICATE----- +MIIFOjCCAyKgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBxMQswCQYDVQQGEwJVUzER +MA8GA1UECAwITmV3IFlvcmsxETAPBgNVBAcMCE5ldyBZb3JrMRAwDgYDVQQKDAdN +b25nb0RCMRAwDgYDVQQLDAdtb25nb2RiMRgwFgYDVQQDDA93d3cubW9uZ29kYi5j +b20wHhcNMjAwNzE2MDk1MTU2WhcNMjMwNDEyMDk1MTU2WjB0MQswCQYDVQQGEwJV +UzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNVBAcMCE5ldyBZb3JrMRAwDgYDVQQK +DAdNb25nb0RCMQ4wDAYDVQQLDAVjbG91ZDEdMBsGA1UEAwwUbW1zLWF1dG9tYXRp +b24tYWdlbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQConOR7Nspl +YYlBaYFAyLNd+gtPSE0psiA2HUL1+YDBuEJFRsmzOIwAyNPg8n1iyhU4UkMLVuwU +lIGHXVimI9KpKYGeW7TSnnmzWza/nx/lpAZzhQiYjHtmUIv9XUUUYZce95Px3n1+ +j220//bnIjjf5W57AFsfMJ3gT5+whqJgaRzu4bL5H5L7wLxWiwSk6aMzBGh6xamS +RYJSTmIZNjGx4eyDNPgtR7e553FMti8c9KKHoxBBYuAaxjbuXSP9fe+wKP5mx1Ft +FjQoZ2z8SPq8AQwx7wgKvzidax+rX1yLmoGCFAcqIOqW0f+zrc2AI9z14W+OOPkO +HWn/79RImDSrAgMBAAGjgdkwgdYwHQYDVR0OBBYEFHBzrx2BuPVbct4EA2XJdjid +Dmy7MB8GA1UdIwQYMBaAFBPa2c3gzFGEXSx8pUYzTgLk8tlsMAkGA1UdEwQCMAAw +CwYDVR0PBAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMDIGA1UdEQQrMCmCJ2Vj +Mi0zLTgyLTEwOS0zMC5jb21wdXRlLTEuYW1hem9uYXdzLmNvbTAzBglghkgBhvhC +AQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRpZmljYXRlMA0GCSqG +SIb3DQEBCwUAA4ICAQAzsfthEzhenuXAmZVPfXpcty7KPj/zA3ymNPzphyJKzQCc +FcKqMWsxL+NyK2xF68fOOV3fqz7rgNvOLfMreQZ9rwZm2RuM9/8m1Htpoti5CeX5 +AFmbBTEnxXRrSR1Lz/ENtSHQPMqSkRwsl3fITCBaPUP0e5oNbT2IWkp0G1LRMnId +y3igicSYTR5aJtiniqqGcCFyfiCiETlhCFEilBwr1oqEFm9rTHxvm7LsqE16W+PM +c9D+ezC3bR19QdqlpRlHm85dDHgTTL3NcXLa3rXK9A5+jKMwVf+UD20sVEBPbZKM +C9czO82dSrgAnsgb1ud8DM/+qD+XChO9ympgnuru555CwE3B7KeTaL0WhPMRPVFh +2KUaY8FZ49TvJhdHh5n4C2MJa4Qsq+RWnBWv36esn49z4jtXvJiR9+Be/+JlQKUl +yarxjx2lFybunUxrKFdQX8FI0PLJ6msoxNLFuOl58oaSNCA8Ui/CAJ22Ufdh1WkU +1nVmifh3mAaowAoNgqw6QJsCfrpc51g36WqtxpM1Z8prruuEXL8K4ZRnN4QFPkoA +Of/cMATCShljLwr+dl06jtfdIriEHfpeZcV7RG29DxUVu8/P+ylPYUyBUZ2GCr7n +AuzWf5o2uHOUr6qna8gzovurUzBKH+wvnezfJhgWW8C8JY4bE2i2GCgNvQkZ5w== +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQConOR7NsplYYlB +aYFAyLNd+gtPSE0psiA2HUL1+YDBuEJFRsmzOIwAyNPg8n1iyhU4UkMLVuwUlIGH +XVimI9KpKYGeW7TSnnmzWza/nx/lpAZzhQiYjHtmUIv9XUUUYZce95Px3n1+j220 +//bnIjjf5W57AFsfMJ3gT5+whqJgaRzu4bL5H5L7wLxWiwSk6aMzBGh6xamSRYJS +TmIZNjGx4eyDNPgtR7e553FMti8c9KKHoxBBYuAaxjbuXSP9fe+wKP5mx1FtFjQo +Z2z8SPq8AQwx7wgKvzidax+rX1yLmoGCFAcqIOqW0f+zrc2AI9z14W+OOPkOHWn/ +79RImDSrAgMBAAECggEAcpY9CCdSINfKKXQD7Pz4OLOHIBgoqF9vWJdGPFeVUxFf +qCjVRkD1lErnAwaIg6yGA0KUYY5u3gWWiWG8rxvFPEUC25XDKyeb2XHxoQQI700r +PTJ5hwJhkkTG/iZ2ncU8qETkfAkSDAJ5MfqJ1sYBFNec32Z8hpPJlvlFsvesPgvW +xGl13TinIl9TtLVu5OU5f3hFD+sODUsS1QQtwzXiOSF7JU/4gA4hJEPiSD4tbZwy +moVjglPVaasVqZDsLqz9RL8mL53QYmzGcSRr5egQsgps5x2IN8sEU7BiUej3qt/k +aUzUuPyQjkH/gdrX71eEY+NC+Zv8e/RSy0z3zdWmcQKBgQDQJ7j+7asO0E5Hibk/ +862dGLVIVxn17CWUPeubgYNLEMZAyqCr85mnnpU89tc8mUOyglZkfqvvdl2sOKK6 +zy4yaiu9Yq3sciXUDEKiSJsZf+GDmK9mQ5LLjogWdq7uB/2uR/DkEbry2gFskUdO +3GIT1QROt8T1XM4YzF5Hu49ddQKBgQDPXmtgmrOLtH9UwtWiGX08vQKv2Oa+jnLK +f+Z3mNDinIjSY2l3KXecpvF20J7TXK0zQWk84p6GQF55sT3PL3oxk/OOwfmwZC0m +FGEL/BGIkS+Hwxc9UfGZ7wck66YiRl2RXF1XF4RpRuBKdXGreORsjl0Aqt34fvzy +CpZe9DZlnwKBgGTn/LxIRrZFsMzpLM6duDoBsk/BOaqHsaftZHvcCuOm3BSopb71 +tjUVoU8OckTEH5c3q93Hsl3BSaOlSO26ZbC220FRxvJqW4Ax+VNmUxnHbnE24UB3 +3X+kNsB9BEwLv6Ru544IMlJr8GjK/IB0QW9PwmjOmUJAnQBUghfQCq3JAoGANy0q +aRQAviWS09zbtzwNBMJOGrgd/YotpRAPJLd2rTV1enWVNG3GM9p/2Vt9R0Qbmc3H +0LmD8Ljj6oFsrto1K0fwwIWAiJy/HqjBgczaZXosKXWRk3FgVdMyFXLWS7xpXSo0 +c94AD3saZvWE/1k1fmUK/gh484vmhginJjDY4IUCgYB2aZ/e8MoJFTu0ADzJBaKz +L+BYwimpdfj2e5UZ/mg95zDGiHAfxe1wZ99rLWZZq+jaRdoaL97K/kh12EDYltvr +EqLjvAISbuWUvSZmJajbpnb9knKZINhunSKcPT9roqD3iitaZ5QgYncAZgloDG0x ++oMJvhAlzNLO7gAbcMuEfw== +-----END PRIVATE KEY----- diff --git a/controllers/operator/testdata/certificates/cert_rfc2253 b/controllers/operator/testdata/certificates/cert_rfc2253 new file mode 100644 index 000000000..92715cffd --- /dev/null +++ b/controllers/operator/testdata/certificates/cert_rfc2253 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDZDCCAkwCCQD0zPhiubeUgjANBgkqhkiG9w0BAQsFADB0MQswCQYDVQQGEwJJ +TjESMBAGA1UECAwJTmV3IERlbGhpMRIwEAYDVQQHDAlOZXcgRGVsaGkxDDAKBgNV +BAsMA1RTRTEWMBQGA1UECgwNTW9uZ29EQi1hZ2VudDEXMBUGA1UEAwwObW1zLWFn +ZW50LWNlcnQwHhcNMjEwOTI4MTUzNjA5WhcNMjIwOTI4MTUzNjA5WjB0MQswCQYD +VQQGEwJJTjESMBAGA1UECAwJTmV3IERlbGhpMRIwEAYDVQQHDAlOZXcgRGVsaGkx +DDAKBgNVBAsMA1RTRTEWMBQGA1UECgwNTW9uZ29EQi1hZ2VudDEXMBUGA1UEAwwO +bW1zLWFnZW50LWNlcnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC6 +oh/i2rT90EOLlUHsqnkdaqyG4CiXe44lzI2zRHunO928lkyoBPLHLTYUriCdm2WO +ljaSZguSmGDfNCCPAufqVhbtusvS9dz5QBNwyRGEY8S246+EAtmqSBeb11OjQpbT +1mGqjDMEcFR/1NXdcPXmvKml3L60Clauuxeb4EsnmWfXh7QSGGnYK8ybQobF6ddE +qrygJHCGzh53hKCwAhq15h+fccaKgBixOPq9eqPx+kvDmsHMEc2ScF6S2KZ+Z/dV +cUnWgY9lWymm2MgfIorwXId4IiGfQT3fQXwUR38yP19+3lGUYDGcD7LW37LqZUI5 +IFVFkPuihO0YpeIH/ogxAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAD3zys7SWzO0 +eP3oG77798OhILyp9E6415bdrY7vrloxpFd6YB4JvEVxQbdIs+k4BgxbOtXt2DqX +K1xMF4MASZpmCL0QynqBfknZhzx9DTqJAli30Xbk7zSS5jaBWZ3v0QxMkV8x0mtS +E9owEOLO6ikTXYgjkllAmbQdYgMnOCos62RDcRbD5Ht5Su5ghokTx5a6eWDB3bPR +YQazxu4jHXZ8M0iDOpl3xGh0GxRaosb+f8U6prZoj+YGpa18H2YmwEygvhSzsUMq +PeHOzlBN/Pur/5N6sRHtP15rRTSdV91BE29zjgDEVKkZNGdb4AruFrJcj+zLS/vO +cNBafZbYWmc= +-----END CERTIFICATE----- diff --git a/controllers/operator/testdata/certificates/certificate_then_key b/controllers/operator/testdata/certificates/certificate_then_key new file mode 100644 index 000000000..28d364921 --- /dev/null +++ b/controllers/operator/testdata/certificates/certificate_then_key @@ -0,0 +1,79 @@ +-----BEGIN CERTIFICATE----- +MIIEgjCCA2qgAwIBAgIRAJm5VGPguqlg4byYvf0RVFEwDQYJKoZIhvcNAQELBQAw +FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0yMDA0MDMwODMyNDVaFw0yMTA0MDMw +ODMyNDVaMIGLMQswCQYDVQQGEwJVUzELMAkGA1UECBMCTlkxCzAJBgNVBAcTAk5Z +MR0wGwYDVQQKExRtbXMtYXV0b21hdGlvbi1hZ2VudDEkMCIGA1UECxMbTW9uZ29E +QiBLdWJlcm5ldGVzIE9wZXJhdG9yMR0wGwYDVQQDExRtbXMtYXV0b21hdGlvbi1h +Z2VudDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKqkwhcQNPvRLITz +IN1Fbod1lpIn2VthrGIYI6J02h7mAW+cULIF7qKcsc1FnqqMY4vL9X0XY2+lBtlI +3L79Gn4pRmQugjqvupJKFQiVbAyZOZgc9d3MefyrRTmzpf+QBVQ6pLg8jDqAhTub +jPMWJ8+OW5RRW+C3Vu8wjgQuQDrDp2xE4/j2FzW6G+4+gpXJ2JtrOOa1IIUGDFLq +3EUeU8BMXKUNZITKeraxbnXTiLEt//QqbFS6WooRxGRBWEcGM6Qqd5kv0bDJpJjl +HU2QoQVoZD2p/cV/nyVs4x4pXalZ68cfuMkD+UR8M9YUHy0QOvfJ3CmWeWsZxyEO +nmGGc/4u15fVkltlCcdMfDMdDGBPo5LjZbiyCItQjKVU9xvHerzHvD7ZPM4P8W95 +c94xVmCmO4JIhCLV3cnTTiY6M4CHSZglldgZdSbMjYUc/fmG1LxHlR6mWurzo0Pa +ZJrix9foBK11W3zi2X799ZNro/4y6u7r4equAqI9zdlcoA+Pstn0J6dgVwKE6C9W +EZOhAZNoiWJtkAWzA6/8vncuDeTEnRh41gNWc0l1Iem2vrJ8zLIvCK+NzMUBYCLn +M+qKagp0Y/PHMvdudgMbFtLNiZvqVJbtAkdW/uDgr4SJ+dxNTZDc++clLodycs2m +8UNq5j1kSf87OrkW6+LWKjnhlz0TAgMBAAGjVjBUMA4GA1UdDwEB/wQEAwIFoDAT +BgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdEQQYMBaCFG1t +cy1hdXRvbWF0aW9uLWFnZW50MA0GCSqGSIb3DQEBCwUAA4IBAQBSIgTFY0F3dc0X +4CZbseynPDd/BRwgQlC5wXsbD6kYmnDhdpPBstlWjj4l6HtKvVsVsZxcghT9SI0l +H1m4imhh/scfVqjkBTTPikyJTfgUnN1RIJlDN+XzRz4HxvP1qVIdTlNfDW3avAvG +Cb2iHwEwH3Xv+s+3m4/FsGm/rtRTDerafkYrtCdCyY77k7/tquFRn68mvZnthiwn +zv3iFWHtdg7idbFYxyCmLXiBNzw54cEqdvehPV9UwbnupM/KEDZTL2Edcxl5gxWV +JAG35yxcvdf2zYbGwf/BWpOwMZmzDP06IxgpAEfPBj7g83Fjj6pn2MM+dQIomFB+ +/m6HRiJE +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCqpMIXEDT70SyE +8yDdRW6HdZaSJ9lbYaxiGCOidNoe5gFvnFCyBe6inLHNRZ6qjGOLy/V9F2NvpQbZ +SNy+/Rp+KUZkLoI6r7qSShUIlWwMmTmYHPXdzHn8q0U5s6X/kAVUOqS4PIw6gIU7 +m4zzFifPjluUUVvgt1bvMI4ELkA6w6dsROP49hc1uhvuPoKVydibazjmtSCFBgxS +6txFHlPATFylDWSEynq2sW5104ixLf/0KmxUulqKEcRkQVhHBjOkKneZL9GwyaSY +5R1NkKEFaGQ9qf3Ff58lbOMeKV2pWevHH7jJA/lEfDPWFB8tEDr3ydwplnlrGcch +Dp5hhnP+LteX1ZJbZQnHTHwzHQxgT6OS42W4sgiLUIylVPcbx3q8x7w+2TzOD/Fv +eXPeMVZgpjuCSIQi1d3J004mOjOAh0mYJZXYGXUmzI2FHP35htS8R5Ueplrq86ND +2mSa4sfX6AStdVt84tl+/fWTa6P+Muru6+HqrgKiPc3ZXKAPj7LZ9CenYFcChOgv +VhGToQGTaIlibZAFswOv/L53Lg3kxJ0YeNYDVnNJdSHptr6yfMyyLwivjczFAWAi +5zPqimoKdGPzxzL3bnYDGxbSzYmb6lSW7QJHVv7g4K+EifncTU2Q3PvnJS6HcnLN +pvFDauY9ZEn/Ozq5Fuvi1io54Zc9EwIDAQABAoICAEMUFP+/7TP328o/UHHqszIo +dRHq/DRBxuOgnZFk4cE3pOTcy5PPZSki83m/nkloelEf0dZkdUAT3QdY7v1cvSdO +zk7fQW4UWgDbgj0nj5u8N7ml2LhhgqpiIQo3pk85q/6aNtn9Yxo0Hyt5UATWdrvO +OA2rlbRWHaRUr97Q14rCEnQq+HqLMkB6cjRK+kYrXCxsD6gRF0FzSTDnBcNd0opK ++jgfdZ4FgguC3+sNRjRv4qd2bbM4thKEPXEzhqIUvAQSdYUQGRuniD5aAhTVf5aC +nLTot8sFCehKT1Ux6ZGCuX5C5/6Mw1W6hR3oNwEd2jBBd3wZnI0PSwmhl3y6v6lM +pmMmO9Tl4fG7bVlRJsktxLsCDanBSRaqn1lsbjmyic3Q82H43VebZV37e/JOYUkH +iXd1gX6a3O/x6om2IJmt48Hm71TJGxLh+NeyEtLa6eOupOK+56qU1ByJB1S/sBz5 +IkbWMaKRXu86XOK+mdnI2jURJ7W14N2zCslE6IkU7Ehx9Bg4qxvyR6/z8B7IzYzh +bWj+ilKjQjoSphGh9sDE3ZXuqIa1HP9U/n5r+03ulCPwuoGwZtwL9QCXk05Iedie +1a2rRSy8QKGVk4mHDRtgFP61jtj00u5XMeGvVMVtEXcs6UlRfK6q9K994aeoPTL9 +IyhygaqFcmrpff2QmVLBAoIBAQDBGmIa4iuMwKB3KLVSiddD9Un+5eOW3gvpmDS+ +v+Y0zyFe5QebV6UVXTXIIsVKPtXdKW6yaCGHAWwBQRemMHX+bNJCigyRKz9HrENu +fZxP80mn5Ev+LaQOjZUMCiu0hNF/dSBoxpk1NV3aKEs3re0Xv2PKvnHMtws7DgeI +ePpJ3HRnEwsHY5kPQnL9DFcwX3Nq9wcgKmM0VCItjhJZiJlHqJiM9f15RgUlThXC +tCON1EPDPvyRfu27RKIggcEQ7SEwLEYlB05EUPp5rEUaM6UeNsZvm/DzfpH6NkjZ +SSDU/mx+v36QVhareYwRJpE1tEIezLluGdSld4LQHyDoFlQDAoIBAQDiOZ/kJ/AS +QCHzU1MyEtGpGwJKoFk9j1cKXFJrfVME2k7Ys2nCdo4mtcJmJzRYhK1rBVPFPdmO +V/nnfZGVHypE1qdfgZKRiBdJTLhILwePHWcDDmWW16nxcIjGfAEMJik8TuuWER2a +B7uwWfDcXpn5FfsRRW28FcE5emVWm+x/CrZCDMdO12xElOFMM2qohC15tAoOlNMI +KsfknK+OlFUXVagqDpgzl8nOoHV3aaM00+ChL32nam97hNnTrMHzrGZUCM9PlbCV +WkIvKwEvEQQfAhoUIV5muNGm0G68LFKYYJbXBuehNQ2MfX6xWJAf4a1rtL7rDtYa +UQ4VQg8moQ2xAoIBACT4Kx4gfNv2qQIHLie+MhNVq7P8SUVB/5/aPwbh8G3d1fK4 +AGvSLM3ZSYmmdoUPYJx16TaIzxpswEPBNYjgsEZkiSCqE1vbnsLXDRXjQIDiABD/ +mTjxff43RvjGHbXy07UGNI06sGxKakxw+G2Rg9nPD4jqSxk5VhIZToHnP2vSpApz +z+G7RLtyKled/DdLnuo0nw2eb9292clE8OhpSYc5lPMvyTZlnGiW+X2MRV5K7Co+ +LdahKVx3+F4m2VKnQ3pYj5lZO7fClSGkRJqOlqchL36AqXHEoqf3qpzG7l041Iaz +nMR/ZtmvbIyACL7yYtJIuZuFoHuJVOcJfqBQXgcCggEBANR/PGmb+i2qgDmH84X9 +l2M5M5XVuP3SPvhEcEb3mZvdVGLJZHZ91lkWMlyyRsE/H4Z/ooiL6GeEzAFeOfnR +JGs1FlLn6z04kGcR4agsRPVxsOl2BIcEXWWlR1Tp9jHrRqCXoUN9IEknKm4kjdLy +Kb+HniZDCSi7Zp0PE1GfdS6AaWLxjeXJBLIHBvoE8hMI1Y6URz4bHX92b/2WEHHl +c2hP1X5r5xvPYIjuwGhCmkNtIntFmMpBeCaWS+ZBSI4TSqt0+wbOnOgtuC2GP75u +RWi7GLQABCSJRqVi9CFdoNfxIr8ohTswEmH9H5yGjBrmaXfad9tkPEjMCmZ9fq3S +aoECggEBAIzguZBBHohHaXPTMYC06M80YmMsV6kuDzA6dwCceMKDcQf+CNiwTOI9 +iv5zVWCkSpRlsYgbYgbsnZ7OV59mr83w9V7vsJB+fFWAFCDj9DcUz7xUYqFhxIk+ +qXUbJKOwW5icau5C5HQhlJAG9PmWrz0Pi+U4fofjWVHZwB3nUuPIiAigRLNEDXia +lZeZ2ZwDE7tfbyVhZageUMhFNsGJEV9S0aECq8mgHjHZUy7s3dwdCMeIsfqqJJk3 +FsaDbij2x09H2GqqtGbko3M/SVN5oCpAnkQpUfq767e27t4R3XuYtmPSEv73pCDh +8Ah6kGlS8cImcSWBf7/tAei7L5xLEqo= +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/controllers/operator/testdata/certificates/just_certificate b/controllers/operator/testdata/certificates/just_certificate new file mode 100644 index 000000000..36ac36c57 --- /dev/null +++ b/controllers/operator/testdata/certificates/just_certificate @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEgjCCA2qgAwIBAgIRAJm5VGPguqlg4byYvf0RVFEwDQYJKoZIhvcNAQELBQAw +FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0yMDA0MDMwODMyNDVaFw0yMTA0MDMw +ODMyNDVaMIGLMQswCQYDVQQGEwJVUzELMAkGA1UECBMCTlkxCzAJBgNVBAcTAk5Z +MR0wGwYDVQQKExRtbXMtYXV0b21hdGlvbi1hZ2VudDEkMCIGA1UECxMbTW9uZ29E +QiBLdWJlcm5ldGVzIE9wZXJhdG9yMR0wGwYDVQQDExRtbXMtYXV0b21hdGlvbi1h +Z2VudDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKqkwhcQNPvRLITz +IN1Fbod1lpIn2VthrGIYI6J02h7mAW+cULIF7qKcsc1FnqqMY4vL9X0XY2+lBtlI +3L79Gn4pRmQugjqvupJKFQiVbAyZOZgc9d3MefyrRTmzpf+QBVQ6pLg8jDqAhTub +jPMWJ8+OW5RRW+C3Vu8wjgQuQDrDp2xE4/j2FzW6G+4+gpXJ2JtrOOa1IIUGDFLq +3EUeU8BMXKUNZITKeraxbnXTiLEt//QqbFS6WooRxGRBWEcGM6Qqd5kv0bDJpJjl +HU2QoQVoZD2p/cV/nyVs4x4pXalZ68cfuMkD+UR8M9YUHy0QOvfJ3CmWeWsZxyEO +nmGGc/4u15fVkltlCcdMfDMdDGBPo5LjZbiyCItQjKVU9xvHerzHvD7ZPM4P8W95 +c94xVmCmO4JIhCLV3cnTTiY6M4CHSZglldgZdSbMjYUc/fmG1LxHlR6mWurzo0Pa +ZJrix9foBK11W3zi2X799ZNro/4y6u7r4equAqI9zdlcoA+Pstn0J6dgVwKE6C9W +EZOhAZNoiWJtkAWzA6/8vncuDeTEnRh41gNWc0l1Iem2vrJ8zLIvCK+NzMUBYCLn +M+qKagp0Y/PHMvdudgMbFtLNiZvqVJbtAkdW/uDgr4SJ+dxNTZDc++clLodycs2m +8UNq5j1kSf87OrkW6+LWKjnhlz0TAgMBAAGjVjBUMA4GA1UdDwEB/wQEAwIFoDAT +BgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdEQQYMBaCFG1t +cy1hdXRvbWF0aW9uLWFnZW50MA0GCSqGSIb3DQEBCwUAA4IBAQBSIgTFY0F3dc0X +4CZbseynPDd/BRwgQlC5wXsbD6kYmnDhdpPBstlWjj4l6HtKvVsVsZxcghT9SI0l +H1m4imhh/scfVqjkBTTPikyJTfgUnN1RIJlDN+XzRz4HxvP1qVIdTlNfDW3avAvG +Cb2iHwEwH3Xv+s+3m4/FsGm/rtRTDerafkYrtCdCyY77k7/tquFRn68mvZnthiwn +zv3iFWHtdg7idbFYxyCmLXiBNzw54cEqdvehPV9UwbnupM/KEDZTL2Edcxl5gxWV +JAG35yxcvdf2zYbGwf/BWpOwMZmzDP06IxgpAEfPBj7g83Fjj6pn2MM+dQIomFB+ +/m6HRiJE +-----END CERTIFICATE----- \ No newline at end of file diff --git a/controllers/operator/testdata/certificates/just_key b/controllers/operator/testdata/certificates/just_key new file mode 100644 index 000000000..82d723a15 --- /dev/null +++ b/controllers/operator/testdata/certificates/just_key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCqpMIXEDT70SyE +8yDdRW6HdZaSJ9lbYaxiGCOidNoe5gFvnFCyBe6inLHNRZ6qjGOLy/V9F2NvpQbZ +SNy+/Rp+KUZkLoI6r7qSShUIlWwMmTmYHPXdzHn8q0U5s6X/kAVUOqS4PIw6gIU7 +m4zzFifPjluUUVvgt1bvMI4ELkA6w6dsROP49hc1uhvuPoKVydibazjmtSCFBgxS +6txFHlPATFylDWSEynq2sW5104ixLf/0KmxUulqKEcRkQVhHBjOkKneZL9GwyaSY +5R1NkKEFaGQ9qf3Ff58lbOMeKV2pWevHH7jJA/lEfDPWFB8tEDr3ydwplnlrGcch +Dp5hhnP+LteX1ZJbZQnHTHwzHQxgT6OS42W4sgiLUIylVPcbx3q8x7w+2TzOD/Fv +eXPeMVZgpjuCSIQi1d3J004mOjOAh0mYJZXYGXUmzI2FHP35htS8R5Ueplrq86ND +2mSa4sfX6AStdVt84tl+/fWTa6P+Muru6+HqrgKiPc3ZXKAPj7LZ9CenYFcChOgv +VhGToQGTaIlibZAFswOv/L53Lg3kxJ0YeNYDVnNJdSHptr6yfMyyLwivjczFAWAi +5zPqimoKdGPzxzL3bnYDGxbSzYmb6lSW7QJHVv7g4K+EifncTU2Q3PvnJS6HcnLN +pvFDauY9ZEn/Ozq5Fuvi1io54Zc9EwIDAQABAoICAEMUFP+/7TP328o/UHHqszIo +dRHq/DRBxuOgnZFk4cE3pOTcy5PPZSki83m/nkloelEf0dZkdUAT3QdY7v1cvSdO +zk7fQW4UWgDbgj0nj5u8N7ml2LhhgqpiIQo3pk85q/6aNtn9Yxo0Hyt5UATWdrvO +OA2rlbRWHaRUr97Q14rCEnQq+HqLMkB6cjRK+kYrXCxsD6gRF0FzSTDnBcNd0opK ++jgfdZ4FgguC3+sNRjRv4qd2bbM4thKEPXEzhqIUvAQSdYUQGRuniD5aAhTVf5aC +nLTot8sFCehKT1Ux6ZGCuX5C5/6Mw1W6hR3oNwEd2jBBd3wZnI0PSwmhl3y6v6lM +pmMmO9Tl4fG7bVlRJsktxLsCDanBSRaqn1lsbjmyic3Q82H43VebZV37e/JOYUkH +iXd1gX6a3O/x6om2IJmt48Hm71TJGxLh+NeyEtLa6eOupOK+56qU1ByJB1S/sBz5 +IkbWMaKRXu86XOK+mdnI2jURJ7W14N2zCslE6IkU7Ehx9Bg4qxvyR6/z8B7IzYzh +bWj+ilKjQjoSphGh9sDE3ZXuqIa1HP9U/n5r+03ulCPwuoGwZtwL9QCXk05Iedie +1a2rRSy8QKGVk4mHDRtgFP61jtj00u5XMeGvVMVtEXcs6UlRfK6q9K994aeoPTL9 +IyhygaqFcmrpff2QmVLBAoIBAQDBGmIa4iuMwKB3KLVSiddD9Un+5eOW3gvpmDS+ +v+Y0zyFe5QebV6UVXTXIIsVKPtXdKW6yaCGHAWwBQRemMHX+bNJCigyRKz9HrENu +fZxP80mn5Ev+LaQOjZUMCiu0hNF/dSBoxpk1NV3aKEs3re0Xv2PKvnHMtws7DgeI +ePpJ3HRnEwsHY5kPQnL9DFcwX3Nq9wcgKmM0VCItjhJZiJlHqJiM9f15RgUlThXC +tCON1EPDPvyRfu27RKIggcEQ7SEwLEYlB05EUPp5rEUaM6UeNsZvm/DzfpH6NkjZ +SSDU/mx+v36QVhareYwRJpE1tEIezLluGdSld4LQHyDoFlQDAoIBAQDiOZ/kJ/AS +QCHzU1MyEtGpGwJKoFk9j1cKXFJrfVME2k7Ys2nCdo4mtcJmJzRYhK1rBVPFPdmO +V/nnfZGVHypE1qdfgZKRiBdJTLhILwePHWcDDmWW16nxcIjGfAEMJik8TuuWER2a +B7uwWfDcXpn5FfsRRW28FcE5emVWm+x/CrZCDMdO12xElOFMM2qohC15tAoOlNMI +KsfknK+OlFUXVagqDpgzl8nOoHV3aaM00+ChL32nam97hNnTrMHzrGZUCM9PlbCV +WkIvKwEvEQQfAhoUIV5muNGm0G68LFKYYJbXBuehNQ2MfX6xWJAf4a1rtL7rDtYa +UQ4VQg8moQ2xAoIBACT4Kx4gfNv2qQIHLie+MhNVq7P8SUVB/5/aPwbh8G3d1fK4 +AGvSLM3ZSYmmdoUPYJx16TaIzxpswEPBNYjgsEZkiSCqE1vbnsLXDRXjQIDiABD/ +mTjxff43RvjGHbXy07UGNI06sGxKakxw+G2Rg9nPD4jqSxk5VhIZToHnP2vSpApz +z+G7RLtyKled/DdLnuo0nw2eb9292clE8OhpSYc5lPMvyTZlnGiW+X2MRV5K7Co+ +LdahKVx3+F4m2VKnQ3pYj5lZO7fClSGkRJqOlqchL36AqXHEoqf3qpzG7l041Iaz +nMR/ZtmvbIyACL7yYtJIuZuFoHuJVOcJfqBQXgcCggEBANR/PGmb+i2qgDmH84X9 +l2M5M5XVuP3SPvhEcEb3mZvdVGLJZHZ91lkWMlyyRsE/H4Z/ooiL6GeEzAFeOfnR +JGs1FlLn6z04kGcR4agsRPVxsOl2BIcEXWWlR1Tp9jHrRqCXoUN9IEknKm4kjdLy +Kb+HniZDCSi7Zp0PE1GfdS6AaWLxjeXJBLIHBvoE8hMI1Y6URz4bHX92b/2WEHHl +c2hP1X5r5xvPYIjuwGhCmkNtIntFmMpBeCaWS+ZBSI4TSqt0+wbOnOgtuC2GP75u +RWi7GLQABCSJRqVi9CFdoNfxIr8ohTswEmH9H5yGjBrmaXfad9tkPEjMCmZ9fq3S +aoECggEBAIzguZBBHohHaXPTMYC06M80YmMsV6kuDzA6dwCceMKDcQf+CNiwTOI9 +iv5zVWCkSpRlsYgbYgbsnZ7OV59mr83w9V7vsJB+fFWAFCDj9DcUz7xUYqFhxIk+ +qXUbJKOwW5icau5C5HQhlJAG9PmWrz0Pi+U4fofjWVHZwB3nUuPIiAigRLNEDXia +lZeZ2ZwDE7tfbyVhZageUMhFNsGJEV9S0aECq8mgHjHZUy7s3dwdCMeIsfqqJJk3 +FsaDbij2x09H2GqqtGbko3M/SVN5oCpAnkQpUfq767e27t4R3XuYtmPSEv73pCDh +8Ah6kGlS8cImcSWBf7/tAei7L5xLEqo= +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/controllers/operator/testdata/certificates/key_then_certificate b/controllers/operator/testdata/certificates/key_then_certificate new file mode 100644 index 000000000..f44aa0696 --- /dev/null +++ b/controllers/operator/testdata/certificates/key_then_certificate @@ -0,0 +1,79 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCqpMIXEDT70SyE +8yDdRW6HdZaSJ9lbYaxiGCOidNoe5gFvnFCyBe6inLHNRZ6qjGOLy/V9F2NvpQbZ +SNy+/Rp+KUZkLoI6r7qSShUIlWwMmTmYHPXdzHn8q0U5s6X/kAVUOqS4PIw6gIU7 +m4zzFifPjluUUVvgt1bvMI4ELkA6w6dsROP49hc1uhvuPoKVydibazjmtSCFBgxS +6txFHlPATFylDWSEynq2sW5104ixLf/0KmxUulqKEcRkQVhHBjOkKneZL9GwyaSY +5R1NkKEFaGQ9qf3Ff58lbOMeKV2pWevHH7jJA/lEfDPWFB8tEDr3ydwplnlrGcch +Dp5hhnP+LteX1ZJbZQnHTHwzHQxgT6OS42W4sgiLUIylVPcbx3q8x7w+2TzOD/Fv +eXPeMVZgpjuCSIQi1d3J004mOjOAh0mYJZXYGXUmzI2FHP35htS8R5Ueplrq86ND +2mSa4sfX6AStdVt84tl+/fWTa6P+Muru6+HqrgKiPc3ZXKAPj7LZ9CenYFcChOgv +VhGToQGTaIlibZAFswOv/L53Lg3kxJ0YeNYDVnNJdSHptr6yfMyyLwivjczFAWAi +5zPqimoKdGPzxzL3bnYDGxbSzYmb6lSW7QJHVv7g4K+EifncTU2Q3PvnJS6HcnLN +pvFDauY9ZEn/Ozq5Fuvi1io54Zc9EwIDAQABAoICAEMUFP+/7TP328o/UHHqszIo +dRHq/DRBxuOgnZFk4cE3pOTcy5PPZSki83m/nkloelEf0dZkdUAT3QdY7v1cvSdO +zk7fQW4UWgDbgj0nj5u8N7ml2LhhgqpiIQo3pk85q/6aNtn9Yxo0Hyt5UATWdrvO +OA2rlbRWHaRUr97Q14rCEnQq+HqLMkB6cjRK+kYrXCxsD6gRF0FzSTDnBcNd0opK ++jgfdZ4FgguC3+sNRjRv4qd2bbM4thKEPXEzhqIUvAQSdYUQGRuniD5aAhTVf5aC +nLTot8sFCehKT1Ux6ZGCuX5C5/6Mw1W6hR3oNwEd2jBBd3wZnI0PSwmhl3y6v6lM +pmMmO9Tl4fG7bVlRJsktxLsCDanBSRaqn1lsbjmyic3Q82H43VebZV37e/JOYUkH +iXd1gX6a3O/x6om2IJmt48Hm71TJGxLh+NeyEtLa6eOupOK+56qU1ByJB1S/sBz5 +IkbWMaKRXu86XOK+mdnI2jURJ7W14N2zCslE6IkU7Ehx9Bg4qxvyR6/z8B7IzYzh +bWj+ilKjQjoSphGh9sDE3ZXuqIa1HP9U/n5r+03ulCPwuoGwZtwL9QCXk05Iedie +1a2rRSy8QKGVk4mHDRtgFP61jtj00u5XMeGvVMVtEXcs6UlRfK6q9K994aeoPTL9 +IyhygaqFcmrpff2QmVLBAoIBAQDBGmIa4iuMwKB3KLVSiddD9Un+5eOW3gvpmDS+ +v+Y0zyFe5QebV6UVXTXIIsVKPtXdKW6yaCGHAWwBQRemMHX+bNJCigyRKz9HrENu +fZxP80mn5Ev+LaQOjZUMCiu0hNF/dSBoxpk1NV3aKEs3re0Xv2PKvnHMtws7DgeI +ePpJ3HRnEwsHY5kPQnL9DFcwX3Nq9wcgKmM0VCItjhJZiJlHqJiM9f15RgUlThXC +tCON1EPDPvyRfu27RKIggcEQ7SEwLEYlB05EUPp5rEUaM6UeNsZvm/DzfpH6NkjZ +SSDU/mx+v36QVhareYwRJpE1tEIezLluGdSld4LQHyDoFlQDAoIBAQDiOZ/kJ/AS +QCHzU1MyEtGpGwJKoFk9j1cKXFJrfVME2k7Ys2nCdo4mtcJmJzRYhK1rBVPFPdmO +V/nnfZGVHypE1qdfgZKRiBdJTLhILwePHWcDDmWW16nxcIjGfAEMJik8TuuWER2a +B7uwWfDcXpn5FfsRRW28FcE5emVWm+x/CrZCDMdO12xElOFMM2qohC15tAoOlNMI +KsfknK+OlFUXVagqDpgzl8nOoHV3aaM00+ChL32nam97hNnTrMHzrGZUCM9PlbCV +WkIvKwEvEQQfAhoUIV5muNGm0G68LFKYYJbXBuehNQ2MfX6xWJAf4a1rtL7rDtYa +UQ4VQg8moQ2xAoIBACT4Kx4gfNv2qQIHLie+MhNVq7P8SUVB/5/aPwbh8G3d1fK4 +AGvSLM3ZSYmmdoUPYJx16TaIzxpswEPBNYjgsEZkiSCqE1vbnsLXDRXjQIDiABD/ +mTjxff43RvjGHbXy07UGNI06sGxKakxw+G2Rg9nPD4jqSxk5VhIZToHnP2vSpApz +z+G7RLtyKled/DdLnuo0nw2eb9292clE8OhpSYc5lPMvyTZlnGiW+X2MRV5K7Co+ +LdahKVx3+F4m2VKnQ3pYj5lZO7fClSGkRJqOlqchL36AqXHEoqf3qpzG7l041Iaz +nMR/ZtmvbIyACL7yYtJIuZuFoHuJVOcJfqBQXgcCggEBANR/PGmb+i2qgDmH84X9 +l2M5M5XVuP3SPvhEcEb3mZvdVGLJZHZ91lkWMlyyRsE/H4Z/ooiL6GeEzAFeOfnR +JGs1FlLn6z04kGcR4agsRPVxsOl2BIcEXWWlR1Tp9jHrRqCXoUN9IEknKm4kjdLy +Kb+HniZDCSi7Zp0PE1GfdS6AaWLxjeXJBLIHBvoE8hMI1Y6URz4bHX92b/2WEHHl +c2hP1X5r5xvPYIjuwGhCmkNtIntFmMpBeCaWS+ZBSI4TSqt0+wbOnOgtuC2GP75u +RWi7GLQABCSJRqVi9CFdoNfxIr8ohTswEmH9H5yGjBrmaXfad9tkPEjMCmZ9fq3S +aoECggEBAIzguZBBHohHaXPTMYC06M80YmMsV6kuDzA6dwCceMKDcQf+CNiwTOI9 +iv5zVWCkSpRlsYgbYgbsnZ7OV59mr83w9V7vsJB+fFWAFCDj9DcUz7xUYqFhxIk+ +qXUbJKOwW5icau5C5HQhlJAG9PmWrz0Pi+U4fofjWVHZwB3nUuPIiAigRLNEDXia +lZeZ2ZwDE7tfbyVhZageUMhFNsGJEV9S0aECq8mgHjHZUy7s3dwdCMeIsfqqJJk3 +FsaDbij2x09H2GqqtGbko3M/SVN5oCpAnkQpUfq767e27t4R3XuYtmPSEv73pCDh +8Ah6kGlS8cImcSWBf7/tAei7L5xLEqo= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEgjCCA2qgAwIBAgIRAJm5VGPguqlg4byYvf0RVFEwDQYJKoZIhvcNAQELBQAw +FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0yMDA0MDMwODMyNDVaFw0yMTA0MDMw +ODMyNDVaMIGLMQswCQYDVQQGEwJVUzELMAkGA1UECBMCTlkxCzAJBgNVBAcTAk5Z +MR0wGwYDVQQKExRtbXMtYXV0b21hdGlvbi1hZ2VudDEkMCIGA1UECxMbTW9uZ29E +QiBLdWJlcm5ldGVzIE9wZXJhdG9yMR0wGwYDVQQDExRtbXMtYXV0b21hdGlvbi1h +Z2VudDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKqkwhcQNPvRLITz +IN1Fbod1lpIn2VthrGIYI6J02h7mAW+cULIF7qKcsc1FnqqMY4vL9X0XY2+lBtlI +3L79Gn4pRmQugjqvupJKFQiVbAyZOZgc9d3MefyrRTmzpf+QBVQ6pLg8jDqAhTub +jPMWJ8+OW5RRW+C3Vu8wjgQuQDrDp2xE4/j2FzW6G+4+gpXJ2JtrOOa1IIUGDFLq +3EUeU8BMXKUNZITKeraxbnXTiLEt//QqbFS6WooRxGRBWEcGM6Qqd5kv0bDJpJjl +HU2QoQVoZD2p/cV/nyVs4x4pXalZ68cfuMkD+UR8M9YUHy0QOvfJ3CmWeWsZxyEO +nmGGc/4u15fVkltlCcdMfDMdDGBPo5LjZbiyCItQjKVU9xvHerzHvD7ZPM4P8W95 +c94xVmCmO4JIhCLV3cnTTiY6M4CHSZglldgZdSbMjYUc/fmG1LxHlR6mWurzo0Pa +ZJrix9foBK11W3zi2X799ZNro/4y6u7r4equAqI9zdlcoA+Pstn0J6dgVwKE6C9W +EZOhAZNoiWJtkAWzA6/8vncuDeTEnRh41gNWc0l1Iem2vrJ8zLIvCK+NzMUBYCLn +M+qKagp0Y/PHMvdudgMbFtLNiZvqVJbtAkdW/uDgr4SJ+dxNTZDc++clLodycs2m +8UNq5j1kSf87OrkW6+LWKjnhlz0TAgMBAAGjVjBUMA4GA1UdDwEB/wQEAwIFoDAT +BgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdEQQYMBaCFG1t +cy1hdXRvbWF0aW9uLWFnZW50MA0GCSqGSIb3DQEBCwUAA4IBAQBSIgTFY0F3dc0X +4CZbseynPDd/BRwgQlC5wXsbD6kYmnDhdpPBstlWjj4l6HtKvVsVsZxcghT9SI0l +H1m4imhh/scfVqjkBTTPikyJTfgUnN1RIJlDN+XzRz4HxvP1qVIdTlNfDW3avAvG +Cb2iHwEwH3Xv+s+3m4/FsGm/rtRTDerafkYrtCdCyY77k7/tquFRn68mvZnthiwn +zv3iFWHtdg7idbFYxyCmLXiBNzw54cEqdvehPV9UwbnupM/KEDZTL2Edcxl5gxWV +JAG35yxcvdf2zYbGwf/BWpOwMZmzDP06IxgpAEfPBj7g83Fjj6pn2MM+dQIomFB+ +/m6HRiJE +-----END CERTIFICATE----- \ No newline at end of file diff --git a/controllers/operator/testdata/mdb-sharded-multi-cluster-complex-expected-replicasets.yaml b/controllers/operator/testdata/mdb-sharded-multi-cluster-complex-expected-replicasets.yaml new file mode 100644 index 000000000..267a5b0ab --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-multi-cluster-complex-expected-replicasets.yaml @@ -0,0 +1,237 @@ +{ + "replicaSets": [ + { + "_id": "mdb-sh-complex-config", + "members": [ + { + "_id": "0", + "host": "mdb-sh-complex-config-0-0", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "1", + "host": "mdb-sh-complex-config-0-1", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "2", + "host": "mdb-sh-complex-config-1-0", + "priority": 10, + "tags": { }, + "votes": 2 + }, + { + "_id": "3", + "host": "mdb-sh-complex-config-1-1", + "priority": 10, + "tags": { }, + "votes": 1 + }, + { + "_id": "4", + "host": "mdb-sh-complex-config-2-0", + "priority": 5, + "tags": { }, + "votes": 1 + } + ], + "protocolVersion": "1" + }, + { + "_id": "mdb-sh-complex-0", + "members": [ + { + "_id": "0", + "host": "mdb-sh-complex-0-0-0", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "1", + "host": "mdb-sh-complex-0-0-1", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "2", + "host": "mdb-sh-complex-0-1-0", + "priority": 10, + "tags": { }, + "votes": 1 + }, + { + "_id": "3", + "host": "mdb-sh-complex-0-1-1", + "priority": 10, + "tags": { }, + "votes": 1 + }, + { + "_id": "4", + "host": "mdb-sh-complex-0-2-0", + "priority": 5, + "tags": { }, + "votes": 1 + } + ], + "protocolVersion": "1" + }, + { + "_id": "mdb-sh-complex-1", + "members": [ + { + "_id": "0", + "host": "mdb-sh-complex-1-0-0", + "priority": 100, + "tags": { }, + "votes": 1 + }, + { + "_id": "1", + "host": "mdb-sh-complex-1-1-0", + "priority": 110, + "tags": { }, + "votes": 1 + }, + { + "_id": "2", + "host": "mdb-sh-complex-1-1-1", + "priority": 0, + "tags": { }, + "votes": 0 + }, + { + "_id": "3", + "host": "mdb-sh-complex-1-2-0", + "priority": 120, + "tags": { }, + "votes": 1 + }, + { + "_id": "4", + "host": "mdb-sh-complex-1-2-1", + "priority": 121, + "tags": { }, + "votes": 2 + }, + { + "_id": "5", + "host": "mdb-sh-complex-1-2-2", + "priority": 122, + "tags": { }, + "votes": 1 + } + ], + "protocolVersion": "1" + }, + { + "_id": "mdb-sh-complex-2", + "members": [ + { + "_id": "0", + "host": "mdb-sh-complex-2-0-0", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "1", + "host": "mdb-sh-complex-2-0-1", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "2", + "host": "mdb-sh-complex-2-1-0", + "priority": 210, + "tags": { }, + "votes": 1 + }, + { + "_id": "3", + "host": "mdb-sh-complex-2-1-1", + "priority": 211, + "tags": { }, + "votes": 2 + }, + { + "_id": "4", + "host": "mdb-sh-complex-2-1-2", + "priority": 212, + "tags": { }, + "votes": 3 + }, + { + "_id": "5", + "host": "mdb-sh-complex-2-3-0", + "priority": 0, + "tags": { }, + "votes": 1 + }, + { + "_id": "6", + "host": "mdb-sh-complex-2-4-0", + "priority": 12, + "tags": { }, + "votes": 3 + }, + { + "_id": "7", + "host": "mdb-sh-complex-2-4-1", + "priority": 0, + "tags": { }, + "votes": 0 + } + ], + "protocolVersion": "1" + }, + { + "_id": "mdb-sh-complex-3", + "members": [ + { + "_id": "0", + "host": "mdb-sh-complex-3-0-0", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "1", + "host": "mdb-sh-complex-3-0-1", + "priority": 1, + "tags": { }, + "votes": 1 + }, + { + "_id": "2", + "host": "mdb-sh-complex-3-1-0", + "priority": 10, + "tags": { }, + "votes": 1 + }, + { + "_id": "3", + "host": "mdb-sh-complex-3-1-1", + "priority": 10, + "tags": { }, + "votes": 1 + }, + { + "_id": "4", + "host": "mdb-sh-complex-3-2-0", + "priority": 5, + "tags": { }, + "votes": 1 + } + ], + "protocolVersion": "1" + } + ] +} diff --git a/controllers/operator/testdata/mdb-sharded-multi-cluster-complex-expected-shardmap.yaml b/controllers/operator/testdata/mdb-sharded-multi-cluster-complex-expected-shardmap.yaml new file mode 100644 index 000000000..e26936629 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-multi-cluster-complex-expected-shardmap.yaml @@ -0,0 +1,436 @@ +0: + agent: + logLevel: DEBUG # applied to all shard processes in all clusters + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 2 # each shard will have 2 members in this cluster + memberConfig: # Everywhere we don't specify a memberConfig, we explicitly set the default values in this file + - votes: 1 + priority: "1" + - votes: 1 + priority: "1" + podSpec: + persistence: # set in spec.shard.clusterSpecList[cluster-0].podSpec.persistence + multiple: + journal: + storage: "10G" + data: + storage: "20G" + logs: + storage: "5G" + # statefulset override applied for cluster-0 from spec.shard.clusterSpecList[cluster-0].statefulSet + # We don't have the global sidecar container here because shardSpecificPodSpec has a replace policy (and not merge) + # it is applied to the whole shard (0) + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.0 + memory: 2.0G + limits: + cpu: 2.0 + memory: 5.0G + - clusterName: cluster-1 + members: 2 + memberConfig: # votes and priorities for two processes of each shard's replica set deployed in this cluster; notice two elements for 2 members + - votes: 1 + priority: "10" + - votes: 1 + priority: "10" + statefulSet: + spec: + template: + spec: + # We don't have the global sidecar container here because shardSpecificPodSpec has a replace policy (and not merge) + # it is applied to the whole shard (0) + containers: + - name: mongodb-enterprise-database + resources: + requests: # applied in spec.shard.clusterSpecList[cluster-1].statefulSet (requests only) + cpu: 2.1 + memory: 2.1G + limits: # from deprecated shardSpecificPodSpec.podTemplate + cpu: 2.3 + memory: 5.3G + podSpec: + persistence: + single: + storage: 14G # overridden from shardSpecificPodSpec.persistence + - clusterName: cluster-2 + members: 1 + memberConfig: # overridden from spec.shard.clusterSpecList[cluster-1].memberConfig + - votes: 1 + priority: "5" + podSpec: + persistence: + single: + storage: 14G # overridden from shardSpecificPodSpec.persistence + statefulSet: + spec: + template: + spec: + # We don't have the global sidecar container here because shardSpecificPodSpec has a replace policy (and not merge) + # it is applied to the whole shard (0) + containers: + - name: mongodb-enterprise-database # From shard.clusterSpecList[cluster-2].statefulSet + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + - name: sidecar-cluster-2 + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] +1: + agent: + logLevel: DEBUG # applied to all shard processes in all clusters + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 1 + memberConfig: + - votes: 1 + priority: "100" # Defined in shardOverride for mdb-sh-complex-1, cluster 0 + # In this shard, we set the field spec.ShardOverrides["mdb-sh-complex-1"].PodSpec.Persistence + # But we also set spec.ShardOverrides["mdb-sh-complex-1"].ClusterSpecList[clutser-0].PodSpec.Persistence, it + # has the highest priority + podSpec: + persistence: + single: + storage: 15G + statefulSet: # statefulset override applied for cluster-0 from spec.shard.clusterSpecList[cluster-0].statefulSet + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database # overriden from shardOverrides["mdb-sh-complex-1"].statefulSet + resources: + requests: + cpu: 2.8 + memory: 2.8G + limits: + cpu: 2.8 + memory: 5.8G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - clusterName: cluster-2 + members: 3 + memberConfig: + - votes: 1 + priority: "120" + - votes: 2 + priority: "121" + - votes: 1 + priority: "122" + podSpec: + persistence: # overridden in spec.ShardOverrides["mdb-sh-complex-1"].PodSpec.Persistence + multiple: + journal: + storage: "130G" + data: + storage: "140G" + logs: + storage: "110G" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database # overriden from shardOverrides["mdb-sh-complex-1"].statefulSet + resources: + requests: + cpu: 2.8 + memory: 2.8G + limits: + cpu: 2.8 + memory: 5.8G + - name: sidecar-cluster-2 + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - clusterName: cluster-1 + members: 2 + memberConfig: + - votes: 1 + priority: "110" + - votes: 0 # one is just secondary + priority: "0" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database # overriden from shardOverrides["mdb-sh-complex-1"].statefulSet.clusterSpecList["cluster-1"].statefulSet + resources: + requests: + cpu: 2.9 + memory: 2.9G + limits: + cpu: 2.9 + memory: 5.9G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + podSpec: + persistence: # overridden in spec.ShardOverrides["mdb-sh-complex-1"].PodSpec.Persistence + multiple: + journal: + storage: "130G" + data: + storage: "140G" + logs: + storage: "110G" +2: # for shard 2 there is dedicated shardOverride + agent: # Overridden by shardOverrides[2].agent + logLevel: INFO # applied to all shard processes in all clusters + # Overridden by shardOverrides[2].additionalMongodConfig + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 150 + clusterSpecList: + - clusterName: cluster-0 + members: 2 + memberConfig: + - votes: 1 + priority: "1" + - votes: 1 + priority: "1" + podSpec: + persistence: # overridden from spec.shard.clusterSpecList[cluster-0].podSpec.persistence + multiple: + journal: + storage: "10G" + data: + storage: "20G" + logs: + storage: "5G" + statefulSet: # statefulset override applied for cluster-0 from spec.shard.clusterSpecList[cluster-0].statefulSet + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.0 + memory: 2.0G + limits: + cpu: 2.0 + memory: 5.0G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - clusterName: cluster-1 + # Overridden by shardOverrides[2].clusterSpecList[”cluster-1"].members/memberConfig + members: 3 + memberConfig: # votes and priorities for two processes of each shard's replica set deployed in this cluster; notice three elements for 3 members + - votes: 1 + priority: "210" + - votes: 2 + priority: "211" + - votes: 3 + priority: "212" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: # spec.shard.clusterSpecList[cluster-1].statefulSet provides values only for requests, not limits + requests: + cpu: 2.1 + memory: 2.1G + # Note: when adding this container to the expected shardmap, the JSONDiff tool used for visual diffs in + # the unit test was not working if they were not in this exact order + # (db, sidecar global, sidecar shard 2) although the displayed diff was empty + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - name: sidecar-shard-2-cluster-1 # added from shardOverride[shardIdx==2].clusterSpecList[clusterName==cluster-1].statefulSet... + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + podSpec: + persistence: + single: + storage: 12G # overridden from shardPodSpec.persistence + - clusterName: cluster-analytics + members: 1 + memberConfig: + - votes: 1 + priority: "0" + podSpec: + persistence: # overridden from shardOverride[shard idx = 2].persistence + multiple: + journal: + storage: "30G" + data: + storage: "40G" + logs: + storage: "10G" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 4 + memory: 5G + limits: + cpu: 4 + memory: 20G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + nodeAffinity: # only members of this shard in this analytical cluster will have different node affinity to deploy on nodes with HDD + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: disktype + operator: In + values: + - hdd + - clusterName: cluster-analytics-2 + members: 2 + memberConfig: + - votes: 3 + priority: "12" + - votes: 3 + priority: "12" + podSpec: + persistence: # defined at top level in shardPodSpec + single: + storage: 12G + # Ensure ShardOverrides.PodSpec is correctly propagated too + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] +3: + agent: + logLevel: DEBUG # applied to all shard processes in all clusters + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 2 # each shard will have 2 members in this cluster + memberConfig: + - votes: 1 + priority: "1" + - votes: 1 + priority: "1" + podSpec: + persistence: # overridden from spec.shard.clusterSpecList[cluster-0].podSpec.persistence + multiple: + journal: + storage: "10G" + data: + storage: "20G" + logs: + storage: "5G" + statefulSet: # statefulset override applied for cluster-0 from spec.shard.clusterSpecList[cluster-0].statefulSet + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.0 + memory: 2.0G + limits: + cpu: 2.0 + memory: 5.0G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - clusterName: cluster-1 + members: 2 + memberConfig: # votes and priorities for two processes of each shard's replica set deployed in this cluster; notice two elements for 2 members + - votes: 1 + priority: "10" + - votes: 1 + priority: "10" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: # spec.shard.clusterSpecList[cluster-1].statefulSet provides values only for requests, not limits + cpu: 2.1 + memory: 2.1G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + podSpec: + persistence: + single: + storage: 12G # overridden from shardPodSpec.persistence + - clusterName: cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "5" + podSpec: + persistence: + single: + storage: 12G # overridden from shardPodSpec.persistence + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + - name: sidecar-cluster-2 + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] diff --git a/controllers/operator/testdata/mdb-sharded-multi-cluster-complex.yaml b/controllers/operator/testdata/mdb-sharded-multi-cluster-complex.yaml new file mode 100644 index 000000000..e0baf7b66 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-multi-cluster-complex.yaml @@ -0,0 +1,360 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-sh-complex + namespace: my-namespace +spec: + shardCount: 4 + # we don't specify mongodsPerShardCount, mongosCount and configServerCount as they don't make sense for multi-cluster + topology: MultiCluster + type: ShardedCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + mongos: + agent: + logLevel: DEBUG # applied to all mongos in all clusters + clusterSpecList: + - clusterName: cluster-0 + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: "3" + memory: "500M" + limits: + cpu: "3" + memory: "1G" + - clusterName: cluster-1 + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: "2" + memory: "300M" + limits: + cpu: "2" + memory: "1G" + configSrvPodSpec: + persistence: # settings applicable to all pods in all clusters + single: + storage: 10G + configSrv: + agent: + logLevel: DEBUG # applied to all agent processes in all clusters + additionalMongodConfig: # applied to all config server processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 2 + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.0 + memory: 2.0G + limits: + cpu: 2.0 + memory: 5.0G + podSpec: # we change defaults defined in spec.configSrvPodSpec + persistence: + single: + storage: 15G # only this cluster will have storage set to 15G + - clusterName: cluster-1 + members: 2 + memberConfig: + - votes: 2 + priority: "10" # Primary is preferred in cluster-1 + - votes: 1 + priority: "10" + # we don't specify podSpec, so it's taken from spec.configSrvPodSpec + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.1 + memory: 2.1G + limits: + cpu: 2.1 + memory: 5.1G + - clusterName: cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "5" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + shardPodSpec: + # default configuration for all shards in all clusters + persistence: # applicable to all shards over all clusters + single: + storage: 12G + podTemplate: + spec: + containers: + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + shardSpecificPodSpec: + - persistence: # shard of index 0 + single: + storage: 14G + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.3 + memory: 2.3G + limits: + cpu: 2.3 + memory: 5.3G + + shard: + agent: + logLevel: DEBUG # applied to all shard processes in all clusters + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 2 # each shard will have 2 members in this cluster + podSpec: + persistence: # applicable to all shards in this cluster + multiple: + journal: + storage: "10G" + data: + storage: "20G" + logs: + storage: "5G" + statefulSet: # statefulset override applicable only to this member cluster + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.0 + memory: 2.0G + limits: + cpu: 2.0 + memory: 5.0G + - clusterName: cluster-1 + members: 2 + memberConfig: # votes and priorities for two processes of each shard's replica set deployed in this cluster; notice two elements for 2 members + - votes: 1 + priority: "10" # Higher priority: we prefer to have primaries in this cluster + - votes: 1 + priority: "10" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.1 + memory: 2.1G + - clusterName: cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "5" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + - name: sidecar-cluster-2 + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + shardOverrides: + - shardNames: ["mdb-sh-complex-2"] # this override will apply to only shard #2 + additionalMongodConfig: # config applied to this shard over all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 150 # we want to configure profiling for this shard differently + agent: + logLevel: INFO # we don't want agent debug logs for this shard + # for this shard we override multi-cluster distribution; this is authoritative + # it will override shard.clusterSpecList elements entirely, but the contents of matching member clusters will be merged + clusterSpecList: + - clusterName: cluster-0 # all other fields are optional, if not provided the fields from matching member cluster from shard.clusterSpecList will be taken by default + - clusterName: cluster-1 + # we change only member count + members: 3 + memberConfig: # we increase the number of members so we need to specify member config for additional process + - votes: 1 + priority: "210" + - votes: 2 + priority: "211" + - votes: 3 + priority: "212" + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar-shard-2-cluster-1 + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + + # we don't provide entry for clusterName: cluster-2, so it won't be deployed there + - clusterName: cluster-analytics # we deploy a member of this shard on a new cluster, not present in other ClusterSpecLists + members: 1 + statefulSet: # we provide extra CPU for member of shard #2 in cluster-analytics + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 4 + memory: 5G + limits: + cpu: 4 + memory: 20G + nodeAffinity: # only members of this shard in this analytical cluster will have different node affinity to deploy on nodes with HDD + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: disktype + operator: In + values: + - hdd + podSpec: + persistence: # applicable to only shard #2 in cluster-analytics + multiple: + journal: + storage: "30G" # we assign additional disk resources on this cluster only + data: + storage: "40G" + logs: + storage: "10G" + memberConfig: + - votes: 1 + priority: "0" # we don't want this node to be elected primary at all + # we use this cluster to verify that top level specs are correctly propagated (Podtemplate and persistence), despite it not being present in other clusterSpecLists + - clusterName: cluster-analytics-2 + members: 2 + memberConfig: + - votes: 3 + priority: "12" + - votes: 3 + priority: "12" + - shardNames: ["mdb-sh-complex-1"] # this override will apply to only shard of index 1 + podSpec: + persistence: # applicable to only shard #1 in all clusters + multiple: + journal: + storage: "130G" + data: + storage: "140G" + logs: + storage: "110G" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.8 + memory: 2.8G + limits: + cpu: 2.8 + memory: 5.8G + + # for this shard we override multi-cluster distribution (members, votes and priorities) + # it will override shard.clusterSpecList elements entirely, but the contents of matching member clusters will be merged + clusterSpecList: + - clusterName: cluster-0 # all other fields are optional, if not provided the fields from matching member cluster from shard.clusterSpecList will be taken by default + members: 1 + memberConfig: # we increase the number of members, so we need to specify member config for additional process + - votes: 1 + priority: "100" + podSpec: + persistence: # applicable to only cluster-0 in shard 1 + single: + storage: 15G + - clusterName: cluster-2 # we deliberately change the order here + members: 3 + memberConfig: + - votes: 1 + priority: "120" + - votes: 2 + priority: "121" + - votes: 1 + priority: "122" + - clusterName: cluster-1 + members: 2 + memberConfig: + - votes: 1 + priority: "110" + - votes: 0 # one is just secondary + priority: "0" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.9 + memory: 2.9G + limits: + cpu: 2.9 + memory: 5.9G diff --git a/controllers/operator/testdata/mdb-sharded-multi-cluster-configsrv-mongos-expected-config.yaml b/controllers/operator/testdata/mdb-sharded-multi-cluster-configsrv-mongos-expected-config.yaml new file mode 100644 index 000000000..0608b1d8c --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-multi-cluster-configsrv-mongos-expected-config.yaml @@ -0,0 +1,89 @@ +agent: + logLevel: DEBUG +additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 +clusterSpecList: + - clusterName: cluster-0 + members: 2 + memberConfig: + - votes: 1 + priority: "100" + - votes: 2 + priority: "100" + podSpec: + persistence: + single: + storage: 15G + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: # From top level pod template + cpu: 2.12 + memory: 2.12G + limits: # From cluster-specific sts configuration + cpu: 2.0 + memory: 5.0G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + - clusterName: cluster-1 + members: 2 + memberConfig: + - votes: 2 + priority: "10" + - votes: 1 + priority: "10" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.1 + memory: 2.1G + limits: + cpu: 2.1 + memory: 5.1G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + podSpec: + persistence: # from configSrvPodSpec + single: + storage: 10G + - clusterName: cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "5" + podSpec: + persistence: # from configSrvPodSpec + single: + storage: 10G + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] diff --git a/controllers/operator/testdata/mdb-sharded-multi-cluster-configsrv-mongos.yaml b/controllers/operator/testdata/mdb-sharded-multi-cluster-configsrv-mongos.yaml new file mode 100644 index 000000000..c868999c6 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-multi-cluster-configsrv-mongos.yaml @@ -0,0 +1,211 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-sh-complex + namespace: my-namespace +spec: + shardCount: 2 + # we don't specify mongodsPerShardCount, mongosCount and configServerCount as they don't make sense for multi-cluster + topology: MultiCluster + type: ShardedCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + mongosPodSpec: # Even though some settings are not relevant for mongos, we use the same ones as for cfg srv to simplify testing + persistence: # settings applicable to all pods in all clusters + single: + storage: 10G + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.12 + memory: 2.12G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + mongos: + agent: + logLevel: DEBUG # applied to all agent processes in all clusters + additionalMongodConfig: # applied to all config server processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 2 + memberConfig: + - votes: 1 + priority: "100" # Primary is preferred in cluster-0 + - votes: 2 + priority: "100" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: # We only specify limits here, requests will be merged from top level podTemplate + cpu: 2.0 + memory: 5.0G + podSpec: # we change defaults defined in spec.configSrvPodSpec + persistence: + single: + storage: 15G # only this cluster will have storage set to 15G + podTemplate: # PodSpec.PodTemplate is ignored for sharded clusters + spec: + containers: + - name: should-be-ignored + resources: + requests: + cpu: 2.12 + memory: 2.12G + - clusterName: cluster-1 + # we don't specify podSpec, so it's taken from spec.configSrvPodSpec + members: 2 + memberConfig: + - votes: 2 + priority: "10" + - votes: 1 + priority: "10" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.1 + memory: 2.1G + limits: + cpu: 2.1 + memory: 5.1G + - clusterName: cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "5" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + + configSrvPodSpec: + persistence: # settings applicable to all pods in all clusters + single: + storage: 10G + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.12 + memory: 2.12G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + configSrv: + agent: + logLevel: DEBUG # applied to all agent processes in all clusters + additionalMongodConfig: # applied to all config server processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: cluster-0 + members: 2 + memberConfig: + - votes: 1 + priority: "100" # Primary is preferred in cluster-0 + - votes: 2 + priority: "100" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: # We only specify limits here, requests will be merged from top level podTemplate + cpu: 2.0 + memory: 5.0G + podSpec: # we change defaults defined in spec.configSrvPodSpec + persistence: + single: + storage: 15G # only this cluster will have storage set to 15G + podTemplate: # PodSpec.PodTemplate is ignored for sharded clusters + spec: + containers: + - name: should-be-ignored + resources: + requests: + cpu: 2.12 + memory: 2.12G + - clusterName: cluster-1 + # we don't specify podSpec, so it's taken from spec.configSrvPodSpec + members: 2 + memberConfig: + - votes: 2 + priority: "10" + - votes: 1 + priority: "10" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.1 + memory: 2.1G + limits: + cpu: 2.1 + memory: 5.1G + - clusterName: cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "5" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.2 + memory: 2.2G + limits: + cpu: 2.2 + memory: 5.2G + + + shard: + clusterSpecList: + - clusterName: cluster-0 + members: 2 # each shard will have 2 members in this cluster + - clusterName: cluster-1 + members: 2 + - clusterName: cluster-2 + members: 1 diff --git a/controllers/operator/testdata/mdb-sharded-single-cluster-configsrv-mongos-expected-config.yaml b/controllers/operator/testdata/mdb-sharded-single-cluster-configsrv-mongos-expected-config.yaml new file mode 100644 index 000000000..9d3b16573 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-single-cluster-configsrv-mongos-expected-config.yaml @@ -0,0 +1,30 @@ +agent: + startupOptions: + dialTimeoutSeconds: "40" +additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 +clusterSpecList: + - clusterName: __default + members: 2 + memberConfig: # When we don't specify a memberConfig, we explicitly set default values + - priority: "1" + votes: 1 + - priority: "1" + votes: 1 + podSpec: + persistence: + single: + storage: 10G + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.3 + memory: 2.3G diff --git a/controllers/operator/testdata/mdb-sharded-single-cluster-configsrv-mongos.yaml b/controllers/operator/testdata/mdb-sharded-single-cluster-configsrv-mongos.yaml new file mode 100644 index 000000000..666458c13 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-single-cluster-configsrv-mongos.yaml @@ -0,0 +1,60 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-sh-complex + namespace: my-namespace +spec: + shardCount: 3 + mongodsPerShardCount: 2 + configServerCount: 2 + mongosCount: 2 + topology: SingleCluster + type: ShardedCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + + mongosPodSpec: # Even though some settings are not relevant for mongos, we use the same ones as for cfg srv to simplify testing + persistence: # settings applicable to all pods in all clusters + single: + storage: 10G + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.3 + memory: 2.3G + mongos: + agent: + startupOptions: + dialTimeoutSeconds: "40" + additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + configSrvPodSpec: + persistence: # settings applicable to all pods in all clusters + single: + storage: 10G + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.3 + memory: 2.3G + + configSrv: + agent: + startupOptions: + dialTimeoutSeconds: "40" + additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 \ No newline at end of file diff --git a/controllers/operator/testdata/mdb-sharded-single-with-overrides-expected-shardmap.yaml b/controllers/operator/testdata/mdb-sharded-single-with-overrides-expected-shardmap.yaml new file mode 100644 index 000000000..270e46196 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-single-with-overrides-expected-shardmap.yaml @@ -0,0 +1,101 @@ +0: + agent: + logLevel: DEBUG # applied to all shard processes in all clusters + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: __default + members: 2 # mongodsPerShardCount is 2 + memberConfig: # From shard override + - votes: 1 + priority: "100" + - votes: 1 + priority: "100" + podSpec: + persistence: # shard of index 0, persistence overridden in shardSpecificPodSpec + single: + storage: 14G + # shard 0 pod template applied with shardSpecificPodSpec + # No sidecar global container here because shardSpecificPodSpec has a replace policy (and not merge) + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.3 + memory: 2.3G + limits: + cpu: 2.3 + memory: 5.3G +1: + agent: + logLevel: DEBUG # applied to all shard processes in all clusters + additionalMongodConfig: # applied to all shard processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + clusterSpecList: + - clusterName: __default + members: 1 # override for shard 1 + memberConfig: + - votes: 1 + priority: "100" + podSpec: + persistence: # shard override + multiple: + journal: + storage: "130G" + data: + storage: "140G" + logs: + storage: "110G" + statefulSet: # statefulset override applied to the shard + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.8 + memory: 2.8G + limits: + cpu: 2.8 + memory: 5.8G + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] +2: + agent: + logLevel: INFO # from shard override + additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 150 + clusterSpecList: + - clusterName: __default + members: 2 # mongodsPerShardCount is 2 + memberConfig: # When we don't specify a memberConfig, we explicitly set default values + - priority: "1" + votes: 1 + - priority: "1" + votes: 1 + podSpec: + persistence: + single: + storage: 12G + statefulSet: # statefulset override applied to the shard + spec: + template: + spec: + containers: + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] diff --git a/controllers/operator/testdata/mdb-sharded-single-with-overrides.yaml b/controllers/operator/testdata/mdb-sharded-single-with-overrides.yaml new file mode 100644 index 000000000..90301f8f2 --- /dev/null +++ b/controllers/operator/testdata/mdb-sharded-single-with-overrides.yaml @@ -0,0 +1,111 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-sh-complex + namespace: my-namespace +spec: + shardCount: 3 + mongodsPerShardCount: 2 + topology: SingleCluster + type: ShardedCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + mongos: + agent: + logLevel: DEBUG # applied to all mongos in all clusters + configSrvPodSpec: + persistence: # settings applicable to all pods in all clusters + single: + storage: 10G + configSrv: + agent: + logLevel: DEBUG # applied to all agent processes in all clusters + additionalMongodConfig: # applied to all config server processes in all clusters + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + + shardPodSpec: + # default configuration for all shards in all clusters + persistence: + single: + storage: 12G + podTemplate: + spec: + containers: + - name: sidecar-global + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + shardSpecificPodSpec: # deprecated way of overriding shards + - persistence: # shard of index 0 + single: + storage: 14G + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.3 + memory: 2.3G + limits: + cpu: 2.3 + memory: 5.3G + + shard: # applied to all shard processes in all clusters + agent: + logLevel: DEBUG + additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 100 + + shardOverrides: + - shardNames: [ "mdb-sh-complex-0" ] # this override will apply to only shard #0 + memberConfig: # we prefer to have primaries in this cluster + - votes: 1 + priority: "100" + - votes: 1 + priority: "100" + + - shardNames: ["mdb-sh-complex-1"] # this override will apply to only shard #1 + podSpec: + persistence: + multiple: + journal: + storage: "130G" + data: + storage: "140G" + logs: + storage: "110G" + members: 1 + memberConfig: + - votes: 1 + priority: "100" + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 2.8 + memory: 2.8G + limits: + cpu: 2.8 + memory: 5.8G + + - shardNames: ["mdb-sh-complex-2"] # this override will apply to only shard #2 + additionalMongodConfig: + operationProfiling: + mode: slowOp + slowOpThresholdMs: 150 # we want to configure profiling for this shard differently + agent: + logLevel: INFO # we don't want agent debug logs for this shard diff --git a/controllers/operator/testdata/version_manifest.json b/controllers/operator/testdata/version_manifest.json new file mode 100644 index 000000000..a5a8150be --- /dev/null +++ b/controllers/operator/testdata/version_manifest.json @@ -0,0 +1,192 @@ +{ + "updated": 1576540800042, + "versions": [ + { + "builds": [ + { + "architecture": "amd64", + "gitVersion": "1c1c76aeca21c5983dc178920f5052c298db616c", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-2.6.0.tgz" + }, + { + "architecture": "amd64", + "gitVersion": "1c1c76aeca21c5983dc178920f5052c298db616c", + "platform": "osx", + "url": "/osx/mongodb-osx-x86_64-2.6.0.tgz" + }, + { + "architecture": "amd64", + "gitVersion": "1c1c76aeca21c5983dc178920f5052c298db616c", + "platform": "windows", + "url": "/win32/mongodb-win32-x86_64-2.6.0.zip" + }, + { + "architecture": "amd64", + "gitVersion": "1c1c76aeca21c5983dc178920f5052c298db616c", + "platform": "windows", + "url": "/win32/mongodb-win32-x86_64-2008plus-2.6.0.zip", + "win2008plus": true + } + ], + "name": "2.6.0" + }, + { + "builds": [ + { + "architecture": "amd64", + "flavor": "ubuntu", + "gitVersion": "1c1c76aeca21c5983dc178920f5052c298db616c modules: enterprise", + "maxOsVersion": "13.04", + "minOsVersion": "12.04", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-ubuntu1204-2.6.0.tgz" + }, + { + "architecture": "amd64", + "gitVersion": "1c1c76aeca21c5983dc178920f5052c298db616c modules: enterprise", + "platform": "windows", + "url": "/win32/mongodb-win32-x86_64-enterprise-windows-64-2.6.0.zip", + "win2008plus": true, + "winVCRedistDll": "msvcr100.dll", + "winVCRedistOptions": [ + "/q", + "/norestart" + ], + "winVCRedistUrl": "http://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x64.exe", + "winVCRedistVersion": "10.0.40219.325" + } + ], + "name": "2.6.0-ent" + }, + { + "builds": [ + { + "architecture": "amd64", + "flavor": "ubuntu", + "gitVersion": "a57d8e71e6998a2d0afde7edc11bd23e5661c915", + "maxOsVersion": "15.04", + "minOsVersion": "14.04", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-ubuntu1404-3.6.0.tgz" + }, + { + "architecture": "amd64", + "flavor": "ubuntu", + "gitVersion": "a57d8e71e6998a2d0afde7edc11bd23e5661c915", + "maxOsVersion": "17.04", + "minOsVersion": "16.04", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-ubuntu1604-3.6.0.tgz" + } + ], + "name": "3.6.0" + }, + { + "builds": [ + { + "architecture": "amd64", + "flavor": "amazon", + "gitVersion": "a57d8e71e6998a2d0afde7edc11bd23e5661c915", + "maxOsVersion": "", + "minOsVersion": "2013.03", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-amzn64-3.6.0.tgz" + }, + { + "architecture": "amd64", + "flavor": "rhel", + "gitVersion": "a57d8e71e6998a2d0afde7edc11bd23e5661c915", + "maxOsVersion": "7.0", + "minOsVersion": "6.2", + "modules": [ + "enterprise" + ], + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-rhel62-3.6.0.tgz" + } + ], + "name": "3.6.0-ent" + }, + { + "builds": [ + { + "architecture": "ppc64le", + "gitVersion": "a0bbbff6ada159e19298d37946ac8dc4b497eadf", + "platform": "linux", + "url": "/linux/mongodb-linux-ppc64le-enterprise-rhel71-4.2.2.tgz", + "flavor": "rhel", + "maxOsVersion": "8.0", + "minOsVersion": "7.0", + "modules": [ + "enterprise" + ] + }, + { + "architecture": "amd64", + "gitVersion": "a0bbbff6ada159e19298d37946ac8dc4b497eadf", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-rhel62-4.2.2.tgz", + "flavor": "rhel", + "maxOsVersion": "7.0", + "minOsVersion": "6.2", + "modules": [ + "enterprise" + ] + }, + { + "architecture": "s390x", + "gitVersion": "a0bbbff6ada159e19298d37946ac8dc4b497eadf", + "platform": "linux", + "url": "/linux/mongodb-linux-s390x-enterprise-rhel72-4.2.2.tgz", + "flavor": "rhel", + "maxOsVersion": "8.0", + "minOsVersion": "7.0", + "modules": [ + "enterprise" + ] + }, + { + "architecture": "amd64", + "gitVersion": "a0bbbff6ada159e19298d37946ac8dc4b497eadf", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-ubuntu1604-4.2.2.tgz", + "flavor": "ubuntu", + "maxOsVersion": "17.04", + "minOsVersion": "16.04", + "modules": [ + "enterprise" + ] + }, + { + "architecture": "amd64", + "gitVersion": "a0bbbff6ada159e19298d37946ac8dc4b497eadf", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-suse12-4.2.2.tgz", + "flavor": "suse", + "maxOsVersion": "13", + "minOsVersion": "12", + "modules": [ + "enterprise" + ] + }, + { + "architecture": "amd64", + "gitVersion": "a0bbbff6ada159e19298d37946ac8dc4b497eadf", + "platform": "linux", + "url": "/linux/mongodb-linux-x86_64-enterprise-debian92-4.2.2.tgz", + "flavor": "debian", + "maxOsVersion": "10.0", + "minOsVersion": "9.1", + "modules": [ + "enterprise" + ] + } + ], + "name": "4.2.2-ent" + } + ] +} diff --git a/controllers/operator/watch/config_change_handler.go b/controllers/operator/watch/config_change_handler.go new file mode 100644 index 000000000..f3cac99b0 --- /dev/null +++ b/controllers/operator/watch/config_change_handler.go @@ -0,0 +1,131 @@ +package watch + +import ( + "context" + "fmt" + "reflect" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + corev1 "k8s.io/api/core/v1" +) + +// Type is an enum for all kubernetes types watched by controller for changes for configuration +type Type string + +const ( + ConfigMap Type = "ConfigMap" + Secret Type = "Secret" + MongoDB Type = "MongoDB" +) + +// the Object watched by controller. Includes its type and namespace+name +type Object struct { + ResourceType Type + Resource types.NamespacedName +} + +func (w Object) String() string { + return fmt.Sprintf("%s (%s)", w.Resource, w.ResourceType) +} + +// ResourcesHandler is a special implementation of 'handler.EventHandler' that checks if the event for +// K8s Resource must trigger reconciliation for any Operator managed Resource (MongoDB, MongoDBOpsManager). This is +// done via consulting the 'TrackedResources' map. The map is stored in the relevant reconciler which puts pairs +// [K8s_resource_name -> operator_managed_resource_name] there as +// soon as reconciliation happens for the Resource +type ResourcesHandler struct { + ResourceType Type + ResourceWatcher *ResourceWatcher +} + +// Note that we implement Create in addition to Update to be able to handle cases when config map or secret is deleted +// and then created again. +func (c *ResourcesHandler) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + c.doHandle(e.Object.GetNamespace(), e.Object.GetName(), q) +} + +func (c *ResourcesHandler) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { + if !shouldHandleUpdate(e) { + return + } + c.doHandle(e.ObjectOld.GetNamespace(), e.ObjectOld.GetName(), q) +} + +// shouldHandleUpdate return true if the update event must be handled. This shouldn't happen if data for watched +// ConfigMap or Secret hasn't changed +func shouldHandleUpdate(e event.UpdateEvent) bool { + switch v := e.ObjectOld.(type) { + case *corev1.ConfigMap: + return !reflect.DeepEqual(v.Data, e.ObjectNew.(*corev1.ConfigMap).Data) + case *corev1.Secret: + return !reflect.DeepEqual(v.Data, e.ObjectNew.(*corev1.Secret).Data) + } + return true +} + +func (c *ResourcesHandler) doHandle(namespace, name string, q workqueue.RateLimitingInterface) { + configMapOrSecret := Object{ + ResourceType: c.ResourceType, + Resource: types.NamespacedName{Name: name, Namespace: namespace}, + } + + for _, v := range c.ResourceWatcher.GetWatchedResources()[configMapOrSecret] { + zap.S().Infof("%s has been modified -> triggering reconciliation for dependent Resource %s", configMapOrSecret, v) + q.Add(reconcile.Request{NamespacedName: v}) + } +} + +// Seems we don't need to react on config map/secret removal.. +func (c *ResourcesHandler) Delete(ctx context.Context, _ event.DeleteEvent, _ workqueue.RateLimitingInterface) { +} + +func (c *ResourcesHandler) Generic(ctx context.Context, _ event.GenericEvent, _ workqueue.RateLimitingInterface) { +} + +// ConfigMapEventHandler is an EventHandler implementation that is used to watch for events on a given ConfigMap and ConfigMapNamespace +// The handler will force a panic on Update and Delete. +// As of right now it is only used to watch for events for the configmap pertaining the member list of multi-cluster. +type ConfigMapEventHandler struct { + ConfigMapName string + ConfigMapNamespace string +} + +func (m ConfigMapEventHandler) Create(ctx context.Context, e event.CreateEvent, _ workqueue.RateLimitingInterface) { +} + +func (m ConfigMapEventHandler) Update(ctx context.Context, e event.UpdateEvent, _ workqueue.RateLimitingInterface) { + if m.isMemberListCM(e.ObjectOld) { + switch v := e.ObjectOld.(type) { + case *corev1.ConfigMap: + changelog := cmp.Diff(v.Data, e.ObjectNew.(*corev1.ConfigMap).Data) + errMsg := fmt.Sprintf("%s/%s has changed! Will kill the pod to source the changes! Changelog: %s", m.ConfigMapNamespace, m.ConfigMapName, changelog) + panic(errMsg) + } + } +} + +func (m ConfigMapEventHandler) Delete(ctx context.Context, e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + if m.isMemberListCM(e.Object) { + errMsg := fmt.Sprintf("%s/%s has been deleted! Note we will need the configmap otherwise the operator will not work", m.ConfigMapNamespace, m.ConfigMapName) + panic(errMsg) + } +} + +func (m ConfigMapEventHandler) Generic(ctx context.Context, e event.GenericEvent, _ workqueue.RateLimitingInterface) { +} + +func (m ConfigMapEventHandler) isMemberListCM(o client.Object) bool { + name := o.GetName() + ns := o.GetNamespace() + if name == m.ConfigMapName && ns == m.ConfigMapNamespace { + return true + } + return false +} diff --git a/controllers/operator/watch/config_change_handler_test.go b/controllers/operator/watch/config_change_handler_test.go new file mode 100644 index 000000000..2469396ce --- /dev/null +++ b/controllers/operator/watch/config_change_handler_test.go @@ -0,0 +1,68 @@ +package watch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/event" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestShouldHandleUpdate(t *testing.T) { + t.Run("Update shouldn't happen if ConfigMaps data hasn't changed", func(t *testing.T) { + oldObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "ns", + }, + Data: map[string]string{"testKey": "testValue"}, + } + newObj := oldObj.DeepCopy() + newObj.ResourceVersion = "4243" + + assert.False(t, shouldHandleUpdate(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObj})) + }) + t.Run("Update should happen if the data has changed for ConfigMap", func(t *testing.T) { + oldObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "ns", + }, + Data: map[string]string{"testKey": "testValue"}, + } + newObj := oldObj.DeepCopy() + newObj.ResourceVersion = "4243" + newObj.Data["secondKey"] = "secondValue" + + assert.True(t, shouldHandleUpdate(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObj})) + }) + t.Run("Update shouldn't happen if Secrets data hasn't changed", func(t *testing.T) { + oldObj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "ns", + }, + Data: map[string][]byte{"testKey": []byte("testValue")}, + } + newObj := oldObj.DeepCopy() + newObj.ResourceVersion = "4243" + + assert.False(t, shouldHandleUpdate(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObj})) + }) + t.Run("Update should happen if the data has changed for Secret", func(t *testing.T) { + oldObj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "name", + Namespace: "ns", + }, + Data: map[string][]byte{"testKey": []byte("testValue")}, + } + newObj := oldObj.DeepCopy() + newObj.ResourceVersion = "4243" + newObj.Data["secondKey"] = []byte("secondValue") + + assert.True(t, shouldHandleUpdate(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObj})) + }) +} diff --git a/controllers/operator/watch/predicates.go b/controllers/operator/watch/predicates.go new file mode 100644 index 000000000..9999c4315 --- /dev/null +++ b/controllers/operator/watch/predicates.go @@ -0,0 +1,180 @@ +package watch + +import ( + "reflect" + + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + appsv1 "k8s.io/api/apps/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + userv1 "github.com/10gen/ops-manager-kubernetes/api/v1/user" + "github.com/10gen/ops-manager-kubernetes/pkg/handler" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func PredicatesForUser() predicate.Funcs { + return predicate.Funcs{ + // don't update users on status changes + UpdateFunc: func(e event.UpdateEvent) bool { + oldResource := e.ObjectOld.(*userv1.MongoDBUser) + newResource := e.ObjectNew.(*userv1.MongoDBUser) + + oldSpecAnnotation := oldResource.GetAnnotations()[util.LastAchievedSpec] + newSpecAnnotation := newResource.GetAnnotations()[util.LastAchievedSpec] + + // don't handle an update to just the previous spec annotation if they are not the same. + // this prevents the operator triggering reconciliations on resource that it is updating itself. + if !reflect.DeepEqual(oldSpecAnnotation, newSpecAnnotation) { + return false + } + + return reflect.DeepEqual(oldResource.GetStatus(), newResource.GetStatus()) + }, + } +} + +func PredicatesForOpsManager() predicate.Funcs { + return predicate.Funcs{ + // don't update ops manager on status changes + UpdateFunc: func(e event.UpdateEvent) bool { + oldResource := e.ObjectOld.(*omv1.MongoDBOpsManager) + newResource := e.ObjectNew.(*omv1.MongoDBOpsManager) + + oldSpecAnnotation := oldResource.GetAnnotations()[util.LastAchievedSpec] + newSpecAnnotation := newResource.GetAnnotations()[util.LastAchievedSpec] + + // don't handle an update to just the previous spec annotation if they are not the same. + // this prevents the operator triggering reconciliations on resource that it is updating itself. + if !reflect.DeepEqual(oldSpecAnnotation, newSpecAnnotation) { + return false + } + // check if any one of the vault annotations are different in revision + if vault.IsVaultSecretBackend() { + + for _, e := range oldResource.GetSecretsMountedIntoPod() { + if oldResource.GetAnnotations()[e] != newResource.GetAnnotations()[e] { + return true + } + } + + for _, e := range oldResource.Spec.AppDB.GetSecretsMountedIntoPod() { + if oldResource.GetAnnotations()[e] != newResource.GetAnnotations()[e] { + return true + } + } + return false + } + + return reflect.DeepEqual(oldResource.GetStatus(), newResource.GetStatus()) + }, + } +} + +func PredicatesForMongoDB(resourceType mdbv1.ResourceType) predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(createEvent event.CreateEvent) bool { + resource := createEvent.Object.(*mdbv1.MongoDB) + return resource.Spec.ResourceType == resourceType + }, + DeleteFunc: func(deleteEvent event.DeleteEvent) bool { + resource := deleteEvent.Object.(*mdbv1.MongoDB) + return resource.Spec.ResourceType == resourceType + }, + GenericFunc: func(genericEvent event.GenericEvent) bool { + resource := genericEvent.Object.(*mdbv1.MongoDB) + return resource.Spec.ResourceType == resourceType + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldResource := e.ObjectOld.(*mdbv1.MongoDB) + newResource := e.ObjectNew.(*mdbv1.MongoDB) + + oldSpecAnnotation := oldResource.GetAnnotations()[util.LastAchievedSpec] + newSpecAnnotation := newResource.GetAnnotations()[util.LastAchievedSpec] + + // don't handle an update to just the previous spec annotation if they are not the same. + // this prevents the operator triggering reconciliations on resource that it is updating itself. + if !reflect.DeepEqual(oldSpecAnnotation, newSpecAnnotation) { + return false + } + + // check if any one of the vault annotations are different in revision + if vault.IsVaultSecretBackend() { + vaultReconcile := false + credentialsAnnotation := newResource.Spec.Credentials + if oldResource.GetAnnotations()[credentialsAnnotation] != newResource.GetAnnotations()[credentialsAnnotation] { + vaultReconcile = true + } + for _, e := range oldResource.GetSecretsMountedIntoDBPod() { + if oldResource.GetAnnotations()[e] != newResource.GetAnnotations()[e] { + vaultReconcile = true + } + } + return newResource.Spec.ResourceType == resourceType && vaultReconcile + } + + // ignore events that aren't related to our target Resource and any changes done to the status + // (it's the controller that has made those changes, not user) + return newResource.Spec.ResourceType == resourceType && + reflect.DeepEqual(oldResource.GetStatus(), newResource.GetStatus()) + }, + } +} + +func PredicatesForStatefulSet() predicate.Funcs { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldSts := e.ObjectOld.(*appsv1.StatefulSet) + newSts := e.ObjectNew.(*appsv1.StatefulSet) + + val, ok := newSts.Annotations["type"] + + if ok && val == "Replicaset" { + if !reflect.DeepEqual(oldSts.Status, newSts.Status) && (newSts.Status.ReadyReplicas < *newSts.Spec.Replicas) { + return true + } + } + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + } +} + +// PredicatesForMultiStatefulSet is the predicate functions for the custom Statefulset Event +// handler used for Multicluster reconciler +func PredicatesForMultiStatefulSet() predicate.Funcs { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldSts := e.ObjectOld.(*appsv1.StatefulSet) + newSts := e.ObjectNew.(*appsv1.StatefulSet) + + // check if it is owned by MultiCluster CR first + if _, ok := newSts.Annotations[handler.MongoDBMultiResourceAnnotation]; !ok { + return false + } + + return !reflect.DeepEqual(oldSts.Status, newSts.Status) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + _, ok := e.Object.GetAnnotations()[handler.MongoDBMultiResourceAnnotation] + return ok + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + } +} diff --git a/controllers/operator/watch/predicates_test.go b/controllers/operator/watch/predicates_test.go new file mode 100644 index 000000000..cadb3de7f --- /dev/null +++ b/controllers/operator/watch/predicates_test.go @@ -0,0 +1,91 @@ +package watch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/event" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/api/v1/user" +) + +func TestPredicatesForUser(t *testing.T) { + t.Run("No reconciliation for MongoDBUser if statuses are not equal", func(t *testing.T) { + oldUser := &user.MongoDBUser{ + Status: user.MongoDBUserStatus{}, + } + newUser := oldUser.DeepCopy() + newUser.Status.Phase = status.PhasePending + assert.False(t, PredicatesForUser().Update(event.UpdateEvent{ObjectOld: oldUser, ObjectNew: newUser})) + }) + t.Run("Reconciliation happens for MongoDBUser if statuses are equal", func(t *testing.T) { + oldUser := &user.MongoDBUser{ + Status: user.MongoDBUserStatus{}, + } + newUser := oldUser.DeepCopy() + newUser.Spec.Username = "test" + assert.True(t, PredicatesForUser().Update(event.UpdateEvent{ObjectOld: oldUser, ObjectNew: newUser})) + }) +} + +func TestPredicatesForOpsManager(t *testing.T) { + t.Run("No reconciliation for MongoDBOpsManager if statuses are not equal", func(t *testing.T) { + oldOm := omv1.NewOpsManagerBuilder().Build() + newOm := oldOm.DeepCopy() + newOm.Spec.Replicas = 2 + newOm.Status.OpsManagerStatus = omv1.OpsManagerStatus{Warnings: []status.Warning{"warning"}} + assert.False(t, PredicatesForOpsManager().Update(event.UpdateEvent{ObjectOld: oldOm, ObjectNew: newOm})) + }) + t.Run("Reconciliation happens for MongoDBOpsManager if statuses are equal", func(t *testing.T) { + oldOm := omv1.NewOpsManagerBuilder().Build() + newOm := oldOm.DeepCopy() + newOm.Spec.Replicas = 2 + assert.True(t, PredicatesForOpsManager().Update(event.UpdateEvent{ObjectOld: oldOm, ObjectNew: newOm})) + }) +} + +func TestPredicatesForMongoDB(t *testing.T) { + t.Run("Creation event is handled", func(t *testing.T) { + standalone := mdbv1.NewStandaloneBuilder().Build() + assert.True(t, PredicatesForMongoDB(mdbv1.Standalone).Create(event.CreateEvent{Object: standalone})) + }) + t.Run("Creation event is not handled", func(t *testing.T) { + rs := mdbv1.NewReplicaSetBuilder().Build() + assert.False(t, PredicatesForMongoDB(mdbv1.Standalone).Create(event.CreateEvent{Object: rs})) + }) + t.Run("Delete event is handled", func(t *testing.T) { + sc := mdbv1.NewClusterBuilder().Build() + assert.True(t, PredicatesForMongoDB(mdbv1.ShardedCluster).Delete(event.DeleteEvent{Object: sc})) + }) + t.Run("Delete event is not handled", func(t *testing.T) { + rs := mdbv1.NewReplicaSetBuilder().Build() + assert.False(t, PredicatesForMongoDB(mdbv1.ShardedCluster).Delete(event.DeleteEvent{Object: rs})) + }) + t.Run("Update event is handled, statuses not changed", func(t *testing.T) { + oldMdb := mdbv1.NewStandaloneBuilder().Build() + newMdb := oldMdb.DeepCopy() + newMdb.Spec.Version = "4.2.0" + assert.True(t, PredicatesForMongoDB(mdbv1.Standalone).Update( + event.UpdateEvent{ObjectOld: oldMdb, ObjectNew: newMdb}), + ) + }) + t.Run("Update event is not handled, statuses changed", func(t *testing.T) { + oldMdb := mdbv1.NewStandaloneBuilder().Build() + newMdb := oldMdb.DeepCopy() + newMdb.Status.Version = "4.2.0" + assert.False(t, PredicatesForMongoDB(mdbv1.Standalone).Update( + event.UpdateEvent{ObjectOld: oldMdb, ObjectNew: newMdb}), + ) + }) + t.Run("Update event is not handled, different types", func(t *testing.T) { + oldMdb := mdbv1.NewStandaloneBuilder().Build() + newMdb := oldMdb.DeepCopy() + newMdb.Spec.Version = "4.2.0" + assert.False(t, PredicatesForMongoDB(mdbv1.ShardedCluster).Update( + event.UpdateEvent{ObjectOld: oldMdb, ObjectNew: newMdb}), + ) + }) +} diff --git a/controllers/operator/watch/resource_watcher.go b/controllers/operator/watch/resource_watcher.go new file mode 100644 index 000000000..8643bc4eb --- /dev/null +++ b/controllers/operator/watch/resource_watcher.go @@ -0,0 +1,160 @@ +package watch + +import ( + "sync" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" +) + +func NewResourceWatcher() *ResourceWatcher { + return &ResourceWatcher{ + watchedResources: map[Object][]types.NamespacedName{}, + } +} + +type ResourceWatcher struct { + mapLock sync.RWMutex + watchedResources map[Object][]types.NamespacedName +} + +// GetWatchedResources returns map of watched resources. +// It is returning deep copy, because underlying map can be modified concurrently. +func (r *ResourceWatcher) GetWatchedResources() map[Object][]types.NamespacedName { + r.mapLock.RLock() + defer r.mapLock.RUnlock() + + watchedResourcesCopy := map[Object][]types.NamespacedName{} + for obj, namespaces := range r.watchedResources { + namespacesCopy := make([]types.NamespacedName, len(namespaces)) + copy(namespacesCopy, namespaces) + watchedResourcesCopy[obj] = namespacesCopy + } + + return watchedResourcesCopy +} + +// RegisterWatchedMongodbResources adds the secret/configMap -> mongodb resource pair to internal reconciler map. This allows +// to start watching for the events for this secret/configMap and trigger reconciliation for all depending mongodb resources +func (r *ResourceWatcher) RegisterWatchedMongodbResources(mongodbResourceNsName types.NamespacedName, configMap string, secret string) { + defaultNamespace := mongodbResourceNsName.Namespace + + r.AddWatchedResourceIfNotAdded(configMap, defaultNamespace, ConfigMap, mongodbResourceNsName) + r.AddWatchedResourceIfNotAdded(secret, defaultNamespace, Secret, mongodbResourceNsName) +} + +// GetWatchedResourcesOfType returns all watched resources of the given type in the specified namespace. +// if the specified namespace is the zero value, resources in all namespaces will be returned. +func (r *ResourceWatcher) GetWatchedResourcesOfType(wType Type, ns string) []types.NamespacedName { + r.mapLock.RLock() + defer r.mapLock.RUnlock() + + var res []types.NamespacedName + for k := range r.watchedResources { + if k.ResourceType != wType { + continue + } + if k.Resource.Namespace == ns || ns == "" { + res = append(res, k.Resource) + } + } + return res +} + +// RegisterWatchedTLSResources adds the CA configMap and a slice of TLS secrets to the list of watched resources. +func (r *ResourceWatcher) RegisterWatchedTLSResources(mongodbResourceNsName types.NamespacedName, caConfigMap string, tlsSecrets []string) { + defaultNamespace := mongodbResourceNsName.Namespace + + if caConfigMap != "" { + r.AddWatchedResourceIfNotAdded(caConfigMap, defaultNamespace, ConfigMap, mongodbResourceNsName) + } + + for _, tlsSecret := range tlsSecrets { + r.AddWatchedResourceIfNotAdded(tlsSecret, defaultNamespace, Secret, mongodbResourceNsName) + } +} + +// RemoveDependentWatchedResources stops watching resources related to the input resource +func (r *ResourceWatcher) RemoveDependentWatchedResources(resourceNsName types.NamespacedName) { + r.RemoveAllDependentWatchedResources(resourceNsName.Namespace, resourceNsName) +} + +// AddWatchedResourceIfNotAdded adds the given resource to the list of watched +// resources. A watched resource is a resource that, when changed, will trigger +// a reconciliation for its dependent resource. +func (r *ResourceWatcher) AddWatchedResourceIfNotAdded(name, namespace string, wType Type, dependentResourceNsName types.NamespacedName) { + r.mapLock.Lock() + defer r.mapLock.Unlock() + key := Object{ + ResourceType: wType, + Resource: types.NamespacedName{ + Name: name, + Namespace: namespace, + }, + } + if _, ok := r.watchedResources[key]; !ok { + r.watchedResources[key] = make([]types.NamespacedName, 0) + } + found := false + for _, v := range r.watchedResources[key] { + if v == dependentResourceNsName { + found = true + } + } + if !found { + r.watchedResources[key] = append(r.watchedResources[key], dependentResourceNsName) + zap.S().Debugf("Watching %s to trigger reconciliation for %s", key, dependentResourceNsName) + } +} + +// unsafeRemoveWatchedResources stop watching resources with input namespace and watched type, if any. +// This function is not thread safe, use locking outside. +func (r *ResourceWatcher) unsafeRemoveWatchedResources(namespace string, wType Type, dependentResourceNsName types.NamespacedName) { + for key := range r.watchedResources { + if key.ResourceType == wType && key.Resource.Namespace == namespace { + index := -1 + for i, v := range r.watchedResources[key] { + if v == dependentResourceNsName { + index = i + } + } + + if index == -1 { + continue + } + + zap.S().Infof("Removing %s from resources dependent on %s", dependentResourceNsName, key) + + if index == 0 { + if len(r.watchedResources[key]) == 1 { + delete(r.watchedResources, key) + continue + } + r.watchedResources[key] = r.watchedResources[key][index+1:] + continue + } + + if index == len(r.watchedResources[key]) { + r.watchedResources[key] = r.watchedResources[key][:index] + continue + } + + r.watchedResources[key] = append(r.watchedResources[key][:index], r.watchedResources[key][index+1:]...) + } + } +} + +// RemoveAllDependentWatchedResources stop watching resources with input namespace and dependent resource +func (r *ResourceWatcher) RemoveAllDependentWatchedResources(namespace string, dependentResourceNsName types.NamespacedName) { + r.mapLock.Lock() + defer r.mapLock.Unlock() + + watchedResourceTypes := map[Type]bool{} + for resource := range r.watchedResources { + watchedResourceTypes[resource.ResourceType] = true + } + + for resourceType := range watchedResourceTypes { + r.unsafeRemoveWatchedResources(namespace, resourceType, dependentResourceNsName) + } +} diff --git a/controllers/operator/workflow/disabled.go b/controllers/operator/workflow/disabled.go new file mode 100644 index 000000000..6201cab38 --- /dev/null +++ b/controllers/operator/workflow/disabled.go @@ -0,0 +1,18 @@ +package workflow + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// disabledStatus indicates that the subresource is not enabled +type disabledStatus struct { + *okStatus +} + +func Disabled() *disabledStatus { + return &disabledStatus{okStatus: &okStatus{requeue: false}} +} + +func (d disabledStatus) Phase() status.Phase { + return status.PhaseDisabled +} diff --git a/controllers/operator/workflow/failed.go b/controllers/operator/workflow/failed.go new file mode 100644 index 000000000..8c4583aa4 --- /dev/null +++ b/controllers/operator/workflow/failed.go @@ -0,0 +1,89 @@ +package workflow + +import ( + "time" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/apierrors" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// failedStatus indicates that the reconciliation process must be suspended and CR should get "Pending" status +type failedStatus struct { + commonStatus + retryInSeconds int + // err contains error with stacktrace + err error +} + +func Failed(err error, params ...interface{}) *failedStatus { + return &failedStatus{commonStatus: newCommonStatus(err.Error(), params...), err: err, retryInSeconds: 10} +} + +func (f *failedStatus) WithWarnings(warnings []status.Warning) *failedStatus { + f.warnings = warnings + return f +} + +func (f *failedStatus) WithRetry(retryInSeconds int) *failedStatus { + f.retryInSeconds = retryInSeconds + return f +} + +func (f *failedStatus) ReconcileResult() (reconcile.Result, error) { + return reconcile.Result{RequeueAfter: time.Second * time.Duration(f.retryInSeconds)}, nil +} + +func (f *failedStatus) WithAdditionalOptions(options []status.Option) *failedStatus { + f.options = options + return f +} + +func (f *failedStatus) IsOK() bool { + return false +} + +func (f *failedStatus) Merge(other Status) Status { + switch v := other.(type) { + // errors are concatenated + case *failedStatus: + return mergedFailed(f, v) + case *invalidStatus: + return other + } + return f +} + +func (f *failedStatus) OnErrorPrepend(msg string) Status { + f.prependMsg(msg) + return f +} + +func (f *failedStatus) StatusOptions() []status.Option { + // don't display any message on the MongoDB resource if the error is transient. + options := f.statusOptions() + return options +} + +func (f *failedStatus) Phase() status.Phase { + if apierrors.IsTransientMessage(f.msg) { + return status.PhasePending + } + return status.PhaseFailed +} + +// Log does not take the f.msg but instead takes f.err to make sure we print the actual stack trace of the error. +func (f *failedStatus) Log(log *zap.SugaredLogger) { + log.Errorf("%+v", f.err) +} + +func mergedFailed(p1, p2 *failedStatus) *failedStatus { + msg := p1.msg + ", " + p2.msg + p := Failed(xerrors.Errorf(msg)) + p.warnings = append(p1.warnings, p2.warnings...) + return p +} diff --git a/controllers/operator/workflow/invalid.go b/controllers/operator/workflow/invalid.go new file mode 100644 index 000000000..6e09d5a03 --- /dev/null +++ b/controllers/operator/workflow/invalid.go @@ -0,0 +1,80 @@ +package workflow + +import ( + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// invalidStatus indicates that the reconciliation process must be suspended and CR should get "Pending" status +type invalidStatus struct { + commonStatus + targetPhase status.Phase +} + +func Invalid(msg string, params ...interface{}) *invalidStatus { + return &invalidStatus{commonStatus: newCommonStatus(msg, params...), targetPhase: status.PhaseFailed} +} + +func (f *invalidStatus) WithWarnings(warnings []status.Warning) *invalidStatus { + f.warnings = warnings + return f +} + +// WithTargetPhase allows to override the default phase for "invalid" (Failed) to another one. +// Most of all it may be Pending +func (f *invalidStatus) WithTargetPhase(targetPhase status.Phase) *invalidStatus { + f.targetPhase = targetPhase + return f +} + +func (f *invalidStatus) ReconcileResult() (reconcile.Result, error) { + // We don't requeue validation failures + return reconcile.Result{}, nil +} + +func (f *invalidStatus) IsOK() bool { + return false +} + +func (f *invalidStatus) Merge(other Status) Status { + switch v := other.(type) { + // errors are concatenated + case *invalidStatus: + return mergedInvalid(f, v) + } + // Invalid spec error dominates over anything else - there's no point in retrying until the spec is fixed + return f +} + +func (f *invalidStatus) OnErrorPrepend(msg string) Status { + f.prependMsg(msg) + return f +} + +func (f *invalidStatus) StatusOptions() []status.Option { + options := f.statusOptions() + // Add any specific options here + return options +} + +func (f *invalidStatus) Phase() status.Phase { + return f.targetPhase +} + +func (f *invalidStatus) Log(log *zap.SugaredLogger) { + log.Error(stringutil.UpperCaseFirstChar(f.msg)) +} + +func mergedInvalid(p1, p2 *invalidStatus) *invalidStatus { + p := Invalid("%s, %s", p1.msg, p2.msg) + p.warnings = append(p1.warnings, p2.warnings...) + // Choosing one of the non-empty target phases + p.targetPhase = p2.targetPhase + if p1.targetPhase != "" { + p.targetPhase = p1.targetPhase + } + return p +} diff --git a/controllers/operator/workflow/ok.go b/controllers/operator/workflow/ok.go new file mode 100644 index 000000000..573e819f1 --- /dev/null +++ b/controllers/operator/workflow/ok.go @@ -0,0 +1,67 @@ +package workflow + +import ( + "time" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +// okStatus indicates that the reconciliation process must be suspended and CR should get "Pending" status +type okStatus struct { + commonStatus + requeue bool + requeueAfter time.Duration +} + +func OK() *okStatus { + return &okStatus{requeueAfter: util.TWENTY_FOUR_HOURS} +} + +func (o *okStatus) WithWarnings(warnings []status.Warning) *okStatus { + o.warnings = warnings + return o +} + +func (o *okStatus) WithAdditionalOptions(options ...status.Option) *okStatus { + o.options = options + return o +} + +func (o *okStatus) ReconcileResult() (reconcile.Result, error) { + return reconcile.Result{Requeue: o.requeue, RequeueAfter: o.requeueAfter}, nil +} + +func (o *okStatus) IsOK() bool { + return !o.requeue +} + +func (o *okStatus) Merge(other Status) Status { + // any other status takes precedence over OK + return other +} + +func (o *okStatus) OnErrorPrepend(_ string) Status { + return o +} + +func (o *okStatus) StatusOptions() []status.Option { + return o.statusOptions() +} + +func (o *okStatus) Log(_ *zap.SugaredLogger) { + // Doing no logging - the reconciler will do instead +} + +func (o *okStatus) Phase() status.Phase { + return status.PhaseRunning +} + +func (o *okStatus) Requeue() Status { + o.requeueAfter = 0 + o.requeue = true + return o +} diff --git a/controllers/operator/workflow/pending.go b/controllers/operator/workflow/pending.go new file mode 100644 index 000000000..f8aec051c --- /dev/null +++ b/controllers/operator/workflow/pending.go @@ -0,0 +1,95 @@ +package workflow + +import ( + "time" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// pendingStatus indicates that the reconciliation process must be suspended and CR should get "Pending" status +type pendingStatus struct { + commonStatus + retryInSeconds int + requeue bool +} + +func Pending(msg string, params ...interface{}) *pendingStatus { + return &pendingStatus{commonStatus: newCommonStatus(msg, params...), retryInSeconds: 10} +} + +func (p *pendingStatus) WithWarnings(warnings []status.Warning) *pendingStatus { + p.warnings = warnings + return p +} + +func (p *pendingStatus) WithRetry(retryInSeconds int) *pendingStatus { + p.retryInSeconds = retryInSeconds + return p +} + +func (p *pendingStatus) WithResourcesNotReady(resourcesNotReady []status.ResourceNotReady) *pendingStatus { + p.resourcesNotReady = resourcesNotReady + return p +} + +func (p *pendingStatus) WithAdditionalOptions(options ...status.Option) *pendingStatus { + p.options = options + return p +} + +func (p *pendingStatus) ReconcileResult() (reconcile.Result, error) { + return reconcile.Result{RequeueAfter: time.Second * time.Duration(p.retryInSeconds), Requeue: p.requeue}, nil +} + +func (p *pendingStatus) IsOK() bool { + return false +} + +func (p *pendingStatus) Merge(other Status) Status { + switch v := other.(type) { + // Pending messages are just merged together + case *pendingStatus: + return mergedPending(p, v) + case *failedStatus: + return v + } + return p +} + +func (p *pendingStatus) OnErrorPrepend(msg string) Status { + p.prependMsg(msg) + return p +} + +func (p *pendingStatus) StatusOptions() []status.Option { + options := p.statusOptions() + // Add any custom options here + return options +} + +func (p *pendingStatus) Phase() status.Phase { + return status.PhasePending +} + +func (p *pendingStatus) Log(log *zap.SugaredLogger) { + log.Info(stringutil.UpperCaseFirstChar(p.msg)) +} + +func mergedPending(p1, p2 *pendingStatus) *pendingStatus { + p := Pending("%s, %s", p1.msg, p2.msg) + p.warnings = append(p1.warnings, p2.warnings...) + p.resourcesNotReady = make([]status.ResourceNotReady, 0) + p.resourcesNotReady = append(p.resourcesNotReady, p1.resourcesNotReady...) + p.resourcesNotReady = append(p.resourcesNotReady, p2.resourcesNotReady...) + return p +} + +func (p *pendingStatus) Requeue() Status { + p.requeue = true + p.retryInSeconds = 0 + return p +} diff --git a/controllers/operator/workflow/status.go b/controllers/operator/workflow/status.go new file mode 100644 index 000000000..f7f8d93d1 --- /dev/null +++ b/controllers/operator/workflow/status.go @@ -0,0 +1,75 @@ +package workflow + +import ( + "fmt" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/apierrors" + + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// Status serves as a container holding the status of the custom resource +// The main reason why it's needed is to allow to pass the information about the state of the resource back to the +// calling functions up to the top-level 'reconcile' avoiding multiple return parameters and 'if' statements +type Status interface { + // Merge performs the Merge of current status with the status returned from the other operation and returns the + // new status + Merge(other Status) Status + + // IsOK returns true if there was no signal to interrupt reconciliation process + IsOK() bool + + // OnErrorPrepend prepends the msg in the case of an error reconcileStatus + OnErrorPrepend(msg string) Status + + // StatusOptions returns options that can be used to populate the CR status + StatusOptions() []status.Option + + // Phase is the phase the status should get + Phase() status.Phase + + // ReconcileResult returns the result of reconciliation to be returned by main controller + ReconcileResult() (reconcile.Result, error) + + // Log performs logging of the status at some level if necessary + Log(log *zap.SugaredLogger) +} + +type commonStatus struct { + msg string + warnings []status.Warning + resourcesNotReady []status.ResourceNotReady + options []status.Option +} + +func newCommonStatus(msg string, params ...interface{}) commonStatus { + return commonStatus{msg: fmt.Sprintf(msg, params...)} +} + +func (c *commonStatus) prependMsg(msg string) { + c.msg = msg + " " + c.msg +} + +func (c commonStatus) statusOptions() []status.Option { + // don't display any message on the MongoDB resource if the error is transient. + msg := c.msg + if apierrors.IsTransientMessage(msg) { + msg = "" + } + options := []status.Option{ + status.NewMessageOption(msg), + status.NewWarningsOption(c.warnings), + status.NewResourcesNotReadyOption(c.resourcesNotReady), + } + return append(options, c.options...) +} + +func ContainsPVCOption(options []status.Option) bool { + if _, exists := status.GetOption(options, status.PVCStatusOption{}); exists { + return true + } + return false +} diff --git a/controllers/operator/workflow/status_test.go b/controllers/operator/workflow/status_test.go new file mode 100644 index 000000000..a95194350 --- /dev/null +++ b/controllers/operator/workflow/status_test.go @@ -0,0 +1,22 @@ +package workflow + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/xerrors" +) + +func TestOnErrorPrepend(t *testing.T) { + result := Pending("my message") + decoratedResult := result.OnErrorPrepend("some prefix").(*pendingStatus) + assert.Equal(t, "some prefix my message", decoratedResult.msg) + + failedResult := Failed(xerrors.Errorf("my failed result")) + failedDecoratedResult := failedResult.OnErrorPrepend("failed wrapper").(*failedStatus) + assert.Equal(t, "failed wrapper my failed result", failedDecoratedResult.msg) + + failedValidationResult := Invalid("my failed validation") + failedDecoratedValidationResult := failedValidationResult.OnErrorPrepend("failed wrapper").(*invalidStatus) + assert.Equal(t, "failed wrapper my failed validation", failedDecoratedValidationResult.msg) +} diff --git a/controllers/operator/workflow/unsupported.go b/controllers/operator/workflow/unsupported.go new file mode 100644 index 000000000..b249cec00 --- /dev/null +++ b/controllers/operator/workflow/unsupported.go @@ -0,0 +1,18 @@ +package workflow + +import ( + "github.com/10gen/ops-manager-kubernetes/api/v1/status" +) + +// unsupportedStatus indicates that the subresource is not supported by the current Operator +type unsupportedStatus struct { + *okStatus +} + +func Unsupported(msg string, params ...interface{}) *unsupportedStatus { + return &unsupportedStatus{okStatus: &okStatus{requeue: false, commonStatus: newCommonStatus(msg, params...)}} +} + +func (d unsupportedStatus) Phase() status.Phase { + return status.PhaseUnsupported +} diff --git a/controllers/operator/workflow/workflow.go b/controllers/operator/workflow/workflow.go new file mode 100644 index 000000000..e02adc597 --- /dev/null +++ b/controllers/operator/workflow/workflow.go @@ -0,0 +1,21 @@ +package workflow + +// RunInGivenOrder will execute N functions, passed as varargs as `funcs`. The order of execution will depend on the result +// of the evaluation of the `shouldRunInOrder` boolean value. If `shouldRunInOrder` is true, the functions will be executed in order; if +// `shouldRunInOrder` is false, the functions will be executed in reverse order (from last to first) +func RunInGivenOrder(shouldRunInOrder bool, funcs ...func() Status) Status { + if shouldRunInOrder { + for _, fn := range funcs { + if status := fn(); !status.IsOK() { + return status + } + } + } else { + for i := len(funcs) - 1; i >= 0; i-- { + if status := funcs[i](); !status.IsOK() { + return status + } + } + } + return OK() +} diff --git a/deploy/crd-and-csv-generation.md b/deploy/crd-and-csv-generation.md new file mode 100644 index 000000000..56e24bd9e --- /dev/null +++ b/deploy/crd-and-csv-generation.md @@ -0,0 +1,58 @@ +# Generating CRDs with the operator-sdk tool + +# prequisites +* The latest version of the [operator-sdk](https://github.com/operator-framework/operator-sdk/releases) + +We can generate crds with the following command + +```bash +operator-sdk generate crds +``` + +This defaults to creating the crds in `deploy/crds/` + + +# Generating the CSV + +Generating the CSV file can be done with the following command + +```bash +operator-sdk generate csv --operator-name mongodb-enterprise --csv-version 1.5.4 --apis-dir api --from-version 1.5.3 +``` + +The default directory the CR examples comes from is `deploy/crds`, but it can be specified +with `--crd-dir `. Note: the CRDs need to exist in this directory for the examples to be shown. + +In order for the deployments, roles and service accounts to be shown, they need to appear +as as yaml files in the `deploy` directory. + + +# Annotation Tips + +* For embedded structs, using `json:",inline"`. This will inherit all of the json tags on the embedded struct. +* Exclude fields from the CRD with `json:"-"` +* Any comments above a field or type will appear as the description of that field or object in the CRD. +* Add printer columns +```golang +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="The current state of the MongoDB User." +``` +* Add a subresource +```golang +// +kubebuilder:subresource:status +``` +* Make a field `// +optional` or `// +required` + +# Leaving dev comments + +Dev comments can be left by having a space between the description and the comment. +```golang + +// Description of MongodbSpec +type MongoDbSpec struct { + + // DEV COMMENT, won't show up in CRD + + // This is the description of Service + Service string `json:"service,omitempty"` + +``` diff --git a/deploy/crds/samples/ops-manager.yaml b/deploy/crds/samples/ops-manager.yaml new file mode 100644 index 000000000..4eb04d120 --- /dev/null +++ b/deploy/crds/samples/ops-manager.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + # the number of Ops Manager instances to run. Set to value bigger + # than 1 to get high availability and upgrades without downtime + replicas: 3 + + # the version of Ops Manager distro to use + version: 5.0.2 + + # optional. Specify the custom cluster domain of the Kubernetes cluster if it's different from the default one ('cluster.local'). + # This affects the urls generated by the Operator. + # This field is also used for Application Database url + clusterDomain: mycompany.net + + # the name of the secret containing admin user credentials. + # Either remove the secret or change the password using Ops Manager UI after the Ops Manager + # resource is created! + adminCredentials: ops-manager-admin-secret + + # optional. The Ops Manager configuration. All the values must be of type string + configuration: + mms.fromEmailAddr: "admin@example.com" + + # the application database backing Ops Manager. Replica Set is the only supported type + # Application database has the SCRAM-SHA authentication mode always enabled + applicationDatabase: + members: 3 + # optional. Configures the version of MongoDB used as an application database. + # The bundled MongoDB binary will be used if omitted and no download from the Internet will happen + version: 4.2.6-ent + podSpec: + cpu: '0.25' diff --git a/deploy/crds/samples/replica-set-scram-user.yaml b/deploy/crds/samples/replica-set-scram-user.yaml new file mode 100644 index 000000000..7f29cf81c --- /dev/null +++ b/deploy/crds/samples/replica-set-scram-user.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-scram-user +spec: + passwordSecretKeyRef: + name: my-scram-secret # the name of the secret that stores this user's password + key: password # the key in the secret that stores the password + username: my-scram-user + db: admin + mongodbResourceRef: + name: my-scram-enabled-replica-set # The name of the MongoDB resource this user will be added to + roles: + - db: admin + name: clusterAdmin + - db: admin + name: userAdminAnyDatabase + - db: admin + name: readWrite diff --git a/deploy/crds/samples/replica-set.yaml b/deploy/crds/samples/replica-set.yaml new file mode 100644 index 000000000..bac8695b1 --- /dev/null +++ b/deploy/crds/samples/replica-set.yaml @@ -0,0 +1,20 @@ +# +# This is a minimal config. To see all the options available, refer to the +# "extended" directory +# +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.2.1-ent + type: ReplicaSet + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false diff --git a/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.7.0.zip b/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.7.0.zip new file mode 100644 index 000000000..5ea30aa85 Binary files /dev/null and b/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.7.0.zip differ diff --git a/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.7.1.zip b/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.7.1.zip new file mode 100644 index 000000000..73fcd118b Binary files /dev/null and b/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.7.1.zip differ diff --git a/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.8.0.zip b/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.8.0.zip new file mode 100644 index 000000000..d0211bdb1 Binary files /dev/null and b/deploy/redhat_connect_zip_files/mongodb-enterprise.v1.8.0.zip differ diff --git a/scripts/dev/__init__.py b/docker/Dockerfile similarity index 100% rename from scripts/dev/__init__.py rename to docker/Dockerfile diff --git a/docker/cluster-cleaner/Chart.yaml b/docker/cluster-cleaner/Chart.yaml new file mode 100644 index 000000000..1d113f537 --- /dev/null +++ b/docker/cluster-cleaner/Chart.yaml @@ -0,0 +1,3 @@ +name: cluster-cleaner +description: The background cleaner +version: 0.14 diff --git a/docker/cluster-cleaner/Dockerfile b/docker/cluster-cleaner/Dockerfile new file mode 100644 index 000000000..0fcb72fe2 --- /dev/null +++ b/docker/cluster-cleaner/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3-slim-buster + +ADD https://storage.googleapis.com/kubernetes-release/release/v1.13.3/bin/linux/amd64/kubectl /usr/bin +RUN chmod +x /usr/bin/kubectl + +COPY scripts/* / diff --git a/docker/cluster-cleaner/Makefile b/docker/cluster-cleaner/Makefile new file mode 100644 index 000000000..801767e2f --- /dev/null +++ b/docker/cluster-cleaner/Makefile @@ -0,0 +1,24 @@ +IMAGE_VERSION=0.14 + +.PHONY: all +all: build push install + +build: + docker build --platform linux/amd64 -t dev/cluster-cleaner:$(IMAGE_VERSION) . + +push: + $(aws ecr get-login --no-include-email --region us-east-1 || true) + docker tag dev/cluster-cleaner:$(IMAGE_VERSION) 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:$(IMAGE_VERSION) + docker tag dev/cluster-cleaner:$(IMAGE_VERSION) 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:latest + + docker push 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:$(IMAGE_VERSION) + docker push 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:latest + +install: build push + kubectl create namespace cluster-cleaner || true + helm template . \ + --set cleanerVersion=$(IMAGE_VERSION) \ + --set namespace=cluster-cleaner\ + --set cleanerNamespace=cluster-cleaner > cluster-cleaner.yaml + kubectl apply -f cluster-cleaner.yaml + rm cluster-cleaner.yaml diff --git a/docker/cluster-cleaner/readme.md b/docker/cluster-cleaner/readme.md new file mode 100644 index 000000000..0c4583308 --- /dev/null +++ b/docker/cluster-cleaner/readme.md @@ -0,0 +1,35 @@ +# Cluster-Cleaner + +The `cluster-cleaner` is a series of scripts in a Docker image that it is +supposed to run on the Kubernetes cluster via CronJobs. + +## Installing a new version + +When making changes to any of the scripts, or adding CronJobs, the image needs +to be rebuild, just increase the version number both in `Chart.yaml` and +`Makefile`. The process of publishing a new version is: + +* Make changes to bash scripts and CronJob yaml files +* Run `make build && make push` + +The CronJobs will be installed, and they will always point at the `latest` +version. + +## Running the scripts locally + +These cleaning scripts can be run locally, just make sure you are pointing at +the right Kubernetes cluster and set the required environment variables. + +### Examples + +* To restart Ops Manager: + + OM_NAMESPACE=operator-testing-42-current ./clean-ops-manager.sh + +* To clean failed namespaces: + + DELETE_OLDER_THAN_AMOUNT=10 DELETE_OLDER_THAN_UNIT=minutes ./clean-failed-namespaces.sh + +* To clean old builder Pods: + + DELETE_OLDER_THAN_AMOUNT=1 DELETE_OLDER_THAN_UNIT=days ./delete-old-builder-pods.sh diff --git a/docker/cluster-cleaner/scripts/clean-cluster-roles-and-bindings.sh b/docker/cluster-cleaner/scripts/clean-cluster-roles-and-bindings.sh new file mode 100755 index 000000000..42106df75 --- /dev/null +++ b/docker/cluster-cleaner/scripts/clean-cluster-roles-and-bindings.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh + +echo "Deleting ClusterRoles" + +for role in $(kubectl get clusterrole -o name | grep -E "mongodb-enterprise-operator|mdb-operator|operator-multi-cluster-tests-role-binding|operator-tests-role-binding") ; do + creation_time=$(kubectl get "${role}" -o jsonpath='{.metadata.creationTimestamp}') + + if ! ./is_older_than.py "${creation_time}" 1 hours; then + continue + fi + + kubectl delete "${role}" +done + +echo "Deleting ClusterRoleBinding" +for binding in $(kubectl get clusterrolebinding -o name | grep -E "mongodb-enterprise-operator|mdb-operator|operator-multi-cluster-tests-role-binding|operator-tests-role-binding"); do + creation_time=$(kubectl get "${binding}" -o jsonpath='{.metadata.creationTimestamp}') + + if ! ./is_older_than.py "${creation_time}" 1 hours; then + continue + fi + + kubectl delete "${binding}" +done diff --git a/docker/cluster-cleaner/scripts/clean-failed-namespaces.sh b/docker/cluster-cleaner/scripts/clean-failed-namespaces.sh new file mode 100755 index 000000000..6ee0596f9 --- /dev/null +++ b/docker/cluster-cleaner/scripts/clean-failed-namespaces.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env sh + +delete_resources_safely() { + resource_type="$1" + namespace="$2" + + echo "Attempting normal deletion of $resource_type in $namespace..." + kubectl delete "${resource_type}" --all -n "${namespace}" --wait=true --timeout=10s || true + + # Check if any resources are still stuck + resources=$(kubectl get "$resource_type" -n "${namespace}" --no-headers -o custom-columns=":metadata.name") + + for resource in ${resources}; do + echo "${resource_type}/${resource} is still present, force deleting..." + + kubectl patch "${resource_type}" "${resource}" -n "${namespace}" -p '{"metadata":{"finalizers":null}}' --type=merge || true + kubectl delete "${resource_type}" "${resource}" -n "${namespace}" --force --grace-period=0 || true + done +} + +if [ -z ${DELETE_OLDER_THAN_AMOUNT+x} ] || [ -z ${DELETE_OLDER_THAN_UNIT+x} ]; then + echo "Need to set both 'DELETE_OLDER_THAN_AMOUNT' and 'DELETE_OLDER_THAN_UNIT' environment variables." + exit 1 +fi + +if [ -z ${LABELS+x} ]; then + echo "Need to set 'LABELS' environment variables." + exit 1 +fi + +echo "Deleting namespaces for evg tasks that are older than ${DELETE_OLDER_THAN_AMOUNT} ${DELETE_OLDER_THAN_UNIT} with label ${LABELS}" +echo "Which are:" +kubectl get namespace -l "${LABELS}" -o name +for namespace in $(kubectl get namespace -l "${LABELS}" -o name); do + creation_time=$(kubectl get "${namespace}" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || echo "") + + if [ -z "$creation_time" ]; then + echo "Namespace ${namespace} does not exist or has no creation timestamp, skipping." + continue + fi + + namespace_name=$(echo "${namespace}" | cut -d '/' -f 2) + + if ! ./is_older_than.py "${creation_time}" "${DELETE_OLDER_THAN_AMOUNT}" "${DELETE_OLDER_THAN_UNIT}"; then + echo "Skipping ${namespace_name}, not old enough." + continue + fi + + echo "Deleting ${namespace_name}" + + csrs_in_namespace=$(kubectl get csr -o name | grep "${namespace_name}" || true) + if [ -n "${csrs_in_namespace}" ]; then + kubectl delete "${csrs_in_namespace}" + fi + + delete_resources_safely "mdb" "${namespace_name}" + delete_resources_safely "mdbu" "${namespace_name}" + delete_resources_safely "om" "${namespace_name}" + + echo "Attempting to delete namespace: ${namespace_name}" + + if kubectl get namespace "${namespace_name}" >/dev/null 2>&1; then + kubectl delete namespace "${namespace_name}" --wait=true --timeout=10s || true + else + echo "Namespace ${namespace_name} not found, skipping deletion." + fi + + if kubectl get namespace "${namespace_name}" >/dev/null 2>&1; then + echo "Namespace ${namespace_name} is still stuck, removing finalizers..." + kubectl patch namespace "${namespace_name}" -p '{"metadata":{"finalizers":null}}' --type=merge + + echo "Force deleting namespace: ${namespace_name}" + kubectl delete namespace "${namespace_name}" --wait=true --timeout=30s + else + echo "Namespace ${namespace_name} deleted successfully." + fi +done diff --git a/docker/cluster-cleaner/scripts/clean-ops-manager.sh b/docker/cluster-cleaner/scripts/clean-ops-manager.sh new file mode 100755 index 000000000..674700c4c --- /dev/null +++ b/docker/cluster-cleaner/scripts/clean-ops-manager.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +if [ -z "${OM_NAMESPACE}" ]; then + echo "OM_NAMESPACE env variable is not specified"; + exit 1 +fi + +echo "Removing Ops Manager in ${OM_NAMESPACE}" + +kubectl --namespace "${OM_NAMESPACE}" delete om ops-manager + diff --git a/docker/cluster-cleaner/scripts/construction-site.sh b/docker/cluster-cleaner/scripts/construction-site.sh new file mode 100755 index 000000000..96aacd21a --- /dev/null +++ b/docker/cluster-cleaner/scripts/construction-site.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# +# Builds the `construction-site` namespace and docker config configmap +# + +kubectl create namespace construction-site || true + +kubectl -n construction-site create configmap docker-config --from-literal=config.json='{"credHelpers":{"268558157000.dkr.ecr.us-east-1.amazonaws.com":"ecr-login"}}' diff --git a/docker/cluster-cleaner/scripts/create-cluster-ca-as-configmap.sh b/docker/cluster-cleaner/scripts/create-cluster-ca-as-configmap.sh new file mode 100755 index 000000000..5067c26f9 --- /dev/null +++ b/docker/cluster-cleaner/scripts/create-cluster-ca-as-configmap.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +kube_ca_file="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + +if ! kubectl -n default get configmap/ca-certificates; then + echo "Creating the ca-certificates for this Kubernetes cluster" + kubectl -n default create configmap ca-certificates --from-file=ca-pem=${kube_ca_file} +else + echo "ca-certificates configmap already exists." +fi diff --git a/docker/cluster-cleaner/scripts/delete-old-builder-pods.sh b/docker/cluster-cleaner/scripts/delete-old-builder-pods.sh new file mode 100755 index 000000000..ed1f39763 --- /dev/null +++ b/docker/cluster-cleaner/scripts/delete-old-builder-pods.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +NAMESPACE=construction-site + +if [ -z ${DELETE_OLDER_THAN_AMOUNT+x} ] || [ -z ${DELETE_OLDER_THAN_UNIT+x} ]; then + echo "Need to set both 'DELETE_OLDER_THAN_AMOUNT' and 'DELETE_OLDER_THAN_UNIT' environment variables." + exit 1 +fi + +for pod in $(kubectl -n ${NAMESPACE} get pods -o name); do + creation_time=$(kubectl -n ${NAMESPACE} get "${pod}" -o jsonpath='{.metadata.creationTimestamp}') + status=$(kubectl get "${pod}" -o jsonpath='{.status.phase}' -n "${NAMESPACE}") + + if [[ "${status}" != "Succeeded" ]] && [[ "${status}" != "Failed" ]]; then + # we don't remove pending tasks + continue + fi + + if ! ./is_older_than.py "${creation_time}" "${DELETE_OLDER_THAN_AMOUNT}" "${DELETE_OLDER_THAN_UNIT}"; then + continue + fi + kubectl -n ${NAMESPACE} delete "${pod}" +done diff --git a/docker/cluster-cleaner/scripts/is_older_than.py b/docker/cluster-cleaner/scripts/is_older_than.py new file mode 100755 index 000000000..657dbcb41 --- /dev/null +++ b/docker/cluster-cleaner/scripts/is_older_than.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +# is_older_than.py +# +# Usage: +# +# ./is_older_than.py +# +# Exits with a successful exit code if is older than s of time. +# The format of is a simple iso datetime as returned by +# +# kubectl get namespaces -o jsonpath='{.items[*].metadata.creationTimestamp}' +# +# Example: +# +# 1. Check if a given timestamp is older than 6 hours +# ./is_older_than.py 2019-02-22T11:28:04Z 6 hours +# +# 2. Check if a given timestamp is older than 3 days +# ./is_older_than.py 2019-02-22T11:28:04Z 3 days +# +# 3. Check if Rodrigo is older than 39 years. +# This command will return 1 until my next birthday. +# ./is_older_than.py 1980-25-04T11:00:04Z 39 years +# + +import sys +from datetime import datetime, timedelta + + +def is_older_than(date, amount, unit): + """Checks if datetime is older than `amount` of `unit`""" + date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%SZ") + # gets the following options, same as we use to construct the timedelta object, + # like 'minutes 6' -- it is expected in command line as '6 minutes' + delta_options = {unit: amount} + delta = timedelta(**delta_options) + + return date + delta > datetime.now() + + +if __name__ == "__main__": + date = sys.argv[1] + amount = int(sys.argv[2]) + unit = sys.argv[3] + + sys.exit(is_older_than(date, amount, unit)) diff --git a/docker/cluster-cleaner/templates/job.yaml b/docker/cluster-cleaner/templates/job.yaml new file mode 100644 index 000000000..b131e75fb --- /dev/null +++ b/docker/cluster-cleaner/templates/job.yaml @@ -0,0 +1,159 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cluster-cleaner + namespace: {{ .Values.cleanerNamespace }} + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-cleaner + namespace: {{ .Values.cleanerNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: cluster-cleaner + namespace: {{ .Values.cleanerNamespace }} + +# Remove old failed namespaces, which are older than 20 minutes. +# This CJ runs every 10 minutes, so a failed namespace will live for +# at least 20 minutes, but can live up to 30. +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-cleaner-delete-each-hour + namespace: {{ .Values.cleanerNamespace }} +spec: + # Run every 10 minutes + schedule: "*/10 * * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: cluster-cleaner + restartPolicy: Never + + containers: + - name: cluster-cleaner + image: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:{{ .Values.cleanerVersion }} + imagePullPolicy: Always + command: ["./clean-failed-namespaces.sh"] + env: + - name: DELETE_OLDER_THAN_UNIT + value: "minutes" + - name: DELETE_OLDER_THAN_AMOUNT + value: "20" + - name: LABELS + value: "evg=task,evg/state=failed" + +# Remove old testing namespaces, no matter if they have succeeded or failed. +# This is run every 90 minutes, so every testing namespace, even if it has not +# finished running it will be removed. +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-cleaner-delete-each-hour-all + namespace: {{ .Values.cleanerNamespace }} +spec: + # Run every 10 minutes + schedule: "*/10 * * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: cluster-cleaner + restartPolicy: Never + + containers: + - name: cluster-cleaner + image: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:{{ .Values.cleanerVersion }} + imagePullPolicy: Always + command: ["./clean-failed-namespaces.sh"] + env: + - name: DELETE_OLDER_THAN_UNIT + value: "minutes" + - name: DELETE_OLDER_THAN_AMOUNT + value: "90" + - name: LABELS + value: "evg=task" + +# Clean old builder pods +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-cleaner-delete-builder-pods + namespace: {{ .Values.cleanerNamespace }} +spec: + # Runs every hour + schedule: "0 * * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: cluster-cleaner + restartPolicy: Never + + containers: + - name: cluster-cleaner + image: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:{{ .Values.cleanerVersion }} + imagePullPolicy: Always + command: ["./delete-old-builder-pods.sh"] + env: + - name: DELETE_OLDER_THAN_UNIT + value: "minutes" + - name: DELETE_OLDER_THAN_AMOUNT + value: "20" + +# Clean old certificates +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-cleaner-ca-certificates + namespace: {{ .Values.cleanerNamespace }} +spec: + # Run at 6:17 am every day of the week + schedule: "17 6 * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: cluster-cleaner + restartPolicy: Never + + containers: + - name: cluster-cleaner + image: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:{{ .Values.cleanerVersion }} + imagePullPolicy: Always + command: ["./create-cluster-ca-as-configmap.sh"] + +# Remove old clusterroles +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-cleaner-cluster-roles-and-bindings + namespace: {{ .Values.cleanerNamespace }} +spec: + # Run at 4:25 every day of the week + schedule: "25 4 * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: cluster-cleaner + restartPolicy: Never + + containers: + - name: cluster-cleaner + image: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:{{ .Values.cleanerVersion }} + imagePullPolicy: Always + command: ["./clean-cluster-roles-and-bindings.sh"] diff --git a/docker/cluster-cleaner/templates/ops_manager_cleaner_job.yaml b/docker/cluster-cleaner/templates/ops_manager_cleaner_job.yaml new file mode 100644 index 000000000..05509344a --- /dev/null +++ b/docker/cluster-cleaner/templates/ops_manager_cleaner_job.yaml @@ -0,0 +1,48 @@ +{{ if .Values.namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ops-manager-cleaner + namespace: {{ .Values.namespace }} + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ops-manager-cleaner + namespace: {{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: ops-manager-cleaner + namespace: {{ .Values.namespace }} + +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-cleaner-ops-manager + namespace: {{ .Values.namespace }} +spec: + # Run at 3:00 am every day. + schedule: "0 3 * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: ops-manager-cleaner + restartPolicy: Never + + containers: + - name: cluster-cleaner + image: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/cluster-cleaner:{{ .Values.cleanerVersion }} + imagePullPolicy: Always + command: ["./clean-ops-manager.sh"] + env: + - name: OM_NAMESPACE + value: {{ .Values.namespace }} +{{ end }} diff --git a/docker/mongodb-agent-non-matrix/Dockerfile b/docker/mongodb-agent-non-matrix/Dockerfile new file mode 100644 index 000000000..e1c1caff2 --- /dev/null +++ b/docker/mongodb-agent-non-matrix/Dockerfile @@ -0,0 +1,60 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi9/ubi-minimal + +ARG version + +LABEL name="MongoDB Agent" \ + version="${version}" \ + summary="MongoDB Agent" \ + description="MongoDB Agent" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +# Replace libcurl-minimal and curl-minimal with the full versions +# https://bugzilla.redhat.com/show_bug.cgi?id=1994521 +RUN microdnf install -y libssh libpsl libbrotli \ + && microdnf download curl libcurl \ + && rpm -Uvh --nodeps --replacefiles "*curl*$( uname -i ).rpm" \ + && microdnf remove -y libcurl-minimal curl-minimal + +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 nss_wrapper +# Copy-pasted from https://www.mongodb.com/docs/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ +RUN microdnf install -y --disableplugin=subscription-manager \ + cyrus-sasl cyrus-sasl-gssapi cyrus-sasl-plain krb5-libs openldap openssl xz-libs +# Dependencies for the Agent +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 \ + net-snmp \ + net-snmp-agent-libs +RUN microdnf install -y --disableplugin=subscription-manager \ + hostname tar gzip procps jq \ + && microdnf upgrade -y \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /agent \ + && mkdir -p /var/lib/mongodb-mms-automation \ + && mkdir -p /var/log/mongodb-mms-automation/ \ + && chmod -R +wr /var/log/mongodb-mms-automation/ \ + # ensure that the agent user can write the logs in OpenShift + && touch /var/log/mongodb-mms-automation/readiness.log \ + && chmod ugo+rw /var/log/mongodb-mms-automation/readiness.log + + +COPY --from=base /data/mongodb-agent.tar.gz /agent +COPY --from=base /data/mongodb-tools.tgz /agent +COPY --from=base /data/LICENSE /licenses/LICENSE + +RUN tar xfz /agent/mongodb-agent.tar.gz \ + && mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent /agent/mongodb-agent \ + && chmod +x /agent/mongodb-agent \ + && mkdir -p /var/lib/automation/config \ + && chmod -R +r /var/lib/automation/config \ + && rm /agent/mongodb-agent.tar.gz \ + && rm -r mongodb-mms-automation-agent-* + +RUN tar xfz /agent/mongodb-tools.tgz --directory /var/lib/mongodb-mms-automation/ && rm /agent/mongodb-tools.tgz + +USER 2000 +CMD ["/agent/mongodb-agent", "-cluster=/var/lib/automation/config/automation-config.json"] diff --git a/docker/mongodb-agent-non-matrix/Dockerfile.builder b/docker/mongodb-agent-non-matrix/Dockerfile.builder new file mode 100644 index 000000000..369212e3e --- /dev/null +++ b/docker/mongodb-agent-non-matrix/Dockerfile.builder @@ -0,0 +1,11 @@ +FROM scratch + +ARG agent_version +ARG agent_distro +ARG tools_distro +ARG tools_version + +ADD https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.${agent_distro}.tar.gz /data/mongodb-agent.tar.gz +ADD https://downloads.mongodb.org/tools/db/mongodb-database-tools-${tools_distro}-${tools_version}.tgz /data/mongodb-tools.tgz + +COPY ./docker/mongodb-enterprise-init-database/content/LICENSE /data/LICENSE diff --git a/docker/mongodb-agent/Dockerfile b/docker/mongodb-agent/Dockerfile new file mode 100644 index 000000000..08d8746d8 --- /dev/null +++ b/docker/mongodb-agent/Dockerfile @@ -0,0 +1,64 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi9/ubi-minimal + +ARG version + +LABEL name="MongoDB Agent" \ + version="${version}" \ + summary="MongoDB Agent" \ + description="MongoDB Agent" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/probe.sh /opt/scripts/probe.sh +COPY --from=base /data/readinessprobe /opt/scripts/readinessprobe +COPY --from=base /data/version-upgrade-hook /opt/scripts/version-upgrade-hook +COPY --from=base /data/agent-launcher-lib.sh /opt/scripts/agent-launcher-lib.sh +COPY --from=base /data/agent-launcher.sh /opt/scripts/agent-launcher.sh +COPY --from=base /data/LICENSE /licenses/LICENSE + +# Replace libcurl-minimal and curl-minimal with the full versions +# https://bugzilla.redhat.com/show_bug.cgi?id=1994521 +RUN microdnf install -y libssh libpsl libbrotli \ + && microdnf download curl libcurl \ + && rpm -Uvh --nodeps --replacefiles "*curl*$( uname -i ).rpm" \ + && microdnf remove -y libcurl-minimal curl-minimal + +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 nss_wrapper +# Copy-pasted from https://www.mongodb.com/docs/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ +RUN microdnf install -y --disableplugin=subscription-manager \ + cyrus-sasl cyrus-sasl-gssapi cyrus-sasl-plain krb5-libs openldap openssl xz-libs +# Dependencies for the Agent +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 \ + net-snmp \ + net-snmp-agent-libs +RUN microdnf install -y --disableplugin=subscription-manager \ + hostname tar gzip procps jq \ + && microdnf upgrade -y \ + && rm -rf /var/lib/apt/lists/* + + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz +COPY --from=base /data/mongodb_agent_ubi.tgz /agent/mongodb_agent.tgz + +RUN tar xfz /tools/mongodb_tools.tgz +RUN mv mongodb-database-tools-*/bin/* /tools +RUN chmod +x /tools/* +RUN rm /tools/mongodb_tools.tgz +RUN rm -rf /mongodb-database-tools-* + +RUN tar xfz /agent/mongodb_agent.tgz +RUN mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent /agent/mongodb-agent +RUN chmod +x /agent/mongodb-agent +RUN rm /agent/mongodb_agent.tgz +RUN rm -rf mongodb-mms-automation-agent-* + +RUN mkdir -p /var/lib/automation/config +RUN chmod -R +r /var/lib/automation/config + +USER 2000 + +HEALTHCHECK --timeout=30s CMD ls /opt/scripts/readinessprobe || exit 1 diff --git a/docker/mongodb-agent/Dockerfile.builder b/docker/mongodb-agent/Dockerfile.builder new file mode 100644 index 000000000..da8f45b2d --- /dev/null +++ b/docker/mongodb-agent/Dockerfile.builder @@ -0,0 +1,37 @@ +# the init database image gets supplied by pipeline.py and corresponds to the operator version we want to release +# the agent with. This enables us to release the agent for older operator. +ARG init_database_image +FROM ${init_database_image} as init_database + +FROM public.ecr.aws/docker/library/golang:1.24 as dependency_downloader + +WORKDIR /go/src/github.com/10gen/ops-manager-kubernetes/ + +COPY go.mod go.sum ./ + +RUN go mod download + +FROM public.ecr.aws/docker/library/golang:1.24 as readiness_builder + +WORKDIR /go/src/github.com/10gen/ops-manager-kubernetes/ + +COPY --from=dependency_downloader /go/pkg /go/pkg +COPY go.mod go.sum ./ + +RUN CGO_ENABLED=0 go build -o /readinessprobe github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness +RUN CGO_ENABLED=0 go build -o /version-upgrade-hook github.com/mongodb/mongodb-kubernetes-operator/cmd/versionhook + +FROM scratch +ARG mongodb_tools_url_ubi +ARG mongodb_agent_url_ubi + +COPY --from=readiness_builder /readinessprobe /data/ +COPY --from=readiness_builder /version-upgrade-hook /data/ + +ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz +ADD ${mongodb_agent_url_ubi} /data/mongodb_agent_ubi.tgz + +COPY --from=init_database /probes/probe.sh /data/probe.sh +COPY --from=init_database /scripts/agent-launcher-lib.sh /data/ +COPY --from=init_database /scripts/agent-launcher.sh /data/ +COPY --from=init_database /licenses/LICENSE /data/ diff --git a/docker/mongodb-agent/README.md b/docker/mongodb-agent/README.md new file mode 100644 index 000000000..377f4b938 --- /dev/null +++ b/docker/mongodb-agent/README.md @@ -0,0 +1,4 @@ +# Mongodb-Agent +The agent gets released in a matrix style with the init-database image, which gets tagged with the operator version. +This works by using the multi-stage pattern and build-args. First - retrieve the `init-database:` and retrieve the +binaries from there. Then we continue with the other steps to fully build the image. \ No newline at end of file diff --git a/docker/mongodb-enterprise-appdb-database/4.0/ubi/Dockerfile b/docker/mongodb-enterprise-appdb-database/4.0/ubi/Dockerfile new file mode 100644 index 000000000..1b5d11310 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/4.0/ubi/Dockerfile @@ -0,0 +1,85 @@ +FROM registry.access.redhat.com/ubi8/ubi + +ARG MONGO_VERSION + +LABEL name="MongoDB Enterprise AppDB Database" \ + version=${MONGO_VERSION} \ + summary="MongoDB Enterprise AppDB Database" \ + description="MongoDB Enterprise AppDB Database" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +ADD licenses /licenses + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN groupadd -r mongodb && useradd -r -g mongodb mongodb + + +RUN set -eux; \ + yum install -y --disableplugin=subscription-manager \ + ca-certificates \ + jq \ + ; \ + if ! command -v ps > /dev/null; then \ + yum install -y --disableplugin=subscription-managers procps; \ + fi; + +# grab gosu for easy step-down from root (https://github.com/tianon/gosu/releases) +ENV GOSU_VERSION 1.14 +# grab "js-yaml" for parsing mongod's YAML config files (https://github.com/nodeca/js-yaml/releases) +ENV JSYAML_VERSION 3.13.1 + +RUN set -ex; \ + \ + yum install -y --disableplugin=subscription-manager \ + wget \ + ; \ + if ! command -v gpg > /dev/null; then \ + yum install -y --disableplugin=subscription-manager gnupg dirmngr; \ + elif gpg --version | grep -q '^gpg (GnuPG) 1\.'; then \ +# "This package provides support for HKPS keyservers." (GnuPG 1.x only) + yum install -y --disableplugin=subscription-manager gnupg-curl; \ + fi; \ + \ + dpkgArch="$(arch| sed 's/x86_64/amd64/')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + command -v gpgconf && gpgconf --kill all || :; \ + rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + \ + wget -O /js-yaml.js "https://github.com/nodeca/js-yaml/raw/${JSYAML_VERSION}/dist/js-yaml.js"; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN mkdir /docker-entrypoint-initdb.d + +# # Allow build-time overrides (eg. to build image with MongoDB Enterprise version) +# # Options for MONGO_PACKAGE: mongodb-org OR mongodb-enterprise +# # Options for MONGO_REPO: repo.mongodb.org OR repo.mongodb.com +# # Example: docker build --build-arg MONGO_PACKAGE=mongodb-enterprise --build-arg MONGO_REPO=repo.mongodb.com . +ARG MONGO_PACKAGE=mongodb-org +ARG MONGO_REPO=repo.mongodb.org +ENV MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO} + +ENV MONGO_MAJOR 4.0 + +COPY 4.0/ubi/mongodb-org-4.0.repo /etc/yum.repos.d/mongodb-org-4.0.repo + + +RUN yum install -y mongodb-enterprise-${MONGO_VERSION} mongodb-enterprise-server-${MONGO_VERSION} mongodb-enterprise-shell-${MONGO_VERSION} mongodb-enterprise-mongos-${MONGO_VERSION} mongodb-enterprise-tools-${MONGO_VERSION} + +RUN echo "exclude=mongodb-org,mongodb-org-server,mongodb-org-shell,mongodb-org-mongos,mongodb-org-tools" >> /etc/yum.conf +RUN mkdir -p /data/db /data/configdb \ + && chown -R mongodb:mongodb /data/db /data/configdb +VOLUME /data/db /data/configdb + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 27017 +CMD ["mongod"] diff --git a/docker/mongodb-enterprise-appdb-database/4.0/ubi/mongodb-org-4.0.repo b/docker/mongodb-enterprise-appdb-database/4.0/ubi/mongodb-org-4.0.repo new file mode 100644 index 000000000..18a2fc0e9 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/4.0/ubi/mongodb-org-4.0.repo @@ -0,0 +1,6 @@ +[mongodb-enterprise-4.0] +name=MongoDB Enterprise Repository +baseurl=https://repo.mongodb.com/yum/redhat/$releasever/mongodb-enterprise/4.0/$basearch/ +gpgcheck=1 +enabled=1 +gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc diff --git a/docker/mongodb-enterprise-appdb-database/4.2/ubi/Dockerfile b/docker/mongodb-enterprise-appdb-database/4.2/ubi/Dockerfile new file mode 100644 index 000000000..907d449ce --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/4.2/ubi/Dockerfile @@ -0,0 +1,87 @@ +FROM registry.access.redhat.com/ubi8/ubi + +ARG MONGO_VERSION + +LABEL name="MongoDB Enterprise AppDB Database" \ + version=${MONGO_VERSION} \ + summary="MongoDB Enterprise AppDB Database" \ + description="MongoDB Enterprise AppDB Database" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +ADD licenses /licenses + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN groupadd -r mongodb && useradd -r -g mongodb mongodb + + +RUN set -eux; \ + yum install -y --disableplugin=subscription-manager \ + ca-certificates \ + jq \ + ; \ + if ! command -v ps > /dev/null; then \ + yum install -y --disableplugin=subscription-managers procps; \ + fi; + +# grab gosu for easy step-down from root (https://github.com/tianon/gosu/releases) +ENV GOSU_VERSION 1.14 +# grab "js-yaml" for parsing mongod's YAML config files (https://github.com/nodeca/js-yaml/releases) +ENV JSYAML_VERSION 3.13.1 + +RUN set -ex; \ + \ + yum install -y --disableplugin=subscription-manager \ + wget \ + ; \ + if ! command -v gpg > /dev/null; then \ + yum install -y --disableplugin=subscription-manager gnupg dirmngr; \ + elif gpg --version | grep -q '^gpg (GnuPG) 1\.'; then \ +# "This package provides support for HKPS keyservers." (GnuPG 1.x only) + yum install -y --disableplugin=subscription-manager gnupg-curl; \ + fi; \ + \ + dpkgArch="$(arch| sed 's/x86_64/amd64/')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + command -v gpgconf && gpgconf --kill all || :; \ + rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + \ + wget -O /js-yaml.js "https://github.com/nodeca/js-yaml/raw/${JSYAML_VERSION}/dist/js-yaml.js"; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + + RUN mkdir /docker-entrypoint-initdb.d + +# # Allow build-time overrides (eg. to build image with MongoDB Enterprise version) +# # Options for MONGO_PACKAGE: mongodb-org OR mongodb-enterprise +# # Options for MONGO_REPO: repo.mongodb.org OR repo.mongodb.com +# # Example: docker build --build-arg MONGO_PACKAGE=mongodb-enterprise --build-arg MONGO_REPO=repo.mongodb.com . +ARG MONGO_PACKAGE=mongodb-org +ARG MONGO_REPO=repo.mongodb.org +ENV MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO} + +ENV MONGO_MAJOR 4.2 + +COPY 4.2/ubi/mongodb-org-4.2.repo /etc/yum.repos.d/mongodb-org-4.2.repo + + +RUN cat /etc/yum.repos.d/mongodb-org-4.2.repo + +RUN yum install -y mongodb-enterprise-${MONGO_VERSION} mongodb-enterprise-server-${MONGO_VERSION} mongodb-enterprise-shell-${MONGO_VERSION} mongodb-enterprise-mongos-${MONGO_VERSION} mongodb-enterprise-tools-${MONGO_VERSION} + +RUN echo "exclude=mongodb-org,mongodb-org-server,mongodb-org-shell,mongodb-org-mongos,mongodb-org-tools" >> /etc/yum.conf +RUN mkdir -p /data/db /data/configdb \ + && chown -R mongodb:mongodb /data/db /data/configdb +VOLUME /data/db /data/configdb + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 27017 +CMD ["mongod"] diff --git a/docker/mongodb-enterprise-appdb-database/4.2/ubi/mongodb-org-4.2.repo b/docker/mongodb-enterprise-appdb-database/4.2/ubi/mongodb-org-4.2.repo new file mode 100644 index 000000000..836d01f3b --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/4.2/ubi/mongodb-org-4.2.repo @@ -0,0 +1,6 @@ +[mongodb-enterprise-4.2] +name=MongoDB Enterprise Repository +baseurl=https://repo.mongodb.com/yum/redhat/$releasever/mongodb-enterprise/4.2/$basearch/ +gpgcheck=1 +enabled=1 +gpgkey=https://www.mongodb.org/static/pgp/server-4.2.asc diff --git a/docker/mongodb-enterprise-appdb-database/4.4/ubi/Dockerfile b/docker/mongodb-enterprise-appdb-database/4.4/ubi/Dockerfile new file mode 100644 index 000000000..a93fffc68 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/4.4/ubi/Dockerfile @@ -0,0 +1,91 @@ +FROM registry.access.redhat.com/ubi8/ubi + +ARG MONGO_VERSION + +LABEL name="MongoDB Enterprise AppDB Database" \ + version=${MONGO_VERSION} \ + summary="MongoDB Enterprise AppDB Database" \ + description="MongoDB Enterprise AppDB Database" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +ADD licenses /licenses + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN groupadd -r mongodb && useradd -r -g mongodb mongodb + +#TODO add numactl + +RUN set -eux; \ + yum install -y --disableplugin=subscription-manager \ + ca-certificates \ + jq \ + ; \ + if ! command -v ps > /dev/null; then \ + yum install -y --disableplugin=subscription-managers procps; \ + fi; + +# grab gosu for easy step-down from root (https://github.com/tianon/gosu/releases) +ENV GOSU_VERSION 1.14 +# grab "js-yaml" for parsing mongod's YAML config files (https://github.com/nodeca/js-yaml/releases) +ENV JSYAML_VERSION 3.13.1 + +RUN set -ex; \ + \ + yum install -y --disableplugin=subscription-manager \ + wget \ + ; \ + if ! command -v gpg > /dev/null; then \ + yum install -y --disableplugin=subscription-manager gnupg dirmngr; \ + elif gpg --version | grep -q '^gpg (GnuPG) 1\.'; then \ +# "This package provides support for HKPS keyservers." (GnuPG 1.x only) + yum install -y --disableplugin=subscription-manager gnupg-curl; \ + fi; \ + \ + dpkgArch="$(arch| sed 's/x86_64/amd64/')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + command -v gpgconf && gpgconf --kill all || :; \ + rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + \ + wget -O /js-yaml.js "https://github.com/nodeca/js-yaml/raw/${JSYAML_VERSION}/dist/js-yaml.js"; \ + +# smoke test + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN mkdir /docker-entrypoint-initdb.d + + +# # Allow build-time overrides (eg. to build image with MongoDB Enterprise version) +# # Options for MONGO_PACKAGE: mongodb-org OR mongodb-enterprise +# # Options for MONGO_REPO: repo.mongodb.org OR repo.mongodb.com +# # Example: docker build --build-arg MONGO_PACKAGE=mongodb-enterprise --build-arg MONGO_REPO=repo.mongodb.com . +ARG MONGO_PACKAGE=mongodb-org +ARG MONGO_REPO=repo.mongodb.org +ENV MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO} + +# TODO put everything in a single dockerfile +ENV MONGO_MAJOR 4.4 + +COPY 4.4/ubi/mongodb-org-4.4.repo /etc/yum.repos.d/mongodb-org-4.4.repo + +RUN cat /etc/yum.repos.d/mongodb-org-4.4.repo + +RUN yum install -y mongodb-enterprise-${MONGO_VERSION} mongodb-enterprise-server-${MONGO_VERSION} mongodb-enterprise-shell-${MONGO_VERSION} mongodb-enterprise-mongos-${MONGO_VERSION} mongodb-enterprise-tools-${MONGO_VERSION} + +RUN echo "exclude=mongodb-org,mongodb-org-server,mongodb-org-shell,mongodb-org-mongos,mongodb-org-tools" >> /etc/yum.conf +RUN mkdir -p /data/db /data/configdb \ + && chown -R mongodb:mongodb /data/db /data/configdb +VOLUME /data/db /data/configdb + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 27017 +CMD ["mongod"] diff --git a/docker/mongodb-enterprise-appdb-database/4.4/ubi/mongodb-org-4.4.repo b/docker/mongodb-enterprise-appdb-database/4.4/ubi/mongodb-org-4.4.repo new file mode 100644 index 000000000..0a81c4c40 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/4.4/ubi/mongodb-org-4.4.repo @@ -0,0 +1,6 @@ +[mongodb-enterprise-4.4] +name=MongoDB Enterprise Repository +baseurl=https://repo.mongodb.com/yum/redhat/$releasever/mongodb-enterprise/4.4/$basearch/ +gpgcheck=1 +enabled=1 +gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc diff --git a/docker/mongodb-enterprise-appdb-database/5.0/ubi/Dockerfile b/docker/mongodb-enterprise-appdb-database/5.0/ubi/Dockerfile new file mode 100644 index 000000000..066e88709 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/5.0/ubi/Dockerfile @@ -0,0 +1,85 @@ +FROM registry.access.redhat.com/ubi8/ubi + +ARG MONGO_VERSION + +LABEL name="MongoDB Enterprise AppDB Database" \ + version=${MONGO_VERSION} \ + summary="MongoDB Enterprise AppDB Database" \ + description="MongoDB Enterprise AppDB Database" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +ADD licenses /licenses + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN groupadd -r mongodb && useradd -r -g mongodb mongodb + +RUN set -eux; \ + yum install -y \ + ca-certificates \ + jq \ + ; \ + if ! command -v ps > /dev/null; then \ + yum install -y procps; \ + fi; + +# grab gosu for easy step-down from root (https://github.com/tianon/gosu/releases) +ENV GOSU_VERSION 1.14 +# grab "js-yaml" for parsing mongod's YAML config files (https://github.com/nodeca/js-yaml/releases) +ENV JSYAML_VERSION 3.13.1 + +RUN set -ex; \ + \ + yum install -y \ + wget \ + ; \ + if ! command -v gpg > /dev/null; then \ + yum install -y gnupg dirmngr; \ + elif gpg --version | grep -q '^gpg (GnuPG) 1\.'; then \ +# "This package provides support for HKPS keyservers." (GnuPG 1.x only) + yum install -y gnupg-curl; \ + fi; \ + \ + dpkgArch="$(arch| sed 's/x86_64/amd64/')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + command -v gpgconf && gpgconf --kill all || :; \ + rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + \ + wget -O /js-yaml.js "https://github.com/nodeca/js-yaml/raw/${JSYAML_VERSION}/dist/js-yaml.js"; \ +# smoke test + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN mkdir /docker-entrypoint-initdb.d + +# Allow build-time overrides (eg. to build image with MongoDB Enterprise version) +# Options for MONGO_PACKAGE: mongodb-org OR mongodb-enterprise +# Options for MONGO_REPO: repo.mongodb.org OR repo.mongodb.com +# Example: docker build --build-arg MONGO_PACKAGE=mongodb-enterprise --build-arg MONGO_REPO=repo.mongodb.com . +ARG MONGO_PACKAGE=mongodb-org +ARG MONGO_REPO=repo.mongodb.org +ENV MONGO_PACKAGE=${MONGO_PACKAGE} MONGO_REPO=${MONGO_REPO} + +ENV MONGO_MAJOR 5.0 + +COPY 5.0/ubi/mongodb-org-5.0.repo /etc/yum.repos.d/mongodb-org-5.0.repo +# 08/03/2021, https://github.com/mongodb/mongo/tree/6d9ec525e78465dcecadcff99cce953d380fedc8 + +RUN yum install -y mongodb-enterprise-${MONGO_VERSION} mongodb-enterprise-server-${MONGO_VERSION} mongodb-enterprise-shell-${MONGO_VERSION} mongodb-enterprise-mongos-${MONGO_VERSION} mongodb-enterprise-tools-${MONGO_VERSION} + +RUN echo "exclude=mongodb-org,mongodb-org-server,mongodb-org-shell,mongodb-org-mongos,mongodb-org-tools" >> /etc/yum.conf +RUN mkdir -p /data/db /data/configdb \ + && chown -R mongodb:mongodb /data/db /data/configdb +VOLUME /data/db /data/configdb + +COPY docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 27017 +CMD ["mongod"] diff --git a/docker/mongodb-enterprise-appdb-database/5.0/ubi/mongodb-org-5.0.repo b/docker/mongodb-enterprise-appdb-database/5.0/ubi/mongodb-org-5.0.repo new file mode 100644 index 000000000..76946402d --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/5.0/ubi/mongodb-org-5.0.repo @@ -0,0 +1,6 @@ +[mongodb-enterprise-5.0] +name=MongoDB Enterprise Repository +baseurl=https://repo.mongodb.com/yum/redhat/$releasever/mongodb-enterprise/5.0/$basearch/ +gpgcheck=1 +enabled=1 +gpgkey=https://www.mongodb.org/static/pgp/server-5.0.asc diff --git a/docker/mongodb-enterprise-appdb-database/README.md b/docker/mongodb-enterprise-appdb-database/README.md new file mode 100644 index 000000000..a5f332d2f --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/README.md @@ -0,0 +1,8 @@ +Build enterprise images with + +```bash +docker build -f path/to/dockerfile --build-arg MONGO_PACKAGE=mongodb-enterprise --build-arg MONGO_REPO=repo.mongodb.com --build-arg MONGO_VERSION=4.0.20 . +``` + +within the given directory. + diff --git a/docker/mongodb-enterprise-appdb-database/build_and_push_appdb_database_images.sh b/docker/mongodb-enterprise-appdb-database/build_and_push_appdb_database_images.sh new file mode 100755 index 000000000..7e3a81565 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/build_and_push_appdb_database_images.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +_44_versions=$(jq -rc '.supportedImages."appdb-database".versions[] | select(test("^4.4"))' < ../../release.json | sed 's/-ent//g' | tr '\n' ' ') +_50_versions=$(jq -rc '.supportedImages."appdb-database".versions[] | select(test("^5.0"))' < ../../release.json | sed 's/-ent//g' | tr '\n' ' ') + +echo "4.4 versions: ${_44_versions}" +echo "5.0 versions: ${_50_versions}" + +build_id="b$(date '+%Y%m%dT000000Z')" + +missing_versions="" + +append_missing_version() { + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + missing_versions+="${1} on ${2}"$'\n' + fi +} + +for version in ${_44_versions}; do + docker build \ + -f 4.4/ubi/Dockerfile \ + --build-arg MONGO_PACKAGE=mongodb-enterprise \ + --build-arg "MONGO_VERSION=${version}" \ + --build-arg MONGO_REPO=repo.mongodb.com \ + -t "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent-${build_id}" \ + -t "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent" . + append_missing_version "${version}" "ubi" + + docker push "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent-${build_id}" + docker push "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent" +done + +for version in ${_50_versions}; do + docker build \ + -f 5.0/ubi/Dockerfile \ + --build-arg MONGO_PACKAGE=mongodb-enterprise \ + --build-arg "MONGO_VERSION=${version}" \ + --build-arg MONGO_REPO=repo.mongodb.com \ + -t "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent-${build_id}" \ + -t "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent" . + append_missing_version "${version}" "ubi" + + docker push "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent-${build_id}" + docker push "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:${version}-ent" +done + +echo "Missing versions" +echo "${missing_versions}" diff --git a/docker/mongodb-enterprise-appdb-database/docker-entrypoint.sh b/docker/mongodb-enterprise-appdb-database/docker-entrypoint.sh new file mode 100644 index 000000000..d3d1c08c7 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/docker-entrypoint.sh @@ -0,0 +1,386 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [ "${1:0:1}" = '-' ]; then + set -- mongod "$@" +fi + +originalArgOne="$1" + +# allow the container to be started with `--user` +# all mongo* commands should be dropped to the correct user +if [[ "$originalArgOne" == mongo* ]] && [ "$(id -u)" = '0' ]; then + if [ "$originalArgOne" = 'mongod' ]; then + find /data/configdb /data/db \! -user mongodb -exec chown mongodb '{}' + + fi + + # make sure we can write to stdout and stderr as "mongodb" + # (for our "initdb" code later; see "--logpath" below) + chown --dereference mongodb "/proc/$$/fd/1" "/proc/$$/fd/2" || : + # ignore errors thanks to https://github.com/docker-library/mongo/issues/149 + + exec gosu mongodb "$BASH_SOURCE" "$@" +fi + + +# you should use numactl to start your mongod instances, including the config servers, mongos instances, and any clients. +# https://docs.mongodb.com/manual/administration/production-notes/#configuring-numa-on-linux +if [[ "$originalArgOne" == mongo* ]]; then + numa='numactl --interleave=all' + if $numa true &> /dev/null; then + set -- $numa "$@" + fi +fi + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# see https://github.com/docker-library/mongo/issues/147 (mongod is picky about duplicated arguments) +_mongod_hack_have_arg() { + local checkArg="$1"; shift + local arg + for arg; do + case "$arg" in + "$checkArg"|"$checkArg"=*) + return 0 + ;; + esac + done + return 1 +} +# _mongod_hack_get_arg_val '--some-arg' "$@" +_mongod_hack_get_arg_val() { + local checkArg="$1"; shift + while [ "$#" -gt 0 ]; do + local arg="$1"; shift + case "$arg" in + "$checkArg") + echo "$1" + return 0 + ;; + "$checkArg"=*) + echo "${arg#$checkArg=}" + return 0 + ;; + esac + done + return 1 +} +declare -a mongodHackedArgs +# _mongod_hack_ensure_arg '--some-arg' "$@" +# set -- "${mongodHackedArgs[@]}" +_mongod_hack_ensure_arg() { + local ensureArg="$1"; shift + mongodHackedArgs=( "$@" ) + if ! _mongod_hack_have_arg "$ensureArg" "$@"; then + mongodHackedArgs+=( "$ensureArg" ) + fi +} +# _mongod_hack_ensure_no_arg '--some-unwanted-arg' "$@" +# set -- "${mongodHackedArgs[@]}" +_mongod_hack_ensure_no_arg() { + local ensureNoArg="$1"; shift + mongodHackedArgs=() + while [ "$#" -gt 0 ]; do + local arg="$1"; shift + if [ "$arg" = "$ensureNoArg" ]; then + continue + fi + mongodHackedArgs+=( "$arg" ) + done +} +# _mongod_hack_ensure_no_arg '--some-unwanted-arg' "$@" +# set -- "${mongodHackedArgs[@]}" +_mongod_hack_ensure_no_arg_val() { + local ensureNoArg="$1"; shift + mongodHackedArgs=() + while [ "$#" -gt 0 ]; do + local arg="$1"; shift + case "$arg" in + "$ensureNoArg") + shift # also skip the value + continue + ;; + "$ensureNoArg"=*) + # value is already included + continue + ;; + esac + mongodHackedArgs+=( "$arg" ) + done +} +# _mongod_hack_ensure_arg_val '--some-arg' 'some-val' "$@" +# set -- "${mongodHackedArgs[@]}" +_mongod_hack_ensure_arg_val() { + local ensureArg="$1"; shift + local ensureVal="$1"; shift + _mongod_hack_ensure_no_arg_val "$ensureArg" "$@" + mongodHackedArgs+=( "$ensureArg" "$ensureVal" ) +} + +# _js_escape 'some "string" value' +_js_escape() { + jq --null-input --arg 'str' "$1" '$str' +} + +: "${TMPDIR:=/tmp}" +jsonConfigFile="$TMPDIR/docker-entrypoint-config.json" +tempConfigFile="$TMPDIR/docker-entrypoint-temp-config.json" +_parse_config() { + if [ -s "$tempConfigFile" ]; then + return 0 + fi + + local configPath + if configPath="$(_mongod_hack_get_arg_val --config "$@")" && [ -s "$configPath" ]; then + # if --config is specified, parse it into a JSON file so we can remove a few problematic keys (especially SSL-related keys) + # see https://docs.mongodb.com/manual/reference/configuration-options/ + if grep -vEm1 '^[[:space:]]*(#|$)' "$configPath" | grep -qE '^[[:space:]]*[^=:]+[[:space:]]*='; then + # if the first non-comment/non-blank line of the config file looks like "foo = ...", this is probably the 2.4 and older "ini-style config format" + # https://docs.mongodb.com/v2.4/reference/configuration-options/ + # https://docs.mongodb.com/v2.6/reference/configuration-options/ + # https://github.com/mongodb/mongo/blob/r4.4.2/src/mongo/util/options_parser/options_parser.cpp#L1359-L1375 + # https://stackoverflow.com/a/25518018/433558 + echo >&2 + echo >&2 "WARNING: it appears that '$configPath' is in the older INI-style format (replaced by YAML in MongoDB 2.6)" + echo >&2 ' This script does not parse the older INI-style format, and thus will ignore it.' + echo >&2 + return 1 + fi + mongo --norc --nodb --quiet --eval "load('/js-yaml.js'); printjson(jsyaml.load(cat($(_js_escape "$configPath"))))" > "$jsonConfigFile" + if [ "$(head -c1 "$jsonConfigFile")" != '{' ] || [ "$(tail -c2 "$jsonConfigFile")" != '}' ]; then + # if the file doesn't start with "{" and end with "}", it's *probably* an error ("uncaught exception: YAMLException: foo" for example), so we should print it out + echo >&2 'error: unexpected "js-yaml.js" output while parsing config:' + cat >&2 "$jsonConfigFile" + exit 1 + fi + jq 'del(.systemLog, .processManagement, .net, .security)' "$jsonConfigFile" > "$tempConfigFile" + return 0 + fi + + return 1 +} +dbPath= +_dbPath() { + if [ -n "$dbPath" ]; then + echo "$dbPath" + return + fi + + if ! dbPath="$(_mongod_hack_get_arg_val --dbpath "$@")"; then + if _parse_config "$@"; then + dbPath="$(jq -r '.storage.dbPath // empty' "$jsonConfigFile")" + fi + fi + + if [ -z "$dbPath" ]; then + if _mongod_hack_have_arg --configsvr "$@" || { + _parse_config "$@" \ + && clusterRole="$(jq -r '.sharding.clusterRole // empty' "$jsonConfigFile")" \ + && [ "$clusterRole" = 'configsvr' ] + }; then + # if running as config server, then the default dbpath is /data/configdb + # https://docs.mongodb.com/manual/reference/program/mongod/#cmdoption-mongod-configsvr + dbPath=/data/configdb + fi + fi + + : "${dbPath:=/data/db}" + + echo "$dbPath" +} + +if [ "$originalArgOne" = 'mongod' ]; then + file_env 'MONGO_INITDB_ROOT_USERNAME' + file_env 'MONGO_INITDB_ROOT_PASSWORD' + # pre-check a few factors to see if it's even worth bothering with initdb + shouldPerformInitdb= + if [ "$MONGO_INITDB_ROOT_USERNAME" ] && [ "$MONGO_INITDB_ROOT_PASSWORD" ]; then + # if we have a username/password, let's set "--auth" + _mongod_hack_ensure_arg '--auth' "$@" + set -- "${mongodHackedArgs[@]}" + shouldPerformInitdb='true' + elif [ "$MONGO_INITDB_ROOT_USERNAME" ] || [ "$MONGO_INITDB_ROOT_PASSWORD" ]; then + cat >&2 <<-'EOF' + + error: missing 'MONGO_INITDB_ROOT_USERNAME' or 'MONGO_INITDB_ROOT_PASSWORD' + both must be specified for a user to be created + + EOF + exit 1 + fi + + if [ -z "$shouldPerformInitdb" ]; then + # if we've got any /docker-entrypoint-initdb.d/* files to parse later, we should initdb + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh|*.js) # this should match the set of files we check for below + shouldPerformInitdb="$f" + break + ;; + esac + done + fi + + # check for a few known paths (to determine whether we've already initialized and should thus skip our initdb scripts) + if [ -n "$shouldPerformInitdb" ]; then + dbPath="$(_dbPath "$@")" + for path in \ + "$dbPath/WiredTiger" \ + "$dbPath/journal" \ + "$dbPath/local.0" \ + "$dbPath/storage.bson" \ + ; do + if [ -e "$path" ]; then + shouldPerformInitdb= + break + fi + done + fi + + if [ -n "$shouldPerformInitdb" ]; then + mongodHackedArgs=( "$@" ) + if _parse_config "$@"; then + _mongod_hack_ensure_arg_val --config "$tempConfigFile" "${mongodHackedArgs[@]}" + fi + _mongod_hack_ensure_arg_val --bind_ip 127.0.0.1 "${mongodHackedArgs[@]}" + _mongod_hack_ensure_arg_val --port 27017 "${mongodHackedArgs[@]}" + _mongod_hack_ensure_no_arg --bind_ip_all "${mongodHackedArgs[@]}" + + # remove "--auth" and "--replSet" for our initial startup (see https://docs.mongodb.com/manual/tutorial/enable-authentication/#start-mongodb-without-access-control) + # https://github.com/docker-library/mongo/issues/211 + _mongod_hack_ensure_no_arg --auth "${mongodHackedArgs[@]}" + # "keyFile implies security.authorization" + # https://docs.mongodb.com/manual/reference/configuration-options/#mongodb-setting-security.keyFile + _mongod_hack_ensure_no_arg_val --keyFile "${mongodHackedArgs[@]}" + if [ "$MONGO_INITDB_ROOT_USERNAME" ] && [ "$MONGO_INITDB_ROOT_PASSWORD" ]; then + _mongod_hack_ensure_no_arg_val --replSet "${mongodHackedArgs[@]}" + fi + + # "BadValue: need sslPEMKeyFile when SSL is enabled" vs "BadValue: need to enable SSL via the sslMode flag when using SSL configuration parameters" + tlsMode='disabled' + if _mongod_hack_have_arg '--tlsCertificateKeyFile' "$@"; then + tlsMode='allowTLS' + elif _mongod_hack_have_arg '--sslPEMKeyFile' "$@"; then + tlsMode='allowSSL' + fi + # 4.2 switched all configuration/flag names from "SSL" to "TLS" + if [ "$tlsMode" = 'allowTLS' ] || mongod --help 2>&1 | grep -q -- ' --tlsMode '; then + _mongod_hack_ensure_arg_val --tlsMode "$tlsMode" "${mongodHackedArgs[@]}" + else + _mongod_hack_ensure_arg_val --sslMode "$tlsMode" "${mongodHackedArgs[@]}" + fi + + if stat "/proc/$$/fd/1" > /dev/null && [ -w "/proc/$$/fd/1" ]; then + # https://github.com/mongodb/mongo/blob/38c0eb538d0fd390c6cb9ce9ae9894153f6e8ef5/src/mongo/db/initialize_server_global_state.cpp#L237-L251 + # https://github.com/docker-library/mongo/issues/164#issuecomment-293965668 + _mongod_hack_ensure_arg_val --logpath "/proc/$$/fd/1" "${mongodHackedArgs[@]}" + else + initdbLogPath="$(_dbPath "$@")/docker-initdb.log" + echo >&2 "warning: initdb logs cannot write to '/proc/$$/fd/1', so they are in '$initdbLogPath' instead" + _mongod_hack_ensure_arg_val --logpath "$initdbLogPath" "${mongodHackedArgs[@]}" + fi + _mongod_hack_ensure_arg --logappend "${mongodHackedArgs[@]}" + + pidfile="$TMPDIR/docker-entrypoint-temp-mongod.pid" + rm -f "$pidfile" + _mongod_hack_ensure_arg_val --pidfilepath "$pidfile" "${mongodHackedArgs[@]}" + + "${mongodHackedArgs[@]}" --fork + + mongo=( mongo --host 127.0.0.1 --port 27017 --quiet ) + + # check to see that our "mongod" actually did start up (catches "--help", "--version", MongoDB 3.2 being silly, slow prealloc, etc) + # https://jira.mongodb.org/browse/SERVER-16292 + tries=30 + while true; do + if ! { [ -s "$pidfile" ] && ps "$(< "$pidfile")" &> /dev/null; }; then + # bail ASAP if "mongod" isn't even running + echo >&2 + echo >&2 "error: $originalArgOne does not appear to have stayed running -- perhaps it had an error?" + echo >&2 + exit 1 + fi + if "${mongo[@]}" 'admin' --eval 'quit(0)' &> /dev/null; then + # success! + break + fi + (( tries-- )) + if [ "$tries" -le 0 ]; then + echo >&2 + echo >&2 "error: $originalArgOne does not appear to have accepted connections quickly enough -- perhaps it had an error?" + echo >&2 + exit 1 + fi + sleep 1 + done + + if [ "$MONGO_INITDB_ROOT_USERNAME" ] && [ "$MONGO_INITDB_ROOT_PASSWORD" ]; then + rootAuthDatabase='admin' + + "${mongo[@]}" "$rootAuthDatabase" <<-EOJS + db.createUser({ + user: $(_js_escape "$MONGO_INITDB_ROOT_USERNAME"), + pwd: $(_js_escape "$MONGO_INITDB_ROOT_PASSWORD"), + roles: [ { role: 'root', db: $(_js_escape "$rootAuthDatabase") } ] + }) + EOJS + fi + + export MONGO_INITDB_DATABASE="${MONGO_INITDB_DATABASE:-test}" + + echo + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) echo "$0: running $f"; . "$f" ;; + *.js) echo "$0: running $f"; "${mongo[@]}" "$MONGO_INITDB_DATABASE" "$f"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + "${mongodHackedArgs[@]}" --shutdown + rm -f "$pidfile" + + echo + echo 'MongoDB init process complete; ready for start up.' + echo + fi + + # MongoDB 3.6+ defaults to localhost-only binding + haveBindIp= + if _mongod_hack_have_arg --bind_ip "$@" || _mongod_hack_have_arg --bind_ip_all "$@"; then + haveBindIp=1 + elif _parse_config "$@" && jq --exit-status '.net.bindIp // .net.bindIpAll' "$jsonConfigFile" > /dev/null; then + haveBindIp=1 + fi + if [ -z "$haveBindIp" ]; then + # so if no "--bind_ip" is specified, let's add "--bind_ip_all" + set -- "$@" --bind_ip_all + fi + + unset "${!MONGO_INITDB_@}" +fi + +rm -f "$jsonConfigFile" "$tempConfigFile" + +exec "$@" diff --git a/docker/mongodb-enterprise-appdb-database/licenses/LICENSE b/docker/mongodb-enterprise-appdb-database/licenses/LICENSE new file mode 100644 index 000000000..dc71da876 --- /dev/null +++ b/docker/mongodb-enterprise-appdb-database/licenses/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-database/Dockerfile.builder b/docker/mongodb-enterprise-database/Dockerfile.builder new file mode 100644 index 000000000..af4064127 --- /dev/null +++ b/docker/mongodb-enterprise-database/Dockerfile.builder @@ -0,0 +1,13 @@ +# +## Database image +# +## Contents +# +# * licenses/mongodb-enterprise-database + + +FROM scratch + + + +COPY LICENSE /data/licenses/mongodb-enterprise-database diff --git a/docker/mongodb-enterprise-database/Dockerfile.template b/docker/mongodb-enterprise-database/Dockerfile.template new file mode 100644 index 000000000..5f4aa5abf --- /dev/null +++ b/docker/mongodb-enterprise-database/Dockerfile.template @@ -0,0 +1,58 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM {{ base_image }} + +{% block labels %} + +LABEL name="MongoDB Enterprise Database" \ + version="{{ version }}" \ + summary="MongoDB Enterprise Database Image" \ + description="MongoDB Enterprise Database Image" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +{% endblock %} + + +{% block variables %} +ENV MMS_HOME /mongodb-automation +ENV MMS_LOG_DIR /var/log/mongodb-mms-automation +{% endblock %} + +{% block packages %} +{% endblock %} + +# Set the required perms +RUN mkdir -p "${MMS_LOG_DIR}" \ + && chmod 0775 "${MMS_LOG_DIR}" \ + && mkdir -p /var/lib/mongodb-mms-automation \ + && chmod 0775 /var/lib/mongodb-mms-automation \ + && mkdir -p /data \ + && chmod 0775 /data \ + && mkdir -p /journal \ + && chmod 0775 /journal \ + && mkdir -p "${MMS_HOME}" \ + && chmod -R 0775 "${MMS_HOME}" + +{% block dcar_copy_scripts %} +{% endblock %} + +# USER needs to be set for this image to pass RedHat verification. Some customers have these requirements as well +# It does not matter what number it is, as long as it is set to something. +# However, OpenShift will run the container as a random user, +# and the number in this configuration is not relevant. +USER 2000 + +{% block entrypoint %} +# The docker image doesn't have any scripts so by default does nothing +# The script will be copied in runtime from init containers and the operator is expected +# to override the COMMAND +ENTRYPOINT ["sleep infinity"] +{% endblock %} + +COPY --from=base /data/licenses/mongodb-enterprise-database /licenses/mongodb-enterprise-database + +{% block healthcheck %} +{% endblock %} diff --git a/docker/mongodb-enterprise-database/Dockerfile.ubi b/docker/mongodb-enterprise-database/Dockerfile.ubi new file mode 100644 index 000000000..a94b625c9 --- /dev/null +++ b/docker/mongodb-enterprise-database/Dockerfile.ubi @@ -0,0 +1,41 @@ +{% extends "Dockerfile.template" %} + +{% set base_image = "registry.access.redhat.com/ubi8/ubi-minimal" %} +{% set distro = "ubi" %} + +{% block packages %} +RUN microdnf update -y && rm -rf /var/cache/yum + +# these are the packages needed for the agent +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 nss_wrapper +RUN microdnf install -y --disableplugin=subscription-manager \ + hostname \ + procps + + +# these are the packages needed for MongoDB +# (https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ "RHEL/CentOS 8" tab) +RUN microdnf install -y --disableplugin=subscription-manager \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libcurl \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + jq \ + tar \ + xz-libs \ + findutils + + +{# +TODO: Find public mongodb documentation about this +# mongodb enterprise expects this library /usr/lib64/libsasl2.so.2 but +# cyrus-sasl creates it in /usr/lib64/libsasl2.so.3 instead +#} +RUN ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2 +{% endblock %} diff --git a/docker/mongodb-enterprise-database/LICENSE b/docker/mongodb-enterprise-database/LICENSE new file mode 100644 index 000000000..dc71da876 --- /dev/null +++ b/docker/mongodb-enterprise-database/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-database/README.md b/docker/mongodb-enterprise-database/README.md new file mode 100644 index 000000000..ab8547635 --- /dev/null +++ b/docker/mongodb-enterprise-database/README.md @@ -0,0 +1,44 @@ +# MongoDB Enterprise Database + +This directory hosts a Dockerfile that can be run locally for development purposes (see below) or +as part of a Kubernetes deployment, using the [MongoDB Enterprise Kubernetes Operator](../mongodb-enterprise-operator). + +### Running locally + +You can use `make clean build run` to build and run the container. + +For more details regarding the available options, run `make` or read the provided [Makefile](Makefile). + + +### Other useful commands + +**See the status of all running Automation Agents:** + +```bash +for img in $(docker ps -a -f 'ancestor=dev/mongodb-enterprise-database' | tail -n +2 | awk '{print $1}'); do echo; echo "$img"; echo "---"; docker exec -t "$img" ps -ef; echo "---"; done +``` + +**Connect to a running container:** + +```bash +docker exec -it $(docker ps -a -f 'ancestor=dev/mongodb-enterprise-database' | tail -n +2 | awk '{print $1}') /bin/bash +``` + +## RHEL based Images + +We have provided a second Dockerfile (`Dockerfile_rhel`) based on RHEL7 instead of the `jessie-slim` that the +normal image is based on. The purpose of this second image is to be uploaded to RedHat Container Catalog to be used +in OpenShift with the MongoDb ClusterServiceVersion. See the `openshift` directory in this repo. + +This image can't be built in any host, because it will require the use of a subscription service with Redhat. A RHEL +host, with subscription service enabled, is required. That's the reason behind using the Redhat build service to build +this images with. + +## Building the DCAR database image + +The dcar image needs to be built manually. + +```bash +docker build . -t 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/usaf/mongodb-enterprise-database:1.5.3 +docker push 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/usaf/mongodb-enterprise-database:1.5.3 +``` diff --git a/docker/mongodb-enterprise-init-database/Dockerfile.builder b/docker/mongodb-enterprise-init-database/Dockerfile.builder new file mode 100644 index 000000000..ef340b04f --- /dev/null +++ b/docker/mongodb-enterprise-init-database/Dockerfile.builder @@ -0,0 +1,22 @@ +# Build compilable stuff + +FROM public.ecr.aws/docker/library/golang:1.24 as readiness_builder +COPY . /go/src/github.com/10gen/ops-manager-kubernetes +WORKDIR /go/src/github.com/10gen/ops-manager-kubernetes +RUN CGO_ENABLED=0 go build -o /readinessprobe github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness +RUN CGO_ENABLED=0 go build -o /version-upgrade-hook github.com/mongodb/mongodb-kubernetes-operator/cmd/versionhook + +FROM scratch +ARG mongodb_tools_url_ubi + +COPY --from=readiness_builder /readinessprobe /data/ +COPY --from=readiness_builder /version-upgrade-hook /data/version-upgrade-hook + +ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz + +COPY ./docker/mongodb-enterprise-init-database/content/probe.sh /data/probe.sh + +COPY ./docker/mongodb-enterprise-init-database/content/agent-launcher-lib.sh /data/scripts/ +COPY ./docker/mongodb-enterprise-init-database/content/agent-launcher.sh /data/scripts/ + +COPY ./docker/mongodb-enterprise-init-database/content/LICENSE /data/licenses/ diff --git a/docker/mongodb-enterprise-init-database/Dockerfile.template b/docker/mongodb-enterprise-init-database/Dockerfile.template new file mode 100644 index 000000000..069300dd7 --- /dev/null +++ b/docker/mongodb-enterprise-init-database/Dockerfile.template @@ -0,0 +1,42 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM {{ base_image }} + +ARG version + +{%- if is_appdb %} +LABEL name="MongoDB Enterprise Init AppDB" \ + version="mongodb-enterprise-init-appdb-${version}" \ + summary="MongoDB Enterprise AppDB Init Image" \ + description="Startup Scripts for MongoDB Enterprise Application Database for Ops Manager" \ +{%- else %} +LABEL name="MongoDB Enterprise Init Database" \ + version="mongodb-enterprise-init-database-${version}" \ + summary="MongoDB Enterprise Database Init Image" \ + description="Startup Scripts for MongoDB Enterprise Database" \ +{%- endif %} + release="1" \ + vendor="MongoDB" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/readinessprobe /probes/readinessprobe +COPY --from=base /data/probe.sh /probes/probe.sh +COPY --from=base /data/scripts/ /scripts/ +COPY --from=base /data/licenses /licenses/ + +{%- if is_appdb %} +COPY --from=base /data/version-upgrade-hook /probes/version-upgrade-hook +{%- endif %} + +{% block mongodb_tools %} +{% endblock %} + +RUN tar xfz /tools/mongodb_tools.tgz --directory /tools \ + && rm /tools/mongodb_tools.tgz + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "-r", "/scripts/agent-launcher.sh", "/scripts/agent-launcher-lib.sh", "/probes/readinessprobe", "/probes/probe.sh", "/tools", "/opt/scripts/" ] + +{% block healthcheck %} +{% endblock %} diff --git a/docker/mongodb-enterprise-init-database/Dockerfile.ubi_minimal b/docker/mongodb-enterprise-init-database/Dockerfile.ubi_minimal new file mode 100644 index 000000000..b5400b147 --- /dev/null +++ b/docker/mongodb-enterprise-init-database/Dockerfile.ubi_minimal @@ -0,0 +1,11 @@ +{% extends "Dockerfile.template" %} + +{% set base_image = "registry.access.redhat.com/ubi8/ubi-minimal" %} + +{% block mongodb_tools %} +RUN microdnf -y update --nodocs \ + && microdnf -y install --nodocs tar gzip \ + && microdnf clean all + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz +{% endblock %} diff --git a/docker/mongodb-enterprise-init-database/content/LICENSE b/docker/mongodb-enterprise-init-database/content/LICENSE new file mode 100644 index 000000000..dc71da876 --- /dev/null +++ b/docker/mongodb-enterprise-init-database/content/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-init-database/content/agent-launcher-lib.sh b/docker/mongodb-enterprise-init-database/content/agent-launcher-lib.sh new file mode 100755 index 000000000..eaed81cf0 --- /dev/null +++ b/docker/mongodb-enterprise-init-database/content/agent-launcher-lib.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +# see if jq is available for json logging +use_jq="$(command -v jq)" + +# log stdout as structured json with given log type +json_log() { + if [ "${use_jq}" ]; then + jq --unbuffered --null-input -c --raw-input "inputs | {\"logType\": \"$1\", \"contents\": .}" + else + echo "$1" + fi +} + +# log a given message in json format +script_log() { + echo "$1" | json_log 'agent-launcher-script' &>>"${MDB_LOG_FILE_AGENT_LAUNCHER_SCRIPT}" +} + +# the function reacting on SIGTERM command sent by the container on its shutdown. Makes sure all processes started (including +# mongodb) receive the signal. For MongoDB this results in graceful shutdown of replication (starting from 4.0.9) which may +# take some time. The script waits for all the processes to finish, otherwise the container would terminate as Kubernetes +# waits only for the process with pid #1 to end +cleanup() { + # Important! Keep this in sync with DefaultPodTerminationPeriodSeconds constant from constants.go + termination_timeout_seconds=600 + + script_log "Caught SIGTERM signal. Passing the signal to the automation agent and the mongod processes." + + kill -15 "${agentPid:?}" + wait "${agentPid}" + + mongoPid="$(cat /data/mongod.lock)" + + if [ -n "${mongoPid}" ]; then + kill -15 "${mongoPid}" + + script_log "Waiting until mongod process is shutdown. Note, that if mongod process fails to shutdown in the time specified by the 'terminationGracePeriodSeconds' property (default ${termination_timeout_seconds} seconds) then the container will be killed by Kubernetes." + + # dev note: we cannot use 'wait' for the external processes, seems the spinning loop is the best option + while [ -e "/proc/${mongoPid}" ]; do sleep 0.1; done + fi + + script_log "Mongod and automation agent processes are shutdown" +} + +# ensure_certs_symlinks function checks if certificates and CAs are mounted and creates symlinks to them +ensure_certs_symlinks() { + # the paths inside the pod. Move to parameters if multiple usage is needed + secrets_dir="/var/lib/mongodb-automation/secrets" + custom_ca_dir="${secrets_dir}/ca" + pod_secrets_dir="/mongodb-automation" + + if [ -d "${secrets_dir}/certs" ]; then + script_log "Found certificates in the host, will symlink to where the automation agent expects them to be" + podname=$(hostname) + + if [ ! -f "${secrets_dir}/certs/${podname}-pem" ]; then + script_log "PEM Certificate file does not exist in ${secrets_dir}/certs/${podname}-pem. Check the Secret object with certificates is well formed." + exit 1 + fi + + ln -sf "${secrets_dir}/certs/${podname}-pem" "${pod_secrets_dir}/server.pem" + fi + + if [ -d "${custom_ca_dir}" ]; then + if [ -f "${custom_ca_dir}/ca-pem" ]; then + script_log "Using CA file provided by user" + ln -sf "${custom_ca_dir}/ca-pem" "${pod_secrets_dir}/ca.pem" + else + script_log "Could not find CA file. The name of the entry on the Secret object should be 'ca-pem'" + exit 1 + fi + else + script_log "Using Kubernetes CA file" + + if [[ ! -f "${pod_secrets_dir}/ca.pem" ]]; then + ln -sf "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" "${pod_secrets_dir}/ca.pem" + fi + fi +} + +# download_agent function downloads and unpacks the Mongodb Agent +download_agent() { + pushd /tmp >/dev/null || true + + + if [[ -z "${MDB_AGENT_VERSION-}" ]]; then + AGENT_VERSION="latest" + else + AGENT_VERSION="${MDB_AGENT_VERSION}" + fi + + script_log "Downloading Agent version: ${AGENT_VERSION}" + script_log "Downloading a Mongodb Agent from ${base_url:?}" + curl_opts=( + "${base_url}/download/agent/automation/mongodb-mms-automation-agent-${AGENT_VERSION}.linux_x86_64.tar.gz" + "--location" "--silent" "--retry" "3" "--fail" "-v" + "--output" "automation-agent.tar.gz" + ); + + if [ "${SSL_REQUIRE_VALID_MMS_CERTIFICATES-}" = "false" ]; then + # If we are not expecting valid certs, `curl` should be run with `--insecure` option. + # The default is NOT to accept insecure connections. + curl_opts+=("--insecure") + fi + + if [ -n "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE-}" ]; then + curl_opts+=("--cacert" "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE}") + fi + + if ! curl "${curl_opts[@]}" &>"${MMS_LOG_DIR}/curl.log"; then + script_log "Error while downloading the Mongodb agent" + exit 1 + fi + json_log 'agent-launcher-script' <"${MMS_LOG_DIR}/curl.log" >>"${MDB_LOG_FILE_AGENT_LAUNCHER_SCRIPT}" + rm "${MMS_LOG_DIR}/curl.log" 2>/dev/null || true + + script_log "The Mongodb Agent binary downloaded, unpacking" + tar -xzf automation-agent.tar.gz + AGENT_VERSION=$(find . -name "mongodb-mms-automation-agent-*" | awk -F"-" '{ print $5 }') + mkdir -p "${MMS_HOME}/files" + echo "${AGENT_VERSION}" >"${MMS_HOME}/files/agent-version" + mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent "${MMS_HOME}/files/" + chmod +x "${MMS_HOME}/files/mongodb-mms-automation-agent" + rm -rf automation-agent.tar.gz mongodb-mms-automation-agent-*.linux_x86_64 + script_log "The Automation Agent was deployed at ${MMS_HOME}/files/mongodb-mms-automation-agent" + + popd >/dev/null || true +} diff --git a/docker/mongodb-enterprise-init-database/content/agent-launcher.sh b/docker/mongodb-enterprise-init-database/content/agent-launcher.sh new file mode 100755 index 000000000..7bdf8164c --- /dev/null +++ b/docker/mongodb-enterprise-init-database/content/agent-launcher.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash + +# The "e" switch cannot be used here. There are certain commands, such as type or command in the agent-launcher-lib.sh +# that return "1" as valid case. +set -Eou pipefail + +MDB_STATIC_CONTAINERS_ARCHITECTURE="${MDB_STATIC_CONTAINERS_ARCHITECTURE:-}" +MMS_HOME=${MMS_HOME:-/mongodb-automation} +MMS_LOG_DIR=${MMS_LOG_DIR:-/var/log/mongodb-mms-automation} + +if [ -z "${MDB_STATIC_CONTAINERS_ARCHITECTURE}" ]; then + AGENT_BINARY_PATH="${MMS_HOME}/files/mongodb-mms-automation-agent" +else + AGENT_BINARY_PATH="/agent/mongodb-agent" +fi + +export MDB_LOG_FILE_AGENT_LAUNCHER_SCRIPT="${MMS_LOG_DIR}/agent-launcher-script.log" + +# We start tailing script logs immediately to not miss anything. +# -F flag is equivalent to --follow=name --retry. +# -n0 parameter is instructing tail to show only new lines (by default tail is showing last 10 lines) +tail -F -n0 "${MDB_LOG_FILE_AGENT_LAUNCHER_SCRIPT}" 2> /dev/null & + +source /opt/scripts/agent-launcher-lib.sh + +# all the following MDB_LOG_FILE_* env var should be defined in container's env vars +tail -F -n0 "${MDB_LOG_FILE_AUTOMATION_AGENT_VERBOSE}" 2> /dev/null | json_log 'automation-agent-verbose' & +tail -F -n0 "${MDB_LOG_FILE_AUTOMATION_AGENT_STDERR}" 2> /dev/null | json_log 'automation-agent-stderr' & +tail -F -n0 "${MDB_LOG_FILE_AUTOMATION_AGENT}" 2> /dev/null | json_log 'automation-agent' & +tail -F -n0 "${MDB_LOG_FILE_MONITORING_AGENT}" 2> /dev/null | json_log 'monitoring-agent' & +tail -F -n0 "${MDB_LOG_FILE_BACKUP_AGENT}" 2> /dev/null | json_log 'backup-agent' & +tail -F -n0 "${MDB_LOG_FILE_MONGODB}" 2> /dev/null | json_log 'mongodb' & +tail -F -n0 "${MDB_LOG_FILE_MONGODB_AUDIT}" 2> /dev/null | json_log 'mongodb-audit' & + +# This is the directory corresponding to 'options.downloadBase' in the automation config - the directory where +# the agent will extract MongoDB binaries to +mdb_downloads_dir="/var/lib/mongodb-mms-automation" + +# The path to the automation config file in case the agent is run in headless mode +cluster_config_file="/var/lib/mongodb-automation/cluster-config.json" + +# file required by Automation Agents of authentication is enabled. +touch "${mdb_downloads_dir}/keyfile" +chmod 600 "${mdb_downloads_dir}/keyfile" + +ensure_certs_symlinks + +# Ensure that the user has an entry in /etc/passwd +current_uid=$(id -u) +declare -r current_uid +if ! grep -q "${current_uid}" /etc/passwd ; then + # Adding it here to avoid panics in the automation agent + sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd + echo "mongodb:x:$(id -u):$(id -g):,,,:/mongodb-automation:/bin/bash" >> /tmp/passwd + export LD_PRELOAD=libnss_wrapper.so + export NSS_WRAPPER_PASSWD=/tmp/passwd + export NSS_WRAPPER_GROUP=/etc/group +fi + +# Create a symlink, after the volumes have been mounted +# If the journal directory already exists (this could be the migration of the existing MongoDB database) - we need +# to copy it to the correct location first and remove a directory +if [[ -d /data/journal ]] && [[ ! -L /data/journal ]]; then + script_log "The journal directory /data/journal already exists - moving its content to /journal" + + # Check if /journal is not empty, if so, empty it + if [[ $(ls -A /journal) && ${MDB_CLEAN_JOURNAL:-1} -eq 1 ]]; then + # we can only create a dir under tmp, since its read only + MDB_JOURNAL_BACKUP_DIR="/tmp/journal_backup_$(date +%Y%m%d%H%M%S)" + script_log "The /journal directory is not empty - moving its content to ${MDB_JOURNAL_BACKUP_DIR}" + mkdir -p "${MDB_JOURNAL_BACKUP_DIR}" + mv /journal/* "${MDB_JOURNAL_BACKUP_DIR}" + fi + + + if [[ $(find /data/journal -maxdepth 1 | wc -l) -gt 0 ]]; then + mv /data/journal/* /journal + fi + + rm -rf /data/journal +fi + +ln -sf /journal /data/ +script_log "Created symlink: /data/journal -> $(readlink -f /data/journal)" + +# If it is a migration of the existing MongoDB - then there could be a mongodb.log in a default location - +# let's try to copy it to a new directory +if [[ -f /data/mongodb.log ]] && [[ ! -f "${MDB_LOG_FILE_MONGODB}" ]]; then + script_log "The mongodb log file /data/mongodb.log already exists - moving it to ${MMS_LOG_DIR}" + mv /data/mongodb.log "${MMS_LOG_DIR}" +fi + +base_url="${BASE_URL-}" # If unassigned, set to empty string to avoid set-u errors +base_url="${base_url%/}" # Remove any accidentally defined trailing slashes +declare -r base_url + +if [ -z "${MDB_STATIC_CONTAINERS_ARCHITECTURE}" ]; then + # Download the Automation Agent from Ops Manager + # Note, that it will be skipped if the agent is supposed to be run in headless mode + if [[ -n "${base_url}" ]]; then + download_agent + fi +fi + +# Start the Automation Agent +agentOpts=( + "-mmsGroupId=${GROUP_ID-}" + "-pidfilepath=${MMS_HOME}/mongodb-mms-automation-agent.pid" + "-maxLogFileDurationHrs=24" + "-logLevel=${LOG_LEVEL:-INFO}" +) + +# in multi-cluster mode we need to override the hostname with which, agents +# registers itself, use service FQDN instead of POD FQDN, this mapping is mounted into +# the pod using configmap +hostpath="$(hostname)" + +# We apply the ephemeralPortOffset when using externalDomain in Single Cluster +# or whenever Multi-Cluster is on. +override_file="/opt/scripts/config/${hostpath}" +if [[ -f "${override_file}" ]]; then + override="$(cat "${override_file}")" + agentOpts+=("-overrideLocalHost=${override}") + agentOpts+=("-ephemeralPortOffset=1") +elif [ "${MULTI_CLUSTER_MODE-}" = "true" ]; then + agentOpts+=("-ephemeralPortOffset=1") +fi + +agentOpts+=("-healthCheckFilePath=${MMS_LOG_DIR}/agent-health-status.json") +if [ -z "${MDB_STATIC_CONTAINERS_ARCHITECTURE}" ]; then + agentOpts+=("-useLocalMongoDbTools=true") +else + agentOpts+=("-operatorMode=true") +fi + + +if [[ -n "${base_url}" ]]; then + agentOpts+=("-mmsBaseUrl=${base_url}") +else + agentOpts+=("-cluster=${cluster_config_file}") + # we need to open the web server on localhost even though we don't use it - otherwise Agent doesn't + # produce status information at all (we need it in health file) + agentOpts+=("-serveStatusPort=5000") + script_log "Mongodb Agent is configured to run in \"headless\" mode using local config file" +fi + + + +if [[ -n "${HTTP_PROXY-}" ]]; then + agentOpts+=("-httpProxy=${HTTP_PROXY}") +fi + +if [[ -n "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE-}" ]]; then + agentOpts+=("-httpsCAFile=${SSL_TRUSTED_MMS_SERVER_CERTIFICATE}") +fi + +if [[ "${SSL_REQUIRE_VALID_MMS_CERTIFICATES-}" != "false" ]]; then + # Only set this option when valid certs are required. The default is false + agentOpts+=("-tlsRequireValidMMSServerCertificates=true") +else + agentOpts+=("-tlsRequireValidMMSServerCertificates=false") +fi + +# we can't directly use readarray as this bash version doesn't support +# the delimiter option +splittedAgentFlags=(); +while read -rd,; do + splittedAgentFlags+=("${REPLY}") +done <<<"${AGENT_FLAGS}"; + +AGENT_API_KEY="$(cat "${MMS_HOME}"/agent-api-key/agentApiKey)" +script_log "Launching automation agent with following arguments: ${agentOpts[*]} -mmsApiKey=${AGENT_API_KEY+} ${splittedAgentFlags[*]}" + +agentOpts+=("-mmsApiKey=${AGENT_API_KEY-}") + +rm /tmp/mongodb-mms-automation-cluster-backup.json &> /dev/null || true + +if [ -z "${MDB_STATIC_CONTAINERS_ARCHITECTURE}" ]; then + echo "Skipping creating symlinks because this is not Static Containers Architecture" +else + WAIT_TIME=5 + MAX_WAIT=300 + ELAPSED_TIME=0 + + # Polling loop to wait for the PID value to be non-zero + while [ ${ELAPSED_TIME} -lt ${MAX_WAIT} ]; do + script_log "waiting for mongod_pid being available" + # shellcheck disable=SC2009 + MONGOD_PID=$(ps aux | grep "mongodb_marker" | grep -v grep | awk '{print $2}') || true + + if [ -n "${MONGOD_PID}" ] && [ "${MONGOD_PID}" -ne 0 ]; then + break + fi + + sleep ${WAIT_TIME} + ELAPSED_TIME=$((ELAPSED_TIME + WAIT_TIME)) + done + + # Check if a non-zero PID value is found + if [ -n "${MONGOD_PID}" ] && [ "${MONGOD_PID}" -ne 0 ]; then + echo "Mongod PID: ${MONGOD_PID}" + MONGOD_ROOT="/proc/${MONGOD_PID}/root" + mkdir -p "${mdb_downloads_dir}/mongod" + mkdir -p "${mdb_downloads_dir}/mongod/bin" + ln -sf "${MONGOD_ROOT}/bin/mongo" ${mdb_downloads_dir}/mongod/bin/mongo + ln -sf "${MONGOD_ROOT}/bin/mongod" ${mdb_downloads_dir}/mongod/bin/mongod + ln -sf "${MONGOD_ROOT}/bin/mongos" ${mdb_downloads_dir}/mongod/bin/mongos + + ln -sf "/tools/mongodump" ${mdb_downloads_dir}/mongod/bin/mongodump + ln -sf "/tools/mongorestore" ${mdb_downloads_dir}/mongod/bin/mongorestore + ln -sf "/tools/mongoexport" ${mdb_downloads_dir}/mongod/bin/mongoexport + ln -sf "/tools/mongoimport" ${mdb_downloads_dir}/mongod/bin/mongoimport + else + echo "Mongod PID not found within the specified time." + exit 1 + fi + + agentOpts+=("-binariesFixedPath=${mdb_downloads_dir}/mongod/bin") +fi + +debug="${MDB_AGENT_DEBUG-}" +if [ "${debug}" = "true" ]; then + cd ${mdb_downloads_dir} || true + mkdir -p /var/lib/mongodb-mms-automation/gopath + mkdir -p /var/lib/mongodb-mms-automation/go + curl -LO https://go.dev/dl/go1.20.1.linux-amd64.tar.gz + tar -xzf go1.20.1.linux-amd64.tar.gz + export GOPATH=${mdb_downloads_dir}/gopath + export GOCACHE=${mdb_downloads_dir}/.cache + export PATH=${PATH}:${mdb_downloads_dir}/go/bin + export PATH=${PATH}:${mdb_downloads_dir}/gopath/bin + go install github.com/go-delve/delve/cmd/dlv@latest + export PATH=${PATH}:${mdb_downloads_dir}/gopath/bin + cd ${mdb_downloads_dir} || true + dlv --headless=true --listen=:5006 --accept-multiclient=true --continue --api-version=2 exec "${AGENT_BINARY_PATH}" -- "${agentOpts[@]}" "${splittedAgentFlags[@]}" 2>> "${MDB_LOG_FILE_AUTOMATION_AGENT_STDERR}" > >(json_log "automation-agent-stdout") & +else +# Note, that we do logging in subshell - this allows us to save the correct PID to variable (not the logging one) + "${AGENT_BINARY_PATH}" "${agentOpts[@]}" "${splittedAgentFlags[@]}" 2>> "${MDB_LOG_FILE_AUTOMATION_AGENT_STDERR}" >> >(json_log "automation-agent-stdout") & +fi + +export agentPid=$! +script_log "Launched automation agent, pid=${agentPid}" + +trap cleanup SIGTERM + +wait diff --git a/docker/mongodb-enterprise-init-database/content/probe.sh b/docker/mongodb-enterprise-init-database/content/probe.sh new file mode 100755 index 000000000..137522b0f --- /dev/null +++ b/docker/mongodb-enterprise-init-database/content/probe.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +check_process() { + local check_process=$1 + # shellcheck disable=SC2009 + ps -ax | grep -v " grep " | grep -v jq | grep -v tail | grep "$check_process" + return $? +} + +check_agent_alive() { + check_process 'mongodb-mms-aut' +} + +check_mongod_alive() { + check_process 'mongod' +} + +check_mongos_alive() { + check_process 'mongos' +} + +check_mongo_process_alive() { + # the mongod process pid might not always exist + # 1. when the container is being created the mongod package needs to be + # downloaded. the agent will wait for 1 hour before giving up. + # 2. the mongod process might be getting updated, we'll set a + # failureThreshold on the livenessProbe to a few minutes before we + # give up. + + check_mongod_alive || check_mongos_alive +} + +# One of 2 conditions is sufficient to state that a Pod is "Alive": +# +# 1. There is an agent process running +# 2. There is a `mongod` or `mongos` process running +# +check_agent_alive || check_mongo_process_alive diff --git a/docker/mongodb-enterprise-init-ops-manager/Dockerfile.builder b/docker/mongodb-enterprise-init-ops-manager/Dockerfile.builder new file mode 100644 index 000000000..62fa29cd7 --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/Dockerfile.builder @@ -0,0 +1,14 @@ +# +# Dockerfile for Init Ops Manager Context. +# + +FROM public.ecr.aws/docker/library/golang:1.24 as builder +WORKDIR /go/src +ADD . . +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/mmsconfiguration ./mmsconfiguration +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/backup-daemon-readiness-probe ./backupdaemon_readinessprobe/ + +COPY scripts/docker-entry-point.sh /data/scripts/ +COPY scripts/backup-daemon-liveness-probe.sh /data/scripts/ + +COPY LICENSE /data/licenses/mongodb-enterprise-ops-manager diff --git a/docker/mongodb-enterprise-init-ops-manager/Dockerfile.template b/docker/mongodb-enterprise-init-ops-manager/Dockerfile.template new file mode 100644 index 000000000..30c178fac --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/Dockerfile.template @@ -0,0 +1,25 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM {{ base_image }} + +LABEL name="MongoDB Enterprise Ops Manager Init" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="mongodb-enterprise-init-ops-manager-{{version}}" \ + release="1" \ + summary="MongoDB Enterprise Ops Manager Init Image" \ + description="Startup Scripts for MongoDB Enterprise Ops Manager" + + +COPY --from=base /data/scripts /scripts +COPY --from=base /data/licenses /licenses + +{% block packages %} +{% endblock %} + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "/scripts/docker-entry-point.sh", "/scripts/backup-daemon-liveness-probe.sh", "/scripts/mmsconfiguration", "/scripts/backup-daemon-readiness-probe", "/opt/scripts/" ] + +{% block healthcheck %} +{% endblock %} diff --git a/docker/mongodb-enterprise-init-ops-manager/Dockerfile.ubi_minimal b/docker/mongodb-enterprise-init-ops-manager/Dockerfile.ubi_minimal new file mode 100644 index 000000000..2f16d3d9a --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/Dockerfile.ubi_minimal @@ -0,0 +1,8 @@ +{% extends "Dockerfile.template" %} + +{% set base_image = "registry.access.redhat.com/ubi9/ubi-minimal" %} + +{% block packages %} +RUN microdnf -y update --nodocs \ + && microdnf clean all +{% endblock %} diff --git a/docker/mongodb-enterprise-init-ops-manager/LICENSE b/docker/mongodb-enterprise-init-ops-manager/LICENSE new file mode 100644 index 000000000..dc71da876 --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness.go b/docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness.go new file mode 100644 index 000000000..b6e4554d6 --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" + + "golang.org/x/xerrors" +) + +const ( + healthEndpointPortEnv = "HEALTH_ENDPOINT_PORT" +) + +// HealthResponse represents the response given from the backup daemon health endpoint. +// Sample responses: +// - {"sync_db":"OK","backup_db":"OK","mms_db":"OK"} +// - {"backup_db":"no master","mms_db":"no master"} +type HealthResponse struct { + SyncDb string `json:"sync_db"` + BackupDb string `json:"backup_db"` + MmsDb string `json:"mms_db"` +} + +func main() { + os.Exit(checkHealthEndpoint(&http.Client{ + Timeout: 5 * time.Second, + })) +} + +// checkHealthEndpoint checks the BackupDaemon health endpoint +// and ensures that the AppDB is up and running. +func checkHealthEndpoint(getter httpGetter) int { + hr, err := getHealthResponse(getter) + if err != nil { + fmt.Printf("error getting health response: %s\n", err) + return 1 + } + + fmt.Printf("received response: %+v\n", hr) + if hr.MmsDb == "OK" { + return 0 + } + return 1 +} + +type httpGetter interface { + Get(url string) (*http.Response, error) +} + +// getHealthResponse fetches the health response from the health endpoint. +func getHealthResponse(getter httpGetter) (HealthResponse, error) { + url := fmt.Sprintf("http://localhost:%s/health", os.Getenv(healthEndpointPortEnv)) // nolint:forbidigo + fmt.Printf("attempting GET request to: [%s]\n", url) + resp, err := getter.Get(url) + if err != nil { + return HealthResponse{}, xerrors.Errorf("failed to reach health endpoint: %w", err) + } + + if resp.StatusCode != 200 { + return HealthResponse{}, xerrors.Errorf("received status code [%d] but expected [200]", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return HealthResponse{}, xerrors.Errorf("failed to read response body: %w", err) + } + + hr := HealthResponse{} + if err := json.Unmarshal(body, &hr); err != nil { + return HealthResponse{}, xerrors.Errorf("failed to unmarshal response: %w", err) + } + + return hr, nil +} diff --git a/docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness_test.go b/docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness_test.go new file mode 100644 index 000000000..f88f0ce8c --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/xerrors" +) + +type mockHttpGetter struct { + resp *http.Response + err error +} + +func (m *mockHttpGetter) Get(string) (*http.Response, error) { + return m.resp, m.err +} + +// newMockHttpGetter returns a httpGetter which will return the given status code, the body and error provided. +func newMockHttpGetter(code int, body []byte, err error) *mockHttpGetter { + return &mockHttpGetter{ + resp: &http.Response{ + StatusCode: code, + Body: io.NopCloser(bytes.NewReader(body)), + }, + err: err, + } +} + +func healthStatusResponseToBytes(hr HealthResponse) []byte { + bytes, err := json.Marshal(hr) + if err != nil { + panic(err) + } + return bytes +} + +func TestCheckHealthEndpoint(t *testing.T) { + t.Run("Test 200 Code with invalid body fails", func(t *testing.T) { + m := newMockHttpGetter(200, healthStatusResponseToBytes(HealthResponse{ + SyncDb: "OK", + BackupDb: "OK", + MmsDb: "no master", + }), nil) + code := checkHealthEndpoint(m) + assert.Equal(t, 1, code) + }) + + t.Run("Test 200 Code with valid body succeeds", func(t *testing.T) { + m := newMockHttpGetter(200, healthStatusResponseToBytes(HealthResponse{ + SyncDb: "OK", + BackupDb: "OK", + MmsDb: "OK", + }), nil) + code := checkHealthEndpoint(m) + assert.Equal(t, 0, code) + }) + + t.Run("Test non-200 Code fails", func(t *testing.T) { + m := newMockHttpGetter(300, healthStatusResponseToBytes(HealthResponse{ + SyncDb: "OK", + BackupDb: "OK", + MmsDb: "OK", + }), nil) + code := checkHealthEndpoint(m) + assert.Equal(t, 1, code) + }) + + t.Run("Test error from http client fails", func(t *testing.T) { + m := newMockHttpGetter(200, nil, xerrors.Errorf("error")) + code := checkHealthEndpoint(m) + assert.Equal(t, 1, code) + }) +} diff --git a/docker/mongodb-enterprise-init-ops-manager/go.mod b/docker/mongodb-enterprise-init-ops-manager/go.mod new file mode 100644 index 000000000..e78c82981 --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/go.mod @@ -0,0 +1,14 @@ +module github.com/10gen/ops-manager-kubernetes/docker/mongodb-enterprise-init-ops-manager + +go 1.24.0 + +require ( + github.com/stretchr/testify v1.7.0 + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 +) + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect +) diff --git a/docker/mongodb-enterprise-init-ops-manager/go.sum b/docker/mongodb-enterprise-init-ops-manager/go.sum new file mode 100644 index 000000000..39ddc59de --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration.go b/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration.go new file mode 100755 index 000000000..ad4d7ae9d --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration.go @@ -0,0 +1,235 @@ +package main + +import ( + "errors" + "fmt" + "net" + "os" + "strings" + + "golang.org/x/xerrors" +) + +const ( + mmsJvmParamsVar = "JAVA_MMS_UI_OPTS" + backupDaemonJvmParamsVar = "JAVA_DAEMON_OPTS" + omPropertyPrefix = "OM_PROP_" + lineBreak = "\n" + commentPrefix = "#" + propOverwriteFmt = "%s=\"${%s} %s\"" + backupDaemon = "BACKUP_DAEMON" + // keep in sync with AppDBConnectionStringPath constant from "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" package. + // currently we cannot reference code from outside of docker/mongodb-enterprise-init-ops-manager + // because this folder is set as the docker build context (configured in inventories/init_om.yaml) + appDbConnectionStringPath = "/mongodb-ops-manager/.mongodb-mms-connection-string" + appDbConnectionStringFilePath = appDbConnectionStringPath + "/connectionString" + // keep in sync with MmsMongoUri constant from github.com/10gen/ops-manager-kubernetes/pkg/util + appDbUriKey = "mongo.mongoUri" +) + +func updateConfFile(confFile string) error { + confFilePropertyName := mmsJvmParamsVar + var isBackupDaemon bool + if _, isBackupDaemon = os.LookupEnv(backupDaemon); isBackupDaemon { // nolint:forbidigo + confFilePropertyName = backupDaemonJvmParamsVar + } + + customJvmParamsVar := "CUSTOM_" + confFilePropertyName + jvmParams, jvmParamsEnvVarExists := os.LookupEnv(customJvmParamsVar) // nolint:forbidigo + + if !jvmParamsEnvVarExists || jvmParams == "" { + fmt.Printf("%s not specified, not modifying %s\n", customJvmParamsVar, confFile) + return nil + } + + if isBackupDaemon { + fqdn, err := getHostnameFQDN() + if err == nil { + // We need to add hostname to the Backup daemon + jvmParams += " -Dmms.system.hostname=" + fqdn + } else { + fmt.Printf("was not able to get fqdn of the pod: %s\n", err) + } + } + + newMmsJvmParams := fmt.Sprintf(propOverwriteFmt, confFilePropertyName, confFilePropertyName, jvmParams) + fmt.Printf("Appending %s to %s\n", newMmsJvmParams, confFile) + + return appendLinesToFile(confFile, getJvmParamDocString()+newMmsJvmParams+lineBreak) +} + +// getHostnameFQDN returns the FQDN name for this Pod, which is the Pod's hostname +// and complete Domain. +// +// We use the pods hostname as the base and calculate which one _is the FQDN_ by +// a simple heuristic: +// +// - the longest string with _dots_ in it should be the FQDN. +// The output should match the shell call: hostname -f +func getHostnameFQDN() (string, error) { + // Get the pod's hostname + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + // Look up the pod's hostname in DNS + addresses, err := net.LookupHost(hostname) + if err != nil { + return "", err + } + + longestFQDN := "" + + for _, address := range addresses { + // Get the pod's FQDN from the IP address + fqdnList, err := net.LookupAddr(address) + if err != nil { + return "", err + } + + for _, fqdn := range fqdnList { + // Only consider fqdns with '.' on it + if !strings.Contains(fqdn, ".") { + continue + } + if len(fqdn) > len(longestFQDN) { + longestFQDN = fqdn + } + } + + } + + if longestFQDN == "" { + return "", errors.New("could not find FQDN for this host") + } + + // Remove the trailing ".", If in there + return strings.TrimRight(longestFQDN, "."), nil +} + +func getMmsProperties() (map[string]string, error) { + newProperties := getOmPropertiesFromEnvVars() + + appDbConnectionString, err := os.ReadFile(appDbConnectionStringFilePath) + if err != nil { + return nil, err + } + newProperties[appDbUriKey] = string(appDbConnectionString) + // Enable dualConnectors to allow the kubelet to perform health checks through HTTP + newProperties["mms.https.dualConnectors"] = "true" + + return newProperties, nil +} + +func updatePropertiesFile(propertiesFile string, newProperties map[string]string) error { + lines, err := readLinesFromFile(propertiesFile) + if err != nil { + return err + } + + lines = updateMmsProperties(lines, newProperties) + fmt.Printf("Updating configuration properties file %s\n", propertiesFile) + err = writeLinesToFile(propertiesFile, lines) + return err +} + +func readLinesFromFile(name string) ([]string, error) { + input, err := os.ReadFile(name) + if err != nil { + return nil, xerrors.Errorf("error reading file %s: %w", name, err) + } + return strings.Split(string(input), lineBreak), nil +} + +func writeLinesToFile(name string, lines []string) error { + output := strings.Join(lines, lineBreak) + + err := os.WriteFile(name, []byte(output), 0o775) + if err != nil { + return xerrors.Errorf("error writing to file %s: %w", name, err) + } + return nil +} + +func appendLinesToFile(name string, lines string) error { + f, err := os.OpenFile(name, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return xerrors.Errorf("error opening file %s: %w", name, err) + } + + if _, err = f.WriteString(lines); err != nil { + return xerrors.Errorf("error writing to file %s: %w", name, err) + } + + err = f.Close() + return err +} + +func getOmPropertiesFromEnvVars() map[string]string { + props := map[string]string{} + for _, pair := range os.Environ() { + if !strings.HasPrefix(pair, omPropertyPrefix) { + continue + } + + p := strings.SplitN(pair, "=", 2) + key := strings.Replace(p[0], omPropertyPrefix, "", 1) + key = strings.ReplaceAll(key, "_", ".") + props[key] = p[1] + } + return props +} + +func updateMmsProperties(lines []string, newProperties map[string]string) []string { + seenProperties := map[string]bool{} + + // Overwrite existing properties + for i, line := range lines { + if strings.HasPrefix(line, commentPrefix) || !strings.Contains(line, "=") { + continue + } + + key := strings.Split(line, "=")[0] + if newVal, ok := newProperties[key]; ok { + lines[i] = fmt.Sprintf("%s=%s", key, newVal) + seenProperties[key] = true + } + } + + // Add new properties + for key, val := range newProperties { + if _, ok := seenProperties[key]; !ok { + lines = append(lines, fmt.Sprintf("%s=%s", key, val)) + } + } + return lines +} + +func getJvmParamDocString() string { + commentMarker := strings.Repeat("#", 55) + return fmt.Sprintf("%s\n## This is the custom JVM configuration set by the Operator\n%s\n\n", commentMarker, commentMarker) +} + +func main() { + if len(os.Args) < 3 { + fmt.Printf("Incorrect arguments %s, must specify path to conf file and path to properties file"+lineBreak, os.Args[1:]) + os.Exit(1) + } + confFile := os.Args[1] + propertiesFile := os.Args[2] + if err := updateConfFile(confFile); err != nil { + fmt.Println(err) + os.Exit(1) + } + + newProperties, err := getMmsProperties() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + if err := updatePropertiesFile(propertiesFile, newProperties); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration_test.go b/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration_test.go new file mode 100755 index 000000000..c849ad26b --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "fmt" + "math/rand" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEditMmsConfiguration_UpdateConfFile_Mms(t *testing.T) { + confFile := _createTestConfFile() + t.Setenv("CUSTOM_JAVA_MMS_UI_OPTS", "-Xmx4000m -Xms4000m") + err := updateConfFile(confFile) + assert.NoError(t, err) + updatedContent := _readLinesFromFile(confFile) + assert.Equal(t, updatedContent[7], "JAVA_MMS_UI_OPTS=\"${JAVA_MMS_UI_OPTS} -Xmx4000m -Xms4000m\"") +} + +func TestEditMmsConfiguration_UpdateConfFile_BackupDaemon(t *testing.T) { + confFile := _createTestConfFile() + + t.Setenv("BACKUP_DAEMON", "something") + t.Setenv("CUSTOM_JAVA_DAEMON_OPTS", "-Xmx4000m -Xms4000m") + err := updateConfFile(confFile) + assert.NoError(t, err) +} + +func TestEditMmsConfiguration_GetOmPropertiesFromEnvVars(t *testing.T) { + val := fmt.Sprintf("test%d", rand.Intn(1000)) + key := "OM_PROP_test_edit_mms_configuration_get_om_props" + t.Setenv(key, val) + props := getOmPropertiesFromEnvVars() + assert.Equal(t, props["test.edit.mms.configuration.get.om.props"], val) +} + +func TestEditMmsConfiguration_UpdatePropertiesFile(t *testing.T) { + newProperties := map[string]string{ + "mms.test.prop": "somethingNew", + "mms.test.prop.new": "400", + } + propFile := _createTestPropertiesFile() + err := updatePropertiesFile(propFile, newProperties) + assert.NoError(t, err) + + updatedContent := _readLinesFromFile(propFile) + assert.Equal(t, updatedContent[0], "mms.prop=1234") + assert.Equal(t, updatedContent[1], "mms.test.prop5=") + assert.Equal(t, updatedContent[2], "mms.test.prop=somethingNew") + assert.Equal(t, updatedContent[3], "mms.test.prop.new=400") +} + +func _createTestConfFile() string { + contents := "JAVA_MMS_UI_OPTS=\"${JAVA_MMS_UI_OPTS} -Xmx4352m -Xss328k -Xms4352m -XX:NewSize=600m -Xmn1500m -XX:ReservedCodeCacheSize=128m -XX:-OmitStackTraceInFastThrow\"\n" + contents += "JAVA_DAEMON_OPTS= \"${JAVA_DAEMON_OPTS} -DMONGO.BIN.PREFIX=\"\n\n" + return _writeTempFileWithContent(contents, "conf") +} + +func _createTestPropertiesFile() string { + contents := "mms.prop=1234\nmms.test.prop5=\nmms.test.prop=something" + return _writeTempFileWithContent(contents, "prop") +} + +func _readLinesFromFile(name string) []string { + content, _ := os.ReadFile(name) + return strings.Split(string(content), "\n") +} + +func _writeTempFileWithContent(content string, prefix string) string { + tmpfile, _ := os.CreateTemp("", prefix) + + _, _ = tmpfile.WriteString(content) + + _ = tmpfile.Close() + + return tmpfile.Name() +} diff --git a/docker/mongodb-enterprise-init-ops-manager/scripts/backup-daemon-liveness-probe.sh b/docker/mongodb-enterprise-init-ops-manager/scripts/backup-daemon-liveness-probe.sh new file mode 100755 index 000000000..ea8daa188 --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/scripts/backup-daemon-liveness-probe.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +check_backup_daemon_alive () { + pgrep --exact 'mms-app' || pgrep -f '/mongodb-ops-manager/jdk/bin/mms-app' +} + +check_backup_daemon_alive diff --git a/docker/mongodb-enterprise-init-ops-manager/scripts/docker-entry-point.sh b/docker/mongodb-enterprise-init-ops-manager/scripts/docker-entry-point.sh new file mode 100755 index 000000000..db7751360 --- /dev/null +++ b/docker/mongodb-enterprise-init-ops-manager/scripts/docker-entry-point.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# the function reacting on SIGTERM command sent by the container on its shutdown. Redirects the signal +# to the child process ("tail" in this case) +cleanup () { + echo "Caught SIGTERM signal." + if [[ -n "${child}" ]]; then + kill -TERM "${child}" + else + # Kill all tail processes + echo "Was not able to find child process, killing all tail processes" + pkill -f tail -TERM + fi +} + +# we need to change the Home directory for current bash so that the gen key was found correctly +# (the key is searched in "${HOME}/.mongodb-mms/gen.key") +HOME=${MMS_HOME} +CONFIG_TEMPLATE_DIR=${MMS_HOME}/conf-template +CONFIG_DIR=${MMS_HOME}/conf + +if [ -d "${CONFIG_TEMPLATE_DIR}" ] +then + if [ "$(ls -A "${CONFIG_DIR}")" ]; then + echo "The ${CONFIG_DIR} directory is not empty. Skipping copying files from ${CONFIG_TEMPLATE_DIR}" + echo "This might cause errors when booting up the OpsManager with read-only root filesystem" + else + echo "Copying ${CONFIG_TEMPLATE_DIR} content to ${CONFIG_DIR}" + cp "${CONFIG_TEMPLATE_DIR}"/* "${CONFIG_DIR}" + echo "Done copying ${CONFIG_TEMPLATE_DIR} content to ${CONFIG_DIR}" + fi +else + echo "It seems you're running an older version of the Ops Manager image." + echo "Please pull the latest one." +fi + +# Execute script that updates properties and conf file used to start ops manager +echo "Updating configuration properties file ${MMS_PROP_FILE} and conf file ${MMS_CONF_FILE}" +/opt/scripts/mmsconfiguration "${MMS_CONF_FILE}" "${MMS_PROP_FILE}" + +if [[ -z ${BACKUP_DAEMON+x} ]]; then + echo "Starting Ops Manager" + "${MMS_HOME}/bin/mongodb-mms" start_mms || { + echo "Startup of Ops Manager failed with code $?" + if [[ -f ${MMS_LOG_DIR}/mms0-startup.log ]]; then + echo + echo "mms0-startup.log:" + echo + cat "${MMS_LOG_DIR}/mms0-startup.log" + fi + if [[ -f ${MMS_LOG_DIR}/mms0.log ]]; then + echo + echo "mms0.log:" + echo + cat "${MMS_LOG_DIR}/mms0.log" + fi + if [[ -f ${MMS_LOG_DIR}/mms-migration.log ]]; then + echo + echo "mms-migration.log" + echo + cat "${MMS_LOG_DIR}/mms-migration.log" + fi + exit 1 + } + + tail -F -n 1000 "${MMS_LOG_DIR}/mms0.log" "${MMS_LOG_DIR}/mms0-startup.log" "${MMS_LOG_DIR}/mms-migration.log" & +else + echo "Starting Ops Manager Backup Daemon" + "${MMS_HOME}/bin/mongodb-mms" start_backup_daemon + + tail -F "${MMS_LOG_DIR}/daemon.log" & +fi + +export child=$! +echo "Launched tail, pid=${child}" + +trap cleanup SIGTERM + +wait "${child}" diff --git a/docker/mongodb-enterprise-operator/Dockerfile.builder b/docker/mongodb-enterprise-operator/Dockerfile.builder new file mode 100644 index 000000000..5df5ebcaa --- /dev/null +++ b/docker/mongodb-enterprise-operator/Dockerfile.builder @@ -0,0 +1,50 @@ +# +# Dockerfile for Operator. +# to be called from git root +# docker build . -f docker/mongodb-enterprise-operator/Dockerfile.builder +# + +FROM public.ecr.aws/docker/library/golang:1.24 as builder + +ARG release_version +ARG log_automation_config_diff +ARG use_race + +COPY go.sum go.mod /go/src/github.com/10gen/ops-manager-kubernetes/ +WORKDIR /go/src/github.com/10gen/ops-manager-kubernetes +RUN go mod download + +COPY . /go/src/github.com/10gen/ops-manager-kubernetes + +RUN go version +RUN git version +RUN mkdir /build && \ + if [ $use_race = "true" ]; then \ + echo "Building with race detector" && \ + CGO_ENABLED=1 go build -o /build/mongodb-enterprise-operator \ + -buildvcs=false \ + -race \ + -ldflags=" -X github.com/10gen/ops-manager-kubernetes/pkg/util.OperatorVersion=${release_version} \ + -X github.com/10gen/ops-manager-kubernetes/pkg/util.LogAutomationConfigDiff=${log_automation_config_diff}"; \ + else \ + echo "Building without race detector" && \ + CGO_ENABLED=0 go build -o /build/mongodb-enterprise-operator \ + -buildvcs=false \ + -ldflags="-s -w -X github.com/10gen/ops-manager-kubernetes/pkg/util.OperatorVersion=${release_version} \ + -X github.com/10gen/ops-manager-kubernetes/pkg/util.LogAutomationConfigDiff=${log_automation_config_diff}"; \ + fi + + +ADD https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 /usr/local/bin/jq +RUN chmod +x /usr/local/bin/jq + +RUN mkdir -p /data +RUN cat release.json | jq -r '.supportedImages."mongodb-agent" | { "supportedImages": { "mongodb-agent": . } }' > /data/om_version_mapping.json +RUN chmod +r /data/om_version_mapping.json + +FROM scratch + +COPY --from=builder /build/mongodb-enterprise-operator /data/ +COPY --from=builder /data/om_version_mapping.json /data/om_version_mapping.json + +ADD docker/mongodb-enterprise-operator/licenses /data/licenses/ diff --git a/docker/mongodb-enterprise-operator/Dockerfile.template b/docker/mongodb-enterprise-operator/Dockerfile.template new file mode 100644 index 000000000..0cef5062f --- /dev/null +++ b/docker/mongodb-enterprise-operator/Dockerfile.template @@ -0,0 +1,36 @@ +# +# Base Template Dockerfile for Operator Image. +# + +ARG imagebase +FROM ${imagebase} as base + +FROM {{ base_image }} + +{% block labels %} +LABEL name="MongoDB Enterprise Operator" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="{{ version }}" \ + release="1" \ + summary="MongoDB Enterprise Operator Image" \ + description="MongoDB Enterprise Operator Image" +{% endblock %} + +{% block packages -%} +{% endblock -%} + +{% block static %} +{% endblock %} + + +COPY --from=base /data/mongodb-enterprise-operator /usr/local/bin/mongodb-enterprise-operator +COPY --from=base /data/om_version_mapping.json /usr/local/om_version_mapping.json +COPY --from=base /data/licenses /licenses/ + +USER 2000 + +ENTRYPOINT exec /usr/local/bin/mongodb-enterprise-operator + +{% block healthcheck %} +{% endblock %} diff --git a/docker/mongodb-enterprise-operator/Dockerfile.ubi b/docker/mongodb-enterprise-operator/Dockerfile.ubi new file mode 100644 index 000000000..04e83faf7 --- /dev/null +++ b/docker/mongodb-enterprise-operator/Dockerfile.ubi @@ -0,0 +1,12 @@ +{% extends "Dockerfile.template" %} + +{% set base_image = "registry.access.redhat.com/ubi9/ubi-minimal" %} + +{% block packages -%} +# Building an UBI-based image: https://red.ht/3n6b9y0 +RUN microdnf update \ + --disableplugin=subscription-manager \ + --disablerepo=* --enablerepo=ubi-9-appstream-rpms --enablerepo=ubi-9-baseos-rpms -y \ + && rm -rf /var/cache/yum +RUN microdnf install -y glibc-langpack-en +{% endblock -%} diff --git a/docker/mongodb-enterprise-operator/LICENSE b/docker/mongodb-enterprise-operator/LICENSE new file mode 100644 index 000000000..dc71da876 --- /dev/null +++ b/docker/mongodb-enterprise-operator/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-operator/README.md b/docker/mongodb-enterprise-operator/README.md new file mode 100644 index 000000000..16d751fee --- /dev/null +++ b/docker/mongodb-enterprise-operator/README.md @@ -0,0 +1,21 @@ +# MongoDB Enterprise Kubernetes Operator + +This directory hosts the Dockerfile for the Ops Manager Operator. + +### Building the source-code + +```bash +CGO_ENABLED=0 GOOS=linux GOFLAGS="-mod=vendor" go build -i -o mongodb-enterprise-operator +``` + +### Building the image + +```bash +docker build -t mongodb-enterprise-operator:0.1 . +``` + +### Running locally + +```bash +docker run -e OPERATOR_ENV=local -e MONGODB_ENTERPRISE_DATABASE_IMAGE=mongodb-enterprise-database -e IMAGE_PULL_POLICY=Never mongodb-enterprise-operator:0.1 +``` diff --git a/docker/mongodb-enterprise-operator/content/.keep b/docker/mongodb-enterprise-operator/content/.keep new file mode 100644 index 000000000..f9c716292 --- /dev/null +++ b/docker/mongodb-enterprise-operator/content/.keep @@ -0,0 +1 @@ +This dir needs to exist to support the build process. diff --git a/docker/mongodb-enterprise-operator/licenses/Apache-2.0/LICENSE b/docker/mongodb-enterprise-operator/licenses/Apache-2.0/LICENSE new file mode 100644 index 000000000..15555da25 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/Apache-2.0/LICENSE @@ -0,0 +1,60 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: +(a) You must give any other recipients of the Work or Derivative Works a copy of this License; and +(b) You must cause any modified files to carry prominent notices stating that You changed the files; and +(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/docker/mongodb-enterprise-operator/licenses/Apache-2.0/README b/docker/mongodb-enterprise-operator/licenses/Apache-2.0/README new file mode 100644 index 000000000..802038a74 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/Apache-2.0/README @@ -0,0 +1,49 @@ +# Components licensed under: Apache License Version 2.0 + +## github.com/prometheus/client_model/go + +## k8s.io/apimachinery + +## k8s.io/utils + +## gomodules.xyz/jsonpatch/v2 + +## github.com/xdg/stringprep + +## gopkg.in/yaml.v2 + +## github.com/prometheus/common + +## github.com/matttproud/golang_protobuf_extensions/pbutil + +## k8s.io/kube-openapi/pkg/util/proto + +## github.com/prometheus/procfs + +## github.com/modern-go/reflect2 + +## github.com/googleapis/gnostic + +## github.com/google/gofuzz + +## k8s.io/klog + +## sigs.k8s.io/controller-runtime + +## k8s.io/client-go + +## github.com/golang/groupcache/lru + +## github.com/prometheus/client_golang/prometheus + +## k8s.io/api + +## github.com/modern-go/concurrent + +## github.com/go-logr/logr + +## k8s.io/apiextensions-apiserver/pkg/apis/apiextensions + +## bson@1.1.1 + +## require_optional@1.0.1 diff --git a/docker/mongodb-enterprise-operator/licenses/BSD-2-Clause/LICENSE b/docker/mongodb-enterprise-operator/licenses/BSD-2-Clause/LICENSE new file mode 100644 index 000000000..07c0aac54 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/BSD-2-Clause/LICENSE @@ -0,0 +1,9 @@ +BSD 2-Clause License + +Copyright (c) . All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docker/mongodb-enterprise-operator/licenses/BSD-2-Clause/README b/docker/mongodb-enterprise-operator/licenses/BSD-2-Clause/README new file mode 100644 index 000000000..cfefaad50 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/BSD-2-Clause/README @@ -0,0 +1,3 @@ +# Components licensed under: BSD 2-Clause License + +## github.com/pkg/errors diff --git a/docker/mongodb-enterprise-operator/licenses/BSD-3-Clause/LICENSE b/docker/mongodb-enterprise-operator/licenses/BSD-3-Clause/LICENSE new file mode 100644 index 000000000..bfdf56636 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/BSD-3-Clause/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) [year], [fullname] +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docker/mongodb-enterprise-operator/licenses/BSD-3-Clause/README b/docker/mongodb-enterprise-operator/licenses/BSD-3-Clause/README new file mode 100644 index 000000000..d8ab70737 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/BSD-3-Clause/README @@ -0,0 +1,37 @@ +# Components licensed under: BSD 3-Clause License + +## gopkg.in/fsnotify.v1 + +## golang.org/x/oauth2 + +## golang.org/x/time/rate + +## golang.org/x/crypto/ssh/terminal + +## github.com/pmezard/go-difflib/difflib + +## github.com/gogo/protobuf + +## github.com/spf13/pflag + +## github.com/golang/protobuf + +## github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg + +## github.com/evanphx/json-patch + +## github.com/google/go-cmp/cmp + +## gopkg.in/inf.v0 + +## golang.org/x/net + +## golang.org/x/sys/unix + +## github.com/google/uuid + +## golang.org/x/xerrors + +## github.com/imdario/mergo + +## golang.org/x/text diff --git a/docker/mongodb-enterprise-operator/licenses/ISC/LICENSE b/docker/mongodb-enterprise-operator/licenses/ISC/LICENSE new file mode 100644 index 000000000..4105f19d4 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/ISC/LICENSE @@ -0,0 +1,8 @@ +ISC License + +Copyright (c) 2004-2010 by Internet Systems Consortium, Inc. ("ISC") +Copyright (c) 1995-2003 by Internet Software Consortium + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/docker/mongodb-enterprise-operator/licenses/ISC/README b/docker/mongodb-enterprise-operator/licenses/ISC/README new file mode 100644 index 000000000..eb9dcf0c4 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/ISC/README @@ -0,0 +1,3 @@ +# Components licensed under: ISC License + +## github.com/davecgh/go-spew/spew diff --git a/docker/mongodb-enterprise-operator/licenses/MIT/LICENSE b/docker/mongodb-enterprise-operator/licenses/MIT/LICENSE new file mode 100644 index 000000000..333c3b09d --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/MIT/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docker/mongodb-enterprise-operator/licenses/MIT/README b/docker/mongodb-enterprise-operator/licenses/MIT/README new file mode 100644 index 000000000..1b3ba408a --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/MIT/README @@ -0,0 +1,29 @@ +# Components licensed under: MIT License + +## go.uber.org/multierr + +## github.com/spf13/cast + +## github.com/blang/semver + +## go.uber.org/zap + +## github.com/beorn7/perks/quantile + +## github.com/stretchr/testify/assert + +## go.uber.org/atomic + +## github.com/json-iterator/go + +## sigs.k8s.io/yaml + +## memory-pager@1.5.0 + +## resolve-from@2.0.0 + +## safe-buffer@5.2.0 + +## saslprep@1.0.3 + +## sparse-bitfield@3.0.3 diff --git a/docker/mongodb-enterprise-operator/licenses/MPL-2.0/LICENSE b/docker/mongodb-enterprise-operator/licenses/MPL-2.0/LICENSE new file mode 100644 index 000000000..91a6cb7ca --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/MPL-2.0/LICENSE @@ -0,0 +1,111 @@ +Mozilla Public License Version 2.0 + +1. Definitions +1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. +1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. +1.3. "Contribution" means Covered Software of a particular Contributor. +1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. +1.5. "Incompatible With Secondary Licenses" means +(a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or +(b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. +1.6. "Executable Form" means any form of the work other than Source Code Form. +1.7. "Larger Work" means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. +1.8. "License" means this document. +1.9. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. +1.10. "Modifications" means any of the following: +(a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or +(b) any new file in Source Code Form that contains any Covered Software. +1.11. "Patent Claims" of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. +1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. +1.13. "Source Code Form" means the form of the work preferred for making modifications. +1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. +2. License Grants and Conditions +2.1. Grants +Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and +(b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. +2.2. Effective Date +The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. + +2.3. Limitations on Grant Scope +The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: + +(a) for any code that a Contributor has removed from Covered Software; or +(b) for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or +(c) under Patent Claims infringed by Covered Software in the absence of its Contributions. +This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). + +2.4. Subsequent Licenses +No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). + +2.5. Representation +Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use +This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. + +3. Responsibilities +3.1. Distribution of Source Form +All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. + +3.2. Distribution of Executable Form +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and +(b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. +3.3. Distribution of a Larger Work +You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). + +3.4. Notices +You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms +You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. + +5. Termination +5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. +5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. +6. Disclaimer of Warranty +Covered Software is provided under this License on an "as is" basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. + +7. Limitation of Liability +Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party's negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation +Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. + +9. Miscellaneous +This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. + +10. Versions of the License +10.1. New Versions +Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. + +10.2. Effect of New Versions +You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. + +10.3. Modified Versions +If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses +If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + +This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. diff --git a/docker/mongodb-enterprise-operator/licenses/MPL-2.0/README b/docker/mongodb-enterprise-operator/licenses/MPL-2.0/README new file mode 100644 index 000000000..827fcbf17 --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/MPL-2.0/README @@ -0,0 +1,7 @@ +# Components licensed under: Mozilla Public License Version 2.0 + +## github.com/hashicorp/golang-lru + +## github.com/hashicorp/go-multierror + +## github.com/hashicorp/errwrap diff --git a/docker/mongodb-enterprise-operator/licenses/THIRD-PARTY-NOTICES b/docker/mongodb-enterprise-operator/licenses/THIRD-PARTY-NOTICES new file mode 100644 index 000000000..b014dbeba --- /dev/null +++ b/docker/mongodb-enterprise-operator/licenses/THIRD-PARTY-NOTICES @@ -0,0 +1,18 @@ +Third-party notices +------------------- + +This product uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +Please contact the MongoDB Legal Department if you need a required license to +be added this list, or you would like a copy of the source code from a library in this list. + + https://www.mongodb.com/contact?jmp=docs + +The attached notices are provided for information only. + +Source code for open source libraries can be obtained from MongoDB. + +In the case of components licensed under multiple licenses, +MongoDB has indicated which license it elects by listing the component +under the corresponding license sub-directory. diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile.builder b/docker/mongodb-enterprise-ops-manager/Dockerfile.builder new file mode 100644 index 000000000..ccfb79597 --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/Dockerfile.builder @@ -0,0 +1,20 @@ +# Build compilable stuff + +FROM public.ecr.aws/docker/library/golang:1.24 as readiness_builder +COPY . /go/src/github.com/10gen/ops-manager-kubernetes +WORKDIR /go/src/github.com/10gen/ops-manager-kubernetes + +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/mmsconfiguration ./docker/mongodb-enterprise-init-ops-manager/mmsconfiguration/edit_mms_configuration.go +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/backup-daemon-readiness-probe ./docker/mongodb-enterprise-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness.go + +# Move binaries and scripts +FROM scratch + +COPY --from=readiness_builder /data/scripts/mmsconfiguration /data/scripts/mmsconfiguration +COPY --from=readiness_builder /data/scripts/backup-daemon-readiness-probe /data/scripts/backup-daemon-readiness-probe + +# After v2.0, when non-Static Agent images will be removed, please ensure to copy those files +# into ./docker/mongodb-enterprise-ops-manager directory. Leaving it this way will make the maintenance easier. +COPY ./docker/mongodb-enterprise-init-ops-manager/scripts/docker-entry-point.sh /data/scripts +COPY ./docker/mongodb-enterprise-init-ops-manager/scripts/backup-daemon-liveness-probe.sh /data/scripts +COPY ./docker/mongodb-enterprise-init-ops-manager/LICENSE /data/licenses/mongodb-enterprise-ops-manager diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile.dcar b/docker/mongodb-enterprise-ops-manager/Dockerfile.dcar new file mode 100644 index 000000000..639c7930b --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/Dockerfile.dcar @@ -0,0 +1,25 @@ +{% extends "Dockerfile.ubi" %} + + +{% block packages %} +RUN yum install --disableplugin=subscription-manager \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libcurl \ + libpcap \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + rpm-libs \ + net-tools \ + procps-ng \ + ncurses +{% endblock %} + +{% block healthcheck %} +HEALTHCHECK --timeout=30s CMD ls /mongodb-ops-manager/bin/mongodb-mms || exit 1 +{% endblock %} diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile.template b/docker/mongodb-enterprise-ops-manager/Dockerfile.template new file mode 100644 index 000000000..5e2b10d7d --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/Dockerfile.template @@ -0,0 +1,61 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM {{ base_image }} + +{% block labels %} +LABEL name="MongoDB Enterprise Ops Manager" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="{{ version }}" \ + release="1" \ + summary="MongoDB Enterprise Ops Manager Image" \ + description="MongoDB Enterprise Ops Manager" +{% endblock %} + +ENV MMS_HOME /mongodb-ops-manager +ENV MMS_PROP_FILE ${MMS_HOME}/conf/conf-mms.properties +ENV MMS_CONF_FILE ${MMS_HOME}/conf/mms.conf +ENV MMS_LOG_DIR ${MMS_HOME}/logs +ENV MMS_TMP_DIR ${MMS_HOME}/tmp + +EXPOSE 8080 + +# OpsManager docker image needs to have the MongoDB dependencies because the +# backup daemon is running its database locally +{% block packages %} +{% endblock %} + +COPY --from=base /data/licenses /licenses/ + +COPY --from=base /data/scripts /opt/scripts + + +{% block static %} +RUN curl --fail -L -o ops_manager.tar.gz {{ om_download_url }} \ + && tar -xzf ops_manager.tar.gz \ + && rm ops_manager.tar.gz \ + && mv mongodb-mms* "${MMS_HOME}" +{% endblock %} + +# permissions +RUN chmod -R 0777 "${MMS_LOG_DIR}" \ + && chmod -R 0777 "${MMS_TMP_DIR}" \ + && chmod -R 0775 "${MMS_HOME}/conf" \ + && chmod -R 0775 "${MMS_HOME}/jdk" \ + && mkdir "${MMS_HOME}/mongodb-releases/" \ + && chmod -R 0775 "${MMS_HOME}/mongodb-releases" \ + && chmod -R 0777 "${MMS_CONF_FILE}" \ + && chmod -R 0777 "${MMS_PROP_FILE}" + +# The "${MMS_HOME}/conf" will be populated by the docker-entry-point.sh. +# For now we need to move into the templates directory. +RUN cp -r "${MMS_HOME}/conf" "${MMS_HOME}/conf-template" + +USER 2000 + +# operator to change the entrypoint to: /mongodb-ops-manager/bin/mongodb-mms start_mms (or a wrapper around this) +ENTRYPOINT [ "sleep infinity" ] + +{% block healthcheck %} +{% endblock %} diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile.ubi b/docker/mongodb-enterprise-ops-manager/Dockerfile.ubi new file mode 100644 index 000000000..00cb2f200 --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/Dockerfile.ubi @@ -0,0 +1,30 @@ +{% extends "Dockerfile.template" %} + +{% set base_image = "registry.access.redhat.com/ubi9/ubi-minimal" %} + +{% block packages %} + +# Replace libcurl-minimal and curl-minimal with the full versions +# https://bugzilla.redhat.com/show_bug.cgi?id=1994521 +RUN microdnf install -y libssh libpsl libbrotli \ + && microdnf download curl libcurl \ + && rpm -Uvh --nodeps --replacefiles "*curl*$( uname -i ).rpm" \ + && microdnf remove -y libcurl-minimal curl-minimal + +RUN microdnf install --disableplugin=subscription-manager -y \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libpcap \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + tar \ + rpm-libs \ + net-tools \ + procps-ng \ + ncurses +{% endblock %} diff --git a/docker/mongodb-enterprise-ops-manager/LICENSE b/docker/mongodb-enterprise-ops-manager/LICENSE new file mode 100644 index 000000000..dc71da876 --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-tests/.dockerignore b/docker/mongodb-enterprise-tests/.dockerignore new file mode 100644 index 000000000..300e35ad9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/.dockerignore @@ -0,0 +1,3 @@ +venv/* +__pycache__/* +**/*.pyc diff --git a/docker/mongodb-enterprise-tests/Dockerfile b/docker/mongodb-enterprise-tests/Dockerfile new file mode 100644 index 000000000..57e489cdb --- /dev/null +++ b/docker/mongodb-enterprise-tests/Dockerfile @@ -0,0 +1,55 @@ +# This image is based on latest Python 3.6 release in latest Debian Stretch. +# I had to move away from Alpine as the latest Kubernetes Python module depends +# on `cryptography` which can be installed in Debian but needs to be compiled +# in Alpine, meaning that we would have to install gcc or clang on it, making +# it too slow for the images. +# +# Ref: https://cryptography.io/en/latest/installation/#building-cryptography-on-linux +# +ARG PYTHON_VERSION + +FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim as builder + + +RUN apt-get -qq update \ + && apt-get -y -qq install \ + curl libldap2-dev libsasl2-dev build-essential git + +COPY requirements.txt requirements.txt + +RUN python3 -m venv /venv && . /venv/bin/activate && python3 -m pip install -r requirements.txt + + +FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim + +RUN apt-get -qq update \ + && apt-get -y -qq install \ + curl \ + libldap2-dev \ + libsasl2-dev \ + git \ + openssl + +ENV HELM_NAME "helm-v3.17.1-linux-amd64.tar.gz" +# install Helm +RUN curl --fail --retry 3 -L -o "${HELM_NAME}" "https://get.helm.sh/${HELM_NAME}" \ + && tar -xzf "${HELM_NAME}" \ + && rm "${HELM_NAME}" \ + && mv "linux-amd64/helm" "/usr/local/bin/helm" + +COPY --from=builder /venv /venv + +ENV PATH="/venv/bin:${PATH}" + +RUN mkdir /tests +WORKDIR /tests + +# copying the test files after python build, otherwise pip install will be called each time the tests change +COPY . /tests +# copying the helm_chart directory as well to support installation of the Operator from the test application +COPY helm_chart /helm_chart +COPY release.json /release.json +# we use the public directory to automatically test resources samples +COPY public /ops-manager-kubernetes/public + +ADD multi-cluster-kube-config-creator_linux /usr/local/bin/multi-cluster-kube-config-creator diff --git a/docker/mongodb-enterprise-tests/README.md b/docker/mongodb-enterprise-tests/README.md new file mode 100644 index 000000000..be60699f9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/README.md @@ -0,0 +1,227 @@ +# Enterprise Kubernetes E2E Testing # + +The MongoDB Enterprise Kubernetes Operator uses a declarative testing +framwork based on the Python programming language and the Pytest +module. Its design allow the developer to write tests in a +declarative way and verify the results by using a collection of helper +functions that check the state of the Kubernetes cluster. + +# Quick start # + +The goal of this guide is to allow you to run your tests locally as +regular Python tests. The resulting "experience" should be something +like: + +``` bash +$ pytest -m e2e_replica_set +===================================================================== test session starts =========================================================== +platform linux -- Python 3.6.8, pytest-4.3.1, py-1.8.0, pluggy-0.9.0 -- /home/rvalin/.virtualenvs/operator-tests/bin/python +cachedir: .pytest_cache +rootdir: /home/rvalin/workspace/go/src/github.com/10gen/ops-manager-kubernetes/docker/mongodb-enterprise-tests, inifile: pytest.ini +collected 168 items / 131 deselected / 37 selected +tests/replicaset/replica_set.py::TestReplicaSetCreation::test_replica_set_sts_exists PASSED [ 2%] +tests/replicaset/replica_set.py::TestReplicaSetCreation::test_sts_creation PASSED [ 5%] +tests/replicaset/replica_set.py::TestReplicaSetCreation::test_sts_metadata PASSED [ 8%] +tests/replicaset/replica_set.py::TestReplicaSetCreation::test_sts_replicas PASSED [ 10%] +tests/replicaset/replica_set.py::TestReplicaSetCreation::test_sts_template PASSED [ 13%] + +... + +tests/replicaset/replica_set.py::TestReplicaSetCreation::test_replica_set_was_configured SKIPPED [ 51%] +tests/replicaset/replica_set.py::TestReplicaSetUpdate::test_replica_set_sts_should_exist PASSED [ 54%] +tests/replicaset/replica_set.py::TestReplicaSetUpdate::test_sts_update PASSED [ 56%] +tests/replicaset/replica_set.py::TestReplicaSetUpdate::test_sts_metadata PASSED [ 59%] + +... + +tests/replicaset/replica_set.py::TestReplicaSetUpdate::test_backup PASSED [ 94%] +tests/replicaset/replica_set.py::TestReplicaSetDelete::test_replica_set_sts_doesnt_exist PASSED [ 97%] +tests/replicaset/replica_set.py::TestReplicaSetDelete::test_service_does_not_exist PASSED [100%] +============================================= 36 passed, 1 skipped, 131 deselected, 10 warnings in 89.47 seconds ==================================== + +``` + +This is a full E2E experience running locally that takes about 90 +seconds. + +## Configuring/Installing dependencies ## + +In order to run, the local E2E tests need certain dependant +components: + +* Ops Manager >= 4.0 +* `kubectl` context to use configured Kubernetes Cluster +* Operator `Project` and `Credentials` created +* Kubernetes Namespace created +* Python 3.6 and dependencies + +Most of the required configuration is achieved by using `make` at the +root of the project. [This +document](https://wiki.corp.mongodb.com/display/MMS/Setting+up+local+development+and+E2E+testing) +will guide you on how to do this. + +The result of this is to have a running Ops Manager with +configurations saved on the `~/.operator-dev/om` file. This file will +be read by the E2E testing framework to configure itself. Once you +have done this, you can proceed to complete the Python installation. + +### Installing Python and Dependencies ### + +Run `scripts/dev/install.sh` or `scripts/dev/recreate_python_venv.sh` to install necessary tools and create python virtualenv. + +* After the first run, when coming back to the project it should be +required to `activate` your virtual environment once again. +``` bash +source venv/bin/activate +``` + +There are many mechanisms to achieve this in a more automated way, +I'll leave that to each person to decide the one that suits their needs. + +# Running the E2E Tests Locally # + + +The tests are defined in terms of "markers" in the Python source +files. To run each test, you need to specify which "marker" you want +to run, for instance: + +``` bash +# Run the Replica Set Enterprise Installation E2E Test +pytest -m e2e_replica_set + +# Run the TLS Upgrade E2E Test +pytest -m e2e_replica_set_tls_require_upgrade +``` + +These markers correspond to the name of the `task` in Evergreen. It is +handy as they can be copied over the command line to try the tests +locally, that usually run faster. + +To find the list of available E2E tasks you can do the following: + + grep pytest.mark.e2e * -R | sort | uniq | cut -d "@" -f 2 + +The `@pytest.mark.` syntax is a class annotation we use to +indicate which test classes need to be run. But for now they help us +to call a particular E2E task we are interested in. + + +# Writing New Tests # + +### Create a new Python test file ### + +Create a new Python file that will hold the test, the contents of this +file will be something like: + +``` python +import pytest +from kubetester.kubetester import KubernetesTester +from kubernetes import client + + +@pytest.mark.e2e_my_new_feature +class TestMyNewFeatureShouldPass(KubernetesTester): + """ + name: My New Feature Test 01 + create: + file: my-mdb-object-to-test.yaml + wait_until: in_running_state + """ + + def test_something_about_my_new_object(self): + mdb_object_name = "my-mdb-object-to-test" + mdb = client.CustomObjectsApi().get_namespaced_custom_object( + "mongodb.com", "v1", self.namespace, "mongodb", mdb_object_name + ) + + assert mdb["status"]["phase"] == "Running" +``` + +The `my-mdb-object-to-test.yaml` will reside in one of the `fixtures` +directories and contain something like: + +``` yaml +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-mdb-object-to-test +spec: + version: 4.0.8 + type: Standalone + project: my-project + credentials: my-credentials + persistent: false +``` + +Check the `@pytest.mark` decorator we have set for this test class, this is +what we'll use to run the test locally with: + +``` bash +$ pytest -m e2e_my_new_feature +========================================== test session starts =========================================== +platform linux -- Python 3.6.8, pytest-4.3.1, py-1.8.0, pluggy-0.9.0 -- /home/rvalin/.virtualenvs/operator-tests/bin/python +cachedir: .pytest_cache +rootdir: /home/rvalin/workspace/go/src/github.com/10gen/ops-manager-kubernetes/docker/mongodb-enterprise-tests, inifile: pytest.ini +collected 170 items / 169 deselected / 1 selected +tests/mixed/sample_test.py::TestMyNewFeatureShouldPass::test_something_about_my_new_object PASSED [100%] +=============================== 1 passed, 169 deselected in 40.62 seconds ================================ +``` + +In this run `pytest` was able to find our e2e test with the name +`e2e_my_new_feature`, the same we used in the `pytest.mark` decorator +and it run by finding our Kubernetes environment by reading `kubectl` +configuration. + +### Make sure your test resources are removed! ### + +There are a few handy functions to have your MDB resources removed, +the usual path to do this is to have a new function that will remove +the object for you: + +``` python +def test_mdb_object_is_removed(self): + mdb_object_name = "my-mdb-object-to-test" + delete_opts = client.V1DeleteOptions() + + mdb = client.CustomObjectsApi().delete_namespaced_custom_object( + "mongodb.com", "v1", self.namespace, "mongodb", mdb_object_name, body=delete_opts + ) + + assert mdb["status"] == "Success" +``` + +### Finally, add an entry into Evergreen ### + +This is a 2-step process, which requires adding a new E2E task and add +this new task into a task group. + +First you add a new task, under the `tasks` dictionary in the +`evergreen.yaml` file: + +``` yaml +tasks: +- name: e2e_my_new_feature + commands: + - func: "e2e_test" +``` + +And secondly, add this task into a `task_group`. Find the one that +matches whatever you are testing and add the task in there. If your +task is testing something structural, it should go into +`e2e_core_task_group`, like: + +``` yaml +- name: e2e_core_task_group + max_hosts: 100 + setup_group: + - func: "clone" + - func: "setup_kubectl" + tasks: + - e2e_all_mongodb_resources_parallel + - e2e_standalone_config_map + - .... # more tasks + - e2e_my_new_feature # this is our new task! +``` + +After doing this, your task will be added into the list of tasks +executed by Evergreen on each test run. diff --git a/docker/mongodb-enterprise-tests/kubeobject/__init__.py b/docker/mongodb-enterprise-tests/kubeobject/__init__.py new file mode 100644 index 000000000..13e2f1d1a --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/__init__.py @@ -0,0 +1,25 @@ +import random +from string import ascii_lowercase, digits + +from .customobject import CustomObject # noqa: F401 +from .kubeobject import KubeObject, create_custom_object # noqa: F401 + + +def generate_random_name(prefix="", suffix="", size=63) -> str: + """Generates a random and valid Kubernetes name.""" + max_len = 63 + min_len = 0 + + if size > max_len: + size = max_len + + random_len = size - len(prefix) - len(suffix) + if random_len < min_len: + random_len = min_len + + body = [] + if random_len > 0: + body = [random.choice(ascii_lowercase + digits) for _ in range(random_len - 1)] + body = [random.choice(ascii_lowercase)] + body + + return prefix + "".join(body) + suffix diff --git a/docker/mongodb-enterprise-tests/kubeobject/customobject.py b/docker/mongodb-enterprise-tests/kubeobject/customobject.py new file mode 100644 index 000000000..6dadb942e --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/customobject.py @@ -0,0 +1,362 @@ +from __future__ import annotations + +import copy +from datetime import datetime, timedelta +from typing import Dict, Optional + +import yaml +from kubernetes import client + + +class CustomObject: + """CustomObject is an object mapping to a Custom Resource in Kubernetes. It + includes simple facilities to update the Custom Resource, save it and + reload its state in a object oriented manner. + + It is meant to be used to apply changes to Custom Resources and watch their + state as it is updated by a controller; an Operator in Kubernetes parlance. + + """ + + def __init__( + self, + name: str, + namespace: str, + kind: Optional[str] = None, + plural: Optional[str] = None, + group: Optional[str] = None, + version: Optional[str] = None, + api_client: Optional[client.ApiClient] = None, + ): + self.name = name + self.namespace = namespace + + if any(value is None for value in (plural, kind, group, version)): + # It is possible to have a CustomObject where some of the initial values are set + # to None. For instance when instantiating CustomObject from a yaml file (from_yaml). + # In this case, we need to look for the rest of the parameters from the + # apiextensions Kubernetes API. + crd = get_crd_names( + plural=plural, + kind=kind, + group=group, + version=version, + api_client=api_client, + ) + self.kind = crd.spec.names.kind + self.plural = crd.spec.names.plural + self.group = crd.spec.group + self.version = crd.spec.version + else: + self.kind = kind + self.plural = plural + self.group = group + self.version = version + + # True if this object is backed by a Kubernetes object, this is, it has + # been loaded or saved from/to Kubernetes API. + self.bound = False + + # Set to True if the object needs to be updated every time one of its + # attributes is changed. + self.auto_save = False + + # Set `auto_reload` to `True` if it needs to be reloaded before every + # read of an attribute. This considers the `auto_reload_period` + # attribute at the same time. + self.auto_reload = False + + # If `auto_reload` is set, it will not reload if less time than + # `auto_reload_period` has passed since last read. + self.auto_reload_period = timedelta(seconds=2) + + # Last time this object was updated + self.last_update: datetime = None + + # Sets the API used for this particular type of object + self.api = client.CustomObjectsApi(api_client=api_client) + + if not hasattr(self, "backing_obj"): + self.backing_obj = { + "metadata": {"name": name, "namespace": namespace}, + "kind": self.kind, + "apiVersion": "/".join(filter(None, [group, version])), + "spec": {}, + "status": {}, + } + + def load(self) -> CustomObject: + """Loads this object from the API.""" + + obj = self.api.get_namespaced_custom_object(self.group, self.version, self.namespace, self.plural, self.name) + + self.backing_obj = obj + self.bound = True + + self._register_updated() + return self + + def create(self) -> CustomObject: + """Creates this object in Kubernetes.""" + obj = self.api.create_namespaced_custom_object( + self.group, self.version, self.namespace, self.plural, self.backing_obj + ) + + self.backing_obj = obj + self.bound = True + + self._register_updated() + return self + + def update(self) -> CustomObject: + """Updates the object in Kubernetes. Deleting keys is done by setting them to None""" + return create_or_update(self) + + def patch(self) -> CustomObject: + """Patch the object in Kubernetes. Deleting keys is done by setting them to None""" + obj = self.api.patch_namespaced_custom_object( + self.group, + self.version, + self.namespace, + self.plural, + self.name, + self.backing_obj, + ) + self.backing_obj = obj + + self._register_updated() + + return obj + + def _register_updated(self): + """Register the last time the object was updated from Kubernetes.""" + self.last_update = datetime.now() + + def _reload_if_needed(self): + """Reloads the object is `self.auto_reload` is set to `True` and more than + `self.auto_reload_period` time has passed since last reload.""" + if not self.auto_reload: + return + + if self.last_update is None: + self.reload() + + if datetime.now() - self.last_update > self.auto_reload_period: + self.reload() + + @classmethod + def from_yaml(cls, yaml_file, name=None, namespace=None): + """Creates a `CustomObject` from a yaml file. In this case, `name` and + `namespace` are optional in this function's signature, because they + might be passed as part of the `yaml_file` document. + """ + doc = yaml.safe_load(open(yaml_file)) + + if "metadata" not in doc: + doc["metadata"] = dict() + + if (name is None or name == "") and "name" not in doc["metadata"]: + raise ValueError( + "`name` needs to be passed as part of the function call " + "or exist in the `metadata` section of the yaml document." + ) + + if (namespace is None or namespace == "") and "namespace" not in doc["metadata"]: + raise ValueError( + "`namespace` needs to be passed as part of the function call " + "or exist in the `metadata` section of the yaml document." + ) + + if name is None: + name = doc["metadata"]["name"] + else: + doc["metadata"]["name"] = name + + if namespace is None: + namespace = doc["metadata"]["namespace"] + else: + doc["metadata"]["namespace"] = namespace + + kind = doc["kind"] + api_version = doc["apiVersion"] + if "/" in api_version: + group, version = api_version.split("/") + else: + group = None + version = api_version + + if getattr(cls, "object_names_initialized", False): + obj = cls(name, namespace) + else: + obj = cls(name, namespace, kind=kind, group=group, version=version) + + obj.backing_obj = doc + + return obj + + @classmethod + def define( + cls: CustomObject, + name: str, + kind: Optional[str] = None, + plural: Optional[str] = None, + group: Optional[str] = None, + version: Optional[str] = None, + api_client: Optional[client.ApiClient] = None, + ): + """Defines a new class that will hold a particular type of object. + + This is meant to be used as a quick replacement for + CustomObject if needed, but not extensive control or behaviour + needs to be implemented. If your particular use case requires more + control or more complex behaviour on top of the CustomObject class, + consider subclassing it. + """ + + def __init__(self, name, namespace, **kwargs): + CustomObject.__init__( + self, + name, + namespace, + kind=kind, + plural=plural, + group=group, + version=version, + api_client=api_client, + ) + + def __repr__(self): + return "{klass_name}({name}, {namespace})".format( + klass_name=name, + name=repr(self.name), + namespace=repr(self.namespace), + ) + + return type( + name, + (CustomObject,), + { + "object_names_initialized": True, + "__init__": __init__, + "__repr__": __repr__, + }, + ) + + def delete(self): + """Deletes the object from Kubernetes.""" + body = client.V1DeleteOptions() + + self.api.delete_namespaced_custom_object( + self.group, self.version, self.namespace, self.plural, self.name, body=body + ) + + self._register_updated() + + def reload(self): + """Reloads the object from the Kubernetes API.""" + return self.load() + + def __getitem__(self, key): + self._reload_if_needed() + + return self.backing_obj[key] + + def __contains__(self, key): + self._reload_if_needed() + return key in self.backing_obj + + def __setitem__(self, key, val): + self.backing_obj[key] = val + + if self.bound and self.auto_save: + self.update() + + +def get_crd_names( + plural: Optional[str] = None, + kind: Optional[str] = None, + group: Optional[str] = None, + version: Optional[str] = None, + api_client: Optional[client.ApiClient] = None, +) -> Optional[Dict]: + """Gets the CRD entry that matches all the parameters passed.""" + # + # TODO: Update to `client.ApiextensionsV1Api()` + # + api = client.ApiextensionsV1beta1Api(api_client=api_client) + + if plural == kind == group == version is None: + return None + + crds = api.list_custom_resource_definition() + for crd in crds.items: + found = True + if group != "": + if crd.spec.group != group: + found = False + + if version != "": + if crd.spec.version != version: + found = False + + if kind is not None: + if crd.spec.names.kind != kind: + found = False + + if plural is not None: + if crd.spec.names.plural != plural: + found = False + + if found: + return crd + + +def create_or_update(resource: CustomObject) -> CustomObject: + """ + Tries to create the resource. If resource already exists (resulting in 409 Conflict), + then it updates it instead. If the resource has been modified externally (operator) + we try to do a client-side merge/override + """ + tries = 0 + if not resource.bound: + try: + resource.create() + except client.ApiException as e: + if e.status != 409: + raise e + resource.patch() + else: + while tries < 10: + if tries > 0: # The first try we don't need to do client-side merge apply + # do a client-side-apply + new_back_obj_to_apply = copy.deepcopy(resource.backing_obj) # resource and changes we want to apply + + resource.load() # resource from the server overwrites resource.backing_obj + + # Merge annotations, and labels. + # Client resource takes precedence + # Spec from the given resource is taken, + # since the operator is not supposed to do changes to the spec. + # There can be cases where the obj from the server does not contain annotations/labels, but the object + # we want to apply has them. But that is highly unlikely, and we can add that code in case that happens. + resource["spec"] = new_back_obj_to_apply["spec"] + if "metadata" in resource and "annotations" in resource["metadata"]: + resource["metadata"]["annotations"].update(new_back_obj_to_apply["metadata"]["annotations"]) + if "metadata" in resource and "labels" in resource["metadata"]: + resource["metadata"]["labels"].update(new_back_obj_to_apply["metadata"]["labels"]) + try: + resource.patch() + break + except client.ApiException as e: + if e.status != 409: + raise e + print( + "detected a resource conflict. That means the operator applied a change " + "to the same resource we are trying to change" + "Applying a client-side merge!" + ) + tries += 1 + if tries == 10: + raise Exception("Tried client side merge 10 times and did not succeed") + + return resource diff --git a/docker/mongodb-enterprise-tests/kubeobject/exceptions.py b/docker/mongodb-enterprise-tests/kubeobject/exceptions.py new file mode 100644 index 000000000..234f14361 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/exceptions.py @@ -0,0 +1,2 @@ +class ObjectNotBoundException(Exception): + pass diff --git a/docker/mongodb-enterprise-tests/kubeobject/kubeobject.py b/docker/mongodb-enterprise-tests/kubeobject/kubeobject.py new file mode 100644 index 000000000..891042b60 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/kubeobject.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import copy +import io +import time +from datetime import datetime, timedelta +from typing import Optional, TextIO, Tuple, Union + +import yaml +from box import Box +from kubeobject.exceptions import ObjectNotBoundException +from kubernetes import client +from kubernetes.client.api import ApiextensionsV1Api, CustomObjectsApi + + +class KubeObject(object): + BACKING_OBJ = "__backing_obj" + + def __init__( + self, + group: str, + version: str, + plural: str, + ): + self.init_attributes() + + # And now we initialize it with actual values, so we know over what kind + # of object to operate. + self.__dict__["crd"] = {"plural": plural, "group": group, "version": version} + + def init_attributes(self): + """This is separated from __init__ because here we initialize empty attributes of + KubeObject instance, but we don't incorporate business logic.""" + + # Initialize an "empty" CRD, which means that this object is not bound + # to an specific CRD. + self.__dict__["crd"] = {} + + # A KubeObject is bound when it is pointing at a given CRD on a name/namespace + self.__dict__["bound"] = {} + + # This is the object that will contain the definition of the Custom Object when bound + self.__dict__[KubeObject.BACKING_OBJ] = Box(default_box=True) + + # Set an API to work with. + # TODO: Allow for a better experience; api could be defined from env variables, + # in_cluster or whatever. See if this is needed. Can we run our samples with + # in_cluster, or based on different clusters pointed at by env variables? + self.__dict__["api"] = CustomObjectsApi() + + # Set `auto_reload` to `True` if it needs to be reloaded before every + # read of an attribute. This considers the `auto_reload_period` + # attribute at the same time. + self.__dict__["auto_reload"]: bool = False + + # If `auto_reload` is set, it will not reload if less time than + # `auto_reload_period` has passed since last read. + self.__dict__["auto_reload_period"] = timedelta(seconds=2) + + # Last time this object was updated + self.__dict__["last_update"]: Optional[datetime] = None + + # These attributes need to be set in order to read the object (as in reload) + # back from the API. + self.__dict__["name"]: str = None + self.__dict__["namespace"]: str = None + + def _register_update(self): + self.last_update = datetime.now() + + def _reload_if_needed(self): + if not self.auto_reload or not self.bound: + return + + if self.last_update is None or datetime.now() - self.last_update > self.auto_reload_period: + self.read(name=self.name, namespace=self.namespace) + + def read(self, name: str, namespace: str): + obj = self.api.get_namespaced_custom_object(name=name, namespace=namespace, **self.crd) + + self.__dict__[KubeObject.BACKING_OBJ] = Box(obj, default_box=True) + self.__dict__["bound"] = True + self.__dict__["name"] = obj["metadata"]["name"] + self.__dict__["namespace"] = obj["metadata"]["namespace"] + + self._register_update() + return self + + def update(self): + if not self.bound: + # there's no corresponding object in the Kubernetes cluster + raise ObjectNotBoundException + + obj = self.api.patch_namespaced_custom_object( + name=self.name, + namespace=self.namespace, + **self.crd, + body=self.__dict__[KubeObject.BACKING_OBJ].to_dict(), + ) + + self.__dict__[KubeObject.BACKING_OBJ] = Box(obj, default_box=True) + self._register_update() + + return self + + def delete(self): + if not self.bound: + raise ObjectNotBoundException + + # TODO: body is supposed to be client.V1DeleteOptions() + # but for now we are just passing the empty dict. + + self.api.delete_namespaced_custom_object( + name=self.name, + namespace=self.namespace, + body={}, + **self.crd, + ) + + self._register_update() + # Not bound any more! + self.bound = False + + def create( + self, + namespace: Optional[str] = None, + ) -> KubeObject: + """Attempts to create an object using the Kubernetes API. This object needs to + have been defined first! This is a complete metadata, spec or any other fields + need to have been populated first.""" + api: CustomObjectsApi = self.api + + if namespace is not None: + self.namespace = namespace + + obj = api.create_namespaced_custom_object( + namespace=self.namespace, + **self.crd, + body=self.__dict__[KubeObject.BACKING_OBJ].to_dict(), + ) + + self.__dict__[KubeObject.BACKING_OBJ] = Box(obj, default_box=True) + self.__dict__["bound"] = True + self.__dict__["name"] = obj["metadata"]["name"] + self.__dict__["namespace"] = obj["metadata"]["namespace"] + + # This object has been bound to an existing object in Kube + self.bound = True + self._register_update() + + return self + + def read_from_yaml_file(self, object_definition: TextIO): + return self._read_from(object_definition) + + def read_from_dict(self, object_definition: dict): + return self._read_from(object_definition) + + def _read_from(self, object_definition=Union[TextIO, dict]): + """Populates this object from object_definition. + + * type(io.IOBase): opens the file and reads a yaml doc from it + * type(dict): Uses it as backing_object + """ + if isinstance(object_definition, io.IOBase): + obj = yaml.safe_load(object_definition.read()) + elif isinstance(object_definition, dict): + obj = copy.deepcopy(object_definition) + else: + raise ValueError("argument should be a file-like object or a dict") + + self.__dict__[KubeObject.BACKING_OBJ] = Box(obj, default_box=True) + self.__dict__["bound"] = False + + return self + + def __setattr__(self, item, value): + if item.startswith("__") or item in self.__dict__: + self.__dict__[item] = value + else: + self.__dict__[KubeObject.BACKING_OBJ][item] = value + + def __getattr__(self, item): + self._reload_if_needed() + # if item not in self.__dict__[KubeObject.BACKING_OBJ]: + # raise AttributeError(item) + + return getattr(self.__dict__[KubeObject.BACKING_OBJ], item) + + def __getitem__(self, key): + """Similar to what dot notation (getattr) produces, but this + will get the dictionary that corresponds to that attribute.""" + d = self.__getattr__(key) + if isinstance(d, Box): + return d.to_dict() + + return d + + def wait_for(self, fn): + while True: + try: + # Add self.reload() somehow + if fn(self): + return True + + time.sleep(4) + except Exception: + print("Function fails") + + def to_dict(self): + return self.__dict__[KubeObject.BACKING_OBJ].to_dict() + + +def create_custom_object(name: str, api=None) -> KubeObject: + """This function returns a Class type that can be used to initialize + custom objects of the type name.""" + + # Get the full name from the API + # Kind is not used, but it should be stored somewhere in case we want + # to pretty print this object or something. + _kind, plural, group, version = full_crd_name(name, api) + + # To be able to work with the objects we only need group, version and plural + return KubeObject(group, version, plural) + + +def full_crd_name(name: str, api: Optional[ApiextensionsV1Api] = None) -> Tuple[str, str, str, str]: + """Fetches this CRD from the kubernetes API by name and returns its + name, kind, plural, group and version.""" + if api is None: + # Use default (already configured) client + api = client.ApiextensionsV1Api() + + # The name here is something like: resource.group (dummy.example.com) + response = api.read_custom_resource_definition(name) + + group = response.spec.group + kind = response.spec.names.kind + plural = response.spec.names.plural + version = [v.name for v in response.spec.versions if v.served][0] + + return kind, plural, group, version diff --git a/docker/mongodb-enterprise-tests/kubeobject/test_custom_object.py b/docker/mongodb-enterprise-tests/kubeobject/test_custom_object.py new file mode 100644 index 000000000..eae682f81 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/test_custom_object.py @@ -0,0 +1,282 @@ +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest import mock +from unittest.mock import MagicMock + +import pytest +from freezegun import freeze_time +from kubeobject import CustomObject + +yaml_data0 = """ +--- +apiVersion: dummy.com/v1 +kind: Dummy +metadata: + name: my-dummy-object0 + namespace: my-dummy-namespace +spec: + attrStr: value0 + attrInt: 10 + subDoc: + anotherAttrStr: value1 +""" + +yaml_data1 = """ +--- +apiVersion: dummy.com/v1 +kind: Dummy +spec: + attrStr: value0 + attrInt: 10 + subDoc: + anotherAttrStr: value1 +""" + + +def mocked_custom_api(): + stored_body = [] + + def get_namespaced_custom_object(group, version, namespace, plural, name): + if len(stored_body) > 0: + return stored_body[-1] + return {"name": name} + + def create_namespaced_custom_object(group, version, namespace, plural, body: dict): + body.update({"name": body["metadata"]["name"]}) + stored_body.append(body) + return body + + def patch_namespaced_custom_object(group, version, namespace, plural, name, body: dict): + stored_body.append(body) + return body + + base = MagicMock() + base.get_namespaced_custom_object = MagicMock(side_effect=get_namespaced_custom_object) + base.patch_namespaced_custom_object = MagicMock(side_effect=patch_namespaced_custom_object) + base.create_namespaced_custom_object = MagicMock(side_effect=create_namespaced_custom_object) + + return base + + +def mocked_crd_return_value(): + return SimpleNamespace( + spec=SimpleNamespace( + group="dummy.com", + version="v1", + names=SimpleNamespace(plural="dummies", kind="Dummy"), + ) + ) + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_creation(mocked_get_crd_names, mocked_client): + mocked_client.return_value = mocked_custom_api() + custom = CustomObject( + "my-dummy-object", + "my-dummy-namespace", + kind="Dummy", + group="dummy.com", + version="v1", + ).create() + + # Test that __getitem__ is well implemented + assert custom["name"] == "my-dummy-object" + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_read_from_disk(mocked_get_crd_names, mocked_client): + mocked_client.return_value = mocked_custom_api() + with mock.patch( + "kubeobject.customobject.open", + mock.mock_open(read_data=yaml_data0), + create=True, + ) as m: + custom = CustomObject.from_yaml("some-file.yaml") + m.assert_called_once_with("some-file.yaml") + + assert custom.name == "my-dummy-object0" + + custom.create() + + # Check the values passed are not lost! + assert custom["spec"]["attrStr"] == "value0" + assert custom["spec"]["attrInt"] == 10 + assert custom["spec"]["subDoc"]["anotherAttrStr"] == "value1" + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_read_from_disk_with_dat_from_yaml(mocked_get_crd_names, mocked_client): + mocked_client.return_value = mocked_custom_api() + with mock.patch( + "kubeobject.customobject.open", + mock.mock_open(read_data=yaml_data0), + create=True, + ) as m: + custom = CustomObject.from_yaml("some-file.yaml") + m.assert_called_once_with("some-file.yaml") + + assert custom.name == "my-dummy-object0" + + custom.create() + + # Check the values passed are not lost! + assert custom["kind"] == "Dummy" + assert custom["apiVersion"] == "dummy.com/v1" + assert custom["metadata"]["name"] == "my-dummy-object0" + assert custom["metadata"]["namespace"] == "my-dummy-namespace" + + # TODO: check why "name" is set but "namespace" is not. + assert custom["name"] == "my-dummy-object0" + # assert custom["namespace"] == "my-dummy-namespace" # this one is not set! + + assert custom.name == "my-dummy-object0" + assert custom.namespace == "my-dummy-namespace" + assert custom.plural == "dummies" + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_can_be_subclassed_create(mocked_crd_return_value, mocked_client): + mocked_client.return_value = mocked_custom_api() + + class Subklass(CustomObject): + pass + + a = Subklass( + "my-dummy-object", + "my-dummy-namespace", + kind="Dummy", + group="dummy.com", + version="v1", + ).create() + + assert a.__class__.__name__ == "Subklass" + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_can_be_subclassed_from_yaml(mocked_crd_return_value, mocked_client): + mocked_client.return_value = mocked_custom_api() + + class Subklass(CustomObject): + pass + + with mock.patch( + "kubeobject.customobject.open", + mock.mock_open(read_data=yaml_data0), + create=True, + ) as m: + a = Subklass.from_yaml("some-other-file.yaml") + m.assert_called_once_with("some-other-file.yaml") + + assert a.__class__.__name__ == "Subklass" + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_defined(mocked_crd_return_value, mocked_client): + mocked_client.return_value = mocked_custom_api() + klass = CustomObject.define("Dummy", plural="dummies", group="dummy.com", version="v1") + + k = klass("my-dummy", "default").create() + + assert k.__class__.__bases__ == (CustomObject,) + assert k.__class__.__name__ == "Dummy" + + assert repr(k) == "Dummy('my-dummy', 'default')" + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +def test_defined_wont_require_api_if_all_parameteres_are_provided(mocked_client): + mocked_client.return_value = mocked_custom_api() + BaseKlass = CustomObject.define("Dummy", kind="Dummy", plural="dummies", group="dummy.com", version="v1") + + class SubKlass(BaseKlass): + def get_spec(self): + return self["spec"] + + k = SubKlass("my-dummy", "default").create() + k["spec"] = {"testAttr": "value"} + + k.update() + k.reload() + + assert k.get_spec() == {"testAttr": "value"} + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_custom_object_auto_reload(mocked_get_crd_names, mocked_client): + instance = mocked_custom_api() + mocked_client.return_value = instance + + klass = CustomObject.define("Dummy", plural="dummies", group="dummy.com", version="v1") + k = klass("my-dummy", "default") + + assert k.last_update is None + + k["status"] = "something" + k.create() + + # first call is initialization of the class, second is `create_` + assert len(mocked_client.mock_calls) == 2 + assert k.last_update is not None + + k.auto_reload = True + assert k["status"] == "something" + assert k.last_update is not None + last_update_recorded = k.last_update + + # If getting the status after 1 second, we don't reload + with freeze_time(datetime.now() + timedelta(milliseconds=1000)): + k["status"] + assert k.last_update == last_update_recorded + + assert len(mocked_client.mock_calls) == 2 + + # If getting the status after 2.1 seconds, we reload! + with freeze_time(datetime.now() + timedelta(milliseconds=2100)): + k["status"] + assert k.last_update > last_update_recorded + + assert len(mocked_client.mock_calls) == 3 + + +def test_raises_if_no_name(): + with mock.patch( + "kubeobject.customobject.open", + mock.mock_open(read_data=yaml_data1), + create=True, + ) as _: + with pytest.raises(ValueError, match=r".*needs to be passed as part of the function call.*"): + CustomObject.from_yaml("some-other-file.yaml") + + +def test_raises_if_no_namespace(): + with mock.patch( + "kubeobject.customobject.open", + mock.mock_open(read_data=yaml_data1), + create=True, + ) as _: + with pytest.raises(ValueError, match=r".*needs to be passed as part of the function call.*"): + CustomObject.from_yaml("some-other-file.yaml", name="some-name") + + +@mock.patch("kubeobject.customobject.get_crd_names", return_value=mocked_crd_return_value()) +def test_name_is_set_as_argument(_): + # TODO: what is this test supposed to do? + with mock.patch( + "kubeobject.customobject.open", + mock.mock_open(read_data=yaml_data1), + create=True, + ) as _: + CustomObject.from_yaml("some-other-file.yaml", name="some-name", namespace="some-namespace") + + +@mock.patch("kubeobject.customobject.client.CustomObjectsApi") +def test_called_when_updating(mocked_client): + mocked_client.return_value = mocked_custom_api() + assert True diff --git a/docker/mongodb-enterprise-tests/kubeobject/test_generate_random_name.py b/docker/mongodb-enterprise-tests/kubeobject/test_generate_random_name.py new file mode 100644 index 000000000..e6a155c46 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/test_generate_random_name.py @@ -0,0 +1,70 @@ +from kubeobject import generate_random_name + + +def test_prefix(): + r0 = generate_random_name(prefix="prefix--") + + assert r0.startswith("prefix--") + assert len(r0) > len("prefix--") + + +def test_suffix(): + r0 = generate_random_name(suffix="--suffix") + + assert r0.endswith("--suffix") + assert len(r0) > len("--suffix") + + +def test_size(): + r0 = generate_random_name(size=0) + assert len(r0) == 0 + + r0 = generate_random_name(size=1) + assert len(r0) == 1 + + r0 = generate_random_name(size=100) + assert len(r0) == 63 + + r0 = generate_random_name(size=20) + assert len(r0) == 20 + + +def test_prefix_size(): + r0 = generate_random_name(prefix="prefix--", size=0) + assert len(r0) == len("prefix--") + assert r0.startswith("prefix--") + + r0 = generate_random_name(prefix="prefix--", size=100) + assert len(r0) == 63 + assert r0.startswith("prefix--") + + r0 = generate_random_name(prefix="prefix--", size=20) + assert len(r0) == 20 + assert r0.startswith("prefix--") + + +def test_suffix_size(): + r0 = generate_random_name(suffix="--suffix", size=0) + assert len(r0) == len("--suffix") + assert r0.startswith("--suffix") + + r0 = generate_random_name(suffix="--suffix", size=100) + assert len(r0) == 63 + assert r0.endswith("--suffix") + + r0 = generate_random_name(suffix="--suffix", size=20) + assert len(r0) == 20 + assert r0.endswith("--suffix") + + +def test_prefix_suffix(): + r0 = generate_random_name(prefix="prefix--", suffix="--suffix") + assert len(r0) == 63 + + r0 = generate_random_name(prefix="prefix--", suffix="--suffix", size=0) + assert len(r0) == (len("prefix--") + len("--suffix")) + + r0 = generate_random_name(prefix="prefix--", suffix="--suffix", size=20) + assert len(r0) == 20 + assert r0.startswith("prefix--") + assert r0.endswith("--suffix") diff --git a/docker/mongodb-enterprise-tests/kubeobject/test_kubeobject.py b/docker/mongodb-enterprise-tests/kubeobject/test_kubeobject.py new file mode 100644 index 000000000..60b02cd46 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubeobject/test_kubeobject.py @@ -0,0 +1,245 @@ +import copy +import io +from unittest.mock import Mock, call, patch + +import pytest +import yaml +from box import Box +from kubeobject import KubeObject, create_custom_object +from kubernetes import client, config + +# config.load_kube_config() + + +def test_box0(): + b = Box({"a": "first", "b": "second"}) + + assert b.a == "first" + assert b.b == "second" + + +def test_box1(): + """Adds a dict to a box""" + + b = Box({"a": {}}) + b.a = {"some": "data"} + + assert b.a.some == "data" + + +def test_box2(): + """Tries to use KubeObject as a proxy to a box""" + k = KubeObject("group", "version", "plural") + k.a = "attribute a" + + assert k.a == "attribute a" + assert k.b.to_dict() == {} + + k.b = "now this works" + assert k.b == "now this works" + + k.c = {"name": "some object"} + assert k.c.name == "some object" + + # nested not yet defined attributes default to Box + assert k.d.n.m.k == Box() + + k.d = {"n": {"m": {"k": "deep value"}}} + + # Now we can fetch the deep object using dot notation + assert k.d.n.m.k == "deep value" + + # Also the previous to last key should be a box! + assert isinstance(k.d.n.m, Box) + + # This will get a mutable reference to m + borrow_box = k.d.n.m + borrow_box.x = "and a final attribute" + + assert k.d.n.m.x == "and a final attribute" + + +def test_box3(): + """Tests that if an attribute exists, then calling it directly will not + proxy the call through the getter.""" + k = KubeObject("group", "version", "plural") + + with pytest.raises(TypeError): + # Raises because the default return for empty Box is not a callable + k.whoami() + + whoami = Mock(return_value="I'm you") + + # but if we add it, then we get what we want + k.whoami = whoami + assert k.whoami() == "I'm you" + + whoami.assert_called_once() + + # TODO: Some magic methods are not easy to mock, see: + # + # https://docs.python.org/3/library/unittest.mock.html#mocking-magic-methods + # + # I would like to be able to test that __getattr__ was *not called*, but not + # really sure how to do it. Not that you could change KubeObject's __getattr__ as + # + # KubeObject.__getattr__ = SOMETHING + # + # but we'll break the rest of the tests? better not to contaminate the tests + # for now. + + +def test_box4(): + k = KubeObject("group", "version", "plural") + + k.a = {"hello": "hola"} + + assert k.a.hello == "hola" + + assert isinstance(k.a, Box) + + assert k.a["hello"] == "hola" + + assert isinstance(k.a, Box) + assert isinstance(k["a"], dict) + + +def test_kubeobject0(): + k = KubeObject("group0", "version0", "plural0") + assert k.crd == {"group": "group0", "version": "version0", "plural": "plural0"} + + +class CustomResource: + def __init__(self, name): + self.name = name + + +class MockedCustomObjectsApi: + store = { + "metadata": {"name": "my-dummy-object", "namespace": "default"}, + "spec": {"thisAttribute": "fourty two"}, + } + + @staticmethod + def get_namespaced_custom_object(name, namespace, **kwargs): + return MockedCustomObjectsApi.store + + @staticmethod + def patch_namespaced_custom_object(name, namespace=None, group=None, version=None, plural=None, body=None): + # body needs to be passed always + assert body is not None + MockedCustomObjectsApi.store = body + + return body + + @staticmethod + def delete_namespaced_custom_object(group, version, namespace, plural, name, body): + MockedCustomObjectsApi.store = {} + + +@patch("kubeobject.kubeobject.CustomObjectsApi") +def test_can_read_and_update(patched_custom_objects_api: Mock): + api = Mock(wraps=MockedCustomObjectsApi) + patched_custom_objects_api.return_value = api + + C = KubeObject("example.com", "v1", "dummies") + c = C.read("my-dummy-object", "default") + + # Read API was called once + api.get_namespaced_custom_object.assert_called_once() + + assert c.spec.thisAttribute == "fourty two" + + c.spec.thisAttribute = "fourty three" + c.update() + + # We expect this body to be one sent to the API. + updated_body = copy.deepcopy(MockedCustomObjectsApi.store) + updated_body["spec"]["thisAttribute"] = "fourty three" + api.patch_namespaced_custom_object.assert_called_once_with( + **dict( + name="my-dummy-object", + namespace="default", + plural="dummies", + group="example.com", + version="v1", + body=updated_body, + ) + ) + + read_calls = [ + call( + **dict( + name="my-dummy-object", + namespace="default", + plural="dummies", + group="example.com", + version="v1", + ) + ), + call( + **dict( + name="my-dummy-object", + namespace="default", + plural="dummies", + group="example.com", + version="v1", + ) + ), + ] + c1 = C.read("my-dummy-object", "default") + # We expect the calls to read from API are identical + api.get_namespaced_custom_object.assert_has_calls(read_calls) + + assert c1.spec.thisAttribute == "fourty three" + + +@patch("kubeobject.kubeobject.CustomObjectsApi") +def test_can_read_delete(patched_custom_objects_api: Mock): + api = Mock(wraps=MockedCustomObjectsApi) + patched_custom_objects_api.return_value = api + + C = KubeObject("example.com", "v1", "dummies") + c = C.read("my-dummy-object", "default") + + api.get_namespaced_custom_object.assert_called_once() + + c.delete() + api.delete_namespaced_custom_object.assert_called_once_with( + group="example.com", + version="v1", + namespace="default", + plural="dummies", + name="my-dummy-object", + body={}, + ) + + +@patch("kubeobject.kubeobject.CustomObjectsApi") +def test_read_from_yaml_file(patched_custom_objects_api: Mock): + api = Mock(wraps=MockedCustomObjectsApi) + patched_custom_objects_api.return_value = api + + y = """ +--- +apiVersion: kubeobject.com/v1 +kind: Dummy +metadata: + name: my-dummy-object +spec: + thisAttribute: "eighty one" + """ + # we mock the file with a io.StringIO which acts like a file object. + yaml_file = io.StringIO(y) + + C = KubeObject("example.com", "v1", "dummies") + c = C.read_from_yaml_file(yaml_file) + + assert c.metadata.name == "my-dummy-object" + assert c.spec.thisAttribute == "eighty one" + assert c.apiVersion == "kubeobject.com/v1" + assert c.kind == "Dummy" + + as_dict = yaml.safe_load(y) + + assert c.to_dict() == as_dict diff --git a/docker/mongodb-enterprise-tests/kubetester/__init__.py b/docker/mongodb-enterprise-tests/kubetester/__init__.py new file mode 100644 index 000000000..fe72c1886 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/__init__.py @@ -0,0 +1,494 @@ +import random +import string +import time +from base64 import b64decode +from typing import Any, Callable, Dict, List, Optional + +import kubernetes.client +from kubeobject import CustomObject +from kubernetes import client, utils +from kubetester.kubetester import run_periodically + +# Re-exports +from .kubetester import fixture as find_fixture +from .mongodb import MongoDB +from .security_context import ( + assert_pod_container_security_context, + assert_pod_security_context, +) + + +def create_secret( + namespace: str, + name: str, + data: Dict[str, str], + type: Optional[str] = "Opaque", + api_client: Optional[client.ApiClient] = None, +) -> str: + """Creates a Secret with `name` in `namespace`. String contents are passed as the `data` parameter.""" + secret = client.V1Secret(metadata=client.V1ObjectMeta(name=name), string_data=data, type=type) + + client.CoreV1Api(api_client=api_client).create_namespaced_secret(namespace, secret) + + return name + + +def create_or_update_secret( + namespace: str, + name: str, + data: Dict[str, str], + type: Optional[str] = "Opaque", + api_client: Optional[client.ApiClient] = None, +) -> str: + try: + create_secret(namespace, name, data, type, api_client) + except kubernetes.client.ApiException as e: + if e.status == 409: + update_secret(namespace, name, data, api_client) + + return name + + +def update_secret( + namespace: str, + name: str, + data: Dict[str, str], + api_client: Optional[client.ApiClient] = None, +): + """Updates a secret in a given namespace with the given name and data—handles base64 encoding.""" + secret = client.V1Secret(metadata=client.V1ObjectMeta(name=name), string_data=data) + client.CoreV1Api(api_client=api_client).patch_namespaced_secret(name, namespace, secret) + + +def delete_secret(namespace: str, name: str, api_client: Optional[kubernetes.client.ApiClient] = None): + client.CoreV1Api(api_client=api_client).delete_namespaced_secret(name, namespace) + + +def create_service_account(namespace: str, name: str) -> str: + """Creates a service account with `name` in `namespace`""" + sa = client.V1ServiceAccount(metadata=client.V1ObjectMeta(name=name)) + client.CoreV1Api().create_namespaced_service_account(namespace=namespace, body=sa) + return name + + +def delete_service_account(namespace: str, name: str) -> str: + """Deletes a service account with `name` in `namespace`""" + client.CoreV1Api().delete_namespaced_service_account(namespace=namespace, name=name) + return name + + +def get_service( + namespace: str, name: str, api_client: Optional[kubernetes.client.ApiClient] = None +) -> client.V1ServiceSpec: + """Gets a service with `name` in `namespace. + :return None if the service does not exist + """ + try: + return client.CoreV1Api(api_client=api_client).read_namespaced_service(name, namespace) + except kubernetes.client.ApiException as e: + if e.status == 404: + return None + else: + raise e + + +def delete_pvc(namespace: str, name: str): + """Deletes a persistent volument claim(pvc) with `name` in `namespace`""" + client.CoreV1Api().delete_namespaced_persistent_volume_claim(namespace=namespace, name=name) + + +def create_object_from_dict(data, namespace: str) -> List: + k8s_client = client.ApiClient() + return utils.create_from_dict(k8s_client=k8s_client, data=data, namespace=namespace) + + +def read_configmap(namespace: str, name: str, api_client: Optional[client.ApiClient] = None) -> Dict[str, str]: + return client.CoreV1Api(api_client=api_client).read_namespaced_config_map(name, namespace).data + + +def create_configmap( + namespace: str, + name: str, + data: Dict[str, str], + api_client: Optional[kubernetes.client.ApiClient] = None, +): + configmap = client.V1ConfigMap(metadata=client.V1ObjectMeta(name=name), data=data) + client.CoreV1Api(api_client=api_client).create_namespaced_config_map(namespace, configmap) + + +def update_configmap( + namespace: str, + name: str, + data: Dict[str, str], + api_client: Optional[kubernetes.client.ApiClient] = None, +): + configmap = client.V1ConfigMap(metadata=client.V1ObjectMeta(name=name), data=data) + client.CoreV1Api(api_client=api_client).replace_namespaced_config_map(name, namespace, configmap) + + +def create_or_update_configmap( + namespace: str, + name: str, + data: Dict[str, str], + api_client: Optional[kubernetes.client.ApiClient] = None, +) -> str: + print("Logging inside create_or_update configmap") + try: + create_configmap(namespace, name, data, api_client) + except kubernetes.client.ApiException as e: + if e.status == 409: + update_configmap(namespace, name, data, api_client) + else: + raise Exception(f"failed to create configmap: {e}") + + return name + + +def create_or_update_service( + namespace: str, + service_name: Optional[str] = None, + cluster_ip: Optional[str] = None, + ports: Optional[List[client.V1ServicePort]] = None, + selector=None, + service: Optional[client.V1Service] = None, +) -> str: + print("Logging inside create_or_update_service") + try: + create_service(namespace, service_name, cluster_ip=cluster_ip, ports=ports, selector=selector, service=service) + except kubernetes.client.ApiException as e: + if e.status == 409: + update_service( + namespace, service_name, cluster_ip=cluster_ip, ports=ports, selector=selector, service=service + ) + return service_name + + +def create_service( + namespace: str, + name: str, + cluster_ip: Optional[str] = None, + ports: Optional[List[client.V1ServicePort]] = None, + selector=None, + service: Optional[client.V1Service] = None, +): + if service is None: + if ports is None: + ports = [] + + service = client.V1Service( + metadata=client.V1ObjectMeta(name=name, namespace=namespace), + spec=client.V1ServiceSpec(ports=ports, cluster_ip=cluster_ip, selector=selector), + ) + client.CoreV1Api().create_namespaced_service(namespace, service) + + +def update_service( + namespace: str, + name: str, + cluster_ip: Optional[str] = None, + ports: Optional[List[client.V1ServicePort]] = None, + selector=None, + service: Optional[client.V1Service] = None, +): + if service is None: + if ports is None: + ports = [] + + service = client.V1Service( + metadata=client.V1ObjectMeta(name=name, namespace=namespace), + spec=client.V1ServiceSpec(ports=ports, cluster_ip=cluster_ip, selector=selector), + ) + client.CoreV1Api().patch_namespaced_service(name, namespace, service) + + +def create_statefulset( + namespace: str, + name: str, + service_name: str, + labels: Dict[str, str], + replicas: int = 1, + containers: Optional[List[client.V1Container]] = None, + volumes: Optional[List[client.V1Volume]] = None, +): + if containers is None: + containers = [] + if volumes is None: + volumes = [] + + sts = client.V1StatefulSet( + metadata=client.V1ObjectMeta(name=name, namespace=namespace), + spec=client.V1StatefulSetSpec( + selector=client.V1LabelSelector(match_labels=labels), + replicas=replicas, + service_name=service_name, + template=client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta(labels=labels), + spec=client.V1PodSpec(containers=containers, volumes=volumes), + ), + ), + ) + client.AppsV1Api().create_namespaced_stateful_set(namespace, body=sts) + + +def read_service( + namespace: str, + name: str, + api_client: Optional[client.ApiClient] = None, +) -> client.V1Service: + return client.CoreV1Api(api_client=api_client).read_namespaced_service(name, namespace) + + +def read_secret( + namespace: str, + name: str, + api_client: Optional[client.ApiClient] = None, +) -> Dict[str, str]: + return decode_secret(client.CoreV1Api(api_client=api_client).read_namespaced_secret(name, namespace).data) + + +def delete_pod(namespace: str, name: str, api_client: Optional[kubernetes.client.ApiClient] = None): + client.CoreV1Api(api_client=api_client).delete_namespaced_pod(name, namespace) + + +def create_or_update_namespace( + namespace: str, + labels: dict = None, + annotations: dict = None, + api_client: Optional[kubernetes.client.ApiClient] = None, +): + namespace_resource = client.V1Namespace( + metadata=client.V1ObjectMeta( + name=namespace, + labels=labels, + annotations=annotations, + ) + ) + try: + client.CoreV1Api(api_client=api_client).create_namespace(namespace_resource) + except kubernetes.client.ApiException as e: + if e.status == 409: + client.CoreV1Api(api_client=api_client).patch_namespace(namespace, namespace_resource) + + +def delete_namespace(name: str): + c = client.CoreV1Api() + c.delete_namespace(name, body=c.V1DeleteOptions()) + + +def get_deployments(namespace: str): + return client.AppsV1Api().list_namespaced_deployment(namespace) + + +def delete_deployment(namespace: str, name: str): + client.AppsV1Api().delete_namespaced_deployment(name, namespace) + + +def delete_statefulset( + namespace: str, + name: str, + propagation_policy: str = "Orphan", + api_client: Optional[client.ApiClient] = None, +): + client.AppsV1Api(api_client=api_client).delete_namespaced_stateful_set( + name, namespace, propagation_policy=propagation_policy + ) + + +def get_statefulset( + namespace: str, + name: str, + api_client: Optional[client.ApiClient] = None, +) -> client.V1StatefulSet: + return client.AppsV1Api(api_client=api_client).read_namespaced_stateful_set(name, namespace) + + +def statefulset_is_deleted(namespace: str, name: str, api_client: Optional[client.ApiClient]): + try: + get_statefulset(namespace, name, api_client=api_client) + return False + except client.ApiException as e: + if e.status == 404: + return True + else: + raise e + + +def delete_cluster_role(name: str, api_client: Optional[client.ApiClient] = None): + try: + client.RbacAuthorizationV1Api(api_client=api_client).delete_cluster_role(name) + except client.rest.ApiException as e: + if e.status != 404: + raise e + + +def delete_cluster_role_binding(name: str, api_client: Optional[client.ApiClient] = None): + try: + client.RbacAuthorizationV1Api(api_client=api_client).delete_cluster_role_binding(name) + except client.rest.ApiException as e: + if e.status != 404: + raise e + + +def random_k8s_name(prefix=""): + return prefix + "".join(random.choice(string.ascii_lowercase) for _ in range(10)) + + +def get_pod_when_running( + namespace: str, + label_selector: str, + api_client: Optional[kubernetes.client.ApiClient] = None, +) -> client.V1Pod: + """ + Returns a Pod that matches label_selector. It will block until the Pod is in + Running state. + """ + while True: + time.sleep(3) + + try: + pods = client.CoreV1Api(api_client=api_client).list_namespaced_pod(namespace, label_selector=label_selector) + try: + pod = pods.items[0] + except IndexError: + continue + + if pod.status.phase == "Running": + return pod + + except client.rest.ApiException as e: + # The Pod might not exist in Kubernetes yet so skip any 404 + if e.status != 404: + raise + + +def get_pod_when_ready( + namespace: str, + label_selector: str, + api_client: Optional[kubernetes.client.ApiClient] = None, + default_retry: Optional[int] = 60, +) -> client.V1Pod: + """ + Returns a Pod that matches label_selector. It will block until the Pod is in + Ready state. + """ + cnt = 0 + + while True and cnt < default_retry: + print(f"get_pod_when_ready: namespace={namespace}, label_selector={label_selector}") + + if cnt > 0: + time.sleep(1) + cnt += 1 + try: + pods = client.CoreV1Api(api_client=api_client).list_namespaced_pod(namespace, label_selector=label_selector) + + if len(pods.items) == 0: + continue + + pod = pods.items[0] + + # This might happen when the pod is still pending + if pod.status.conditions is None: + continue + + for condition in pod.status.conditions: + if condition.type == "Ready" and condition.status == "True": + return pod + + except client.rest.ApiException as e: + # The Pod might not exist in Kubernetes yet so skip any 404 + if e.status != 404: + raise + + print(f"bailed on getting pod ready after 10 retries") + + +def is_pod_ready( + namespace: str, + label_selector: str, + api_client: Optional[kubernetes.client.ApiClient] = None, +) -> client.V1Pod: + """ + Checks if a Pod that matches label_selector is ready. It will return False if the pod is not ready, + if it does not exist or there is any other kind of error. + This function is intended to check if installing third party components is needed. + """ + print(f"Checking if pod is ready: namespace={namespace}, label_selector={label_selector}") + try: + pods = client.CoreV1Api(api_client=api_client).list_namespaced_pod(namespace, label_selector=label_selector) + + if len(pods.items) == 0: + return None + + pod = pods.items[0] + + if pod.status.conditions is None: + return None + + for condition in pod.status.conditions: + if condition.type == "Ready" and condition.status == "True": + return pod + except client.rest.ApiException: + return None + + return None + + +def get_default_storage_class() -> str: + default_class_annotations = ( + "storageclass.kubernetes.io/is-default-class", # storage.k8s.io/v1 + "storageclass.beta.kubernetes.io/is-default-class", # storage.k8s.io/v1beta1 + ) + sc: client.V1StorageClass + for sc in client.StorageV1Api().list_storage_class().items: + if sc.metadata.annotations is not None and any( + sc.metadata.annotations.get(a) == "true" for a in default_class_annotations + ): + return sc.metadata.name + + +def decode_secret(data: Dict[str, str]) -> Dict[str, str]: + return {k: b64decode(v).decode("utf-8") for (k, v) in data.items()} + + +def wait_until(fn: Callable[..., Any], timeout=0, **kwargs): + """ + Runs the Callable `fn` until timeout is reached or until it returns True. + """ + return run_periodically(fn, timeout=timeout, **kwargs) + + +def try_load(resource: CustomObject) -> bool: + """ + Tries to load the resource without raising an exception when the resource does not exist. + Returns False if the resource does not exist. + """ + try: + resource.load() + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + else: + return False + + return True + + +def wait_for_webhook(namespace, retries=5, delay=5, service_name="operator-webhook"): + webhook_api = client.AdmissionregistrationV1Api() + core_api = client.CoreV1Api() + + for attempt in range(retries): + try: + core_api.read_namespaced_service(service_name, namespace) + + # make sure the validating_webhook is installed. + webhook_api.read_validating_webhook_configuration("mdbpolicy.mongodb.com") + print("Webhook is ready.") + return True + except kubernetes.client.ApiException as e: + print(f"Attempt {attempt + 1} failed, webhook not ready. Sleeping: {delay}, error: {e}") + time.sleep(delay) + + print("Webhook did not become ready in time.") + return False diff --git a/docker/mongodb-enterprise-tests/kubetester/automation_config_tester.py b/docker/mongodb-enterprise-tests/kubetester/automation_config_tester.py new file mode 100644 index 000000000..68c75198e --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/automation_config_tester.py @@ -0,0 +1,136 @@ +from typing import Dict, List, Optional, Set, Tuple + +from kubetester.kubetester import KubernetesTester + +X509_AGENT_SUBJECT = "CN=mms-automation-agent,OU=MongoDB Kubernetes Operator,O=mms-automation-agent,L=NY,ST=NY,C=US" +SCRAM_AGENT_USER = "mms-automation-agent" + + +class AutomationConfigTester: + """Tester for AutomationConfig. Should be initialized with the + AutomationConfig we will test (`ac`), the expected amount of users, and if it should be + set to `authoritative_set`, which means that the Automation Agent will force the existing + users in MongoDB to be the ones defined in the Automation Config. + """ + + def __init__(self, ac: Optional[Dict] = None): + if ac is None: + ac = KubernetesTester.get_automation_config() + self.automation_config = ac + + def get_replica_set_processes(self, rs_name: str) -> List[Dict]: + """Returns all processes for the specified replica set""" + replica_set = ([rs for rs in self.automation_config["replicaSets"] if rs["_id"] == rs_name])[0] + rs_processes_name = [member["host"] for member in replica_set["members"]] + return [process for process in self.automation_config["processes"] if process["name"] in rs_processes_name] + + def get_replica_set_members(self, rs_name: str) -> List[Dict]: + replica_set = ([rs for rs in self.automation_config["replicaSets"] if rs["_id"] == rs_name])[0] + return sorted(replica_set["members"], key=lambda member: member["_id"]) + + def get_mongos_processes(self): + """ " Returns all mongos processes in deployment. We don't need to filter by sharded cluster name as + we have only a single resource per deployment""" + return [process for process in self.automation_config["processes"] if process["processType"] == "mongos"] + + def get_all_processes(self): + return self.automation_config["processes"] + + def assert_expected_users(self, expected_users: int): + automation_config_users = 0 + + for user in self.automation_config["auth"]["usersWanted"]: + if user["user"] != "mms-backup-agent" and user["user"] != "mms-monitoring-agent": + automation_config_users += 1 + + assert automation_config_users == expected_users + + def assert_authoritative_set(self, authoritative_set: bool): + assert self.automation_config["auth"]["authoritativeSet"] == authoritative_set + + def assert_authentication_mechanism_enabled(self, mechanism: str, active_auth_mechanism: bool = True) -> None: + auth: dict = self.automation_config["auth"] + assert mechanism in auth.get("deploymentAuthMechanisms", []) + if active_auth_mechanism: + assert mechanism in auth.get("autoAuthMechanisms", []) + assert auth["autoAuthMechanism"] == mechanism + + def assert_authentication_mechanism_disabled(self, mechanism: str, check_auth_mechanism: bool = True) -> None: + auth = self.automation_config["auth"] + assert mechanism not in auth.get("deploymentAuthMechanisms", []) + assert mechanism not in auth.get("autoAuthMechanisms", []) + if check_auth_mechanism: + assert auth["autoAuthMechanism"] != mechanism + + def assert_authentication_enabled(self, expected_num_deployment_auth_mechanisms: int = 1) -> None: + assert not self.automation_config["auth"]["disabled"] + + actual_num_deployment_auth_mechanisms = len(self.automation_config["auth"].get("deploymentAuthMechanisms", [])) + assert actual_num_deployment_auth_mechanisms == expected_num_deployment_auth_mechanisms + + def assert_internal_cluster_authentication_enabled(self): + for process in self.automation_config["processes"]: + assert process["args2_6"]["security"]["clusterAuthMode"] == "x509" + + def assert_authentication_disabled(self, remaining_users: int = 0) -> None: + assert self.automation_config["auth"]["disabled"] + self.assert_expected_users(expected_users=remaining_users) + assert len(self.automation_config["auth"].get("deploymentAuthMechanisms", [])) == 0 + + def assert_user_has_roles(self, username: str, roles: Set[Tuple[str, str]]) -> None: + user = [user for user in self.automation_config["auth"]["usersWanted"] if user["user"] == username][0] + actual_roles = {(role["db"], role["role"]) for role in user["roles"]} + assert actual_roles == roles + + def assert_has_user(self, username: str) -> None: + assert username in {user["user"] for user in self.automation_config["auth"]["usersWanted"]} + + def assert_agent_user(self, agent_user: str) -> None: + assert self.automation_config["auth"]["autoUser"] == agent_user + + def assert_replica_sets_size(self, expected_size: int): + assert len(self.automation_config["replicaSets"]) == expected_size + + def assert_processes_size(self, expected_size: int): + assert len(self.automation_config["processes"]) == expected_size + + def assert_sharding_size(self, expected_size: int): + assert len(self.automation_config["sharding"]) == expected_size + + def assert_empty(self): + self.assert_processes_size(0) + self.assert_replica_sets_size(0) + self.assert_sharding_size(0) + + def assert_mdb_option(self, process: Dict, value, *keys): + current = process["args2_6"] + for k in keys[:-1]: + current = current[k] + assert current[keys[-1]] == value + + def get_role_at_index(self, index: int) -> Dict: + roles = self.automation_config["roles"] + assert roles is not None + assert len(roles) > index + return roles[index] + + def assert_has_expected_number_of_roles(self, expected_roles: int): + roles = self.automation_config["roles"] + assert len(roles) == expected_roles + + def assert_expected_role(self, role_index: int, expected_value: Dict): + role = self.automation_config["roles"][role_index] + assert role == expected_value + + def assert_tls_client_certificate_mode(self, mode: str): + assert self.automation_config["tls"]["clientCertificateMode"] == mode + + def reached_version(self, version: int) -> bool: + return self.automation_config["version"] == version + + def get_agent_version(self) -> str: + try: + return self.automation_config["agentVersion"]["name"] + except KeyError: + # the agent version can be empty if the /automationConfig/upgrade endpoint hasn't been called yet + return "" diff --git a/docker/mongodb-enterprise-tests/kubetester/awss3client.py b/docker/mongodb-enterprise-tests/kubetester/awss3client.py new file mode 100644 index 000000000..3d64d63e9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/awss3client.py @@ -0,0 +1,76 @@ +from time import sleep + +import boto3 +from botocore.exceptions import ClientError +from kubetester.kubetester import get_env_var_or_fail + + +class AwsS3Client: + def __init__(self, region: str, **tags): + # these variables are not used in connection as boto3 client uses the env variables though + # it makes sense to fail fast if the env variables are not specified + self.aws_access_key = get_env_var_or_fail("AWS_ACCESS_KEY_ID") + self.aws_secret_access_key = get_env_var_or_fail("AWS_SECRET_ACCESS_KEY") + self.s3_client = boto3.client("s3", region_name=region) + self.tags = tags + + def create_s3_bucket(self, name: str): + self.s3_client.create_bucket(ACL="private", Bucket=name) + self.update_bucket_tags(name=name, **self.tags) + + def update_bucket_tags(self, name: str, **tags): + try: + existing_tags_response = self.s3_client.get_bucket_tagging(Bucket=name) + except ClientError as error: + if error.response["Error"]["Code"] == "NoSuchTagSet": + print(f"Bucket ({name}) does not have any tags") + existing_tags_response = {} + else: + raise error + + existing_tags = existing_tags_response.get("TagSet", []) + new_keys = tags.keys() + new_tags = [{"Key": k, "Value": v} for k, v in tags.items()] + desired_tags = [tag for tag in existing_tags if tag["Key"] not in new_keys] + desired_tags.extend(new_tags) + if len(desired_tags) > 0: + try: + self.s3_client.put_bucket_tagging(Bucket=name, Tagging={"TagSet": desired_tags}) + except ClientError as error: + if error.response["Error"]["Code"] == "InvalidTag": + print(f"Tags {desired_tags} failed input validation") + raise error + + def delete_s3_bucket(self, name: str, attempts: int = 10): + v = self.s3_client.list_objects_v2(Bucket=name) + print(v) + if v is not None and "Contents" in v: + for x in v["Contents"]: + self.s3_client.delete_object(Bucket=name, Key=x["Key"]) + + while attempts > 0: + try: + self.s3_client.delete_bucket(Bucket=name) + break + except ClientError: + print("Can't delete bucket, will try again in 5 seconds") + attempts -= 1 + sleep(5) + + def upload_file(self, file_path: str, bucket: str, object_name: str, public_read: bool = False): + """Upload a file to an S3 bucket. + + Args: + file_name: File to upload + bucket: Bucket to upload to + object_name: S3 object name + + Throws botocore.exceptions.ClientError if upload fails + """ + + extraArgs = {"ACL": "public-read"} if public_read else None + self.s3_client.upload_file(file_path, bucket, object_name, extraArgs) + + +def s3_endpoint(aws_region: str) -> str: + return f"s3.{aws_region}.amazonaws.com" diff --git a/docker/mongodb-enterprise-tests/kubetester/certs.py b/docker/mongodb-enterprise-tests/kubetester/certs.py new file mode 100644 index 000000000..6328b56da --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/certs.py @@ -0,0 +1,881 @@ +""" +Certificate Custom Resource Definition. +""" + +import collections +import copy +import random +import time +from datetime import datetime, timezone +from typing import Dict, Generator, List, Optional + +import kubernetes +from kubeobject import CustomObject +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import create_secret, delete_secret, random_k8s_name, read_secret +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from tests.vaultintegration import ( + store_secret_in_vault, + vault_namespace_name, + vault_sts_name, +) + +ISSUER_CA_NAME = "ca-issuer" + +SUBJECT = { + # Organizational Units matches your namespace (to be overriden by test) + "organizationalUnits": ["TO-BE-REPLACED"], +} + +# Defines properties of a set of servers, like a Shard, or Replica Set holding config servers. +# This is almost equivalent to the StatefulSet created. +SetProperties = collections.namedtuple("SetProperties", ["name", "service", "replicas"]) +SetPropertiesMultiCluster = collections.namedtuple( + "SetProperties", ["name", "service", "replicas", "number_of_clusters"] +) + + +CertificateType = CustomObject.define( + "Certificate", + kind="Certificate", + plural="certificates", + group="cert-manager.io", + version="v1", +) + + +class WaitForConditions: + def is_ready(self): + self.reload() + + if "status" not in self or "conditions" not in self["status"]: + return + + for condition in self["status"]["conditions"]: + if condition["reason"] == self.Reason and condition["status"] == "True" and condition["type"] == "Ready": + return True + + def block_until_ready(self): + while not self.is_ready(): + time.sleep(2) + + +class Certificate(CertificateType, WaitForConditions): + Reason = "Ready" + + +IssuerType = CustomObject.define("Issuer", kind="Issuer", plural="issuers", group="cert-manager.io", version="v1") +ClusterIssuerType = CustomObject.define( + "ClusterIssuer", + kind="ClusterIssuer", + plural="clusterissuers", + group="cert-manager.io", + version="v1", +) + + +class Issuer(IssuerType, WaitForConditions): + Reason = "KeyPairVerified" + + +class ClusterIssuer(ClusterIssuerType, WaitForConditions): + Reason = "KeyPairVerified" + + +def generate_cert( + namespace: str, + pod: str, + dns: str, + issuer: str, + spec: Optional[Dict] = None, + additional_domains: Optional[List[str]] = None, + multi_cluster_mode=False, + api_client: Optional[client.ApiClient] = None, + secret_name: Optional[str] = None, + secret_backend: Optional[str] = None, + vault_subpath: Optional[str] = None, + dns_list: Optional[List[str]] = None, + common_name: Optional[str] = None, + clusterwide: bool = False, +) -> str: + if spec is None: + spec = dict() + + if secret_name is None: + secret_name = "{}-{}".format(pod[0], random_k8s_name(prefix="")[:4]) + + if secret_backend is None: + secret_backend = "Kubernetes" + + cert = Certificate(namespace=namespace, name=secret_name) + + if multi_cluster_mode: + dns_names = dns_list + else: + dns_names = [dns] + + if not multi_cluster_mode: + dns_names.append(pod) + + if additional_domains is not None: + dns_names += additional_domains + + issuerRef = {"name": issuer, "kind": "Issuer"} + if clusterwide: + issuerRef["kind"] = "ClusterIssuer" + + cert["spec"] = { + "dnsNames": dns_names, + "secretName": secret_name, + "issuerRef": issuerRef, + "duration": "240h", + "renewBefore": "120h", + "usages": ["server auth", "client auth"], + } + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + # However, KMIP still enforces it :( + if common_name is not None: + cert["spec"]["commonName"] = common_name + + cert["spec"].update(spec) + cert.api = kubernetes.client.CustomObjectsApi(api_client=api_client) + cert.update() + print(f"Waiting for certificate to become ready: {cert}") + cert.block_until_ready() + + if secret_backend == "Vault": + path = "secret/mongodbenterprise/" + if vault_subpath is None: + raise ValueError("When secret backend is Vault, a subpath must be specified") + path += f"{vault_subpath}/{namespace}/{secret_name}" + + data = read_secret(namespace, secret_name) + store_secret_in_vault(vault_namespace_name(), vault_sts_name(), data, path) + cert.delete() + delete_secret(namespace, secret_name) + + return secret_name + + +def rotate_cert(namespace, certificate_name): + cert = Certificate(name=certificate_name, namespace=namespace) + cert.load() + cert["spec"]["dnsNames"].append("foo") # Append DNS to cert to rotate the certificate + cert.update() + + +def create_tls_certs( + issuer: str, + namespace: str, + resource_name: str, + replicas: int = 3, + replicas_cluster_distribution: Optional[List[int]] = None, + service_name: str = None, + spec: Optional[Dict] = None, + secret_name: Optional[str] = None, + additional_domains: Optional[List[str]] = None, + secret_backend: Optional[str] = None, + vault_subpath: Optional[str] = None, + common_name: Optional[str] = None, + process_hostnames: Optional[List[str]] = None, + clusterwide: bool = False, +) -> str: + """ + :param process_hostnames: set for TLS certificate to contain only given domains + """ + if service_name is None: + service_name = resource_name + "-svc" + + if spec is None: + spec = dict() + + pod_dns = [] + pods = [] + if replicas_cluster_distribution is None: + for pod_idx in range(replicas): + if process_hostnames is not None: + pod_dns.append(process_hostnames[pod_idx]) + else: + pod_dns.append(f"{resource_name}-{pod_idx}.{service_name}.{namespace}.svc.cluster.local") + pods.append(f"{resource_name}-{pod_idx}") + else: + for cluster_idx, pod_count in enumerate(replicas_cluster_distribution): + if process_hostnames is not None: + raise Exception("process_hostnames are not yet implemented for cluster_distribution argument") + for pod_idx in range(pod_count or 0): + pod_dns.append(f"{resource_name}-{cluster_idx}-{pod_idx}-svc.{namespace}.svc.cluster.local") + pods.append(f"{resource_name}-{cluster_idx}-{pod_idx}") + + spec["dnsNames"] = pods + pod_dns + if additional_domains is not None: + spec["dnsNames"] += additional_domains + + cert_secret_name = generate_cert( + namespace=namespace, + pod=pods, + dns=pod_dns, + issuer=issuer, + spec=spec, + secret_name=secret_name, + secret_backend=secret_backend, + vault_subpath=vault_subpath, + common_name=common_name, + clusterwide=clusterwide, + ) + return cert_secret_name + + +def create_ops_manager_tls_certs( + issuer: str, + namespace: str, + om_name: str, + secret_name: Optional[str] = None, + secret_backend: Optional[str] = None, + additional_domains: Optional[List[str]] = None, + api_client: Optional[kubernetes.client.ApiClient] = None, + clusterwide: bool = False, +) -> str: + certs_secret_name = "certs-for-ops-manager" + + if secret_name is not None: + certs_secret_name = secret_name + + domain = f"{om_name}-svc.{namespace}.svc.cluster.local" + central_domain = f"{om_name}-central.{namespace}.svc.cluster.local" + hostnames = [domain, central_domain] + if additional_domains: + hostnames += additional_domains + + spec = {"dnsNames": hostnames} + + return generate_cert( + namespace=namespace, + pod="foo", + dns="", + issuer=issuer, + spec=spec, + secret_name=certs_secret_name, + secret_backend=secret_backend, + vault_subpath="opsmanager", + api_client=api_client, + clusterwide=clusterwide, + ) + + +def create_vault_certs(namespace: str, issuer: str, vault_namespace: str, vault_name: str, secret_name: str): + cert = Certificate(namespace=namespace, name=secret_name) + + cert["spec"] = { + "commonName": f"{vault_name}", + "ipAddresses": [ + "127.0.0.1", + ], + "dnsNames": [ + f"{vault_name}", + f"{vault_name}.{vault_namespace}", + f"{vault_name}.{vault_namespace}.svc", + f"{vault_name}.{vault_namespace}.svc.cluster.local", + ], + "secretName": secret_name, + "issuerRef": {"name": issuer}, + "duration": "240h", + "renewBefore": "120h", + "usages": ["server auth", "digital signature", "key encipherment"], + } + + cert.create().block_until_ready() + data = read_secret(namespace, secret_name) + + # When re-running locally, we need to delete the secrets, if it exists + try: + delete_secret(vault_namespace, secret_name) + except ApiException: + pass + create_secret(vault_namespace, secret_name, data) + return secret_name + + +def create_mongodb_tls_certs( + issuer: str, + namespace: str, + resource_name: str, + bundle_secret_name: str, + replicas: int = 3, + replicas_cluster_distribution: Optional[List[int]] = None, + service_name: str = None, + spec: Optional[Dict] = None, + additional_domains: Optional[List[str]] = None, + secret_backend: Optional[str] = None, + vault_subpath: Optional[str] = None, + process_hostnames: Optional[List[str]] = None, + clusterwide: bool = False, +) -> str: + """ + :param process_hostnames: set for TLS certificate to contain only given domains + """ + cert_and_pod_names = create_tls_certs( + issuer=issuer, + namespace=namespace, + resource_name=resource_name, + replicas=replicas, + replicas_cluster_distribution=replicas_cluster_distribution, + service_name=service_name, + spec=spec, + additional_domains=additional_domains, + secret_name=bundle_secret_name, + secret_backend=secret_backend, + vault_subpath=vault_subpath, + process_hostnames=process_hostnames, + clusterwide=clusterwide, + ) + + return cert_and_pod_names + + +def multi_cluster_service_fqdns( + resource_name: str, + namespace: str, + external_domain: str, + cluster_index: int, + replicas: int, +) -> List[str]: + service_fqdns = [] + + for n in range(replicas): + if external_domain is None: + service_fqdns.append(f"{resource_name}-{cluster_index}-{n}-svc.{namespace}.svc.cluster.local") + else: + service_fqdns.append(f"{resource_name}-{cluster_index}-{n}.{external_domain}") + + return service_fqdns + + +def multi_cluster_external_service_fqdns( + resource_name: str, namespace: str, cluster_index: int, replicas: int +) -> List[str]: + service_fqdns = [] + + for n in range(replicas): + service_fqdns.append(f"{resource_name}-{cluster_index}-{n}-svc-external.{namespace}.svc.cluster.local") + + return service_fqdns + + +def create_multi_cluster_tls_certs( + multi_cluster_issuer: str, + secret_name: str, + central_cluster_client: kubernetes.client.ApiClient, + member_clients: List[MultiClusterClient], + mongodb_multi: Optional[MongoDBMulti] = None, + namespace: Optional[str] = None, + secret_backend: Optional[str] = None, + additional_domains: Optional[List[str]] = None, + service_fqdns: Optional[List[str]] = None, + clusterwide: bool = False, + spec: Optional[dict] = None, +) -> str: + if service_fqdns is None: + service_fqdns = [f"{mongodb_multi.name}-svc.{mongodb_multi.namespace}.svc.cluster.local"] + + for client in member_clients: + cluster_spec = mongodb_multi.get_item_spec(client.cluster_name) + try: + external_domain = cluster_spec["externalAccess"]["externalDomain"] + except KeyError: + external_domain = None + service_fqdns.extend( + multi_cluster_service_fqdns( + mongodb_multi.name, + mongodb_multi.namespace, + external_domain, + client.cluster_index, + cluster_spec["members"], + ) + ) + + if namespace is None: + namespace = mongodb_multi.namespace + + generate_cert( + namespace=namespace, + pod="tmp", + dns="", + issuer=multi_cluster_issuer, + additional_domains=additional_domains, + multi_cluster_mode=True, + api_client=central_cluster_client, + secret_backend=secret_backend, + secret_name=secret_name, + vault_subpath="database", + dns_list=service_fqdns, + spec=spec, + clusterwide=clusterwide, + ) + + return secret_name + + +def create_multi_cluster_agent_certs( + multi_cluster_issuer: str, + secret_name: str, + central_cluster_client: kubernetes.client.ApiClient, + mongodb_multi: MongoDBMulti, + secret_backend: Optional[str] = None, +) -> str: + agents = ["mms-automation-agent"] + subject = copy.deepcopy(SUBJECT) + subject["organizationalUnits"] = [mongodb_multi.namespace] + + spec = { + "subject": subject, + "usages": ["client auth"], + } + spec["dnsNames"] = agents + spec["commonName"] = "mms-automation-agent" + return generate_cert( + namespace=mongodb_multi.namespace, + pod="tmp", + dns="", + issuer=multi_cluster_issuer, + spec=spec, + multi_cluster_mode=True, + api_client=central_cluster_client, + secret_backend=secret_backend, + secret_name=secret_name, + vault_subpath="database", + ) + + +def create_multi_cluster_x509_agent_certs( + multi_cluster_issuer: str, + secret_name: str, + central_cluster_client: kubernetes.client.ApiClient, + mongodb_multi: MongoDBMulti, + secret_backend: Optional[str] = None, +) -> str: + spec = get_agent_x509_subject(mongodb_multi.namespace) + + return generate_cert( + namespace=mongodb_multi.namespace, + pod="tmp", + dns="", + issuer=multi_cluster_issuer, + spec=spec, + multi_cluster_mode=True, + api_client=central_cluster_client, + secret_backend=secret_backend, + secret_name=secret_name, + vault_subpath="database", + ) + + +def create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer: str, + bundle_secret_name: str, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, + mongodb_multi: Optional[MongoDBMulti] = None, + namespace: Optional[str] = None, + additional_domains: Optional[List[str]] = None, + service_fqdns: Optional[List[str]] = None, + clusterwide: bool = False, +) -> str: + # create the "source-of-truth" tls cert in central cluster + create_multi_cluster_tls_certs( + multi_cluster_issuer=multi_cluster_issuer, + central_cluster_client=central_cluster_client, + member_clients=member_cluster_clients, + secret_name=bundle_secret_name, + mongodb_multi=mongodb_multi, + namespace=namespace, + additional_domains=additional_domains, + service_fqdns=service_fqdns, + clusterwide=clusterwide, + ) + + return bundle_secret_name + + +def create_multi_cluster_mongodb_x509_tls_certs( + multi_cluster_issuer: str, + bundle_secret_name: str, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, + mongodb_multi: MongoDBMulti, + additional_domains: Optional[List[str]] = None, + service_fqdns: Optional[List[str]] = None, + clusterwide: bool = False, +) -> str: + spec = get_mongodb_x509_subject(mongodb_multi.namespace) + + # create the "source-of-truth" tls cert in central cluster + create_multi_cluster_tls_certs( + multi_cluster_issuer=multi_cluster_issuer, + central_cluster_client=central_cluster_client, + member_clients=member_cluster_clients, + secret_name=bundle_secret_name, + mongodb_multi=mongodb_multi, + additional_domains=additional_domains, + service_fqdns=service_fqdns, + clusterwide=clusterwide, + spec=spec, + ) + + return bundle_secret_name + + +def create_x509_mongodb_tls_certs( + issuer: str, + namespace: str, + resource_name: str, + bundle_secret_name: str, + replicas: int = 3, + replicas_cluster_distribution: Optional[List[int]] = None, + service_name: str = None, + additional_domains: Optional[List[str]] = None, + secret_backend: Optional[str] = None, + vault_subpath: Optional[str] = None, +) -> str: + spec = get_mongodb_x509_subject(namespace) + + return create_mongodb_tls_certs( + issuer=issuer, + namespace=namespace, + resource_name=resource_name, + bundle_secret_name=bundle_secret_name, + replicas=replicas, + replicas_cluster_distribution=replicas_cluster_distribution, + service_name=service_name, + spec=spec, + additional_domains=additional_domains, + secret_backend=secret_backend, + vault_subpath=vault_subpath, + ) + + +def get_mongodb_x509_subject(namespace): + """ + x509 certificates need a subject, more here: https://wiki.corp.mongodb.com/display/MMS/E2E+Tests+Notes + """ + subject = { + "countries": ["US"], + "provinces": ["NY"], + "localities": ["NY"], + "organizations": ["cluster.local-server"], + "organizationalUnits": [namespace], + } + spec = { + "subject": subject, + "usages": [ + "digital signature", + "key encipherment", + "client auth", + "server auth", + ], + } + return spec + + +def get_agent_x509_subject(namespace): + """ + x509 certificates need a subject, more here: https://wiki.corp.mongodb.com/display/MMS/E2E+Tests+Notes + """ + agents = ["automation", "monitoring", "backup"] + subject = { + "countries": ["US"], + "provinces": ["NY"], + "localities": ["NY"], + "organizations": ["cluster.local-agent"], + "organizationalUnits": [namespace], + } + spec = { + "subject": subject, + "usages": ["digital signature", "key encipherment", "client auth"], + "dnsNames": agents, + "commonName": "mms-automation-agent", + } + return spec + + +def create_agent_tls_certs( + issuer: str, + namespace: str, + name: str, + secret_prefix: Optional[str] = None, + secret_backend: Optional[str] = None, +) -> str: + agents = ["mms-automation-agent"] + subject = copy.deepcopy(SUBJECT) + subject["organizationalUnits"] = [namespace] + + spec = { + "subject": subject, + "usages": ["client auth"], + } + spec["dnsNames"] = agents + spec["commonName"] = "mms-automation-agent" + secret_name = "agent-certs" if secret_prefix is None else f"{secret_prefix}-{name}-agent-certs" + secret = generate_cert( + namespace=namespace, + pod=[], + dns=[], + issuer=issuer, + spec=spec, + secret_name=secret_name, + secret_backend=secret_backend, + vault_subpath="database", + ) + return secret + + +def create_sharded_cluster_certs( + namespace: str, + resource_name: str, + shards: int, + mongod_per_shard: int, + config_servers: int, + mongos: int, + internal_auth: bool = False, + x509_certs: bool = False, + additional_domains: Optional[List[str]] = None, + secret_prefix: Optional[str] = None, + secret_backend: Optional[str] = None, + shard_distribution: Optional[List[int]] = None, + mongos_distribution: Optional[List[int]] = None, + config_srv_distribution: Optional[List[int]] = None, +): + cert_generation_func = create_mongodb_tls_certs + if x509_certs: + cert_generation_func = create_x509_mongodb_tls_certs + + for shard_idx in range(shards): + additional_domains_for_shard = None + if additional_domains is not None: + additional_domains_for_shard = [] + for domain in additional_domains: + if shard_distribution is None: + for pod_idx in range(mongod_per_shard): + additional_domains_for_shard.append(f"{resource_name}-{shard_idx}-{pod_idx}.{domain}") + else: + for cluster_idx, pod_count in enumerate(shard_distribution): + for pod_idx in range(pod_count or 0): + additional_domains_for_shard.append( + f"{resource_name}-{shard_idx}-{cluster_idx}-{pod_idx}.{domain}" + ) + + secret_name = f"{resource_name}-{shard_idx}-cert" + if secret_prefix is not None: + secret_name = secret_prefix + secret_name + cert_generation_func( + issuer=ISSUER_CA_NAME, + namespace=namespace, + resource_name=f"{resource_name}-{shard_idx}", + bundle_secret_name=secret_name, + replicas=mongod_per_shard, + replicas_cluster_distribution=shard_distribution, + service_name=resource_name + "-sh", + additional_domains=additional_domains_for_shard, + secret_backend=secret_backend, + ) + if internal_auth: + cert_generation_func( + issuer=ISSUER_CA_NAME, + namespace=namespace, + resource_name=f"{resource_name}-{shard_idx}-clusterfile", + bundle_secret_name=f"{resource_name}-{shard_idx}-clusterfile", + replicas=mongod_per_shard, + replicas_cluster_distribution=shard_distribution, + service_name=resource_name + "-sh", + additional_domains=additional_domains_for_shard, + secret_backend=secret_backend, + ) + + additional_domains_for_config = None + if additional_domains is not None: + additional_domains_for_config = [] + for domain in additional_domains: + if config_srv_distribution is None: + for pod_idx in range(config_servers): + additional_domains_for_config.append(f"{resource_name}-config-{pod_idx}.{domain}") + else: + for cluster_idx, pod_count in enumerate(config_srv_distribution): + for pod_idx in range(pod_count or 0): + additional_domains_for_config.append(f"{resource_name}-config-{cluster_idx}-{pod_idx}.{domain}") + + secret_name = f"{resource_name}-config-cert" + if secret_prefix is not None: + secret_name = secret_prefix + secret_name + cert_generation_func( + issuer=ISSUER_CA_NAME, + namespace=namespace, + resource_name=resource_name + "-config", + bundle_secret_name=secret_name, + replicas=config_servers, + replicas_cluster_distribution=config_srv_distribution, + service_name=resource_name + "-cs", + additional_domains=additional_domains_for_config, + secret_backend=secret_backend, + ) + if internal_auth: + cert_generation_func( + issuer=ISSUER_CA_NAME, + namespace=namespace, + resource_name=f"{resource_name}-config-clusterfile", + bundle_secret_name=f"{resource_name}-config-clusterfile", + replicas=config_servers, + replicas_cluster_distribution=config_srv_distribution, + service_name=resource_name + "-cs", + additional_domains=additional_domains_for_config, + secret_backend=secret_backend, + ) + + additional_domains_for_mongos = None + if additional_domains is not None: + additional_domains_for_mongos = [] + for domain in additional_domains: + if mongos_distribution is None: + for pod_idx in range(mongos): + additional_domains_for_mongos.append(f"{resource_name}-mongos-{pod_idx}.{domain}") + else: + for cluster_idx, pod_count in enumerate(mongos_distribution): + for pod_idx in range(pod_count or 0): + additional_domains_for_mongos.append(f"{resource_name}-mongos-{cluster_idx}-{pod_idx}.{domain}") + + secret_name = f"{resource_name}-mongos-cert" + if secret_prefix is not None: + secret_name = secret_prefix + secret_name + cert_generation_func( + issuer=ISSUER_CA_NAME, + namespace=namespace, + resource_name=resource_name + "-mongos", + bundle_secret_name=secret_name, + service_name=resource_name + "-svc", + replicas=mongos, + replicas_cluster_distribution=mongos_distribution, + additional_domains=additional_domains_for_mongos, + secret_backend=secret_backend, + ) + + if internal_auth: + cert_generation_func( + issuer=ISSUER_CA_NAME, + namespace=namespace, + resource_name=f"{resource_name}-mongos-clusterfile", + bundle_secret_name=f"{resource_name}-mongos-clusterfile", + replicas=mongos, + replicas_cluster_distribution=mongos_distribution, + service_name=resource_name + "-sh", + additional_domains=additional_domains_for_mongos, + secret_backend=secret_backend, + ) + + +def create_x509_agent_tls_certs(issuer: str, namespace: str, name: str, secret_backend: Optional[str] = None) -> str: + spec = get_agent_x509_subject(namespace) + return generate_cert( + namespace=namespace, + pod=[], + dns=[], + issuer=issuer, + spec=spec, + secret_name="agent-certs", + secret_backend=secret_backend, + vault_subpath="database", + ) + + +def approve_certificate(name: str) -> None: + """Approves the CertificateSigningRequest with the provided name""" + body = client.CertificatesV1beta1Api().read_certificate_signing_request_status(name) + conditions = client.V1beta1CertificateSigningRequestCondition( + last_update_time=datetime.now(timezone.utc).astimezone(), + message="This certificate was approved by E2E testing framework", + reason="E2ETestingFramework", + type="Approved", + ) + + body.status.conditions = [conditions] + client.CertificatesV1beta1Api().replace_certificate_signing_request_approval(name, body) + + +def create_x509_user_cert(issuer: str, namespace: str, path: str): + user_name = "x509-testing-user" + + spec = { + "usages": ["digital signature", "key encipherment", "client auth"], + "commonName": user_name, + } + secret = generate_cert( + namespace=namespace, + pod=user_name, + dns=user_name, + issuer=issuer, + spec=spec, + secret_name="mongodbuser", + ) + cert = KubernetesTester.read_secret(namespace, secret) + with open(path, mode="w") as f: + f.write(cert["tls.key"]) + f.write(cert["tls.crt"]) + f.flush() + + +def create_multi_cluster_x509_user_cert( + multi_cluster_issuer: str, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + path: str, +): + user_name = "x509-testing-user" + spec = { + "usages": ["digital signature", "key encipherment", "client auth"], + "commonName": user_name, + } + secret = generate_cert( + namespace=namespace, + pod="tmp", + dns=user_name, + issuer=multi_cluster_issuer, + api_client=central_cluster_client, + spec=spec, + multi_cluster_mode=True, + secret_name="mongodbuser", + ) + cert = read_secret(namespace, secret, api_client=central_cluster_client) + with open(path, mode="w") as f: + f.write(cert["tls.key"]) + f.write(cert["tls.crt"]) + f.flush() + + +def yield_existing_csrs(csr_names: List[str], timeout: int = 300) -> Generator[str, None, None]: + """Returns certificate names as they start appearing in the Kubernetes API.""" + csr_names = csr_names.copy() + total_csrs = len(csr_names) + seen_csrs = 0 + stop_time = time.time() + timeout + + while len(csr_names) > 0 and time.time() < stop_time: + csr = random.choice(csr_names) + try: + client.CertificatesV1beta1Api().read_certificate_signing_request_status(csr) + except ApiException: + time.sleep(3) + continue + + seen_csrs += 1 + csr_names.remove(csr) + yield csr + + if len(csr_names) == 0: + # All the certificates have been "consumed" and yielded back to the user. + return + + # we didn't find all of the expected csrs after the timeout period + raise AssertionError( + f"Expected to find {total_csrs} csrs, but only found {seen_csrs} after {timeout} seconds. Expected csrs {csr_names}" + ) diff --git a/docker/mongodb-enterprise-tests/kubetester/create_or_replace_from_yaml.py b/docker/mongodb-enterprise-tests/kubetester/create_or_replace_from_yaml.py new file mode 100644 index 000000000..824013212 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/create_or_replace_from_yaml.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import re +from os import path + +import yaml +from kubernetes import client +from kubernetes.client import ApiClient +from kubernetes.utils.create_from_yaml import create_from_yaml_single_item + +""" +This is a modification of 'create_from_yaml.py' from python kubernetes client library +It allows to mimic the 'kubectl apply' operation on yaml file. It performs either create or +patch on each individual object. +""" + + +def create_or_replace_from_yaml(k8s_client: ApiClient, yaml_file: str, namespace: str = "default", **kwargs): + with open(path.abspath(yaml_file)) as f: + yml_document_all = yaml.safe_load_all(f) + # Load all documents from a single YAML file + for yml_document in yml_document_all: + create_or_patch_from_dict(k8s_client, yml_document, namespace=namespace, **kwargs) + + +def create_or_patch_from_dict(k8s_client, yml_document, namespace="default", **kwargs): + # If it is a list type, will need to iterate its items + if "List" in yml_document["kind"]: + # Could be "List" or "Pod/Service/...List" + # This is a list type. iterate within its items + kind = yml_document["kind"].replace("List", "") + for yml_object in yml_document["items"]: + # Mitigate cases when server returns a xxxList object + # See kubernetes-client/python#586 + if kind != "": + yml_object["apiVersion"] = yml_document["apiVersion"] + yml_object["kind"] = kind + create_or_replace_from_yaml_single_item(k8s_client, yml_object, namespace, **kwargs) + else: + # Try to create the object or patch if it already exists + create_or_replace_from_yaml_single_item(k8s_client, yml_document, namespace, **kwargs) + + +def create_or_replace_from_yaml_single_item(k8s_client, yml_object, namespace="default", **kwargs): + try: + create_from_yaml_single_item(k8s_client, yml_object, verbose=False, namespace=namespace, **kwargs) + except client.rest.ApiException: + patch_from_yaml_single_item(k8s_client, yml_object, namespace, **kwargs) + except ValueError: + if get_kind(yml_object) == "custom_resource_definition": + # TODO unfortunately CRD creation results in error before 1.12 python lib and 1.16 K8s + # https://github.com/kubernetes-client/python/issues/1022 + pass + + +def patch_from_yaml_single_item(k8s_client, yml_object, namespace="default", **kwargs): + k8s_api = get_k8s_api(k8s_client, yml_object) + kind = get_kind(yml_object) + # Decide which namespace we are going to put the object in, + # if any + if "namespace" in yml_object["metadata"]: + namespace = yml_object["metadata"]["namespace"] + name = yml_object["metadata"]["name"] + + method = "patch" + if kind == "custom_resource_definition": + # fetching the old CRD to make the replace working (has conflict resolution based on 'resourceVersion') + # TODO this is prone to race conditions - we need to either loop or use patch with json merge + # see https://github.com/helm/helm/pull/6092/files#diff-a483d6c0863082c3df21f4aad513afe2R663 + resource = client.ApiextensionsV1Api().read_custom_resource_definition(name) + + yml_object["metadata"]["resourceVersion"] = resource.metadata.resource_version + method = "replace" + + namespaced = hasattr(k8s_api, "{}_namespaced_{}".format("create", kind)) + url_path = get_url_path(namespaced, method) + + if namespaced: + # Note that patch the deployment can result in + # "Invalid value: \"\": may not be specified when `value` is not empty","field":"spec.template.spec.containers[0].env[1].valueFrom" + # (https://github.com/kubernetes/kubernetes/issues/46861) if "kubectl apply" was used initially to create the object + # This is safe though if the object was created using python API + getattr(k8s_api, url_path.format(kind))(body=yml_object, namespace=namespace, name=name, **kwargs) + else: + # "patch" endpoints require to specify 'name' attribute + getattr(k8s_api, url_path.format(kind))(body=yml_object, name=name, **kwargs) + + +def get_kind(yml_object): + # Replace CamelCased action_type into snake_case + kind = yml_object["kind"] + kind = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", kind) + kind = re.sub("([a-z0-9])([A-Z])", r"\1_\2", kind).lower() + return kind + + +def get_k8s_api(k8s_client, yml_object): + group, _, version = yml_object["apiVersion"].partition("/") + if version == "": + version = group + group = "core" + # Take care for the case e.g. api_type is "apiextensions.k8s.io" + # Only replace the last instance + group = "".join(group.rsplit(".k8s.io", 1)) + # convert group name from DNS subdomain format to + # python class name convention + group = "".join(word.capitalize() for word in group.split(".")) + fcn_to_call = "{0}{1}Api".format(group, version.capitalize()) + k8s_api = getattr(client, fcn_to_call)(k8s_client) + return k8s_api + + +def get_url_path(namespaced, method: str): + if namespaced: + url_path = method + "_namespaced_{}" + else: + url_path = method + "_{}" + return url_path diff --git a/docker/mongodb-enterprise-tests/kubetester/crypto.py b/docker/mongodb-enterprise-tests/kubetester/crypto.py new file mode 100644 index 000000000..c59341fb7 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/crypto.py @@ -0,0 +1,72 @@ +import base64 +import time +from typing import List, Optional + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID +from kubernetes import client + + +def generate_csr(namespace: str, host: str, servicename: str): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) + + csr = ( + x509.CertificateSigningRequestBuilder() + .subject_name( + x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "New York"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "New York"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Mongodb"), + x509.NameAttribute(NameOID.COMMON_NAME, host), + ] + ) + ) + .add_extension( + x509.SubjectAlternativeName( + [ + x509.DNSName(f"{host}."), + x509.DNSName(f"{host}.{servicename}.{namespace}.svc.cluster.local"), + x509.DNSName(f"{servicename}.{namespace}.svc.cluster.local"), + ] + ), + critical=False, + ) + .sign(key, hashes.SHA256(), default_backend()) + ) + + return ( + csr.public_bytes(serialization.Encoding.PEM), + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ), + ) + + +def get_pem_certificate(name: str) -> Optional[str]: + body = client.CertificatesV1beta1Api().read_certificate_signing_request_status(name) + if body.status.certificate is None: + return None + return base64.b64decode(body.status.certificate) + + +def wait_for_certs_to_be_issued(certificates: List[str]) -> None: + un_issued_certs = set(certificates) + while un_issued_certs: + issued_certs = set() + to_wait = False + for cert in un_issued_certs: + if get_pem_certificate(cert): + issued_certs.add(cert) + else: + print(f"waiting for certificate {cert} to be issued") + to_wait = True + un_issued_certs -= issued_certs + if to_wait: + time.sleep(1) diff --git a/docker/mongodb-enterprise-tests/kubetester/custom_podspec.py b/docker/mongodb-enterprise-tests/kubetester/custom_podspec.py new file mode 100644 index 000000000..dd1e06af0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/custom_podspec.py @@ -0,0 +1,40 @@ +from typing import List, Optional + + +def assert_stateful_set_podspec( + pod_template_spec, + weight: int = 0, + topology_key: str = "", + grace_period_seconds: int = 0, + containers_spec: Optional[List] = None, +) -> None: + assert pod_template_spec.termination_grace_period_seconds == grace_period_seconds + assert ( + pod_template_spec.affinity.pod_anti_affinity.preferred_during_scheduling_ignored_during_execution[0].weight + == weight + ) + assert ( + pod_template_spec.affinity.pod_anti_affinity.preferred_during_scheduling_ignored_during_execution[ + 0 + ].pod_affinity_term.topology_key + == topology_key + ) + if containers_spec is None: + containers_spec = [] + for i, expected_spec in enumerate(containers_spec): + spec = pod_template_spec.containers[i].to_dict() + # compare only the expected keys + for k in expected_spec: + if k == "volume_mounts": + assert_volume_mounts_are_equal(expected_spec[k], spec[k]) + else: + assert expected_spec[k] == spec[k] + + +def assert_volume_mounts_are_equal(volume_mounts_1, volume_mounts_2): + + sorted_vols_1 = sorted(volume_mounts_1, key=lambda m: (m["name"], m["mount_path"])) + sorted_vols_2 = sorted(volume_mounts_2, key=lambda m: (m["name"], m["mount_path"])) + + for vol1, vol2 in zip(sorted_vols_1, sorted_vols_2): + assert vol1 == vol2 diff --git a/docker/mongodb-enterprise-tests/kubetester/git.py b/docker/mongodb-enterprise-tests/kubetester/git.py new file mode 100644 index 000000000..e385e33b8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/git.py @@ -0,0 +1,6 @@ +from git import Repo + + +def clone_and_checkout(url: str, fs_path: str, branch_name: str): + repo = Repo.clone_from(url, fs_path) + repo.git.checkout(branch_name) diff --git a/docker/mongodb-enterprise-tests/kubetester/helm.py b/docker/mongodb-enterprise-tests/kubetester/helm.py new file mode 100644 index 000000000..99bf69cbd --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/helm.py @@ -0,0 +1,206 @@ +import logging +import os +import re +import subprocess +import uuid +from typing import Dict, List, Optional, Tuple + +from tests import test_logger + +logger = test_logger.get_test_logger(__name__) + + +def helm_template( + helm_args: Dict, + helm_chart_path: Optional[str] = "helm_chart", + templates: Optional[str] = None, + helm_options: Optional[List[str]] = None, +) -> str: + """generates yaml file using Helm and returns its name. Provide 'templates' if you need to run + a specific template from the helm chart""" + command_args = _create_helm_args(helm_args, helm_options) + + if templates is not None: + command_args.append("--show-only") + command_args.append(templates) + + args = ("helm", "template", *(command_args), _helm_chart_dir(helm_chart_path)) + logger.info(args) + + yaml_file_name = "{}.yaml".format(str(uuid.uuid4())) + with open(yaml_file_name, "w") as output: + process_run_and_check(" ".join(args), stdout=output, check=True, shell=True) + + return yaml_file_name + + +def helm_install( + name: str, + namespace: str, + helm_args: Dict, + helm_chart_path: Optional[str] = "helm_chart", + helm_options: Optional[List[str]] = None, + custom_operator_version: Optional[str] = None, +): + command_args = _create_helm_args(helm_args, helm_options) + args = [ + "helm", + "upgrade", + "--install", + f"--namespace={namespace}", + *command_args, + name, + _helm_chart_dir(helm_chart_path), + ] + if custom_operator_version: + args.append(f"--version={custom_operator_version}") + logger.info(f"Running helm install command: {' '.join(args)}") + + process_run_and_check(" ".join(args), check=True, capture_output=True, shell=True) + + +def helm_install_from_chart( + namespace: str, + release: str, + chart: str, + version: str = "", + custom_repo: Tuple[str, str] = ("stable", "https://charts.helm.sh/stable"), + helm_args: Optional[Dict[str, str]] = None, + override_path: Optional[str] = None, +): + """Installs a helm chart from a repo. It can accept a new custom_repo to add before the + chart is installed. Also, `helm_args` accepts a dictionary that will be passed as --set + arguments to `helm install`. + + Some charts are clusterwide (like CertManager), and simultaneous installation can + fail. This function tolerates errors when installing the Chart if `stderr` of the + Helm process has the "release: already exists" string on it. + """ + + args = [ + "helm", + "upgrade", + "--install", + release, + f"--namespace={namespace}", + chart, + ] + + if override_path is not None: + args.extend(["-f", f"{override_path}"]) + + if version != "": + args.append("--version=" + version) + + if helm_args is not None: + args += _create_helm_args(helm_args) + + helm_repo_add(custom_repo[0], custom_repo[1]) + + try: + # In shared clusters (Kops: e2e) multiple simultaneous cert-manager + # installations will fail. We tolerate errors in those cases. + process_run_and_check(args, capture_output=True) + except subprocess.CalledProcessError as exc: + stderr = exc.stderr.decode("utf-8") + if "release: already exists" in stderr or "Error: UPGRADE FAILED: another operation" in stderr: + logger.info(f"Helm chart '{chart}' already installed in cluster.") + else: + raise + + +def helm_repo_add(repo_name: str, url: str): + """ + Adds a new repo to Helm. + """ + helm_repo = f"helm repo add {repo_name} {url}".split() + logger.info(helm_repo) + process_run_and_check(helm_repo, capture_output=True) + + +def process_run_and_check(args, **kwargs): + try: + completed_process = subprocess.run(args, **kwargs) + completed_process.check_returncode() + except subprocess.CalledProcessError as exc: + if exc.stdout is not None: + stdout = exc.stdout.decode("utf-8") + logger.error(f"stdout: {stdout}") + if exc.stderr is not None: + stderr = exc.stderr.decode("utf-8") + logger.error(f"stderr: {stderr}") + logger.error(f"output: {exc.output}") + raise + + +def helm_upgrade( + name: str, + namespace: str, + helm_args: Dict, + helm_chart_path: Optional[str] = "helm_chart", + helm_options: Optional[List[str]] = None, + helm_override_path: Optional[bool] = False, + custom_operator_version: Optional[str] = None, +): + if not helm_chart_path: + logger.warning("Helm chart path is empty, defaulting to 'helm_chart'") + helm_chart_path = "helm_chart" + command_args = _create_helm_args(helm_args, helm_options) + args = [ + "helm", + "upgrade", + "--install", + f"--namespace={namespace}", + *command_args, + name, + ] + if custom_operator_version: + args.append(f"--version={custom_operator_version}") + if helm_override_path: + args.append(helm_chart_path) + else: + args.append(_helm_chart_dir(helm_chart_path)) + + command = " ".join(args) + logger.debug("Running helm upgrade command:") + logger.debug(command) + process_run_and_check(command, check=True, capture_output=True, shell=True) + + +def helm_uninstall(name): + args = ("helm", "uninstall", name) + logger.info(args) + process_run_and_check(" ".join(args), check=True, capture_output=True, shell=True) + + +def _create_helm_args(helm_args: Dict[str, str], helm_options: Optional[List[str]] = None) -> List[str]: + command_args = [] + for key, value in helm_args.items(): + command_args.append("--set") + + if "," in value: + # helm lists are defined with {}, hence matching this means we don't have to escape. + if not re.match("^{.+}$", value): + # Commas in values, but no lists, should be escaped + value = value.replace(",", "\,") + + # and when commas are present, we should quote "key=value" + key = '"' + key + value = value + '"' + + command_args.append("{}={}".format(key, value)) + + if "useRunningOperator" in helm_args: + logger.info("Operator will not be installed this time, passing --dry-run") + command_args.append("--dry-run") + + command_args.append("--create-namespace") + + if helm_options: + command_args.extend(helm_options) + + return command_args + + +def _helm_chart_dir(default: Optional[str] = "helm_chart") -> str: + return os.environ.get("HELM_CHART_DIR", default) diff --git a/docker/mongodb-enterprise-tests/kubetester/http.py b/docker/mongodb-enterprise-tests/kubetester/http.py new file mode 100644 index 000000000..187db5b66 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/http.py @@ -0,0 +1,37 @@ +from typing import Tuple + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + + +def get_retriable_session(proto: str, tls_verify: bool) -> requests.Session: + """ + Returns a request Session object with a retry mechanism. + + This is required to overcome a DNS resolution problem that we have + experienced in the Evergreen hosts. This can also probably alleviate + problems arising from request throttling. + """ + + s = requests.Session() + + s.verify = tls_verify + retries = Retry( + total=5, + backoff_factor=2, + ) + s.mount(proto + "://", HTTPAdapter(max_retries=retries)) + + return s + + +def get_retriable_https_session(*, tls_verify: bool) -> requests.Session: + return get_retriable_session("https", tls_verify) + + +def https_endpoint_is_reachable(url: str, auth: Tuple[str], *, tls_verify: bool) -> bool: + """ + Checks that `url` is reachable, using `auth` basic credentials. + """ + return get_retriable_https_session(tls_verify=tls_verify).get(url, auth=auth).status_code == 200 diff --git a/docker/mongodb-enterprise-tests/kubetester/kmip.py b/docker/mongodb-enterprise-tests/kubetester/kmip.py new file mode 100644 index 000000000..128d7ac51 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/kmip.py @@ -0,0 +1,201 @@ +# ---------------------------------------------------------------------------- +# This file contains the implementation of the KMIP Server (PyKMIP) deployment +# +# The deployment has been outlined in the outdated Enterprise Kubernetes Operator +# guide, that might be found here: +# https://docs.google.com/document/d/12Y5h7XDFedcgpSIWRxMgcjZClL6kZdIwdxPRotkuKck/edit# +# ----------------------------------------------------------- + +from typing import Dict, Optional + +from kubernetes import client +from kubetester import ( + create_or_update_configmap, + create_or_update_secret, + create_or_update_service, + create_statefulset, + read_configmap, + read_secret, +) +from kubetester.certs import create_tls_certs +from kubetester.kubetester import KubernetesTester + + +class KMIPDeployment(object): + """ + A KMIP Server deployment class. Deploys PyKMIP in the cluster. + """ + + def __init__(self, namespace, issuer, root_cert_secret, ca_configmap: str): + self.namespace = namespace + self.issuer = issuer + self.root_cert_secret = root_cert_secret + self.ca_configmap = ca_configmap + self.statefulset_name = "kmip" + self.labels = { + "app": "kmip", + } + + def deploy(self): + """ + Deploys a PyKMIP Server and returns the name of the deployed StatefulSet. + """ + service_name = f"{self.statefulset_name}-svc" + + cert_secret_name = self._create_tls_certs_kmip( + self.issuer, + self.namespace, + self.statefulset_name, + "kmip-certs", + 1, + service_name, + ) + + create_or_update_service( + self.namespace, + service_name, + cluster_ip=None, + ports=[client.V1ServicePort(name="kmip", port=5696)], + selector=self.labels, + ) + + self._create_kmip_config_map(self.namespace, "kmip-config", self._default_configuration()) + + create_statefulset( + self.namespace, + self.statefulset_name, + service_name, + self.labels, + containers=[ + client.V1Container( + # We need this awkward copy step as PyKMIP uses /etc/pykmip as a tmp directory. When booting up + # it stores there some intermediate configuration files. So it must have write access to the whole + # /etc/pykmip directory. Very awkward... + args=[ + "bash", + "-c", + "cp /etc/pykmip-conf/server.conf /etc/pykmip/server.conf & /tmp/configure.sh & mkdir -p /var/log/pykmip & touch /var/log/pykmip/server.log & tail -f /var/log/pykmip/server.log", + ], + name="kmip", + image="beergeek1679/pykmip:0.6.0", + image_pull_policy="IfNotPresent", + ports=[ + client.V1ContainerPort( + container_port=5696, + name="kmip", + ) + ], + volume_mounts=[ + client.V1VolumeMount( + name="certs", + mount_path="/data/pki", + read_only=True, + ), + client.V1VolumeMount( + name="config", + mount_path="/etc/pykmip-conf", + read_only=True, + ), + ], + ) + ], + volumes=[ + client.V1Volume( + name="certs", + secret=client.V1SecretVolumeSource( + secret_name=cert_secret_name, + ), + ), + client.V1Volume( + name="config", + config_map=client.V1ConfigMapVolumeSource(name="kmip-config"), + ), + ], + ) + return self + + def status(self): + return KMIPDeploymentStatus(self) + + def _create_tls_certs_kmip( + self, + issuer: str, + namespace: str, + resource_name: str, + bundle_secret_name: str, + replicas: int = 3, + service_name: str = None, + spec: Optional[Dict] = None, + ) -> str: + ca = read_configmap(namespace, self.ca_configmap) + cert_secret_name = create_tls_certs( + issuer, + namespace, + resource_name, + replicas=replicas, + service_name=service_name, + spec=spec, + additional_domains=[service_name], + ) + secret = read_secret(namespace, cert_secret_name) + create_or_update_secret( + namespace, + bundle_secret_name, + { + "server.key": secret["tls.key"], + "server.cert": secret["tls.crt"], + "ca.cert": ca["ca-pem"], + }, + ) + + return bundle_secret_name + + def _default_configuration(self) -> Dict: + return { + "hostname": "kmip-0", + "port": 5696, + "certificate_path": "/data/pki/server.cert", + "key_path": "/data/pki/server.key", + "ca_path": "/data/pki/ca.cert", + "auth_suite": "TLS1.2", + "enable_tls_client_auth": True, + "policy_path": "/data/policies", + "logging_level": "DEBUG", + "database_path": "/data/db/pykmip.db", + } + + def _create_kmip_config_map(self, namespace: str, name: str, config_dict: Dict) -> None: + """ + _create_configuration_config_map converts a dictionary of options into the server.conf + file that the kmip server uses to start. + """ + equals_separated = [k + "=" + str(v) for (k, v) in config_dict.items()] + config_file_contents = "[server]\n" + "\n".join(equals_separated) + create_or_update_configmap( + namespace, + name, + { + "server.conf": config_file_contents, + }, + ) + + +class KMIPDeploymentStatus: + """ + A class designed to check the KMIP Server deployment status. + """ + + def __init__(self, deployment: KMIPDeployment): + self.deployment = deployment + + def assert_is_running(self): + """ + Waits and assert if the KMIP server is running. + :return: raises an error if the server is not running within the timeout. + """ + KubernetesTester.wait_for_condition_stateful_set( + self.deployment.namespace, + self.deployment.statefulset_name, + "status.current_replicas", + 1, + ) diff --git a/docker/mongodb-enterprise-tests/kubetester/kubetester.py b/docker/mongodb-enterprise-tests/kubetester/kubetester.py new file mode 100644 index 000000000..dd2e96dd0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/kubetester.py @@ -0,0 +1,1685 @@ +import json +import os +import random +import re +import ssl +import string +import sys +import tarfile +import tempfile +import time +import warnings +from base64 import b64decode, b64encode +from typing import Dict, List, Optional + +import jsonpatch +import kubernetes.client +import pymongo +import pytest +import requests +import semver +import yaml +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from kubeobject import CustomObject +from kubernetes import client, config +from kubernetes.client.rest import ApiException +from kubernetes.stream import stream +from kubetester.crypto import wait_for_certs_to_be_issued +from requests.auth import HTTPBasicAuth, HTTPDigestAuth + +SSL_CA_CERT = "/var/run/secrets/kubernetes.io/serviceaccount/..data/ca.crt" +EXTERNALLY_MANAGED_TAG = "EXTERNALLY_MANAGED_BY_KUBERNETES" +MAX_TAG_LEN = 32 + +DEPRECATION_WARNING = "This feature has been DEPRECATED and should only be used in testing environments." +AGENT_WARNING = "The Operator is generating TLS x509 certificates for agent authentication. " + DEPRECATION_WARNING +MEMBER_AUTH_WARNING = ( + "The Operator is generating TLS x509 certificates for internal cluster authentication. " + DEPRECATION_WARNING +) +SERVER_WARNING = "The Operator is generating TLS certificates for server authentication. " + DEPRECATION_WARNING + +plural_map = { + "MongoDB": "mongodb", + "MongoDBUser": "mongodbusers", + "MongoDBOpsManager": "opsmanagers", + "MongoDBMultiCluster": "mongodbmulticluster", +} + + +def running_locally(): + return os.getenv("POD_NAME", "local") == "local" + + +def is_multi_cluster(): + return len(os.getenv("MEMBER_CLUSTERS", "")) > 0 + + +def is_default_architecture_static() -> bool: + return os.getenv("MDB_DEFAULT_ARCHITECTURE", "non-static") == "static" + + +def get_default_architecture() -> str: + return "static" if is_default_architecture_static() else "non-static" + + +def assert_statefulset_architecture(statefulset: client.V1StatefulSet, architecture: str): + """ + Asserts that the statefulset is configured with the expected architecture. + """ + agent_container = next((c for c in statefulset.spec.template.spec.containers if c.name == "mongodb-agent"), None) + if architecture == "non-static": + # In non-static architecture expect agent container to not be present + assert agent_container is None + else: + # In static architecture we expect agent container to be present + # and contain static environment variable which + # instructs the agent launcher script to not download binaries + assert agent_container is not None + static_env_var = next( + (env for env in agent_container.env if env.name == "MDB_STATIC_CONTAINERS_ARCHITECTURE"), None + ) + assert static_env_var.value == "true" + + +skip_if_static_containers = pytest.mark.skipif( + is_default_architecture_static(), + reason="Skip if this test is executed using the Static Containers architecture", +) + +skip_if_local = pytest.mark.skipif(running_locally(), reason="Only run in Kubernetes cluster") +skip_if_multi_cluster = pytest.mark.skipif(is_multi_cluster(), reason="Only run in Kubernetes single cluster") +# time to sleep between retries +SLEEP_TIME = 2 +# no timeout (loop forever) +INFINITY = -1 + + +class KubernetesTester(object): + """ + KubernetesTester is the base class for all python tests. It deliberately doesn't have object state + as it is not expected to have more than one concurrent instance running. All tests must be run in separate + Kubernetes namespaces and use separate Ops Manager groups. + The class provides some common utility methods used by all children and also performs some common + create/update/delete actions for Kubernetes objects based on the docstrings of subclasses + """ + + init = None + group_id = None + + @classmethod + def setup_env(cls): + """Optionally override this in a test instance to create an appropriate test environment.""" + pass + + @classmethod + def teardown_env(cls): + """Optionally override this in a test instance to destroy the test environment.""" + pass + + @classmethod + def create_config_map(cls, namespace, name, data): + """Create a config map in a given namespace with the given name and data.""" + config_map = cls.clients("client").V1ConfigMap( + metadata=cls.clients("client").V1ObjectMeta(name=name), data=data + ) + cls.clients("corev1").create_namespaced_config_map(namespace, config_map) + + @classmethod + def patch_config_map(cls, namespace, name, data): + """Patch a config map in a given namespace with the given name and data.""" + config_map = cls.clients("client").V1ConfigMap(data=data) + cls.clients("corev1").patch_namespaced_config_map(name, namespace, config_map) + + @classmethod + def create_secret(cls, namespace: str, name: str, data: Dict[str, str]): + """ + Deprecated: use kubetester.create_secret instead. + + Create a secret in a given namespace with the given name and data—handles base64 encoding. + """ + secret = cls.clients("client").V1Secret( + metadata=cls.clients("client").V1ObjectMeta(name=name), string_data=data + ) + + try: + cls.clients("corev1").create_namespaced_secret(namespace, secret) + except client.rest.ApiException as e: + if e.status == 409 and running_locally(): + pass + + @classmethod + def update_secret(cls, namespace: str, name: str, data: Dict[str, str]): + """ + Deprecated: use kubetester.update_secret instead. + + Updates a secret in a given namespace with the given name and data—handles base64 encoding. + """ + secret = cls.clients("client").V1Secret( + metadata=cls.clients("client").V1ObjectMeta(name=name), string_data=data + ) + cls.clients("corev1").patch_namespaced_secret(name, namespace, secret) + + @classmethod + def delete_secret(cls, namespace: str, name: str): + """Delete a secret in a given namespace with the given name.""" + cls.clients("corev1").delete_namespaced_secret(name, namespace) + + @classmethod + def delete_csr(cls, name: str): + cls.clients("certificates").delete_certificate_signing_request(name) + + @classmethod + def read_secret(cls, namespace: str, name: str) -> Dict[str, str]: + """ + Deprecated: use kubetester.read_secret instead. + """ + data = cls.clients("corev1").read_namespaced_secret(name, namespace).data + return decode_secret(data=data) + + @classmethod + def decode_secret(cls, data: Dict[str, str]) -> Dict[str, str]: + return {k: b64decode(v).decode("utf-8") for (k, v) in data.items()} + + @classmethod + def read_configmap(cls, namespace: str, name: str, api_client: Optional[client.ApiClient] = None) -> Dict[str, str]: + corev1 = cls.clients("corev1") + if api_client is not None: + corev1 = client.CoreV1Api(api_client=api_client) + + return corev1.read_namespaced_config_map(name, namespace).data + + @classmethod + def read_pod(cls, namespace: str, name: str) -> Dict[str, str]: + """Reads a Pod and returns its contents""" + return cls.clients("corev1").read_namespaced_pod(name, namespace) + + @classmethod + def read_pod_logs( + cls, + namespace: str, + name: str, + container: str = None, + api_client: Optional[client.ApiClient] = None, + ) -> str: + return cls.clients("corev1", api_client=api_client).read_namespaced_pod_log( + name=name, namespace=namespace, container=container + ) + + @classmethod + def read_operator_pod(cls, namespace: str) -> Dict[str, str]: + label_selector = "app.kubernetes.io/name=mongodb-enterprise-operator" + return cls.read_pod_labels(namespace, label_selector).items[0] + + @classmethod + def read_pod_labels(cls, namespace: str, label_selector: Optional[str] = None) -> Dict[str, str]: + """Reads a Pod by labels.""" + return cls.clients("corev1").list_namespaced_pod(namespace=namespace, label_selector=label_selector) + + @classmethod + def update_configmap( + cls, + namespace: str, + name: str, + data: Dict[str, str], + api_client: Optional[client.ApiClient] = None, + ): + """Updates a ConfigMap in a given namespace with the given name and data—handles base64 encoding.""" + configmap = cls.clients("client", api_client=api_client).V1ConfigMap( + metadata=cls.clients("client", api_client=api_client).V1ObjectMeta(name=name), + data=data, + ) + cls.clients("corev1").patch_namespaced_config_map(name, namespace, configmap) + + @classmethod + def delete_configmap(cls, namespace: str, name: str, api_client: Optional[client.ApiClient] = None): + """Delete a ConfigMap in a given namespace with the given name.""" + cls.clients("corev1", api_client=api_client).delete_namespaced_config_map(name, namespace) + + @classmethod + def delete_service(cls, namespace: str, name: str): + """Delete a Service in a given namespace with the given name.""" + cls.clients("corev1").delete_namespaced_service(name, namespace) + + @classmethod + def create_namespace(cls, namespace_name): + """Create a namespace with the given name.""" + namespace = cls.clients("client").V1Namespace(metadata=cls.clients("client").V1ObjectMeta(name=namespace_name)) + cls.clients("corev1").create_namespace(namespace) + + @classmethod + def create_pod(cls, namespace: str, body: Dict): + cls.clients("corev1").create_namespaced_pod(body=body, namespace=namespace) + + @classmethod + def delete_pod(cls, namespace: str, name: str): + """Delete a Pod in a given namespace with the given name.""" + cls.clients("corev1").delete_namespaced_pod(name, namespace) + + @classmethod + def create_deployment(cls, namespace: str, body: Dict): + cls.clients("appsv1").create_namespaced_deployment(body=body, namespace=namespace) + + @classmethod + def create_service( + cls, + namespace: str, + body: Dict, + api_client: Optional[kubernetes.client.ApiClient] = None, + ): + cls.clients("corev1", api_client=api_client).create_namespaced_service(body=body, namespace=namespace) + + @classmethod + def create_or_update_pvc( + cls, + namespace: str, + body: Dict, + storage_class_name: str = "gp2", + api_client: Optional[kubernetes.client.ApiClient] = None, + ): + if storage_class_name is not None: + body["spec"]["storageClassName"] = storage_class_name + try: + cls.clients("corev1", api_client=api_client).create_namespaced_persistent_volume_claim( + body=body, namespace=namespace + ) + except client.rest.ApiException as e: + if e.status == 409: + cls.clients("corev1", api_client=api_client).patch_namespaced_persistent_volume_claim( + body=body, name=body["metadata"]["name"], namespace=namespace + ) + + @classmethod + def delete_pvc(cls, namespace: str, name: str): + cls.clients("corev1").delete_namespaced_persistent_volume_claim(name, namespace=namespace) + + @classmethod + def delete_namespace(cls, name): + """Delete the specified namespace.""" + cls.clients("corev1").delete_namespace(name, body=cls.clients("client").V1DeleteOptions()) + + @classmethod + def delete_deployment(cls, namespace: str, name): + cls.clients("appsv1").delete_namespaced_deployment(name, namespace) + + @staticmethod + def clients(name, api_client: Optional[client.ApiClient] = None): + return { + "client": client, + "corev1": client.CoreV1Api(api_client=api_client), + "appsv1": client.AppsV1Api(api_client=api_client), + "storagev1": client.StorageV1Api(api_client=api_client), + "customv1": client.CustomObjectsApi(api_client=api_client), + "certificates": client.CertificatesV1beta1Api(api_client=api_client), + "namespace": KubernetesTester.get_namespace(), + }[name] + + @classmethod + def teardown_class(cls): + "Tears down testing class, make sure pytest ends after tests are run." + cls.teardown_env() + sys.stdout.flush() + + @classmethod + def doc_string_to_init(cls, doc_string) -> dict: + result = yaml.safe_load(doc_string) + for m in ["create", "update"]: + if m in result and "patch" in result[m]: + result[m]["patch"] = json.loads(result[m]["patch"]) + return result + + @classmethod + def setup_class(cls): + "Will setup class (initialize kubernetes objects)" + print("\n") + KubernetesTester.load_configuration() + # Loads the subclass doc + if cls.init is None and cls.__doc__: + cls.init = cls.doc_string_to_init(cls.__doc__) + + if cls.init: + cls.prepare(cls.init, KubernetesTester.get_namespace()) + + cls.setup_env() + + @staticmethod + def load_configuration(): + "Loads kubernetes client configuration from kubectl config or incluster." + try: + config.load_kube_config() + except Exception: + config.load_incluster_config() + + @staticmethod + def get_namespace(): + return get_env_var_or_fail("NAMESPACE") + + @staticmethod + def get_om_group_name(): + return KubernetesTester.get_namespace() + + @staticmethod + def get_om_base_url(): + return get_env_var_or_fail("OM_HOST") + + @staticmethod + def get_om_user(): + return get_env_var_or_fail("OM_USER") + + @staticmethod + def get_om_api_key(): + return get_env_var_or_fail("OM_API_KEY") + + @staticmethod + def get_om_org_id(): + "Gets Organization ID. Makes sure to return None if it is not present" + + org_id = None + # Do not fail if OM_ORGID is not set + try: + org_id = get_env_var_or_fail("OM_ORGID") + except ValueError: + pass + + if isinstance(org_id, str) and org_id.strip() == "": + org_id = None + + return org_id + + @staticmethod + def get_om_group_id(group_name=None, org_id=None): + # doing some "caching" for the group id on the first invocation + if (KubernetesTester.group_id is None) or group_name or org_id: + group_name = group_name or KubernetesTester.get_om_group_name() + + org_id = org_id or KubernetesTester.get_om_org_id() + + group = KubernetesTester.query_group(group_name, org_id) + + KubernetesTester.group_id = group["id"] + + # Those are saved here so we can access om at the end of the test run and retrieve diagnostic data easily. + if os.environ.get("OM_PROJECT_ID", ""): + os.environ["OM_PROJECT_ID"] = os.environ["OM_PROJECT_ID"] + "," + KubernetesTester.group_id + else: + os.environ["OM_PROJECT_ID"] = KubernetesTester.group_id + + return KubernetesTester.group_id + + @classmethod + def prepare(cls, test_setup, namespace): + allowed_actions = ["create", "create_many", "update", "delete", "noop", "wait"] + for action in [action for action in allowed_actions if action in test_setup]: + rules = test_setup[action] + + if not isinstance(rules, list): + rules = [rules] + + for rule in rules: + KubernetesTester.execute(action, rule, namespace) + + if "wait_for_condition" in rule: + cls.wait_for_condition_string(rule["wait_for_condition"]) + elif "wait_for_message" in rule: + cls.wait_for_status_message(rule) + else: + cls.wait_condition(rule) + + @staticmethod + def execute(action, rules, namespace): + "Execute function with name `action` with arguments `rules` and `namespace`" + getattr(KubernetesTester, action)(rules, namespace) + + @staticmethod + def wait_for(seconds): + "Will wait for a given amount of seconds." + time.sleep(int(seconds)) + + @staticmethod + def wait(rules, namespace): + KubernetesTester.name = rules["resource"] + KubernetesTester.wait_until(rules["until"], rules.get("timeout", 0)) + + @staticmethod + def create(section, namespace): + "creates a custom object from filename" + resource = yaml.safe_load(open(fixture(section["file"]))) + KubernetesTester.create_custom_resource_from_object( + namespace, + resource, + exception_reason=section.get("exception", None), + patch=section.get("patch", None), + ) + + @staticmethod + def create_many(section, namespace): + "creates multiple custom objects from a yaml list" + resources = yaml.safe_load(open(fixture(section["file"]))) + for res in resources: + name, kind = KubernetesTester.create_custom_resource_from_object( + namespace, + res, + exception_reason=section.get("exception", None), + patch=section.get("patch", None), + ) + + @staticmethod + def create_mongodb_from_file(namespace, file_path): + name, kind = KubernetesTester.create_custom_resource_from_file(namespace, file_path) + KubernetesTester.namespace = namespace + KubernetesTester.name = name + KubernetesTester.kind = kind + + @staticmethod + def create_custom_resource_from_file(namespace, file_path): + with open(file_path) as f: + resource = yaml.safe_load(f) + return KubernetesTester.create_custom_resource_from_object(namespace, resource) + + @staticmethod + def create_mongodb_from_object(namespace, resource, exception_reason=None, patch=None): + name, kind = KubernetesTester.create_custom_resource_from_object(namespace, resource, exception_reason, patch) + KubernetesTester.namespace = namespace + KubernetesTester.name = name + KubernetesTester.kind = kind + + @staticmethod + def create_custom_resource_from_object( + namespace, + resource, + exception_reason=None, + patch=None, + api_client: Optional[client.ApiClient] = None, + ): + name, kind, group, version, res_type = get_crd_meta(resource) + if patch: + patch = jsonpatch.JsonPatch(patch) + resource = patch.apply(resource) + + KubernetesTester.namespace = namespace + KubernetesTester.name = name + KubernetesTester.kind = kind + + # For some long-running actions (e.g. creation of OpsManager) we may want to reuse already existing CR + if os.getenv("SKIP_EXECUTION") == "true": + print("Skipping creation as 'SKIP_EXECUTION' env variable is not empty") + return + + print("Creating resource {} {} {}".format(kind, name, "(" + res_type + ")" if kind == "MongoDb" else "")) + + # TODO move "wait for exception" logic to a generic function and reuse for create/update/delete + try: + KubernetesTester.clients("customv1", api_client=api_client).create_namespaced_custom_object( + group, version, namespace, plural(kind), resource + ) + except ApiException as e: + if e.status == 409: + KubernetesTester.clients("customv1", api_client=api_client).patch_namespaced_custom_object( + group, version, namespace, plural(kind), name, resource + ) + else: + if isinstance(e.body, str): + # In Kubernetes v1.16+ the result body is a json string that needs to be parsed, according to + # whatever exception_reason was passed. + try: + body_json = json.loads(e.body) + except json.decoder.JSONDecodeError: + # The API did not return a JSON string + pass + else: + reason = validation_reason_from_exception(exception_reason) + + if reason is not None: + field = exception_reason.split()[0] + for cause in body_json["details"]["causes"]: + if cause["reason"] == reason and cause["field"] == field: + return None, None + + if exception_reason: + assert e.reason == exception_reason or exception_reason in e.body, "Real exception is: {}".format(e) + print( + '"{}" exception raised while creating the resource - this is expected!'.format(exception_reason) + ) + return None, None + + print("Failed to create a resource ({}): \n {}".format(e, resource)) + raise + + else: + if exception_reason: + raise AssertionError("Expected ApiException, but create operation succeeded!") + + print("Created resource {} {} {}".format(kind, name, "(" + res_type + ")" if kind == "MongoDb" else "")) + return name, kind + + @staticmethod + def update(section, namespace): + """ + Updates the resource in the "file" section, applying the jsonpatch in "patch" section. + + Python API client (patch_namespaced_custom_object) will send a "merge-patch+json" by default. + This means that the resulting objects, after the patch is the union of the old and new objects. The + patch can only change attributes or add, but not delete, as it is the case with "json-patch+json" + requests. The json-patch+json requests are the ones used by `kubectl edit` and `kubectl patch`. + + # TODO: + A fix for this has been merged already (https://github.com/kubernetes-client/python/issues/862). The + Kubernetes Python module should be updated when the client is regenerated (version 10.0.1 or so) + + # TODO 2 (fixed in 10.0.1): As of 10.0.0 the patch gets completely broken: https://github.com/kubernetes-client/python/issues/866 + ("reason":"UnsupportedMediaType","code":415) + So we still should be careful with "remove" operation - better use "replace: null" + """ + resource = yaml.safe_load(open(fixture(section["file"]))) + + patch = section.get("patch") + KubernetesTester.patch_custom_resource_from_object(namespace, resource, patch) + + @staticmethod + def patch_custom_resource_from_object(namespace, resource, patch): + name, kind, group, version, res_type = get_crd_meta(resource) + KubernetesTester.namespace = namespace + KubernetesTester.name = name + KubernetesTester.kind = kind + + if patch is not None: + patch = jsonpatch.JsonPatch(patch) + resource = patch.apply(resource) + + # For some long-running actions (e.g. update of OpsManager) we may want to reuse already existing CR + if os.getenv("SKIP_EXECUTION") == "true": + print("Skipping creation as 'SKIP_EXECUTION' env variable is not empty") + return + + try: + # TODO currently if the update doesn't pass (e.g. patch is incorrect) - we don't fail here... + KubernetesTester.clients("customv1").patch_namespaced_custom_object( + group, version, namespace, plural(kind), name, resource + ) + except Exception: + print("Failed to update a resource ({}): \n {}".format(sys.exc_info()[0], resource)) + raise + print("Updated resource {} {} {}".format(kind, name, "(" + res_type + ")" if kind == "MongoDb" else "")) + + @staticmethod + def delete(section, namespace): + "delete custom object" + delete_name = section.get("delete_name") + loaded_yaml = yaml.safe_load(open(fixture(section["file"]))) + + resource = None + if delete_name is None: + resource = loaded_yaml + else: + # remove the element by name in the case of a list of elements + resource = [res for res in loaded_yaml if res["metadata"]["name"] == delete_name][0] + + name, kind, group, version, _ = get_crd_meta(resource) + + KubernetesTester.delete_custom_resource(namespace, name, kind, group, version) + + @staticmethod + def delete_custom_resource(namespace, name, kind, group="mongodb.com", version="v1"): + print("Deleting resource {} {}".format(kind, name)) + + KubernetesTester.namespace = namespace + KubernetesTester.name = name + KubernetesTester.kind = kind + + del_options = KubernetesTester.clients("client").V1DeleteOptions() + + KubernetesTester.clients("customv1").delete_namespaced_custom_object( + group, version, namespace, plural(kind), name, body=del_options + ) + print("Deleted resource {} {}".format(kind, name)) + + @staticmethod + def noop(section, namespace): + "noop action" + pass + + @staticmethod + def get_namespaced_custom_object(namespace, name, kind, group="mongodb.com", version="v1"): + return KubernetesTester.clients("customv1").get_namespaced_custom_object( + group, version, namespace, plural(kind), name + ) + + @staticmethod + def get_resource(): + """Assumes a single resource in the test environment""" + return KubernetesTester.get_namespaced_custom_object( + KubernetesTester.namespace, KubernetesTester.name, KubernetesTester.kind + ) + + @staticmethod + def in_error_state(): + return KubernetesTester.check_phase( + KubernetesTester.namespace, + KubernetesTester.kind, + KubernetesTester.name, + "Failed", + ) + + @staticmethod + def in_updated_state(): + return KubernetesTester.check_phase( + KubernetesTester.namespace, + KubernetesTester.kind, + KubernetesTester.name, + "Updated", + ) + + @staticmethod + def in_pending_state(): + return KubernetesTester.check_phase( + KubernetesTester.namespace, + KubernetesTester.kind, + KubernetesTester.name, + "Pending", + ) + + @staticmethod + def in_running_state(): + return KubernetesTester.check_phase( + KubernetesTester.namespace, + KubernetesTester.kind, + KubernetesTester.name, + "Running", + ) + + @staticmethod + def in_failed_state(): + return KubernetesTester.check_phase( + KubernetesTester.namespace, + KubernetesTester.kind, + KubernetesTester.name, + "Failed", + ) + + @staticmethod + def wait_for_status_message(rule): + timeout = int(rule.get("timeout", INFINITY)) + + def wait_for_status(): + res = KubernetesTester.get_namespaced_custom_object( + KubernetesTester.namespace, KubernetesTester.name, KubernetesTester.kind + ) + expected_message = rule["wait_for_message"] + message = res.get("status", {}).get("message", "") + if isinstance(expected_message, re.Pattern): + return expected_message.match(message) + return expected_message in message + + return KubernetesTester.wait_until(wait_for_status, timeout) + + @staticmethod + def is_deleted(namespace, name, kind="MongoDB"): + try: + KubernetesTester.get_namespaced_custom_object(namespace, name, kind) + return False + except ApiException: # ApiException is thrown when the object does not exist + return True + + @staticmethod + def check_phase(namespace, kind, name, phase): + resource = KubernetesTester.get_namespaced_custom_object(namespace, name, kind) + if "status" not in resource: + return False + if resource["metadata"]["generation"] != resource["status"]["observedGeneration"]: + # If generations don't match - we're observing a previous state and the Operator + # hasn't managed to take action yet. + return False + return resource["status"]["phase"] == phase + + @classmethod + def wait_condition(cls, action): + """Waits for a condition to occur before proceeding, + or for some amount of time, both can appear in the file, + will always wait for the condition and then for some amount of time. + """ + if "wait_until" not in action: + return + + print("Waiting until {}".format(action["wait_until"])) + wait_phases = [a.strip() for a in action["wait_until"].split(",") if a != ""] + for phase in wait_phases: + # Will wait for each action passed as a , separated list + # waiting on average the same amount of time for each phase + # totaling `timeout` + cls.wait_until(phase, int(action.get("timeout", 0)) / len(wait_phases)) + + @classmethod + def wait_until(cls, action, timeout=0, **kwargs): + func = None + # if passed a function directly, we can use it + if callable(action): + func = action + else: # otherwise find a function of that name + func = getattr(cls, action) + return run_periodically(func, timeout=timeout, **kwargs) + + @classmethod + def wait_for_condition_string(cls, condition): + """Waits for a given condition from the cluster + Example: + 1. statefulset/my-replica-set -> status.current_replicas == 5 + """ + type_, name, attribute, expected_value = parse_condition_str(condition) + + if type_ not in ["sts", "statefulset"]: + raise NotImplementedError("Only StatefulSets can be tested with condition strings for now") + + return cls.wait_for_condition_stateful_set(cls.get_namespace(), name, attribute, expected_value) + + @classmethod + def wait_for_condition_stateful_set(cls, namespace, name, attribute, expected_value): + appsv1 = KubernetesTester.clients("appsv1") + namespace = KubernetesTester.get_namespace() + ready_to_go = False + while not ready_to_go: + try: + sts = appsv1.read_namespaced_stateful_set(name, namespace) + ready_to_go = get_nested_attribute(sts, attribute) == expected_value + except ApiException: + pass + + if ready_to_go: + return + + time.sleep(0.5) + + def setup_method(self): + self.client = client + self.corev1 = client.CoreV1Api() + self.appsv1 = client.AppsV1Api() + self.certificates = client.CertificatesV1beta1Api() + self.customv1 = client.CustomObjectsApi() + self.namespace = KubernetesTester.get_namespace() + self.name = None + self.kind = None + + @staticmethod + def create_group(org_id, group_name): + """ + Creates the group with specified name and organization id in Ops Manager, returns its ID + """ + url = build_om_groups_endpoint(KubernetesTester.get_om_base_url()) + response = KubernetesTester.om_request("post", url, {"name": group_name, "orgId": org_id}) + + return response.json()["id"] + + @staticmethod + def ensure_group(org_id, group_name): + try: + return KubernetesTester.get_om_group_id(group_name=group_name, org_id=org_id) + except Exception as e: + print(f"Caught exception: {e}") + return KubernetesTester.create_group(org_id, group_name) + + @staticmethod + def query_group(group_name, org_id=None): + """ + Obtains the group id of the group with specified name. + Note, that the logic used imitates the logic used by the Operator, 'getByName' returns all groups in all + organizations which may be inconvenient for local development as may result in "may groups exist" exception + """ + if org_id is None: + # If no organization is passed, then look for all organizations + org_id = KubernetesTester.find_organizations(group_name) + + if not isinstance(org_id, list): + org_id = [org_id] + + if len(org_id) != 1: + raise Exception('{} organizations with name "{}" found instead of 1!'.format(len(org_id), group_name)) + + group_ids = KubernetesTester.find_groups_in_organization(org_id[0], group_name) + if len(group_ids) != 1: + raise Exception( + f'{len(group_ids)} groups with name "{group_name}" found inside organization "{org_id[0]}" instead of 1!' + ) + url = build_om_group_endpoint(KubernetesTester.get_om_base_url(), group_ids[0]) + response = KubernetesTester.om_request("get", url) + + return response.json() + + @staticmethod + def remove_group(group_id): + url = build_om_group_endpoint(KubernetesTester.get_om_base_url(), group_id) + KubernetesTester.om_request("delete", url) + + @staticmethod + def remove_group_by_name(group_name): + orgid = KubernetesTester.get_om_org_id() + project_id = KubernetesTester.query_group(group_name, orgid)["id"] + KubernetesTester.remove_group(project_id) + + @staticmethod + def create_organization(org_name): + """ + Creates the organization with specified name in Ops Manager, returns its ID + """ + url = build_om_org_endpoint(KubernetesTester.get_om_base_url()) + response = KubernetesTester.om_request("post", url, {"name": org_name}) + + return response.json()["id"] + + @staticmethod + def find_organizations(org_name): + """ + Finds all organization with specified name, iterates over max 200 pages to find all matching organizations + (aligned with 'ompaginator.TraversePages'). + + If the Organization ID has been defined, return that instead. This is required to avoid 500's in Cloud Manager. + Returns the list of ids. + """ + org_id = KubernetesTester.get_om_org_id() + if org_id is not None: + return [org_id] + + ids = [] + page = 1 + while True: + url = build_om_org_list_endpoint(KubernetesTester.get_om_base_url(), page) + json = KubernetesTester.om_request("get", url).json() + + # Add organization id if its name is the searched one + ids.extend([org["id"] for org in json["results"] if org["name"] == org_name]) + + if not any(link["rel"] == "next" for link in json["links"]): + break + page += 1 + + return ids + + @staticmethod + def remove_organization(org_id): + """ + Removes the organization with specified id from Ops Manager + """ + url = build_om_one_org_endpoint(KubernetesTester.get_om_base_url(), org_id) + KubernetesTester.om_request("delete", url) + + @staticmethod + def get_groups_in_organization_first_page(org_id): + """ + :return: the first page of groups (100 items for OM 4.0 and 500 for OM 4.1) + """ + url = build_om_groups_in_org_endpoint(KubernetesTester.get_om_base_url(), org_id, 1) + response = KubernetesTester.om_request("get", url) + + return response.json() + + @staticmethod + def find_groups_in_organization(org_id, group_name): + """ + Finds all group with specified name, iterates over max 200 pages to find all matching groups inside the + organization (aligned with 'ompaginator.TraversePages') + Returns the list of ids. + """ + + max_pages = 200 + ids = [] + for i in range(1, max_pages): + url = build_om_groups_in_org_endpoint(KubernetesTester.get_om_base_url(), org_id, i) + json = KubernetesTester.om_request("get", url).json() + # Add group id if its name is the searched one + ids.extend([group["id"] for group in json["results"] if group["name"] == group_name]) + + if not any(link["rel"] == "next" for link in json["links"]): + break + + if len(ids) == 0: + print( + "Group name {} not found in organization with id {} (in {} pages)".format(group_name, org_id, max_pages) + ) + + return ids + + @staticmethod + def get_automation_config(group_id=None, group_name=None): + if group_id is None: + group_id = KubernetesTester.get_om_group_id(group_name=group_name) + + url = build_automation_config_endpoint(KubernetesTester.get_om_base_url(), group_id) + response = KubernetesTester.om_request("get", url) + + return response.json() + + @staticmethod + def get_monitoring_config(group_id=None): + if group_id is None: + group_id = KubernetesTester.get_om_group_id() + url = build_monitoring_config_endpoint(KubernetesTester.get_om_base_url(), group_id) + response = KubernetesTester.om_request("get", url) + + return response.json() + + @staticmethod + def get_backup_config(group_id=None): + if group_id is None: + group_id = KubernetesTester.get_om_group_id() + url = build_backup_config_endpoint(KubernetesTester.get_om_base_url(), group_id) + response = KubernetesTester.om_request("get", url) + + return response.json() + + @staticmethod + def put_automation_config(config): + url = build_automation_config_endpoint(KubernetesTester.get_om_base_url(), KubernetesTester.get_om_group_id()) + response = KubernetesTester.om_request("put", url, config) + + return response + + @staticmethod + def put_monitoring_config(config, group_id=None): + if group_id is None: + group_id = KubernetesTester.get_om_group_id() + url = build_monitoring_config_endpoint(KubernetesTester.get_om_base_url(), group_id) + response = KubernetesTester.om_request("put", url, config) + + return response + + @staticmethod + def get_hosts(): + url = build_hosts_endpoint(KubernetesTester.get_om_base_url(), KubernetesTester.get_om_group_id()) + response = KubernetesTester.om_request("get", url) + + return response.json() + + @staticmethod + def om_request(method, endpoint, json_object=None): + headers = {"Content-Type": "application/json"} + auth = build_auth(KubernetesTester.get_om_user(), KubernetesTester.get_om_api_key()) + + response = requests.request(method, endpoint, auth=auth, headers=headers, json=json_object) + + if response.status_code >= 300: + raise Exception( + "Error sending request to Ops Manager API. {} ({}).\n Request details: {} {} (data: {})".format( + response.status_code, response.text, method, endpoint, json_object + ) + ) + + return response + + @staticmethod + def om_version() -> Optional[Dict[str, str]]: + "Gets the X-MongoDB-Service-Version" + response = KubernetesTester.om_request( + "get", + "{}/api/public/v1.0/groups".format(KubernetesTester.get_om_base_url()), + ) + + version = response.headers.get("X-MongoDB-Service-Version") + if version is None: + return None + + return dict(attr.split("=", 1) for attr in version.split("; ")) + + @staticmethod + def check_om_state_cleaned(): + """Checks that OM state is cleaned: Automation config is empty, monitoring hosts are removed""" + + config = KubernetesTester.get_automation_config() + assert len(config["replicaSets"]) == 0, "ReplicaSets not empty: {}".format(config["replicaSets"]) + assert len(config["sharding"]) == 0, "Sharding not empty: {}".format(config["sharding"]) + assert len(config["processes"]) == 0, "Processes not empty: {}".format(config["processes"]) + + hosts = KubernetesTester.get_hosts() + assert len(hosts["results"]) == 0, "Hosts not empty: ({} hosts left)".format(len(hosts["results"])) + + @staticmethod + def is_om_state_cleaned(): + config = KubernetesTester.get_automation_config() + hosts = KubernetesTester.get_hosts() + + return ( + len(config["replicaSets"]) == 0 + and len(config["sharding"]) == 0 + and len(config["processes"]) == 0 + and len(hosts["results"]) == 0 + ) + + @staticmethod + def mongo_resource_deleted(check_om_state=True): + # First we check that the MDB resource is removed + # This depends on global state set by "create_custom_resouce", this means + # that it can't be called independently, or, calling the remove function without + # calling the "create" function first. + # Should not depend in the global state of KubernetesTester + + deleted_in_k8 = KubernetesTester.is_deleted( + KubernetesTester.namespace, KubernetesTester.name, KubernetesTester.kind + ) + + # Then we check that the resource was removed in Ops Manager if specified + return deleted_in_k8 if not check_om_state else (deleted_in_k8 and KubernetesTester.is_om_state_cleaned()) + + @staticmethod + def mongo_resource_deleted_no_om(): + """ + Waits until the MDB resource dissappears but won't wait for OM state to be removed, as sometimes + OM will just fail on us and make the test fail. + """ + return KubernetesTester.mongo_resource_deleted(False) + + @staticmethod + def build_mongodb_uri_for_rs(hosts): + return "mongodb://{}".format(",".join(hosts)) + + @staticmethod + def random_k8s_name(prefix="test-"): + """Deprecated: user kubetester.random_k8s_name instead.""" + return prefix + "".join(random.choice(string.ascii_lowercase) for _ in range(5)) + + @staticmethod + def random_om_project_name() -> str: + """Generates the name for the projects with our common namespace (and project) convention so that + GC process could remove it if it's left for some reasons. Always has a whitespace. + """ + current_seconds_epoch = int(time.time()) + prefix = f"a-{current_seconds_epoch}-" + + return "{} {}".format( + KubernetesTester.random_k8s_name(prefix), + KubernetesTester.random_k8s_name(""), + ) + + @staticmethod + def run_command_in_pod_container( + pod_name: str, + namespace: str, + cmd: List[str], + container: str = "mongodb-enterprise-database", + api_client: Optional[kubernetes.client.ApiClient] = None, + ) -> str: + api_client = client.CoreV1Api(api_client=api_client) + api_response = stream( + api_client.connect_get_namespaced_pod_exec, + pod_name, + namespace, + container=container, + command=cmd, + stdout=True, + stderr=True, + ) + return api_response + + @staticmethod + def copy_file_inside_pod(pod_name, src_path, dest_path, namespace="default"): + """ + This function copies a file inside the pod from localhost. (Taken from: https://stackoverflow.com/questions/59703610/copy-file-from-pod-to-host-by-using-kubernetes-python-client) + :param api_instance: coreV1Api() + :param name: pod name + :param ns: pod namespace + :param source_file: Path of the file to be copied into pod + """ + + api_client = client.CoreV1Api() + try: + exec_command = ["tar", "xvf", "-", "-C", "/"] + api_response = stream( + api_client.connect_get_namespaced_pod_exec, + pod_name, + namespace, + command=exec_command, + stderr=True, + stdin=True, + stdout=True, + tty=False, + _preload_content=False, + ) + + with tempfile.TemporaryFile() as tar_buffer: + with tarfile.open(fileobj=tar_buffer, mode="w") as tar: + tar.add(src_path, dest_path) + + tar_buffer.seek(0) + commands = [] + commands.append(tar_buffer.read()) + + while api_response.is_open(): + api_response.update(timeout=1) + if api_response.peek_stdout(): + print("STDOUT: %s" % api_response.read_stdout()) + if api_response.peek_stderr(): + print("STDERR: %s" % api_response.read_stderr()) + if commands: + c = commands.pop(0) + api_response.write_stdin(c.decode()) + else: + break + api_response.close() + except ApiException as e: + raise Exception("Failed to copy file to the pod: {}".format(e)) + + @staticmethod + def approve_certificate(name: str): + warnings.warn( + DeprecationWarning( + "KubernetesTester.approve_certificate is deprecated, use kubetester.certs.approve_certificate instead!" + ) + ) + # TODO: remove this method entirely + from kubetester.certs import approve_certificate + + return approve_certificate(name) + + def generate_certfile( + self, + csr_name: str, + certificate_request_fixture: str, + server_pem_fixture: str, + namespace: Optional[str] = None, + ): + """ + generate_certfile create a temporary file object that is created from a certificate request fixture + as well as a fixture containing the server pem key. This file can be used to pass to a MongoClient + when using MONGODB-X509 authentication + + :param csr_name: The name of the CSR that is to be created + :param certificate_request_fixture: a fixture containing the contents of the certificate request + :param server_pem_fixture: a fixture containing the server pem key file + :return: A File object containing the key and certificate + """ + with open(fixture(certificate_request_fixture), "r") as f: + encoded_request = b64encode(f.read().encode("utf-8")).decode("utf-8") + + if namespace is None: + namespace = self.namespace + + csr_body = client.V1beta1CertificateSigningRequest( + metadata=client.V1ObjectMeta(name=csr_name, namespace=namespace), + spec=client.V1beta1CertificateSigningRequestSpec( + groups=["system:authenticated"], + usages=["digital signature", "key encipherment", "client auth"], + request=encoded_request, + ), + ) + + client.CertificatesV1beta1Api().create_certificate_signing_request(csr_body) + self.approve_certificate(csr_name) + wait_for_certs_to_be_issued([csr_name]) + csr = client.CertificatesV1beta1Api().read_certificate_signing_request(csr_name) + certificate = b64decode(csr.status.certificate) + + tmp = tempfile.NamedTemporaryFile() + with open(fixture(server_pem_fixture), "r+b") as f: + key = f.read() + tmp.write(key) + tmp.write(certificate) + tmp.flush() + + return tmp + + @staticmethod + def list_storage_class() -> List[client.V1StorageClass]: + """Returns a list of all the Storage classes in this cluster.""" + return KubernetesTester.clients("storagev1").list_storage_class().items + + @staticmethod + def get_storage_class_provisioner_enabled() -> str: + """Returns 'a' provisioner that is known to exist in this cluster.""" + # If there's no storageclass in this cluster, then the following + # will raise a KeyError. + return KubernetesTester.list_storage_class()[0].provisioner + + @staticmethod + def create_storage_class(name: str, provisioner: Optional[str] = None) -> None: + """Creates a new StorageClass which is a duplicate of an existing one.""" + if provisioner is None: + provisioner = KubernetesTester.get_storage_class_provisioner_enabled() + + sc0 = KubernetesTester.list_storage_class()[0] + + sc = client.V1StorageClass( + metadata=client.V1ObjectMeta( + name=name, + annotations={"storageclass.kubernetes.io/is-default-class": "true"}, + ), + provisioner=provisioner, + volume_binding_mode=sc0.volume_binding_mode, + reclaim_policy=sc0.reclaim_policy, + ) + KubernetesTester.clients("storagev1").create_storage_class(sc) + + @staticmethod + def storage_class_make_not_default(name: str): + """Changes the 'default' annotation from a storage class.""" + sv1 = KubernetesTester.clients("storagev1") + sc = sv1.read_storage_class(name) + sc.metadata.annotations["storageclass.kubernetes.io/is-default-class"] = "false" + sv1.patch_storage_class(name, sc) + + @staticmethod + def make_default_gp2_storage_class(): + """ + gp2 is an aws-ebs storage class, make sure to only use that on aws based tests + """ + classes = KubernetesTester.list_storage_class() + + for sc in classes: + if sc.metadata.name == "gp2": + # The required class already exist, no need to create it. + return + + KubernetesTester.create_storage_class("gp2") + KubernetesTester.storage_class_make_not_default("standard") + + @staticmethod + def yield_existing_csrs(csr_names, timeout=300): + warnings.warn( + DeprecationWarning( + "KubernetesTester.yield_existing_csrs is deprecated, use kubetester.certs.yield_existing_csrs instead!" + ) + ) + # TODO: remove this method entirely + from kubetester.certs import yield_existing_csrs + + return yield_existing_csrs(csr_names, timeout) + + # TODO eventually replace all usages of this function with "ReplicaSetTester(mdb_resource, 3).assert_connectivity()" + @staticmethod + def wait_for_rs_is_ready(hosts, wait_for=60, check_every=5, ssl=False): + "Connects to a given replicaset and wait a while for a primary and secondaries." + client = KubernetesTester.check_hosts_are_ready(hosts, ssl) + + check_times = wait_for / check_every + + while (client.primary is None or len(client.secondaries) < len(hosts) - 1) and check_times >= 0: + time.sleep(check_every) + check_times -= 1 + + return client.primary, client.secondaries + + @staticmethod + def check_hosts_are_ready(hosts, ssl=False): + mongodburi = KubernetesTester.build_mongodb_uri_for_rs(hosts) + options = {} + if ssl: + options = {"ssl": True, "tlsCAFile": SSL_CA_CERT} + client = pymongo.MongoClient(mongodburi, **options, serverSelectionTimeoutMs=300000) + + # The ismaster command is cheap and does not require auth. + client.admin.command("ismaster") + + return client + + def _get_pods(self, podname, qty=3): + return [podname.format(i) for i in range(qty)] + + @staticmethod + def check_single_pvc( + namespace: str, + volume, + expected_name, + expected_claim_name, + expected_size, + storage_class=None, + labels: Optional[Dict[str, str]] = None, + api_client: Optional[client.ApiClient] = None, + ): + assert volume.name == expected_name + assert volume.persistent_volume_claim.claim_name == expected_claim_name + + pvc = client.CoreV1Api(api_client=api_client).read_namespaced_persistent_volume_claim( + expected_claim_name, namespace + ) + assert pvc.status.phase == "Bound" + assert pvc.spec.resources.requests["storage"] == expected_size + + assert getattr(pvc.spec, "storage_class_name") == storage_class + if labels is not None: + pvc_labels = pvc.metadata.labels + for k in labels: + assert k in pvc_labels and pvc_labels[k] == labels[k] + + @staticmethod + def get_mongo_server_sans(host: str) -> List[str]: + cert_bytes = ssl.get_server_certificate((host, 27017)).encode("ascii") + cert = x509.load_pem_x509_certificate(cert_bytes, default_backend()) + ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName) + return ext.value.get_values_for_type(x509.DNSName) + + @staticmethod + def get_csr_sans(csr_name: str) -> List[str]: + """ + Return all of the subject alternative names for a given Kubernetes + certificate signing request. + """ + csr = client.CertificatesV1beta1Api().read_certificate_signing_request_status(csr_name) + base64_csr_request = csr.spec.request + csr_pem_string = b64decode(base64_csr_request) + csr = x509.load_pem_x509_csr(csr_pem_string, default_backend()) + ext = csr.extensions.get_extension_for_class(x509.SubjectAlternativeName) + return ext.value.get_values_for_type(x509.DNSName) + + +# Some general functions go here + + +def get_group(doc): + return doc["apiVersion"].split("/")[0] + + +def get_version(doc): + return doc["apiVersion"].split("/")[1] + + +def get_kind(doc): + return doc["kind"] + + +def get_name(doc): + return doc["metadata"]["name"] + + +def get_type(doc): + return doc.get("spec", {}).get("type") + + +def get_crd_meta(doc): + return get_name(doc), get_kind(doc), get_group(doc), get_version(doc), get_type(doc) + + +def plural(name): + """Returns the plural of the name, in the case of `mongodb` the plural is the same.""" + return plural_map[name] + + +def parse_condition_str(condition): + """ + Returns a condition into constituent parts: + >>> parse_condition_str('sts/my-replica-set -> status.current_replicas == 3') + >>> 'sts', 'my-replica-set', '{.status.currentReplicas}', '3' + """ + type_name, condition = condition.split("->") + type_, name = type_name.split("/") + type_ = type_.strip() + name = name.strip() + + test, expected = condition.split("==") + test = test.strip() + expected = expected.strip() + if expected.isdigit(): + expected = int(expected) + + return type_, name, test, expected + + +def get_nested_attribute(obj, attrs): + """Returns the `attrs` attribute descending into this object following the . notation. + Assume you have a class Some() and: + >>> class Some: pass + >>> a = Some() + >>> b = Some() + >>> c = Some() + >>> a.b = b + >>> b.c = c + >>> c.my_string = 'hello!' + >>> get_nested_attribute(a, 'b.c.my_string') + 'hello!' + """ + + attrs = list(reversed(attrs.split("."))) + while attrs: + obj = getattr(obj, attrs.pop()) + + return obj + + +def current_milliseconds(): + return int(round(time.time() * 1000)) + + +def run_periodically(fn, *args, **kwargs): + """ + Calls `fn` until it succeeds or until the `timeout` is reached, every `sleep_time` seconds. + If `timeout` is negative or zero, it never times out. + Callable fn can return single bool (condition result) or tuple[bool, str] ([condition result, status message]). + + >>> run_periodically(lambda: time.sleep(5), timeout=3, sleep_time=2) + False + >>> run_periodically(lambda: time.sleep(2), timeout=5, sleep_time=2) + True + """ + sleep_time = kwargs.get("sleep_time", SLEEP_TIME) + timeout = kwargs.get("timeout", INFINITY) + msg = kwargs.get("msg", None) + + start_time = current_milliseconds() + end = start_time + (timeout * 1000) + callable_name = fn.__name__ + attempts = 0 + + while current_milliseconds() < end or timeout <= 0: + attempts += 1 + fn_result = fn() + fn_condition_msg = None + if isinstance(fn_result, bool): + fn_condition = fn_result + elif isinstance(fn_result, tuple) and len(fn_result) == 2: + fn_condition = fn_result[0] + fn_condition_msg = fn_result[1] + else: + raise Exception("Invalid fn return type. Fn have to return either bool or a tuple[bool, str].") + + if fn_condition: + print( + "{} executed successfully after {} seconds and {} attempts".format( + callable_name, (current_milliseconds() - start_time) / 1000, attempts + ) + ) + return True + if msg is not None: + condition_msg = f": {fn_condition_msg}" if fn_condition_msg is not None else "" + print(f"waiting for {msg}{condition_msg}...") + time.sleep(sleep_time) + + raise AssertionError( + "Timed out executing {} after {} seconds and {} attempt(s)".format( + callable_name, (current_milliseconds() - start_time) / 1000, attempts + ) + ) + + +def get_env_var_or_fail(name): + """ + Gets a configuration option from an Environment variable. If not found, will try to find + this option in one of the configuration files for the user. + """ + value = os.getenv(name) + + if value is None: + raise ValueError("Environment variable `{}` needs to be set.".format(name)) + + if isinstance(value, str): + value = value.strip() + + return value + + +def build_auth(user, api_key): + return HTTPDigestAuth(user, api_key) + + +def build_agent_auth(group_id, api_key): + return HTTPBasicAuth(group_id, api_key) + + +def build_om_groups_endpoint(base_url): + return "{}/api/public/v1.0/groups".format(base_url) + + +def build_om_group_endpoint(base_url, group_id): + return "{}/api/public/v1.0/groups/{}".format(base_url, group_id) + + +def build_om_org_endpoint(base_url): + return "{}/api/public/v1.0/orgs".format(base_url) + + +def build_om_org_list_endpoint(base_url: string, page_num: int): + return "{}/api/public/v1.0/orgs?itemsPerPage=500&pageNum={}".format(base_url, page_num) + + +def build_om_org_list_by_name_endpoint(base_url: string, name: string): + return "{}/api/public/v1.0/orgs?name={}".format(base_url, name) + + +def build_om_one_org_endpoint(base_url, org_id): + return "{}/api/public/v1.0/orgs/{}".format(base_url, org_id) + + +def build_om_groups_in_org_endpoint(base_url, org_id, page_num): + return "{}/api/public/v1.0/orgs/{}/groups?itemsPerPage=500&pageNum={}".format(base_url, org_id, page_num) + + +def build_om_groups_in_org_by_name_endpoint(base_url: string, org_id: string, name: string): + return "{}/api/public/v1.0/orgs/{}/groups?name={}".format(base_url, org_id, name) + + +def build_automation_config_endpoint(base_url, group_id): + return "{}/api/public/v1.0/groups/{}/automationConfig".format(base_url, group_id) + + +def build_monitoring_config_endpoint(base_url, group_id): + return "{}/api/public/v1.0/groups/{}/automationConfig/monitoringAgentConfig".format(base_url, group_id) + + +def build_backup_config_endpoint(base_url, group_id): + return "{}/api/public/v1.0/groups/{}/automationConfig/backupAgentConfig".format(base_url, group_id) + + +def build_hosts_endpoint(base_url, group_id): + return "{}/api/public/v1.0/groups/{}/hosts".format(base_url, group_id) + + +def ensure_nested_objects(resource: CustomObject, keys: List[str]): + curr_dict = resource + for k in keys: + if k not in curr_dict: + curr_dict[k] = {} + curr_dict = curr_dict[k] + + +def fixture(filename): + """ + Returns a relative path to a filename in one of the fixture's directories + """ + root_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests") + + fixture_dirs = [] + + for dirpath, dirnames, filenames in os.walk(root_dir): + if dirpath.endswith("/fixtures"): + fixture_dirs.append(dirpath) + + found = None + for dirs in fixture_dirs: + full_path = os.path.join(dirs, filename) + if os.path.exists(full_path) and os.path.isfile(full_path): + if found is not None: + warnings.warn("Fixtures with the same name were found: {}".format(full_path)) + found = full_path + + if found is None: + raise Exception("Fixture file {} not found".format(filename)) + + return found + + +def build_list_of_hosts( + mdb_resource, + namespace, + members, + servicename=None, + clustername: str = "cluster.local", + port=27017, +): + if servicename is None: + servicename = "{}-svc".format(mdb_resource) + + return [ + build_host_fqdn(hostname(mdb_resource, idx), namespace, servicename, clustername, port) + for idx in range(members) + ] + + +def build_host_fqdn( + hostname: str, + namespace: str, + servicename: str, + clustername: str = "cluster.local", + port=27017, +) -> str: + return "{hostname}.{servicename}.{namespace}.svc.{clustername}:{port}".format( + hostname=hostname, + servicename=servicename, + namespace=namespace, + clustername=clustername, + port=port, + ) + + +def build_svc_fqdn(service: str, namespace: str, clustername: str = "cluster.local") -> str: + return "{}.{}.svc.{}".format(service, namespace, clustername) + + +def hostname(hostname, idx): + return "{}-{}".format(hostname, idx) + + +def get_pods(podname_format, qty=3): + return [podname_format.format(i) for i in range(qty)] + + +def decode_secret(data: Dict[str, str]) -> Dict[str, str]: + return {k: b64decode(v).decode("utf-8") for (k, v) in data.items()} + + +def validation_reason_from_exception(exception_msg): + reasons = [ + ("in body is required", "FieldValueRequired"), + ("in body should be one of", "FieldValueNotSupported"), + ("in body must be of type", "FieldValueInvalid"), + ] + + for reason in reasons: + if reason[0] in exception_msg: + return reason[1] + + +def create_testing_namespace( + evergreen_task_id: str, + name: str, + api_client: Optional[kubernetes.client.ApiClient] = None, + istio_label: Optional[bool] = False, +) -> str: + """creates the namespace that is used by the test. Marks it with necessary labels and annotations so that + it would be handled by configuration scripts correctly (cluster cleaner, dumping the diagnostics information) + """ + + labels = {"evg": "task"} + if istio_label: + labels.update({"istio-injection": "enabled"}) + + annotations = {"evg/task": f"https://evergreen.mongodb.com/task/{evergreen_task_id}"} + + from kubetester import create_or_update_namespace + + create_or_update_namespace(name, labels, annotations, api_client=api_client) + + return name + + +def fcv_from_version(version: str) -> str: + parsed_version = semver.VersionInfo.parse(version) + return f"{parsed_version.major}.{parsed_version.minor}" + + +def ensure_ent_version(mdb_version: str) -> str: + if "-ent" not in mdb_version: + return mdb_version + "-ent" + return mdb_version diff --git a/docker/mongodb-enterprise-tests/kubetester/ldap.py b/docker/mongodb-enterprise-tests/kubetester/ldap.py new file mode 100644 index 000000000..c00a21b05 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/ldap.py @@ -0,0 +1,168 @@ +import time +from dataclasses import dataclass +from typing import Optional + +import ldap +import ldap.modlist + +LDAP_BASE = "dc=example,dc=org" +LDAP_AUTHENTICATION_MECHANISM = "PLAIN" + + +@dataclass(init=True) +class OpenLDAP: + host: str + admin_password: str + ldap_base: str = LDAP_BASE + + @property + def servers(self): + return self.host.partition("//")[2] + + +@dataclass(init=True) +class LDAPUser: + uid: str + password: str + ldap_base: str = LDAP_BASE + ou: str = None + + @property + def username(self): + return build_dn(uid=self.uid, ou=self.ou, base=self.ldap_base) + + +def create_user( + server: OpenLDAP, + user: LDAPUser, + ca_path: Optional[str] = None, + ou: Optional[str] = None, + o: Optional[str] = None, +): + """Creates a new user in the LDAP database. It might include an optional organizational unit (ou).""" + con = ldap_initialize(server, ca_path) + + modlist = { + "objectClass": [b"top", b"account", b"simpleSecurityObject"], + "userPassword": [str.encode(user.password)], + "uid": [str.encode(user.uid)], + } + ldapmodlist = ldap.modlist.addModlist(modlist) + + dn = build_dn(uid=user.uid, ou=ou, o=o, base=server.ldap_base) + try: + con.add_s(dn, ldapmodlist) + except ldap.ALREADY_EXISTS as e: + pass + + +def ensure_organization(server: OpenLDAP, o: str, ca_path: Optional[str] = None): + """If an organizational unit with the provided name does not exists, it creates one.""" + con = ldap_initialize(server, ca_path) + + result = con.search_s(server.ldap_base, ldap.SCOPE_SUBTREE, filterstr="o=" + o) + if result is None: + raise Exception(f"Error when trying to check for organization {o} in the ldap server") + if len(result) != 0: + return + modlist = {"objectClass": [b"top", b"organization"], "o": [str.encode(o)]} + + ldapmodlist = ldap.modlist.addModlist(modlist) + + dn = build_dn(o=o, base=server.ldap_base) + con.add_s(dn, ldapmodlist) + + +def ensure_organizational_unit(server: OpenLDAP, ou: str, o: Optional[str] = None, ca_path: Optional[str] = None): + """If an organizational unit with the provided name does not exists, it creates one.""" + con = ldap_initialize(server, ca_path) + + result = con.search_s(server.ldap_base, ldap.SCOPE_SUBTREE, filterstr="ou=" + ou) + if result is None: + raise Exception(f"Error when trying to check for organizationalUnit {ou} in the ldap server") + if len(result) != 0: + return + modlist = {"objectClass": [b"top", b"organizationalUnit"], "ou": [str.encode(ou)]} + + ldapmodlist = ldap.modlist.addModlist(modlist) + + dn = build_dn(ou=ou, o=o, base=server.ldap_base) + con.add_s(dn, ldapmodlist) + + +def ensure_group( + server: OpenLDAP, + cn: str, + ou: str, + o: Optional[str] = None, + ca_path: Optional[str] = None, +): + """If a group with the provided name does not exists, it creates a group in the LDAP database, + that also belongs to an organizational unit. By default, it adds the admin user to it.""" + con = ldap_initialize(server, ca_path) + + result = con.search_s(server.ldap_base, ldap.SCOPE_SUBTREE, filterstr="cn=" + cn) + if result is None: + raise Exception(f"Error when trying to check for group {cn} in the ldap server") + if len(result) != 0: + return + unique_member = build_dn(base=server.ldap_base, uid="admin", ou=ou, o=o) + modlist = { + "objectClass": [b"top", b"groupOfUniqueNames"], + "cn": str.encode(cn), + "uniqueMember": str.encode(unique_member), + } + ldapmodlist = ldap.modlist.addModlist(modlist) + + dn = build_dn(base=server.ldap_base, cn=cn, ou=ou, o=o) + + con.add_s(dn, ldapmodlist) + + +def add_user_to_group( + server: OpenLDAP, + user: str, + group_cn: str, + ou: str, + o: Optional[str] = None, + ca_path: Optional[str] = None, +): + """Adds a new uniqueMember to a group, this is equivalent to add a user to the group.""" + con = ldap_initialize(server, ca_path) + + unique_member = build_dn(uid=user, ou=ou, o=o, base=server.ldap_base) + modlist = {"uniqueMember": [str.encode(unique_member)]} + ldapmodlist = ldap.modlist.modifyModlist({}, modlist) + + dn = build_dn(cn=group_cn, ou=ou, o=o, base=server.ldap_base) + try: + con.modify_s(dn, ldapmodlist) + except ldap.TYPE_OR_VALUE_EXISTS as e: + pass + + +def ldap_initialize(server: OpenLDAP, ca_path: Optional[str] = None, retries=5): + con = ldap.initialize(server.host) + + if server.host.startswith("ldaps://") and ca_path is not None: + con.set_option(ldap.OPT_X_TLS_CACERTFILE, ca_path) + con.set_option(ldap.OPT_X_TLS_NEWCTX, 0) + + dn_admin = build_dn(cn="admin", base=server.ldap_base) + r = retries + while r >= 0: + try: + con.simple_bind_s(dn_admin, server.admin_password) + return con + except ldap.SERVER_DOWN as e: + r -= 1 + time.sleep(5) + + +def build_dn(base: Optional[str] = None, **kwargs): + """Builds a distinguished name from arguments.""" + dn = ",".join("{}={}".format(k, v) for k, v in kwargs.items() if v is not None) + if base is not None: + dn += "," + base + + return dn diff --git a/docker/mongodb-enterprise-tests/kubetester/mongodb.py b/docker/mongodb-enterprise-tests/kubetester/mongodb.py new file mode 100644 index 000000000..d796398ed --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/mongodb.py @@ -0,0 +1,632 @@ +from __future__ import annotations + +import os +import re +import time +import urllib.parse +from enum import Enum +from typing import Dict, List, Optional, Tuple + +import semver +from kubeobject import CustomObject +from kubernetes import client +from kubetester.kubetester import ( + KubernetesTester, + build_host_fqdn, + ensure_ent_version, + ensure_nested_objects, + is_default_architecture_static, +) +from kubetester.omtester import OMContext, OMTester +from opentelemetry import trace +from tests import test_logger + +from .mongotester import ( + MongoTester, + ReplicaSetTester, + ShardedClusterTester, + StandaloneTester, +) + +logger = test_logger.get_test_logger(__name__) + + +class Phase(Enum): + Running = 1 + Pending = 2 + Failed = 3 + Updated = 4 + Disabled = 5 + Unsupported = 6 + + +class MongoDBCommon: + def wait_for(self, fn, timeout=None, should_raise=True): + if timeout is None: + timeout = 600 + initial_timeout = timeout + + wait = 3 + while timeout > 0: + try: + self.reload() + except Exception as e: + print(f"Caught error: {e} while waiting for {fn.__name__}") + pass + if fn(self): + return True + timeout -= wait + time.sleep(wait) + + if should_raise: + raise Exception("Timeout ({}) reached while waiting for {}".format(initial_timeout, self)) + + def get_generation(self) -> int: + return self.backing_obj["metadata"]["generation"] + + +class MongoDB(CustomObject, MongoDBCommon): + def __init__(self, *args, **kwargs): + with_defaults = { + "plural": "mongodb", + "kind": "MongoDB", + "group": "mongodb.com", + "version": "v1", + } + with_defaults.update(kwargs) + super(MongoDB, self).__init__(*args, **with_defaults) + + @classmethod + def from_yaml(cls, yaml_file, name=None, namespace=None) -> MongoDB: + resource = super().from_yaml(yaml_file=yaml_file, name=name, namespace=namespace) + custom_mdb_prev_version = os.getenv("CUSTOM_MDB_VERSION") + custom_mdb_version = os.getenv("CUSTOM_MDB_VERSION") + if custom_mdb_prev_version is not None and semver.compare(resource.get_version(), custom_mdb_prev_version) < 0: + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + elif custom_mdb_version is not None and semver.compare(resource.get_version(), custom_mdb_version) < 0: + resource.set_version(ensure_ent_version(custom_mdb_version)) + return resource + + def assert_state_transition_happens(self, last_transition, timeout=None): + def transition_changed(mdb: MongoDB): + return mdb.get_status_last_transition_time() != last_transition + + self.wait_for(transition_changed, timeout, should_raise=True) + + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=None, ignore_errors=False): + intermediate_events = ( + "haven't reached READY", + "Some agents failed to register", + # Sometimes Cloud-QA timeouts so we anticipate to this + "Error sending GET request to", + # "Get https://cloud-qa.mongodb.com/api/public/v1.0/groups/5f186b406c835e37e6160aef/automationConfig: + # read tcp 10.244.0.6:33672->75.2.105.99:443: read: connection reset by peer" + "read: connection reset by peer", + # Ops Manager must be recovering from an Upgrade, and it is + # currently DOWN. + "connect: connection refused", + "MongoDB version information is not yet available", + # Enabling authentication is a lengthy process where the agents might not reach READY in time. + # That can cause a failure and a restart of the reconcile. + "Failed to enable Authentication", + # Sometimes agents need longer to register with OM. + "some agents failed to register or the Operator", + ) + + start_time = time.time() + + self.wait_for( + lambda s: in_desired_state( + current_state=self.get_status_phase(), + desired_state=phase, + current_generation=self.get_generation(), + observed_generation=self.get_status_observed_generation(), + current_message=self.get_status_message(), + msg_regexp=msg_regexp, + ignore_errors=ignore_errors, + intermediate_events=intermediate_events, + ), + timeout, + should_raise=True, + ) + + end_time = time.time() + span = trace.get_current_span() + span.set_attribute("meko_resource", self.__class__.__name__) + span.set_attribute("meko_action", "assert_phase") + span.set_attribute("meko_desired_phase", phase.name) + span.set_attribute("meko_time_needed", end_time - start_time) + logger.debug( + f"Reaching phase {phase.name} for resource {self.__class__.__name__} took {end_time - start_time}s" + ) + + def assert_abandons_phase(self, phase: Phase, timeout=None): + """This method can be racy by nature, it assumes that the operator is slow enough that its phase transition + happens during the time we call this method. If there is not a lot of work, then the phase can already finished + transitioning during the modification call before calling this method. + """ + return self.wait_for(lambda s: s.get_status_phase() != phase, timeout, should_raise=True) + + def assert_backup_reaches_status(self, expected_status: str, timeout: int = 600): + def reaches_backup_status(mdb: MongoDB) -> bool: + try: + return mdb["status"]["backup"]["statusName"] == expected_status + except KeyError: + return False + + self.wait_for(reaches_backup_status, timeout=timeout, should_raise=True) + + def assert_status_resource_not_ready(self, name: str, kind: str = "StatefulSet", msg_regexp=None, idx=0): + """Checks the element in 'resources_not_ready' field by index 'idx'""" + assert self.get_status_resources_not_ready()[idx]["kind"] == kind + assert self.get_status_resources_not_ready()[idx]["name"] == name + assert re.search(msg_regexp, self.get_status_resources_not_ready()[idx]["message"]) is not None + + @property + def type(self) -> str: + return self["spec"]["type"] + + def tester( + self, + ca_path: Optional[str] = None, + srv: bool = False, + use_ssl: Optional[bool] = None, + service_names: list[str] = None, + ): + """Returns a Tester instance for this type of deployment.""" + if self.type == "ReplicaSet" and "clusterSpecList" in self["spec"]: + raise ValueError("A MongoDB class is being used to represent a MongoDBMulti instance!") + + if self.type == "ReplicaSet": + return ReplicaSetTester( + mdb_resource_name=self.name, + replicas_count=self["status"]["members"], + ssl=self.is_tls_enabled() if use_ssl is None else use_ssl, + srv=srv, + ca_path=ca_path, + namespace=self.namespace, + external_domain=self.get_external_domain(), + cluster_domain=self.get_cluster_domain(), + ) + elif self.type == "ShardedCluster": + return ShardedClusterTester( + mdb_resource_name=self.name, + mongos_count=self["spec"].get("mongosCount", 0), + ssl=self.is_tls_enabled() if use_ssl is None else use_ssl, + srv=srv, + ca_path=ca_path, + namespace=self.namespace, + cluster_domain=self.get_cluster_domain(), + multi_cluster=self.is_multicluster(), + service_names=service_names, + external_domain=self.get_external_domain(), + ) + elif self.type == "Standalone": + return StandaloneTester( + mdb_resource_name=self.name, + ssl=self.is_tls_enabled() if use_ssl is None else use_ssl, + ca_path=ca_path, + namespace=self.namespace, + external_domain=self.get_external_domain(), + cluster_domain=self.get_cluster_domain(), + ) + + def assert_connectivity(self, ca_path: Optional[str] = None, cluster_domain: str = "cluster.local"): + return self.tester(ca_path=ca_path).assert_connectivity() + + def set_architecture_annotation(self): + if "annotations" not in self["metadata"]: + self["metadata"]["annotations"] = {} + if is_default_architecture_static(): + self["metadata"]["annotations"].update({"mongodb.com/v1.architecture": "static"}) + else: + self["metadata"]["annotations"].update({"mongodb.com/v1.architecture": "non-static"}) + + def trigger_architecture_migration(self): + self.load() + if "annotations" not in self["metadata"]: + self["metadata"]["annotations"] = {} + if is_default_architecture_static(): + self["metadata"]["annotations"].update({"mongodb.com/v1.architecture": "non-static"}) + self.update() + else: + self["metadata"]["annotations"].update({"mongodb.com/v1.architecture": "static"}) + self.update() + + def trigger_sts_restart(self, component=""): + """ + Adds or changes a label from the pod template to trigger a rolling restart of that StatefulSet. + Leave component to empty if a ReplicaSet deployment is used. + Set component to either "shard", "config", "mongos" to trigger a restart of the respective StatefulSet. + """ + pod_spec = "podSpec" + if component == "shard": + pod_spec = "shardPodSpec" + elif component == "config": + pod_spec = "configSrvPodSpec" + elif component == "mongos": + pod_spec = "mongosPodSpec" + + self.load() + self["spec"][pod_spec] = { + "podTemplate": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": str(time.time())}}} + } + self.update() + + def assert_connectivity_from_connection_string(self, cnx_string: str, tls: bool, ca_path: Optional[str] = None): + """ + Tries to connect to a database using a connection string only. + """ + return MongoTester(cnx_string, tls, ca_path).assert_connectivity() + + def __repr__(self): + # FIX: this should be __unicode__ + return "MongoDB ({})| status: {}| message: {}".format( + self.name, self.get_status_phase(), self.get_status_message() + ) + + def configure( + self, + om: MongoDBOpsManager, + project_name: str, + api_client: Optional[client.ApiClient] = None, + ) -> MongoDB: + if "project" in self["spec"]: + del self["spec"]["project"] + + ensure_nested_objects(self, ["spec", "opsManager", "configMapRef"]) + + self["spec"]["opsManager"]["configMapRef"]["name"] = om.get_or_create_mongodb_connection_config_map( + self.name, project_name, self.namespace, api_client=api_client + ) + # Note that if the MongoDB object is created in a different namespace than the Operator + # then the secret needs to be copied there manually + self["spec"]["credentials"] = om.api_key_secret(self.namespace, api_client=api_client) + return self + + def configure_backup(self, mode: str = "enabled") -> MongoDB: + ensure_nested_objects(self, ["spec", "backup"]) + self["spec"]["backup"]["mode"] = mode + return self + + def configure_custom_tls( + self, + issuer_ca_configmap_name: str, + tls_cert_secret_name: str, + ): + ensure_nested_objects(self, ["spec", "security", "tls"]) + self["spec"]["security"] = { + "certsSecretPrefix": tls_cert_secret_name, + "tls": {"enabled": True, "ca": issuer_ca_configmap_name}, + } + + def build_list_of_hosts(self): + """Returns the list of full_fqdn:27017 for every member of the mongodb resource""" + return [ + build_host_fqdn( + f"{self.name}-{idx}", + self.namespace, + self.get_service(), + self.get_cluster_domain(), + 27017, + ) + for idx in range(self.get_members()) + ] + + def read_statefulset(self) -> client.V1StatefulSet: + return client.AppsV1Api().read_namespaced_stateful_set(self.name, self.namespace) + + def read_configmap(self) -> Dict[str, str]: + return KubernetesTester.read_configmap(self.namespace, self.config_map_name) + + def mongo_uri(self, user_name: Optional[str] = None, password: Optional[str] = None) -> str: + """Returns the mongo uri for the MongoDB resource. The logic matches the one in 'types.go'""" + proto = "mongodb://" + auth = "" + params = {"connectTimeoutMS": "20000", "serverSelectionTimeoutMS": "20000"} + if "SCRAM" in self.get_authentication_modes(): + auth = "{}:{}@".format( + urllib.parse.quote(user_name, safe=""), + urllib.parse.quote(password, safe=""), + ) + params["authSource"] = "admin" + if self.get_version().startswith("3.6"): + params["authMechanism"] = "SCRAM-SHA-1" + else: + params["authMechanism"] = "SCRAM-SHA-256" + + hosts = ",".join(self.build_list_of_hosts()) + + if self.get_resource_type() == "ReplicaSet": + params["replicaSet"] = self.name + + if self.is_tls_enabled(): + params["ssl"] = "true" + + query_params = ["{}={}".format(key, params[key]) for key in sorted(params.keys())] + joined_params = "&".join(query_params) + return proto + auth + hosts + "/?" + joined_params + + def get_members(self) -> int: + return self["spec"]["members"] + + def get_version(self) -> str: + try: + return self["spec"]["version"] + except KeyError: + custom_mdb_version = os.getenv("CUSTOM_MDB_VERSION", "6.0.10") + return custom_mdb_version + + def get_service(self) -> str: + try: + return self["spec"]["service"] + except KeyError: + return "{}-svc".format(self.name) + + def get_cluster_domain(self) -> Optional[str]: + try: + return self["spec"]["clusterDomain"] + except KeyError: + return "cluster.local" + + def get_resource_type(self) -> str: + return self["spec"]["type"] + + def is_tls_enabled(self): + """Checks if this object is TLS enabled.""" + try: + return self["spec"]["security"]["tls"]["enabled"] + except KeyError: + return False + + def set_version(self, version: str): + self["spec"]["version"] = version + return self + + def get_authentication(self) -> Optional[Dict]: + try: + return self["spec"]["security"]["authentication"] + except KeyError: + return {} + + def get_authentication_modes(self) -> Optional[Dict]: + try: + return self.get_authentication()["modes"] + except KeyError: + return {} + + def get_status_phase(self) -> Optional[Phase]: + try: + return Phase[self["status"]["phase"]] + except KeyError: + return None + + def get_status_fcv(self) -> Optional[str]: + try: + return self["status"]["featureCompatibilityVersion"] + except KeyError: + return None + + def get_status_last_transition_time(self) -> Optional[str]: + return self["status"]["lastTransition"] + + def get_status_message(self) -> Optional[str]: + try: + return self["status"]["message"] + except KeyError: + return None + + def get_status_observed_generation(self) -> Optional[int]: + try: + return self["status"]["observedGeneration"] + except KeyError: + return None + + def get_status_members(self) -> Optional[str]: + try: + return self["status"]["members"] + except KeyError: + return None + + def get_status_resources_not_ready(self) -> Optional[List[Dict]]: + try: + return self["status"]["resourcesNotReady"] + except KeyError: + return None + + def get_om_tester(self) -> OMTester: + """Returns the OMTester instance based on MongoDB connectivity parameters""" + config_map = self.read_configmap() + secret = KubernetesTester.read_secret(self.namespace, self["spec"]["credentials"]) + return OMTester(OMContext.build_from_config_map_and_secret(config_map, secret)) + + def get_automation_config_tester(self, **kwargs): + """This is just a shortcut for getting automation config tester for replica set""" + return self.get_om_tester().get_automation_config_tester(**kwargs) + + def get_external_domain(self): + multi_cluster_external_domain = ( + self["spec"] + .get("mongos", {}) + .get("clusterSpecList", [{}])[0] + .get("externalAccess", {}) + .get("externalDomain", None) + ) + return self["spec"].get("externalAccess", {}).get("externalDomain", None) or multi_cluster_external_domain + + @property + def config_map_name(self) -> str: + if "opsManager" in self["spec"]: + return self["spec"]["opsManager"]["configMapRef"]["name"] + return self["spec"]["project"] + + def shard_replicaset_names(self) -> List[str]: + return ["{}-{}".format(self.name, i) for i in range(1, self["spec"]["shardCount"])] + + def shard_statefulset_name(self, shard_idx: int, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-{shard_idx}-{cluster_idx}" + return f"{self.name}-{shard_idx}" + + def shard_pod_name(self, shard_idx: int, member_idx: int, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-{shard_idx}-{cluster_idx}-{member_idx}" + return f"{self.name}-{shard_idx}-{member_idx}" + + def shard_service_name(self) -> str: + return f"{self.name}-sh" + + def shard_hostname( + self, shard_idx: int, member_idx: int, cluster_idx: Optional[int] = None, port: int = 27017 + ) -> str: + if self.is_multicluster(): + return f"{self.name}-{shard_idx}-{cluster_idx}-{member_idx}-svc.{self.namespace}.svc.cluster.local:{port}" + return f"{self.name}-{shard_idx}-{member_idx}.{self.shard_service_name()}.{self.namespace}.svc.cluster.local:{port}" + + def shard_pvc_name(self, shard_idx: int, member_idx: int, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"data-{self.name}-{shard_idx}-{cluster_idx}-{member_idx}" + return f"data-{self.name}-{shard_idx}-{member_idx}" + + def shard_members_in_cluster(self, cluster_name: str) -> int: + if "shardOverrides" in self["spec"]: + raise Exception("Shard overrides logic is not supported") + + if self.is_multicluster(): + for cluster_spec_item in self["spec"]["shard"]["clusterSpecList"]: + if cluster_spec_item["clusterName"] == cluster_name: + return cluster_spec_item["members"] + + return self["spec"].get("mongodsPerShardCount", 0) + + def config_srv_statefulset_name(self, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-config-{cluster_idx}" + return f"{self.name}-config" + + def config_srv_replicaset_name(self) -> str: + return f"{self.name}-config" + + def config_srv_pod_name(self, member_idx: int, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-config-{cluster_idx}-{member_idx}" + return f"{self.name}-config-{member_idx}" + + def config_srv_members_in_cluster(self, cluster_name: str) -> int: + if self.is_multicluster(): + for cluster_spec_item in self["spec"]["configSrv"]["clusterSpecList"]: + if cluster_spec_item["clusterName"] == cluster_name: + return cluster_spec_item["members"] + + return self["spec"].get("configServerCount", 0) + + def config_srv_pvc_name(self, member_idx: int, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"data-{self.name}-config-{cluster_idx}-{member_idx}" + return f"data-{self.name}-config-{member_idx}" + + def mongos_statefulset_name(self, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-mongos-{cluster_idx}" + return f"{self.name}-mongos" + + def mongos_pod_name(self, member_idx: int, cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-mongos-{cluster_idx}-{member_idx}" + return f"{self.name}-mongos-{member_idx}" + + def mongos_hostname(self, member_idx: int, cluster_idx: Optional[int] = None) -> str: + service_name = self.mongos_service_name(member_idx, cluster_idx) + if self.is_multicluster(): + return f"{service_name}.{self.namespace}.svc.cluster.local" + + return f"{self.mongos_pod_name(member_idx, cluster_idx)}.{service_name}.{self.namespace}.svc.cluster.local" + + def mongos_service_name(self, member_idx: Optional[int], cluster_idx: Optional[int] = None) -> str: + if self.is_multicluster(): + return f"{self.name}-mongos-{cluster_idx}-{member_idx}-svc" + else: + return f"{self.name}-svc" + + def mongos_members_in_cluster(self, cluster_name: str) -> int: + if self.is_multicluster(): + for cluster_spec_item in self["spec"]["mongos"]["clusterSpecList"]: + if cluster_spec_item["clusterName"] == cluster_name: + return cluster_spec_item["members"] + + return self["spec"].get("mongosCount", 0) + + def is_multicluster(self) -> bool: + return self["spec"].get("topology", None) == "MultiCluster" + + class Types: + REPLICA_SET = "ReplicaSet" + SHARDED_CLUSTER = "ShardedCluster" + STANDALONE = "Standalone" + + +def get_pods(podname, qty) -> List[str]: + return [podname.format(i) for i in range(qty)] + + +def in_desired_state( + current_state: Phase, + desired_state: Phase, + current_generation: int, + observed_generation: int, + current_message: str, + msg_regexp: Optional[str] = None, + ignore_errors=False, + intermediate_events: Tuple = (), +) -> bool: + """Returns true if the current_state is equal to desired state, fails fast if got into Failed error. + Optionally checks if the message matches the specified regexp expression""" + if current_state is None: + return False + + if current_generation != observed_generation: + # We shouldn't check the status further if the Operator hasn't started working on the new spec yet + return False + + if current_state == Phase.Failed and not desired_state == Phase.Failed and not ignore_errors: + found = False + for event in intermediate_events: + if event in current_message: + found = True + + if not found: + raise AssertionError(f'Got into Failed phase while waiting for Running! ("{current_message}")') + + is_in_desired_state = current_state == desired_state + if msg_regexp is not None: + print("msg_regexp: " + str(msg_regexp)) + regexp = re.compile(msg_regexp) + is_in_desired_state = is_in_desired_state and current_message is not None and regexp.match(current_message) + + return is_in_desired_state + + +def generic_replicaset( + namespace: str, + version: str, + name: Optional[str] = None, + ops_manager: Optional[MongoDBOpsManager] = None, +) -> MongoDB: + if name is None: + name = KubernetesTester.random_k8s_name("rs-") + + rs = MongoDB(namespace=namespace, name=name) + rs["spec"] = { + "members": 3, + "type": "ReplicaSet", + "persistent": False, + "version": version, + } + + if ops_manager is None: + rs["spec"]["credentials"] = "my-credentials" + rs["spec"]["opsManager"] = {"configMapRef": {"name": "my-project"}} + else: + rs.configure(ops_manager, KubernetesTester.random_k8s_name("project-")) + + return rs diff --git a/docker/mongodb-enterprise-tests/kubetester/mongodb_multi.py b/docker/mongodb-enterprise-tests/kubetester/mongodb_multi.py new file mode 100644 index 000000000..349c5b60e --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/mongodb_multi.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from typing import Dict, List, Optional + +import kubernetes.client +import pytest +from kubernetes import client +from kubetester import MongoDB +from kubetester.mongotester import MongoTester, MultiReplicaSetTester + + +class MultiClusterClient: + def __init__( + self, + api_client: kubernetes.client.ApiClient, + cluster_name: str, + cluster_index: Optional[int] = None, + ): + self.api_client = api_client + self.cluster_name = cluster_name + self.cluster_index = cluster_index + + def apps_v1_api(self) -> kubernetes.client.AppsV1Api: + return kubernetes.client.AppsV1Api(self.api_client) + + def core_v1_api(self) -> kubernetes.client.CoreV1Api: + return kubernetes.client.CoreV1Api(self.api_client) + + def read_namespaced_stateful_set(self, name: str, namespace: str): + return self.apps_v1_api().read_namespaced_stateful_set(name, namespace) + + def list_namespaced_stateful_set(self, namespace: str): + return self.apps_v1_api().list_namespaced_stateful_set(namespace) + + def read_namespaced_service(self, name: str, namespace: str): + return self.core_v1_api().read_namespaced_service(name, namespace) + + def read_namespaced_config_map(self, name: str, namespace: str): + return self.core_v1_api().read_namespaced_config_map(name, namespace) + + def read_namespaced_persistent_volume_claim(self, name: str, namespace: str): + return self.core_v1_api().read_namespaced_persistent_volume_claim(name, namespace) + + def assert_sts_members_count(self, sts_name: str, namespace: str, expected_shard_members_in_cluster: int): + try: + sts = self.read_namespaced_stateful_set(sts_name, namespace) + assert sts.spec.replicas == expected_shard_members_in_cluster + except kubernetes.client.ApiException as api_exception: + assert ( + 0 == expected_shard_members_in_cluster and api_exception.status == 404 + ), f"expected {expected_shard_members_in_cluster} members, but received {api_exception.status} exception while reading {namespace}:{sts_name}" + + +class MongoDBMulti(MongoDB): + def __init__(self, *args, **kwargs): + with_defaults = { + "plural": "mongodbmulticluster", + "kind": "MongoDBMultiCluster", + "group": "mongodb.com", + "version": "v1", + } + with_defaults.update(kwargs) + super(MongoDBMulti, self).__init__(*args, **with_defaults) + + def read_statefulsets(self, clients: List[MultiClusterClient]) -> Dict[str, client.V1StatefulSet]: + statefulsets = {} + for mcc in clients: + statefulsets[mcc.cluster_name] = mcc.read_namespaced_stateful_set( + f"{self.name}-{mcc.cluster_index}", self.namespace + ) + return statefulsets + + def get_item_spec(self, cluster_name: str) -> Dict: + for spec in sorted( + self["spec"]["clusterSpecList"], + key=lambda x: x["clusterName"], + ): + if spec["clusterName"] == cluster_name: + return spec + + raise ValueError(f"Cluster with name {cluster_name} not found!") + + def read_services(self, clients: List[MultiClusterClient]) -> Dict[str, client.V1Service]: + services = {} + for mcc in clients: + spec = self.get_item_spec(mcc.cluster_name) + for i, item in enumerate(spec): + services[mcc.cluster_name] = mcc.read_namespaced_service( + f"{self.name}-{mcc.cluster_index}-{i}-svc", self.namespace + ) + return services + + def read_headless_services(self, clients: List[MultiClusterClient]) -> Dict[str, client.V1Service]: + services = {} + for mcc in clients: + services[mcc.cluster_name] = mcc.read_namespaced_service( + f"{self.name}-{mcc.cluster_index}-svc", self.namespace + ) + return services + + def read_configmaps(self, clients: List[MultiClusterClient]) -> Dict[str, client.V1ConfigMap]: + configmaps = {} + for mcc in clients: + configmaps[mcc.cluster_name] = mcc.read_namespaced_config_map( + f"{self.name}-hostname-override", self.namespace + ) + return configmaps + + def service_names(self) -> List[str]: + # TODO: this function does not account for previous + # clusters being removed, the indices do not line up + # and as a result the incorrect service name will be returned. + service_names = [] + cluster_specs = sorted( + self["spec"]["clusterSpecList"], + key=lambda x: x["clusterName"], + ) + for i, spec in enumerate(cluster_specs): + for j in range(spec["members"]): + service_names.append(f"{self.name}-{i}-{j}-svc") + return service_names + + def tester( + self, + ca_path: Optional[str] = None, + srv: bool = False, + use_ssl: Optional[bool] = None, + service_names: Optional[List[str]] = None, + port="27017", + external: bool = False, + ) -> MongoTester: + if service_names is None: + service_names = self.service_names() + + return MultiReplicaSetTester( + service_names=service_names, + namespace=self.namespace, + port=port, + external=external, + ssl=self.is_tls_enabled() if use_ssl is None else use_ssl, + ca_path=ca_path, + ) diff --git a/docker/mongodb-enterprise-tests/kubetester/mongodb_user.py b/docker/mongodb-enterprise-tests/kubetester/mongodb_user.py new file mode 100644 index 000000000..c36807031 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/mongodb_user.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional + +from kubeobject import CustomObject +from kubetester import random_k8s_name +from kubetester.mongodb import MongoDB, MongoDBCommon, Phase, in_desired_state + + +class MongoDBUser(CustomObject, MongoDBCommon): + def __init__(self, *args, **kwargs): + with_defaults = { + "plural": "mongodbusers", + "kind": "MongoDBUser", + "group": "mongodb.com", + "version": "v1", + } + with_defaults.update(kwargs) + super(MongoDBUser, self).__init__(*args, **with_defaults) + + @property + def password(self): + return self._password + + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=None, ignore_errors=False): + return self.wait_for( + lambda s: in_desired_state( + current_state=self.get_status_phase(), + desired_state=phase, + current_generation=self.get_generation(), + observed_generation=self.get_status_observed_generation(), + current_message=self.get_status_message(), + msg_regexp=msg_regexp, + ignore_errors=ignore_errors, + ), + timeout, + should_raise=True, + ) + + def get_user_name(self): + return self["spec"]["username"] + + def get_secret_name(self) -> str: + return self["spec"]["passwordSecretKeyRef"]["name"] + + def get_status_phase(self) -> Optional[Phase]: + try: + return Phase[self["status"]["phase"]] + except KeyError: + return None + + def get_status_message(self) -> Optional[str]: + try: + return self["status"]["msg"] + except KeyError: + return None + + def get_status_observed_generation(self) -> Optional[int]: + try: + return self["status"]["observedGeneration"] + except KeyError: + return None + + def add_role(self, role: Role) -> MongoDBUser: + self["spec"]["roles"] = self["spec"].get("roles", []) + self["spec"]["roles"].append({"db": role.db, "name": role.role}) + + def add_roles(self, roles: List[Role]): + for role in roles: + self.add_role(role) + + +@dataclass(init=True) +class Role: + db: str + role: str + + +def generic_user( + namespace: str, + username: str, + db: str = "admin", + password: Optional[str] = None, + mongodb_resource: Optional[MongoDB] = None, +) -> MongoDBUser: + """Returns a generic User with a username and a pseudo-random k8s name.""" + user = MongoDBUser(name=random_k8s_name("user-"), namespace=namespace) + user["spec"] = { + "username": username, + "db": db, + } + + if mongodb_resource is not None: + user["spec"]["mongodbResourceRef"] = {"name": mongodb_resource.name} + + user._password = password + + return user diff --git a/docker/mongodb-enterprise-tests/kubetester/mongotester.py b/docker/mongodb-enterprise-tests/kubetester/mongotester.py new file mode 100644 index 000000000..3a6db776d --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/mongotester.py @@ -0,0 +1,721 @@ +import copy +import inspect +import logging +import random +import string +import threading +import time +from typing import Callable, Dict, List, Optional + +import pymongo +from kubetester import kubetester +from kubetester.kubetester import KubernetesTester +from opentelemetry import trace +from pymongo.errors import OperationFailure, PyMongoError, ServerSelectionTimeoutError +from pytest import fail + +TEST_DB = "test-db" +TEST_COLLECTION = "test-collection" + + +def with_tls(use_tls: bool = False, ca_path: Optional[str] = None) -> Dict[str, str]: + # SSL is set to true by default if using mongodb+srv, it needs to be explicitely set to false + # https://docs.mongodb.com/manual/reference/program/mongo/index.html#cmdoption-mongo-host + options = {"tls": use_tls} + + if use_tls: + options["tlsCAFile"] = kubetester.SSL_CA_CERT if ca_path is None else ca_path + return options + + +def with_scram(username: str, password: str, auth_mechanism: str = "SCRAM-SHA-256") -> Dict[str, str]: + valid_mechanisms = {"SCRAM-SHA-256", "SCRAM-SHA-1"} + if auth_mechanism not in valid_mechanisms: + raise ValueError(f"auth_mechanism must be one of {valid_mechanisms}, but was {auth_mechanism}.") + + return { + "authMechanism": auth_mechanism, + "password": password, + "username": username, + } + + +def with_x509(cert_file_name: str, ca_path: Optional[str] = None) -> Dict[str, str]: + options = with_tls(True, ca_path=ca_path) + options.update( + { + "authMechanism": "MONGODB-X509", + "tlsCertificateKeyFile": cert_file_name, + "tlsAllowInvalidCertificates": False, + } + ) + return options + + +def with_ldap(ssl_certfile: Optional[str] = None, tls_ca_file: Optional[str] = None) -> Dict[str, str]: + options = {} + if tls_ca_file is not None: + options.update(with_tls(True, tls_ca_file)) + if ssl_certfile is not None: + options["tlsCertificateKeyFile"] = ssl_certfile + return options + + +class MongoTester: + """MongoTester is a general abstraction to work with mongo database. It encapsulates the client created in + the constructor. All general methods non-specific to types of mongodb topologies should reside here.""" + + def __init__( + self, + connection_string: str, + use_ssl: bool, + ca_path: Optional[str] = None, + ): + self.default_opts = with_tls(use_ssl, ca_path) + self.default_opts["serverSelectionTimeoutMs"] = "120000" # 2 minutes + self.cnx_string = connection_string + self.client = None + logging.info( + f"Initialized MongoTester with connection string: {connection_string}, TLS: {use_ssl} and CA Path: {ca_path}" + ) + + @property + def client(self): + if self._client is None: + self._client = self._init_client(**self.default_opts) + return self._client + + @client.setter + def client(self, value): + self._client = value + + def _merge_options(self, opts: List[Dict[str, str]]) -> Dict[str, str]: + options = copy.deepcopy(self.default_opts) + for opt in opts: + options.update(opt) + return options + + def _init_client(self, **kwargs): + return pymongo.MongoClient(self.cnx_string, **kwargs) + + def assert_connectivity( + self, + attempts: int = 20, + db: str = "admin", + col: str = "myCol", + opts: Optional[List[Dict[str, any]]] = None, + write_concern: pymongo.WriteConcern = None, + ): + if opts is None: + opts = [] + + options = self._merge_options(opts) + self.client = self._init_client(**options) + + assert attempts > 0 + while True: + attempts -= 1 + try: + logging.warning(f"connected nodes: {self.client.nodes}") + self.client.admin.command("ismaster") + if write_concern: + d = self.client.get_database(name=db, write_concern=write_concern) + c = d.get_collection(name=col) + c.insert_one({}) + if "authMechanism" in options: + # Perform an action that will require auth. + self.client[db][col].insert_one({}) + except PyMongoError: + if attempts == 0: + raise + time.sleep(5) + else: + break + + def assert_no_connection(self, opts: Optional[List[Dict[str, str]]] = None): + try: + if opts: + opts.append({"serverSelectionTimeoutMs": "30000"}) + else: + opts = [{"serverSelectionTimeoutMs": "30000"}] + self.assert_connectivity(opts=opts) + fail() + except ServerSelectionTimeoutError: + pass + + def assert_version(self, expected_version: str): + # version field does not contain -ent suffix in MongoDB + assert self.client.admin.command("buildInfo")["version"] == expected_version.split("-")[0] + if expected_version.endswith("-ent"): + self.assert_is_enterprise() + + def assert_data_size(self, expected_count, test_collection=TEST_COLLECTION): + assert self.client[TEST_DB][test_collection].estimated_document_count() == expected_count + + def assert_is_enterprise(self): + assert "enterprise" in self.client.admin.command("buildInfo")["modules"] + + def assert_scram_sha_authentication( + self, + username: str, + password: str, + auth_mechanism: str, + attempts: int = 20, + ssl: bool = False, + **kwargs, + ) -> None: + assert attempts > 0 + assert auth_mechanism in {"SCRAM-SHA-256", "SCRAM-SHA-1"} + + for i in reversed(range(attempts)): + try: + self._authenticate_with_scram( + username, + password, + auth_mechanism=auth_mechanism, + ssl=ssl, + **kwargs, + ) + return + except OperationFailure as e: + if i == 0: + fail(msg=f"unable to authenticate after {attempts} attempts with error: {e}") + time.sleep(5) + + def assert_scram_sha_authentication_fails( + self, + username: str, + password: str, + retries: int = 20, + ssl: bool = False, + **kwargs, + ): + """ + If a password has changed, it could take some time for the user changes to propagate, meaning + this could return true if we make a CRD change and immediately try to auth as the old user + which still exists. When we change a password, we should eventually no longer be able to auth with + that user's credentials. + """ + for i in range(retries): + try: + self._authenticate_with_scram(username, password, ssl=ssl, **kwargs) + except OperationFailure: + return + time.sleep(5) + fail(f"was still able to authenticate with username={username} password={password} after {retries} attempts") + + def _authenticate_with_scram( + self, + username: str, + password: str, + auth_mechanism: str, + ssl: bool = False, + **kwargs, + ): + + options = self._merge_options( + [ + with_tls(ssl, ca_path=kwargs.get("tlsCAFile")), + with_scram(username, password, auth_mechanism), + ] + ) + + self.client = self._init_client(**options) + # authentication doesn't actually happen until we interact with a database + self.client["admin"]["myCol"].insert_one({}) + + def assert_x509_authentication(self, cert_file_name: str, attempts: int = 20, **kwargs): + assert attempts > 0 + + options = self._merge_options( + [ + with_x509(cert_file_name, kwargs.get("tlsCAFile", kubetester.SSL_CA_CERT)), + ] + ) + + total_attempts = attempts + while True: + attempts -= 1 + try: + self.client = self._init_client(**options) + self.client["admin"]["myCol"].insert_one({}) + return + except OperationFailure: + if attempts == 0: + fail(msg=f"unable to authenticate after {total_attempts} attempts") + time.sleep(5) + + def assert_ldap_authentication( + self, + username: str, + password: str, + db: str = "admin", + collection: str = "myCol", + tls_ca_file: Optional[str] = None, + ssl_certfile: str = None, + attempts: int = 20, + ): + + options = with_ldap(ssl_certfile, tls_ca_file) + total_attempts = attempts + + while True: + attempts -= 1 + try: + client = self._init_client( + **options, + username=username, + password=password, + authSource="$external", + authMechanism="PLAIN", + ) + client[db][collection].insert_one({"data": "I need to exist!"}) + + return + except OperationFailure: + if attempts <= 0: + fail(msg=f"unable to authenticate after {total_attempts} attempts") + time.sleep(5) + + def upload_random_data( + self, + count: int, + generation_function: Optional[Callable] = None, + ): + return upload_random_data(self.client, count, generation_function) + + def assert_deployment_reachable(self, attempts: int = 10): + """See: https://jira.mongodb.org/browse/CLOUDP-68873 + the agents might report being in goal state, the MDB resource + would report no errors but the deployment would be unreachable + The workaround is to use the public API to get the list of + hosts and check the typeName field of each host. + This would be NO_DATA if the hosts are not reachable + See docs: https://docs.opsmanager.mongodb.com/current/reference/api/hosts/get-all-hosts-in-group/#response-document + at the "typeName" field + """ + while True: + hosts_unreachable = 0 + attempts -= 1 + hosts = KubernetesTester.get_hosts() + print(f"hosts: {hosts}") + for host in hosts["results"]: + print(f"current host: {host}") + if host["typeName"] == "NO_DATA": + hosts_unreachable += 1 + if hosts_unreachable == 0: + return + if attempts <= 0: + fail(msg="Some hosts still report NO_DATA state") + time.sleep(10) + + +class StandaloneTester(MongoTester): + def __init__( + self, + mdb_resource_name: str, + ssl: bool = False, + ca_path: Optional[str] = None, + namespace: Optional[str] = None, + port="27017", + external_domain: Optional[str] = None, + cluster_domain: str = "cluster.local", + ): + if namespace is None: + namespace = KubernetesTester.get_namespace() + + self.cnx_string = build_mongodb_connection_uri( + mdb_resource_name, namespace, 1, port, external_domain=external_domain, cluster_domain=cluster_domain + ) + super().__init__(self.cnx_string, ssl, ca_path) + + +class ReplicaSetTester(MongoTester): + def __init__( + self, + mdb_resource_name: str, + replicas_count: int, + ssl: bool = False, + srv: bool = False, + ca_path: Optional[str] = None, + namespace: Optional[str] = None, + port="27017", + external_domain: Optional[str] = None, + cluster_domain: str = "cluster.local", + ): + if namespace is None: + # backward compatibility with docstring tests + namespace = KubernetesTester.get_namespace() + + self.replicas_count = replicas_count + + self.cnx_string = build_mongodb_connection_uri( + mdb_resource_name, + namespace, + replicas_count, + servicename=None, + srv=srv, + port=port, + external_domain=external_domain, + cluster_domain=cluster_domain, + ) + + super().__init__(self.cnx_string, ssl, ca_path) + + def assert_connectivity( + self, + wait_for=60, + check_every=5, + with_srv=False, + attempts: int = 5, + write_concern: pymongo.WriteConcern = None, + opts: Optional[List[Dict[str, str]]] = None, + ): + """For replica sets in addition to is_master() we need to make sure all replicas are up""" + super().assert_connectivity(attempts=attempts, write_concern=write_concern, opts=opts) + + if self.replicas_count == 1: + # On 1 member replica-set, the last member is considered primary and secondaries will be `set()` + assert self.client.is_primary + assert len(self.client.secondaries) == 0 + return + + check_times = wait_for // check_every + + while ( + self.client.primary is None or len(self.client.secondaries) < self.replicas_count - 1 + ) and check_times >= 0: + time.sleep(check_every) + check_times -= 1 + + assert self.client.primary is not None + assert len(self.client.secondaries) == self.replicas_count - 1 + + +class MultiReplicaSetTester(MongoTester): + def __init__( + self, + service_names: List[str], + port: str, + ssl: Optional[bool] = False, + ca_path: Optional[str] = None, + namespace: Optional[str] = None, + external: bool = False, + ): + super().__init__( + build_mongodb_multi_connection_uri(namespace, service_names, port, external=external), + use_ssl=ssl, + ca_path=ca_path, + ) + + +class ShardedClusterTester(MongoTester): + def __init__( + self, + mdb_resource_name: str, + mongos_count: int, + ssl: bool = False, + srv: bool = False, + ca_path: Optional[str] = None, + namespace: Optional[str] = None, + port="27017", + cluster_domain: str = "cluster.local", + multi_cluster: Optional[bool] = False, + service_names: Optional[list[str]] = None, + external_domain: str = None, + ): + mdb_name = mdb_resource_name + "-mongos" + servicename = mdb_resource_name + "-svc" + + if namespace is None: + # backward compatibility with docstring tests + namespace = KubernetesTester.get_namespace() + + if multi_cluster: + self.cnx_string = build_mongodb_multi_connection_uri( + namespace, + service_names=service_names, + port=port, + cluster_domain=cluster_domain, + external=external_domain is not None, + ) + else: + self.cnx_string = build_mongodb_connection_uri( + mdb_name, + namespace, + mongos_count, + port=port, + servicename=servicename, + srv=srv, + cluster_domain=cluster_domain, + external_domain=external_domain, + ) + super().__init__(self.cnx_string, ssl, ca_path) + + def shard_collection(self, shards_pattern, shards_count, key, test_collection=TEST_COLLECTION): + """enables sharding and creates zones to make sure data is spread over shards. + Assumes that the documents have field 'key' with value in [0,10] range""" + for i in range(shards_count): + self.client.admin.command("addShardToZone", shards_pattern.format(i), zone="zone-{}".format(i)) + + for i in range(shards_count): + self.client.admin.command( + "updateZoneKeyRange", + db_namespace(test_collection), + min={key: i * (10 / shards_count)}, + max={key: (i + 1) * (10 / shards_count)}, + zone="zone-{}".format(i), + ) + + self.client.admin.command("enableSharding", TEST_DB) + self.client.admin.command("shardCollection", db_namespace(test_collection), key={key: 1}) + + def prepare_for_shard_removal(self, shards_pattern, shards_count): + """We need to map all the shards to all the zones to let shard be removed (otherwise the balancer gets + stuck as it cannot move chunks from shards being removed)""" + for i in range(shards_count): + for j in range(shards_count): + self.client.admin.command("addShardToZone", shards_pattern.format(i), zone="zone-{}".format(j)) + + def assert_number_of_shards(self, expected_count): + assert len(self.client.admin.command("listShards")["shards"]) == expected_count + + +class BackgroundHealthChecker(threading.Thread): + """BackgroundHealthChecker is the thread which periodically calls the function to check health of some resource. It's + run as a daemon, so usually there's no need to stop it manually. + """ + + def __init__( + self, + health_function, + wait_sec: int = 3, + allowed_sequential_failures: int = 3, + health_function_params=None, + ): + super().__init__() + if health_function_params is None: + health_function_params = {} + self._stop_event = threading.Event() + self.health_function = health_function + self.health_function_params = health_function_params + self.wait_sec = wait_sec + self.allowed_sequential_failures = allowed_sequential_failures + self.exception_number = 0 + self.last_exception = None + self.daemon = True + self.max_consecutive_failure = 0 + self.number_of_runs = 0 + + def run(self): + consecutive_failure = 0 + while not self._stop_event.isSet(): + self.number_of_runs += 1 + try: + self.health_function(**self.health_function_params) + consecutive_failure = 0 + except Exception as e: + print(f"Error in {self.__class__.__name__}: {e})") + self.last_exception = e + consecutive_failure = consecutive_failure + 1 + self.max_consecutive_failure = max(self.max_consecutive_failure, consecutive_failure) + self.exception_number = self.exception_number + 1 + time.sleep(self.wait_sec) + + def stop(self): + self._stop_event.set() + + def assert_healthiness(self, allowed_rate_of_failure: Optional[float] = None): + """ + + `allowed_rate_of_failure` allows you to define a rate of allowed failures, + instead of the default, absolute amount of failures. + + `allowed_rate_of_failure` is a number between 0 and 1 that desribes a "percentage" + of tolerated failures. + + For instance, the following values: + + - 0.1 -- means that 10% of the requests might fail, before + failing the tests. + - 0.9 -- 90% of checks are allowed to fail. + - 0.0 -- very strict: no checks are allowed to fail. + - 1.0 -- very relaxed: all checks can fail. + + """ + print("\nlongest consecutive failures: {}".format(self.max_consecutive_failure)) + print("total exceptions count: {}".format(self.exception_number)) + print("total checks number: {}".format(self.number_of_runs)) + + allowed_failures = self.allowed_sequential_failures + if allowed_rate_of_failure is not None: + allowed_failures = self.number_of_runs * allowed_rate_of_failure + + # Automatically get the caller file information + caller_info = inspect.stack()[1] # Stack frame of the caller + caller_file = caller_info.filename # Get the filename of the caller + caller_function = caller_info.function # Optional: Get the function name + + span = trace.get_current_span() + span.set_attribute("meko_version_change_connection_failures", allowed_failures) + span.set_attribute("meko_caller_file", caller_file) + span.set_attribute("meko_caller_function", caller_function) + span.set_attribute("meko_health_check_failed", True) + + try: + assert self.max_consecutive_failure <= allowed_failures + assert self.number_of_runs > 0 + except AssertionError as e: + span.set_attribute("meko_health_check_failed", True) + span.set_attribute("meko_failure_reason", str(e)) + raise + + +class MongoDBBackgroundTester(BackgroundHealthChecker): + def __init__( + self, + mongo_tester: MongoTester, + wait_sec: int = 3, + allowed_sequential_failures: int = 1, + health_function_params=None, + ): + if health_function_params is None: + health_function_params = {"attempts": 1} + super().__init__( + health_function=mongo_tester.assert_connectivity, + wait_sec=wait_sec, + health_function_params=health_function_params, + allowed_sequential_failures=allowed_sequential_failures, + ) + + +def build_mongodb_connection_uri( + mdb_resource: str, + namespace: str, + members: int, + port: str, + servicename: str = None, + srv: bool = False, + external_domain: str = None, + cluster_domain: str = "cluster.local", +) -> str: + if servicename is None: + servicename = "{}-svc".format(mdb_resource) + + if external_domain: + return build_mongodb_uri(build_list_of_hosts_with_external_domain(mdb_resource, members, external_domain, port)) + if srv: + return build_mongodb_uri(build_host_srv(servicename, namespace, cluster_domain), srv) + else: + return build_mongodb_uri( + build_list_of_hosts(mdb_resource, namespace, members, servicename, port, cluster_domain) + ) + + +def build_mongodb_multi_connection_uri( + namespace: str, + service_names: List[str], + port: str, + external: bool = False, + cluster_domain: str = "cluster.local", +) -> str: + return build_mongodb_uri( + build_list_of_multi_hosts(namespace, service_names, port, external=external, cluster_domain=cluster_domain) + ) + + +def build_list_of_hosts( + mdb_resource: str, namespace: str, members: int, servicename: str, port: str, cluster_domain: str +) -> List[str]: + return [ + build_host_fqdn("{}-{}".format(mdb_resource, idx), namespace, servicename, port, cluster_domain) + for idx in range(members) + ] + + +def build_list_of_hosts_with_external_domain( + mdb_resource: str, members: int, external_domain: str, port: str +) -> List[str]: + return [f"{mdb_resource}-{idx}.{external_domain}:{port}" for idx in range(members)] + + +def build_list_of_multi_hosts( + namespace: str, service_names: List[str], port, external: bool = False, cluster_domain: str = "cluster.local" +) -> List[str]: + if external: + return [f"{service_name}:{port}" for service_name in service_names] + return [ + build_host_service_fqdn(service_name, namespace, port, cluster_domain=cluster_domain) + for service_name in service_names + ] + + +def build_host_service_fqdn(servicename: str, namespace: str, port: int, cluster_domain: str = "cluster.local") -> str: + return f"{servicename}.{namespace}.svc.{cluster_domain}:{port}" + + +def build_host_fqdn(hostname: str, namespace: str, servicename: str, port, cluster_domain: str) -> str: + return "{hostname}.{servicename}.{namespace}.svc.{cluster_domain}:{port}".format( + hostname=hostname, servicename=servicename, namespace=namespace, port=port, cluster_domain=cluster_domain + ) + + +def build_host_srv(servicename: str, namespace: str, cluster_domain: str) -> str: + srv_host = "{servicename}.{namespace}.svc.{cluster_domain}".format( + servicename=servicename, namespace=namespace, cluster_domain=cluster_domain + ) + return srv_host + + +def build_mongodb_uri(hosts, srv: bool = False) -> str: + plus_srv = "" + if srv: + plus_srv = "+srv" + else: + hosts = ",".join(hosts) + + return "mongodb{}://{}".format(plus_srv, hosts) + + +def generate_single_json(): + """Generates a json with two fields. String field contains random characters and has length 100 characters.""" + random_str = "".join([random.choice(string.ascii_lowercase) for _ in range(100)]) + return {"description": random_str, "type": random.uniform(1, 10)} + + +def db_namespace(collection): + """https://docs.mongodb.com/manual/reference/glossary/#term-namespace""" + return "{}.{}".format(TEST_DB, collection) + + +def upload_random_data( + client, + count: int, + generation_function: Optional[Callable] = None, + task_name: Optional[str] = "default", +): + """ + Generates random json documents and uploads them to database. This data can + be later checked for integrity. + """ + + if generation_function is None: + generation_function = generate_single_json + + logging.info("task: {}. Inserting {} fake records to {}.{}".format(task_name, count, TEST_DB, TEST_COLLECTION)) + + target = client[TEST_DB][TEST_COLLECTION] + buf = [] + + for a in range(count): + buf.append(generation_function()) + if len(buf) == 1_000: + target.insert_many(buf) + buf.clear() + if (a + 1) % 10_000 == 0: + logging.info("task: {}. Inserted {} document".format(task_name, a + 1)) + # tail + if len(buf) > 0: + target.insert_many(buf) + + logging.info("task: {}. Task finished, {} documents inserted. ".format(task_name, count)) diff --git a/docker/mongodb-enterprise-tests/kubetester/om_queryable_backups.py b/docker/mongodb-enterprise-tests/kubetester/om_queryable_backups.py new file mode 100644 index 000000000..9f6902aee --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/om_queryable_backups.py @@ -0,0 +1,201 @@ +import logging +import subprocess +import tempfile +import time +from dataclasses import dataclass +from html.parser import HTMLParser + +import requests + + +@dataclass(init=True) +class QueryableBackupParams: + host: str + ca_pem: str + client_pem: str + + +class CsrfHtmlParser(HTMLParser): + def parse_html(self, html): + self._csrf_fields = dict({}) + self.feed(html) + return self._csrf_fields + + def handle_starttag(self, tag, attrs): + if tag == "meta": + attr_name = next((a[1] for a in attrs if a[0] == "name"), None) + attr_content = next((a[1] for a in attrs if a[0] == "content"), None) + if attr_name is not None and attr_name.startswith("csrf"): + self._csrf_fields[attr_name] = attr_content + + +class OMQueryableBackup: + def __init__(self, om_url, project_id): + self._om_url = om_url + self._project_id = project_id + + def _login(self): + endpoint = f"{self._om_url}/user/v1/auth" + headers = {"Content-Type": "application/json"} + data = { + "username": "jane.doe@example.com", + "password": "Passw0rd.", + "reCaptchaResponse": None, + } + + response = requests.post(endpoint, json=data, headers=headers) + if response.status_code != 200: + raise Exception(f"OM login failed with status code: {response.status_code}, content: {response.content}") + + self._auth_cookies = response.cookies + + def _authenticated_http_get(self, url, headers=None): + response = requests.get(url, headers=headers or {}, cookies=self._auth_cookies) + if response.status_code != 200: + raise Exception(f"HTTP GET failed with status code: {response.status_code}, content: {response.content}") + return response + + def _get_snapshots_query_host(self): + return ( + self._authenticated_http_get( + f"{self._om_url}/v2/{self._project_id}/params", + headers={"Accept": "application/json"}, + ) + .json() + .get("snapshotsQueryHost") + ) + + def _get_first_snapshot_id(self): + return ( + self._authenticated_http_get( + f"{self._om_url}/backup/web/snapshot/{self._project_id}/mdb-four-two", + headers={"Accept": "application/json"}, + ) + .json() + .get("entries")[0] + .get("id") + ) + + def _get_csrf_headers(self): + html_response = self._authenticated_http_get(f"{self._om_url}/v2/{self._project_id}").text + csrf_fields = CsrfHtmlParser().parse_html(html_response) + return {f"x-{k}": v for k, v in csrf_fields.items()} + + def _start_query_backup(self, first_snapshot_id, csrf_headers): + response = requests.put( + f"{self._om_url}/backup/web/restore/{self._project_id}/query/{first_snapshot_id}", + headers=csrf_headers, + cookies=self._auth_cookies, + ) + return response.json().get("snapshotQueryId") + + def _download_client_cert(self, snapshot_query_id): + return self._authenticated_http_get( + f"{self._om_url}/backup/web/restore/{self._project_id}/query/{snapshot_query_id}/keypair" + ).text + + def _download_ca(self): + return self._authenticated_http_get(f"{self._om_url}/backup/web/restore/{self._project_id}/query/ca").text + + def _wait_until_ready_to_query(self, timeout: int): + initial_timeout = timeout + ready_statuses = ["waitingForCustomer", "completed"] + last_reached_state = "" + while timeout > 0: + restore_entries = self._authenticated_http_get( + f"{self._om_url}/v2/backup/restore/{self._project_id}", + headers={"Accept": "application/json"}, + ).json() + + if len(restore_entries) > 0 and restore_entries[0].get("progressPhase") in ready_statuses: + time_needed = initial_timeout - timeout + print(f"needed {time_needed} seconds to be able to query backups") + return + last_reached_state = restore_entries[0].get("progressPhase") + time.sleep(3) + timeout -= 3 + + raise Exception( + f"Timeout ({initial_timeout}) reached while waiting for '{ready_statuses}' snapshot query status. " + f"Last reached status: {last_reached_state}" + ) + + def connection_params(self, timeout: int): + """Retrieves the connection config (host, ca / client pem files) used to query a backup snapshot.""" + self._login() + + first_snapshot_id = self._get_first_snapshot_id() + + csrf_headers = self._get_csrf_headers() + + snapshot_query_id = self._start_query_backup(first_snapshot_id, csrf_headers) + + self._wait_until_ready_to_query(timeout) + + return QueryableBackupParams( + host=self._get_snapshots_query_host(), + ca_pem=self._download_ca(), + client_pem=self._download_client_cert(snapshot_query_id), + ) + + +def generate_queryable_pem(namespace: str): + # todo: investigate if cert-manager can be used instead of openssl + openssl_conf = f""" +prompt=no +distinguished_name = qb_req_distinguished_name + +x509_extensions = qb_extensions + +[qb_req_distinguished_name] +C=US +ST=New York +L=New York +O=MongoDB, Inc. +CN=queryable-backup-test.mongodb.com + +[qb_extensions] +basicConstraints=CA:true +subjectAltName=@qb_subject_alt_names + +[qb_subject_alt_names] +DNS.1 = om-backup-svc.{namespace}.svc.cluster.local +""" + + openssl_conf_file = tempfile.NamedTemporaryFile(delete=False, mode="w") + openssl_conf_file.write(openssl_conf) + openssl_conf_file.flush() + + csr_file_path = "/tmp/queryable-backup.csr" + key_file_path = "/tmp/queryable-backup.key" + + args = [ + "openssl", + "req", + "-new", + "-x509", + "-days", + "824", + "-nodes", + "-out", + csr_file_path, + "-newkey", + "rsa:2048", + "-keyout", + key_file_path, + "-config", + openssl_conf_file.name, + ] + + try: + completed_process = subprocess.run(args, capture_output=True) + completed_process.check_returncode() + except subprocess.CalledProcessError as exc: + stdout = exc.stdout.decode("utf-8") + stderr = exc.stderr.decode("utf-8") + logging.info(stdout) + logging.info(stderr) + raise + + with open(csr_file_path, "r") as csr_file, open(key_file_path, "r") as key_file: + return f"{csr_file.read()}\n{key_file.read()}" diff --git a/docker/mongodb-enterprise-tests/kubetester/omtester.py b/docker/mongodb-enterprise-tests/kubetester/omtester.py new file mode 100644 index 000000000..1c713a783 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/omtester.py @@ -0,0 +1,802 @@ +from __future__ import annotations + +import logging +import os +import re +import tempfile +import time +import urllib.parse +from datetime import datetime +from enum import Enum +from typing import Dict, List, Optional + +import pymongo +import pytest +import requests +import semver +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import build_agent_auth, build_auth, run_periodically +from kubetester.mongotester import BackgroundHealthChecker +from kubetester.om_queryable_backups import OMQueryableBackup +from opentelemetry import trace +from requests.adapters import HTTPAdapter, Retry + +from .kubetester import get_env_var_or_fail + + +def running_cloud_manager(): + "Determines if the current test is running against Cloud Manager" + return get_env_var_or_fail("OM_HOST") == "https://cloud-qa.mongodb.com" + + +skip_if_cloud_manager = pytest.mark.skipif(running_cloud_manager(), reason="Do not run in Cloud Manager") + + +class BackupStatus(str, Enum): + """Enum for backup statuses in Ops Manager. Note that 'str' is inherited to fix json serialization issues""" + + STARTED = "STARTED" + STOPPED = "STOPPED" + TERMINATING = "TERMINATING" + + +# todo use @dataclass annotation https://www.python.org/dev/peps/pep-0557/ +class OMContext(object): + def __init__( + self, + base_url, + user, + public_key, + project_name=None, + project_id=None, + org_id=None, + agent_api_key=None, + ): + self.base_url = base_url + self.project_id = project_id + self.group_name = project_name + self.user = user + self.public_key = public_key + self.org_id = org_id + self.agent_api_key = agent_api_key + + @staticmethod + def build_from_config_map_and_secret( + connection_config_map: Dict[str, str], connection_secret: Dict[str, str] + ) -> OMContext: + if "publicApiKey" in connection_secret: + return OMContext( + base_url=connection_config_map["baseUrl"], + project_id=None, + project_name=connection_config_map["projectName"], + org_id=connection_config_map.get("orgId", ""), + user=connection_secret["user"], + public_key=connection_secret["publicApiKey"], + ) + else: + return OMContext( + base_url=connection_config_map["baseUrl"], + project_id=None, + project_name=connection_config_map["projectName"], + org_id=connection_config_map.get("orgId", ""), + user=connection_secret["publicKey"], + public_key=connection_secret["privateKey"], + ) + + +class OMTester(object): + """OMTester is designed to encapsulate communication with Ops Manager. It also provides the + set of assertion methods helping to write tests""" + + def __init__(self, om_context: OMContext): + self.context = om_context + # we only have a group id if we also have a name + if self.context.group_name: + self.ensure_group_id() + + # Those are saved here so we can access om at the end of the test run and retrieve diagnostic data easily. + if self.context.project_id: + if os.environ.get("OM_PROJECT_ID", ""): + os.environ["OM_PROJECT_ID"] = os.environ["OM_PROJECT_ID"] + "," + self.context.project_id + else: + os.environ["OM_PROJECT_ID"] = self.context.project_id + if self.context.public_key: + os.environ["OM_API_KEY"] = self.context.public_key + if self.context.public_key: + os.environ["OM_USER"] = self.context.user + if self.context.base_url: + os.environ["OM_HOST"] = self.context.base_url + + def ensure_group_id(self): + if self.context.project_id is None: + self.context.project_id = self.find_group_id() + + def get_project_events(self): + return self.om_request("get", f"/groups/{self.context.project_id}/events") + + def create_restore_job_snapshot(self, snapshot_id: Optional[str] = None) -> str: + """restores the mongodb cluster to some version using the snapshot. If 'snapshot_id' omitted then the + latest snapshot will be used.""" + cluster_id = self.get_backup_cluster_id() + if snapshot_id is None: + snapshots = self.api_get_snapshots(cluster_id) + snapshot_id = snapshots[-1]["id"] + + return self.api_create_restore_job_from_snapshot(cluster_id, snapshot_id)["id"] + + def create_restore_job_pit(self, pit_milliseconds: int, retry: int = 120): + """creates a restore job to restore the mongodb cluster to some version specified by the parameter.""" + cluster_id = self.get_backup_cluster_id() + while retry > 0: + try: + span = trace.get_current_span() + span.set_attribute(key="meko_pit_retries", value=retry) + self.api_create_restore_job_pit(cluster_id, pit_milliseconds) + return + except Exception as e: + # this exception is usually raised for some time (some oplog slices not received or whatever) + # but eventually is gone and restore job is started. + if "Invalid restore point:" not in str(e): + raise e + retry -= 1 + time.sleep(1) + raise Exception("Failed to create a restore job!") + + def wait_until_backup_snapshots_are_ready( + self, + expected_count: int, + timeout: int = 800, + expected_config_count: int = 1, + is_sharded_cluster: bool = False, + ): + """waits until at least 'expected_count' backup snapshots is in complete state""" + start_time = time.time() + cluster_id = self.get_backup_cluster_id(expected_config_count, is_sharded_cluster) + + if expected_count == 1: + print(f"Waiting until 1 snapshot is ready (can take a while)") + else: + print(f"Waiting until {expected_count} snapshots are ready (can take a while)") + + initial_timeout = timeout + while timeout > 0: + snapshots = self.api_get_snapshots(cluster_id) + if len([s for s in snapshots if s["complete"]]) >= expected_count: + print(f"Snapshots are ready, project: {self.context.group_name}, time: {time.time() - start_time} sec") + span = trace.get_current_span() + span.set_attribute(key="meko_snapshot_time", value=time.time() - start_time) + return + time.sleep(3) + timeout -= 3 + + snapshots = self.api_get_snapshots(cluster_id) + print(f"Current Backup Snapshots: {snapshots}") + + raise Exception( + f"Timeout ({initial_timeout}) reached while waiting for {expected_count} snapshot(s) to be ready for the " + f"project {self.context.group_name} " + ) + + def wait_until_restore_job_is_ready(self, job_id: str, timeout: int = 1500): + """waits until there's one finished restore job in the project""" + start_time = time.time() + cluster_id = self.get_backup_cluster_id() + + print(f"Waiting until restore job with id {job_id} is finished") + + initial_timeout = timeout + while timeout > 0: + job = self.api_get_restore_job_by_id(cluster_id, job_id) + if job["statusName"] == "FINISHED": + print( + f"Restore job is finished, project: {self.context.group_name}, time: {time.time() - start_time} sec" + ) + return + time.sleep(3) + timeout -= 3 + + jobs = self.api_get_restore_jobs(cluster_id) + print(f"Current Restore Jobs: {jobs}") + + raise AssertionError( + f"Timeout ({initial_timeout}) reached while waiting for the restore job to finish for the " + f"project {self.context.group_name} " + ) + + def get_backup_cluster_id(self, expected_config_count: int = 1, is_sharded_cluster: bool = False) -> str: + configs = self.api_read_backup_configs() + assert len(configs) == expected_config_count + + if not is_sharded_cluster: + # we can use the first config as there's only one MongoDB in deployment + return configs[0]["clusterId"] + # retrieve the sharded_replica_set + clusters = self.api_get_clusters()["results"] + for cluster in clusters: + if cluster["typeName"] == "SHARDED_REPLICA_SET": + return cluster["id"] + + def assert_healthiness(self): + self.do_assert_healthiness(self.context.base_url) + # TODO we need to check the login page as well (/user) - does it render properly? + + def assert_om_instances_healthiness(self, pod_urls: str): + """Checks each of the OM urls for healthiness. This is different from 'assert_healthiness' which makes + a call to the service instead""" + for pod_fqdn in pod_urls: + self.do_assert_healthiness(pod_fqdn) + + def assert_version(self, version: str): + """makes the request to a random API url to get headers""" + response = self.om_request("get", "/orgs") + assert f"versionString={version}" in response.headers["X-MongoDB-Service-Version"] + + def assert_test_service(self): + endpoint = self.context.base_url + "/test/utils/systemTime" + response = requests.request("get", endpoint, verify=False) + assert response.status_code == requests.status_codes.codes.OK + + def assert_support_page_enabled(self): + """The method ends successfully if 'mms.helpAndSupportPage.enabled' is set to 'true'. It's 'false' by default. + See mms SupportResource.supportLoggedOut()""" + endpoint = self.context.base_url + "/support" + response = requests.request("get", endpoint, allow_redirects=False, verify=False) + + # logic: if mms.helpAndSupportPage.enabled==true - then status is 307, otherwise 303" + assert response.status_code == 307 + + def assert_group_exists(self): + path = "/groups/" + self.context.project_id + response = self.om_request("get", path) + + assert response.status_code == requests.status_codes.codes.OK + + def assert_daemon_enabled(self, host_fqdn: str, head_db_path: str): + encoded_head_db_path = urllib.parse.quote(head_db_path, safe="") + response = self.om_request( + "get", + f"/admin/backup/daemon/configs/{host_fqdn}/{encoded_head_db_path}", + ) + + assert response.status_code == requests.status_codes.codes.OK + daemon_config = response.json() + assert daemon_config["machine"] == { + "headRootDirectory": head_db_path, + "machine": host_fqdn, + } + assert daemon_config["assignmentEnabled"] + assert daemon_config["configured"] + + def _assert_stores(self, expected_stores: List[Dict], endpoint: str, store_type: str): + response = self.om_request("get", endpoint) + assert response.status_code == requests.status_codes.codes.OK + + existing_stores = {result["id"]: result for result in response.json()["results"]} + + assert len(expected_stores) == len(existing_stores), f"expected:{expected_stores} actual: {existing_stores}." + + for expected in expected_stores: + store_id = expected["id"] + assert store_id in existing_stores, f"existing {store_type} store with id {store_id} not found" + existing = existing_stores[store_id] + for key in expected: + assert expected[key] == existing[key] + + def assert_oplog_stores(self, expected_oplog_stores: List): + """verifies that the list of oplog store configs in OM is equal to the expected one""" + self._assert_stores(expected_oplog_stores, "/admin/backup/oplog/mongoConfigs", "oplog") + + def assert_oplog_s3_stores(self, expected_oplog_s3_stores: List): + """verifies that the list of oplog s3 store configs in OM is equal to the expected one""" + self._assert_stores(expected_oplog_s3_stores, "/admin/backup/oplog/s3Configs", "s3") + + def assert_block_stores(self, expected_block_stores: List): + """verifies that the list of oplog store configs in OM is equal to the expected one""" + self._assert_stores(expected_block_stores, "/admin/backup/snapshot/mongoConfigs", "blockstore") + + def assert_s3_stores(self, expected_s3_stores: List): + """verifies that the list of s3 store configs in OM is equal to the expected one""" + self._assert_stores(expected_s3_stores, "/admin/backup/snapshot/s3Configs", "s3") + + def get_s3_stores(self): + """verifies that the list of s3 store configs in OM is equal to the expected one""" + response = self.om_request("get", "/admin/backup/snapshot/s3Configs") + assert response.status_code == requests.status_codes.codes.OK + return response.json() + + def get_oplog_s3_stores(self): + """verifies that the list of s3 store configs in OM is equal to the expected one""" + response = self.om_request("get", "/admin/backup/oplog/s3Configs") + assert response.status_code == requests.status_codes.codes.OK + return response.json() + + def assert_hosts_empty(self): + self.get_automation_config_tester().assert_empty() + hosts = self.api_get_hosts() + assert len(hosts["results"]) == 0 + + def wait_until_hosts_are_empty(self, timeout=30): + def hosts_are_empty(): + hosts = self.api_get_hosts()["results"] + return len(hosts) == 0 + + run_periodically(fn=hosts_are_empty, timeout=timeout) + + def wait_until_hosts_are_not_empty(self, timeout=30): + def hosts_are_not_empty(): + hosts = self.api_get_hosts()["results"] + return len(hosts) != 0 + + run_periodically(fn=hosts_are_not_empty, timeout=timeout) + + def assert_om_version(self, expected_version: str): + assert self.api_get_om_version() == expected_version + + def check_healthiness(self) -> tuple[str, str]: + return OMTester.request_health(self.context.base_url) + + @staticmethod + def request_health(base_url: str) -> tuple[str, str]: + endpoint = base_url + "/monitor/health" + response = requests.request("get", endpoint, verify=False) + return response.status_code, response.text + + @staticmethod + def do_assert_healthiness(base_url: str): + status_code, _ = OMTester.request_health(base_url) + assert ( + status_code == requests.status_codes.codes.OK + ), "Expected HTTP 200 from Ops Manager but got {} ({})".format(status_code, datetime.now()) + + def om_request(self, method, path, json_object: Optional[Dict] = None, retries=3, agent_endpoint=False): + """performs the digest API request to Ops Manager. Note that the paths don't need to be prefixed with + '/api../v1.0' as the method does it internally.""" + span = trace.get_current_span() + + headers = {"Content-Type": "application/json"} + if agent_endpoint: + auth = build_agent_auth(self.context.project_id, self.context.agent_api_key) + endpoint = f"{self.context.base_url}{path}" + else: + auth = build_auth(self.context.user, self.context.public_key) + endpoint = f"{self.context.base_url}/api/public/v1.0{path}" + + start_time = time.time() + + session = requests.Session() + retry = Retry(backoff_factor=5) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + + pattern = re.compile(r"/[a-f0-9]{24}") + sanitized_path = pattern.sub("/{id}", path) + span.set_attribute(key=f"meko.om.request.resource", value=sanitized_path) + + def om_request(): + try: + response = session.request( + url=endpoint, + method=method, + auth=auth, + headers=headers, + json=json_object, + timeout=30, + verify=False, + ) + except Exception as e: + print("failed connecting to om") + raise e + + span.set_attribute(key=f"meko.om.request.duration", value=time.time() - start_time) + span.set_attribute(key=f"meko.om.request.fullpath", value=path) + + if response.status_code >= 300: + raise Exception( + "Error sending request to Ops Manager API. {} ({}).\n Request details: {} {} (data: {})".format( + response.status_code, response.text, method, endpoint, json_object + ) + ) + return response + + retry_count = retries + last_exception = Exception("Failed unexpectedly while retrying OM request") + while retry_count >= 0: + try: + resp = om_request() + span.set_attribute(key=f"meko.om.request.retries", value=retries - retry_count) + return resp + except Exception as e: + print(f"Encountered exception: {e} on retry number {retries-retry_count}") + span.set_attribute(key=f"meko.om.request.exception", value=str(e)) + last_exception = e + time.sleep(1) + retry_count -= 1 + + raise last_exception + + def get_feature_controls(self): + return self.om_request("get", f"/groups/{self.context.project_id}/controlledFeature").json() + + def find_group_id(self): + """ + Obtains the group id of the group with specified name. + Note, that the logic used repeats the logic used by the Operator. + """ + if self.context.org_id is None or self.context.org_id == "": + # If no organization is passed, then look for all organizations + self.context.org_id = self.api_get_organization_id(self.context.group_name) + if self.context.org_id == "": + raise Exception(f"Organization with name {self.context.group_name} not found!") + + group_id = self.api_get_group_in_organization(self.context.org_id, self.context.group_name) + if group_id == "": + raise Exception( + f"Group with name {self.context.group_name} not found in organization {self.context.org_id}!" + ) + return group_id + + def api_backup_group(self): + group_id = self.find_group_id() + return self.om_request("get", f"/admin/backup/groups/{self.context.project_id}").json() + + def api_get_om_version(self) -> str: + # This can be any API request - we just need the header in the response + response = self.om_request("get", f"/groups/{self.context.project_id}/backupConfigs") + version_header = response.headers["X-MongoDB-Service-Version"] + version = version_header.split("versionString=")[1] + parsed_version = semver.VersionInfo.parse(version) + return f"{parsed_version.major}.{parsed_version.minor}.{parsed_version.patch}" + + def api_get_organization_id(self, org_name: str) -> str: + encoded_org_name = urllib.parse.quote_plus(org_name) + json = self.om_request("get", f"/orgs?name={encoded_org_name}").json() + if len(json["results"]) == 0: + return "" + return json["results"][0]["id"] + + def api_get_group_in_organization(self, org_id: str, group_name: str) -> str: + encoded_group_name = urllib.parse.quote_plus(group_name) + json = self.om_request("get", f"/orgs/{org_id}/groups?name={encoded_group_name}").json() + if len(json["results"]) == 0: + return "" + if len(json["results"]) > 1: + for res in json["results"]: + if res["name"] == group_name: + return res["id"] + raise Exception(f"More than one groups with name {group_name} found!") + return json["results"][0]["id"] + + def api_get_hosts(self) -> Dict: + return self.om_request("get", f"/groups/{self.context.project_id}/hosts").json() + + def get_automation_config_tester(self, **kwargs) -> AutomationConfigTester: + json = self.om_request("get", f"/groups/{self.context.project_id}/automationConfig").json() + return AutomationConfigTester(json, **kwargs) + + def get_backup_config(self) -> List: + return self.om_request("get", f"/groups/{self.context.project_id}/automationConfig/backupAgentConfig").json() + + def get_monitoring_config(self) -> List: + return self.om_request( + "get", f"/groups/{self.context.project_id}/automationConfig/monitoringAgentConfig" + ).json() + + def api_read_backup_configs(self) -> List: + return self.om_request("get", f"/groups/{self.context.project_id}/backupConfigs").json()["results"] + + # Backup states are from here: + # https://github.com/10gen/mms/blob/bcec76f60fc10fd6b7de40ee0f57951b54a4b4a0/server/src/main/com/xgen/cloud/common/brs/_public/model/BackupConfigState.java#L8 + def wait_until_backup_deactivated(self, timeout=30, is_sharded_cluster=False, expected_config_count=1): + def wait_until_backup_deactivated(): + found_backup = False + cluster_id = self.get_backup_cluster_id( + is_sharded_cluster=is_sharded_cluster, + expected_config_count=expected_config_count, + ) + for config in self.api_read_backup_configs(): + if config["clusterId"] == cluster_id: + found_backup = True + # Backup has been deactivated + if config["statusName"] in ["INACTIVE", "TERMINATING", "STOPPED"]: + return True + # Backup does not exist, which we correlate with backup is deactivated + if not found_backup: + return True + return False + + run_periodically(fn=wait_until_backup_deactivated, timeout=timeout) + + def wait_until_backup_running(self, timeout=30, is_sharded_cluster=False, expected_config_count=1): + def wait_until_backup_running(): + cluster_id = self.get_backup_cluster_id( + is_sharded_cluster=is_sharded_cluster, + expected_config_count=expected_config_count, + ) + for config in self.api_read_backup_configs(): + if config["clusterId"] == cluster_id: + if config["statusName"] in ["STARTED", "PROVISIONING"]: + return True + return False + + run_periodically(fn=wait_until_backup_running, timeout=timeout) + + def api_read_backup_snapshot_schedule(self) -> Dict: + backup_configs = self.api_read_backup_configs()[0] + return self.om_request( + "get", + f"/groups/{self.context.project_id}/backupConfigs/{backup_configs['clusterId']}/snapshotSchedule", + ).json() + + def api_read_monitoring_measurements( + self, + host_id: str, + database_name: Optional[str] = None, + project_id: Optional[str] = None, + period: str = "P1DT12H", + ): + """ + Reads a measurement from the measurements and alerts API: + + https://docs.opsmanager.mongodb.com/v4.4/reference/api/measures/get-host-process-system-measurements/ + """ + if database_name is None: + database_name = "admin" + return self.om_request( + "get", + f"/groups/{project_id}/hosts/{host_id}/databases/{database_name}/measurements?granularity=PT30S&period={period}", + ).json()["measurements"] + + def api_read_monitoring_agents(self) -> List: + return self._read_agents("MONITORING") + + def api_read_automation_agents(self) -> List: + return self._read_agents("AUTOMATION") + + def _read_agents(self, agent_type: str, page_num: int = 1): + return self.om_request( + "get", + f"/groups/{self.context.project_id}/agents/{agent_type}?pageNum={page_num}", + ).json()["results"] + + def api_get_snapshots(self, cluster_id: str) -> List: + return self.om_request("get", f"/groups/{self.context.project_id}/clusters/{cluster_id}/snapshots").json()[ + "results" + ] + + def api_get_clusters(self) -> List: + return self.om_request("get", f"/groups/{self.context.project_id}/clusters/").json() + + def api_create_restore_job_pit(self, cluster_id: str, pit_milliseconds: int): + """Creates a restore job that reverts a mongodb cluster to some time defined by 'pit_milliseconds'""" + data = self._restore_job_payload(cluster_id) + data["pointInTimeUTCMillis"] = pit_milliseconds + return self.om_request( + "post", + f"/groups/{self.context.project_id}/clusters/{cluster_id}/restoreJobs", + data, + ) + + def api_create_restore_job_from_snapshot(self, cluster_id: str, snapshot_id: str, retry: int = 3) -> Dict: + """ + Creates a restore job that uses an existing snapshot as the source + + The restore job might fail to be created if + """ + data = self._restore_job_payload(cluster_id) + data["snapshotId"] = snapshot_id + + for r in range(retry): + try: + result = self.om_request( + "post", + f"/groups/{self.context.project_id}/clusters/{cluster_id}/restoreJobs", + data, + ) + except Exception as e: + logging.info(e) + logging.info(f"Could not create restore job, attempt {r + 1}") + time.sleep((r + 1) * 10) + continue + + return result.json()["results"][0] + + raise Exception(f"Could not create restore job after {retry} attempts") + + def api_get_restore_jobs(self, cluster_id: str) -> List: + return self.om_request( + "get", + f"/groups/{self.context.project_id}/clusters/{cluster_id}/restoreJobs", + ).json()["results"] + + def api_get_restore_job_by_id(self, cluster_id: str, id: str) -> Dict: + return self.om_request( + "get", + f"/groups/{self.context.project_id}/clusters/{cluster_id}/restoreJobs/{id}", + ).json() + + def api_remove_group(self): + controlled_features_data = { + "externalManagementSystem": {"name": "mongodb-enterprise-operator"}, + "policies": [], + } + self.om_request( + "put", + f"/groups/{self.context.project_id}/controlledFeature", + controlled_features_data, + ) + self.om_request("put", f"/groups/{self.context.project_id}/automationConfig", {}) + return self.om_request("delete", f"/groups/{self.context.project_id}") + + def _restore_job_payload(self, cluster_id) -> Dict: + return { + "delivery": { + "methodName": "AUTOMATED_RESTORE", + "targetGroupId": self.context.project_id, + "targetClusterId": cluster_id, + }, + } + + def query_backup(self, db_name: str, collection_name: str, timeout: int): + """Query the first backup snapshot and return all records from specified collection.""" + qb = OMQueryableBackup(self.context.base_url, self.context.project_id) + connParams = qb.connection_params(timeout) + + caPem = tempfile.NamedTemporaryFile(delete=False, mode="w") + caPem.write(connParams.ca_pem) + caPem.flush() + + clientPem = tempfile.NamedTemporaryFile(delete=False, mode="w") + clientPem.write(connParams.client_pem) + clientPem.flush() + + dbClient = pymongo.MongoClient( + host=connParams.host, + tls=True, + tlsCAFile=caPem.name, + tlsCertificateKeyFile=clientPem.name, + serverSelectionTimeoutMs=300000, + )[db_name] + collection = dbClient[collection_name] + return list(collection.find()) + + def api_get_preferred_hostnames(self): + return self.om_request("get", f"/group/v2/info/{self.context.project_id}", agent_endpoint=True).json()[ + "preferredHostnames" + ] + + +class OMBackgroundTester(BackgroundHealthChecker): + """ + + Note, that it may return sporadic 500 when the appdb is being restarted, we + won't fail because of this so checking only for + 'allowed_sequential_failures' failures. In practice having + 'allowed_sequential_failures' should work as failures are very rare (1-2 per + appdb upgrade) but let's be safe to avoid e2e flakiness. + + """ + + def __init__( + self, + om_tester: OMTester, + wait_sec: int = 3, + allowed_sequential_failures: int = 3, + ): + super().__init__( + health_function=om_tester.assert_healthiness, + wait_sec=wait_sec, + allowed_sequential_failures=allowed_sequential_failures, + ) + + +# TODO can we move below methods to some other place? + + +def get_agent_cert_names(namespace: str) -> List[str]: + agent_names = ["mms-automation-agent", "mms-backup-agent", "mms-monitoring-agent"] + return ["{}.{}".format(agent_name, namespace) for agent_name in agent_names] + + +def get_rs_cert_names( + mdb_resource: str, + namespace: str, + *, + members: int = 3, + with_internal_auth_certs: bool = False, + with_agent_certs: bool = False, +) -> List[str]: + cert_names = [f"{mdb_resource}-{i}.{namespace}" for i in range(members)] + + if with_internal_auth_certs: + cert_names += [f"{mdb_resource}-{i}-clusterfile.{namespace}" for i in range(members)] + + if with_agent_certs: + cert_names += get_agent_cert_names(namespace) + + return cert_names + + +def get_st_cert_names( + mdb_resource: str, + namespace: str, + *, + with_internal_auth_certs: bool = False, + with_agent_certs: bool = False, +) -> List[str]: + return get_rs_cert_names( + mdb_resource, + namespace, + members=1, + with_internal_auth_certs=with_internal_auth_certs, + with_agent_certs=with_agent_certs, + ) + + +def get_sc_cert_names( + mdb_resource: str, + namespace: str, + *, + num_shards: int = 1, + members: int = 3, + config_members: int = 3, + num_mongos: int = 2, + with_internal_auth_certs: bool = False, + with_agent_certs: bool = False, +) -> List[str]: + names = [] + + for shard_num in range(num_shards): + for member in range(members): + # e.g. test-tls-x509-sc-0-1.developer14 + names.append("{}-{}-{}.{}".format(mdb_resource, shard_num, member, namespace)) + if with_internal_auth_certs: + # e.g. test-tls-x509-sc-0-2-clusterfile.developer14 + names.append("{}-{}-{}-clusterfile.{}".format(mdb_resource, shard_num, member, namespace)) + + for member in range(config_members): + # e.g. test-tls-x509-sc-config-1.developer14 + names.append("{}-config-{}.{}".format(mdb_resource, member, namespace)) + if with_internal_auth_certs: + # e.g. test-tls-x509-sc-config-1-clusterfile.developer14 + names.append("{}-config-{}-clusterfile.{}".format(mdb_resource, member, namespace)) + + for mongos in range(num_mongos): + # e.g.test-tls-x509-sc-mongos-1.developer14 + names.append("{}-mongos-{}.{}".format(mdb_resource, mongos, namespace)) + if with_internal_auth_certs: + # e.g. test-tls-x509-sc-mongos-0-clusterfile.developer14 + names.append("{}-mongos-{}-clusterfile.{}".format(mdb_resource, mongos, namespace)) + + if with_agent_certs: + names.extend(get_agent_cert_names(namespace)) + + return names + + +def should_include_tag(version: Optional[Dict[str, str]]) -> bool: + """Checks if the Ops Manager version API includes the EXTERNALLY_MANAGED tag. + This is, the version of Ops Manager is greater or equals than 4.2.2 or Cloud + Manager. + + """ + + feature_controls_enabled_version = "4.2.2" + if version is None: + return True + + if "versionString" not in version: + return True + + if re.match("^v\d+", version["versionString"]): + # Cloud Manager supports Feature Controls + return False + + match = re.match(r"^(\d{1,2}\.\d{1,2}\.\d{1,2}).*", version["versionString"]) + if match: + version_string = match.group(1) + + # version_string is lower than 4.2.2 + return semver.compare(version_string, feature_controls_enabled_version) < 0 + + return True diff --git a/docker/mongodb-enterprise-tests/kubetester/operator.py b/docker/mongodb-enterprise-tests/kubetester/operator.py new file mode 100644 index 000000000..0fe1931bd --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/operator.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import logging +import time +from typing import Dict, List, Optional + +import requests +from kubernetes import client +from kubernetes.client import V1beta1CustomResourceDefinition, V1Deployment, V1Pod +from kubernetes.client.rest import ApiException +from kubetester import wait_for_webhook +from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml +from kubetester.helm import ( + helm_install, + helm_repo_add, + helm_template, + helm_uninstall, + helm_upgrade, +) + +OPERATOR_CRDS = ( + "mongodb.mongodb.com", + "mongodbusers.mongodb.com", + "opsmanagers.mongodb.com", +) + + +class Operator(object): + """Operator is an abstraction over some Operator and relevant resources. It + allows to create and delete the Operator deployment and K8s resources. + + * `helm_args` corresponds to the --set values passed to helm installation. + * `helm_options` refers to the options passed to the helm command. + + The operator is installed from published Helm Charts. + """ + + def __init__( + self, + namespace: str, + helm_args: Optional[Dict] = None, + helm_options: Optional[List[str]] = None, + helm_chart_path: Optional[str] = "helm_chart", + name: Optional[str] = "mongodb-enterprise-operator", + api_client: Optional[client.api_client.ApiClient] = None, + ): + + # The Operator will be installed from the following repo, so adding it first + helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts") + + if helm_args is None: + helm_args = {} + + helm_args["namespace"] = namespace + helm_args["operator.env"] = "dev" + + # the import is done here to prevent circular dependency + from tests.conftest import local_operator + + if local_operator(): + helm_args["operator.replicas"] = "0" + + self.namespace = namespace + self.helm_arguments = helm_args + self.helm_options = helm_options + self.helm_chart_path = helm_chart_path + self.name = name + self.api_client = api_client + + def install_from_template(self): + """Uses helm to generate yaml specification and then uses python K8s client to apply them to the cluster + This is equal to helm template...| kubectl apply -""" + yaml_file = helm_template(self.helm_arguments, helm_chart_path=self.helm_chart_path) + create_or_replace_from_yaml(client.api_client.ApiClient(), yaml_file) + self._wait_for_operator_ready() + self._wait_operator_webhook_is_ready() + + return self + + def install(self, custom_operator_version: Optional[str] = None) -> Operator: + """Installs the Operator to Kubernetes cluster using 'helm install', waits until it's running""" + helm_install( + "mongodb-enterprise-operator", + self.namespace, + self.helm_arguments, + helm_chart_path=self.helm_chart_path, + helm_options=self.helm_options, + custom_operator_version=custom_operator_version, + ) + self._wait_for_operator_ready() + self._wait_operator_webhook_is_ready() + + return self + + def upgrade(self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None) -> Operator: + """Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running""" + helm_upgrade( + self.name, + self.namespace, + self.helm_arguments, + helm_chart_path=self.helm_chart_path, + helm_options=self.helm_options, + custom_operator_version=custom_operator_version, + ) + self._wait_for_operator_ready() + self._wait_operator_webhook_is_ready(multi_cluster=multi_cluster) + + return self + + def uninstall(self): + helm_uninstall(self.name) + + def delete_operator_deployment(self): + """Deletes the Operator deployment from K8s cluster.""" + client.AppsV1Api(api_client=self.api_client).delete_namespaced_deployment(self.name, self.namespace) + + def list_operator_pods(self) -> List[V1Pod]: + pods = ( + client.CoreV1Api(api_client=self.api_client) + .list_namespaced_pod( + self.namespace, + label_selector="app.kubernetes.io/name={}".format(self.name), + ) + .items + ) + return pods + + def read_deployment(self) -> V1Deployment: + return client.AppsV1Api(api_client=self.api_client).read_namespaced_deployment(self.name, self.namespace) + + def assert_is_running(self): + self._wait_for_operator_ready() + + def _wait_for_operator_ready(self, retries: int = 60): + """waits until the Operator deployment is ready.""" + + # we don't want to wait for the operator if the operator is running locally and not in a pod + from tests.conftest import local_operator + + if local_operator(): + return + + # we need to give some time for the new pod to start instead of the existing one (if any) + time.sleep(4) + retry_count = retries + while retry_count > 0: + pods = self.list_operator_pods() + if len(pods) == 1: + if pods[0].status.phase == "Running" and pods[0].status.container_statuses[0].ready: + return + if pods[0].status.phase == "Failed": + raise Exception("Operator failed to start: {}".format(pods[0].status.phase)) + time.sleep(1) + retry_count = retry_count - 1 + + # Operator hasn't started - printing some debug information + self.print_diagnostics() + + raise Exception(f"Operator hasn't started in specified time after {retries} retries.") + + def _wait_operator_webhook_is_ready(self, retries: int = 10, multi_cluster: bool = False): + + # we don't want to wait for the operator webhook if the operator is running locally and not in a pod + from tests.conftest import get_cluster_domain, local_operator + + if local_operator(): + return + + # in multi-cluster mode the operator and the test pod are in different clusters(test pod won't be able to talk to webhook), + # so we skip this extra check for multi-cluster + from tests.conftest import get_central_cluster_name, get_test_pod_cluster_name + + if multi_cluster and get_central_cluster_name() != get_test_pod_cluster_name(): + print( + f"Skipping waiting for the webhook as we cannot call the webhook endpoint from a test_pod_cluster ({get_test_pod_cluster_name()}) " + f"to central cluster ({get_central_cluster_name()}); sleeping for 10s instead" + ) + # We need to sleep here otherwise the function returns too early and we create a race condition in tests + time.sleep(10) + return + + logging.debug("_wait_operator_webhook_is_ready") + validation_endpoint = "validate-mongodb-com-v1-mongodb" + webhook_endpoint = "https://operator-webhook.{}.svc.{}/{}".format( + self.namespace, get_cluster_domain(), validation_endpoint + ) + headers = {"Content-Type": "application/json"} + + retry_count = retries + 1 + while retry_count > 0: + retry_count -= 1 + + logging.debug("Waiting for operator/webhook to be functional") + try: + response = requests.post(webhook_endpoint, headers=headers, verify=False, timeout=2) + except Exception as e: + logging.warn(e) + time.sleep(2) + continue + + try: + # Let's assume that if we get a json response, then the webhook + # is already in place. + response.json() + except Exception: + logging.warn("Didn't get a json response from webhook") + else: + return + + time.sleep(2) + + raise Exception("Operator webhook didn't start after {} retries".format(retries)) + + def print_diagnostics(self): + logging.info("Operator Deployment: ") + logging.info(self.read_deployment()) + + pods = self.list_operator_pods() + if len(pods) > 0: + logging.info("Operator pods: %d", len(pods)) + logging.info("Operator spec: %s", pods[0].spec) + logging.info("Operator status: %s", pods[0].status) + + def wait_for_webhook(self, retries=5, delay=5): + return wait_for_webhook(namespace=self.namespace, retries=retries, delay=delay) + + def disable_webhook(self): + webhook_api = client.AdmissionregistrationV1Api() + + # break the existing webhook + webhook = webhook_api.read_validating_webhook_configuration("mdbpolicy.mongodb.com") + + # First webhook is for mongodb validations, second is for ops manager + webhook.webhooks[1].client_config.service.name = "a-non-existent-service" + webhook.metadata.uid = "" + webhook_api.replace_validating_webhook_configuration("mdbpolicy.mongodb.com", webhook) + + def restart_operator_deployment(self): + client.AppsV1Api(api_client=self.api_client).patch_namespaced_deployment_scale( + self.name, + self.namespace, + [{"op": "replace", "path": "/spec/replicas", "value": 0}], + ) + + # wait till there are 0 operator pods + count = 0 + while count < 6: + pods = self.list_operator_pods() + if len(pods) == 0: + break + time.sleep(3) + + # scale the resource back to 1 + client.AppsV1Api(api_client=self.api_client).patch_namespaced_deployment_scale( + self.name, + self.namespace, + [{"op": "replace", "path": "/spec/replicas", "value": 1}], + ) + + return self._wait_for_operator_ready() + + +def delete_operator_crds(): + for crd_name in OPERATOR_CRDS: + try: + client.ApiextensionsV1Api().delete_custom_resource_definition(crd_name) + except ApiException as e: + if e.status != 404: + raise e + + +def list_operator_crds() -> List[V1beta1CustomResourceDefinition]: + return sorted( + [ + crd + for crd in client.ApiextensionsV1Api().list_custom_resource_definition().items + if crd.metadata.name in OPERATOR_CRDS + ], + key=lambda crd: crd.metadata.name, + ) diff --git a/docker/mongodb-enterprise-tests/kubetester/opsmanager.py b/docker/mongodb-enterprise-tests/kubetester/opsmanager.py new file mode 100644 index 000000000..866bf9cf4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/opsmanager.py @@ -0,0 +1,1185 @@ +from __future__ import annotations + +import json +import re +import time +from base64 import b64decode +from typing import Any, Callable, Dict, List, Optional + +import kubernetes.client +import requests +from kubeobject import CustomObject +from kubernetes.client.rest import ApiException +from kubetester import ( + create_configmap, + create_or_update_secret, + read_configmap, + read_secret, +) +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import ( + KubernetesTester, + build_list_of_hosts, + is_default_architecture_static, +) +from kubetester.mongodb import MongoDB, MongoDBCommon, Phase, get_pods, in_desired_state +from kubetester.mongotester import MongoTester, MultiReplicaSetTester, ReplicaSetTester +from kubetester.omtester import OMContext, OMTester +from opentelemetry import trace +from requests.auth import HTTPDigestAuth +from tests import test_logger +from tests.conftest import ( + LEGACY_CENTRAL_CLUSTER_NAME, + get_central_cluster_client, + get_member_cluster_api_client, + get_member_cluster_client_map, + is_member_cluster, + multi_cluster_pod_names, + multi_cluster_service_names, +) +from tests.shardedcluster.conftest import read_deployment_state + +logger = test_logger.get_test_logger(__name__) + + +class MongoDBOpsManager(CustomObject, MongoDBCommon): + def __init__(self, *args, **kwargs): + with_defaults = { + "plural": "opsmanagers", + "kind": "MongoDBOpsManager", + "group": "mongodb.com", + "version": "v1", + } + with_defaults.update(kwargs) + super(MongoDBOpsManager, self).__init__(*args, **with_defaults) + + def trigger_architecture_migration(self): + self.load() + + if is_default_architecture_static(): + self["metadata"]["annotations"].update({"mongodb.com/v1.architecture": "non-static"}) + self.update() + else: + self["metadata"]["annotations"].update({"mongodb.com/v1.architecture": "static"}) + self.update() + + def trigger_om_sts_restart(self): + """ + Adds or changes a label from the pod template to trigger a rolling restart of the OpsManager StatefulSet. + """ + self.load() + self["spec"]["statefulSet"] = { + "spec": {"template": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": str(time.time())}}}} + } + self.update() + + def trigger_appdb_sts_restart(self): + """ + Adds or changes a label from the pod template to trigger a rolling restart of the AppDB StatefulSet. + """ + self.load() + self["spec"]["applicationDatabase"] = { + "podSpec": { + "podTemplate": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": str(time.time())}}} + } + } + self.update() + + def appdb_status(self) -> MongoDBOpsManager.AppDbStatus: + return self.AppDbStatus(self) + + def om_status(self) -> MongoDBOpsManager.OmStatus: + return self.OmStatus(self) + + def backup_status(self) -> MongoDBOpsManager.BackupStatus: + return self.BackupStatus(self) + + def assert_reaches(self, fn: Callable[[MongoDBOpsManager], bool], timeout=None): + return self.wait_for(fn, timeout=timeout, should_raise=True) + + def get_appdb_hosts(self): + tester = self.get_om_tester(self.app_db_name()) + tester.assert_group_exists() + return tester.api_get_hosts()["results"] + + def get_appdb_preferred_hostnames(self): + tester = self.get_om_tester(self.app_db_name()) + return tester.api_get_preferred_hostnames() + + def assert_appdb_preferred_hostnames_are_added(self): + def appdb_preferred_hostnames_are_added(): + expected_hostnames = self.get_appdb_hostnames_for_monitoring() + preferred_hostnames = self.get_appdb_preferred_hostnames() + + if len(preferred_hostnames) != len(expected_hostnames): + return False + + for hostname in preferred_hostnames: + if hostname["value"] not in expected_hostnames: + return False + return True + + KubernetesTester.wait_until(appdb_preferred_hostnames_are_added, timeout=120, sleep_time=5) + + def assert_appdb_hostnames_are_correct(self): + def appdb_hostnames_are_correct(): + expected_hostnames = self.get_appdb_hostnames_for_monitoring() + hosts = self.get_appdb_hosts() + + if len(hosts) != len(expected_hostnames): + return False + + for host in hosts: + if host["hostname"] not in expected_hostnames: + return False + return True + + KubernetesTester.wait_until(appdb_hostnames_are_correct, timeout=300, sleep_time=10) + + def assert_appdb_monitoring_group_was_created(self): + tester = self.get_om_tester(self.app_db_name()) + tester.assert_group_exists() + + appdb_hostnames = self.get_appdb_hostnames_for_monitoring() + + def monitoring_agents_have_registered() -> bool: + monitoring_agents = tester.api_read_monitoring_agents() + appdb_monitoring_agents = [a for a in monitoring_agents if a["hostname"] in appdb_hostnames] + expected_number_of_agents = len(appdb_monitoring_agents) == self.get_appdb_members_count() + + expected_number_of_agents_in_standby = ( + len([agent for agent in appdb_monitoring_agents if agent["stateName"] == "STANDBY"]) + == self.get_appdb_members_count() - 1 + ) + expected_number_of_agents_are_active = ( + len([agent for agent in appdb_monitoring_agents if agent["stateName"] == "ACTIVE"]) == 1 + ) + + return ( + expected_number_of_agents + and expected_number_of_agents_in_standby + and expected_number_of_agents_are_active + ) + + KubernetesTester.wait_until(monitoring_agents_have_registered, timeout=600, sleep_time=5) + + def no_automation_agents_have_registered() -> bool: + automation_agents = tester.api_read_automation_agents() + appdb_automation_agents = [a for a in automation_agents if a["hostname"] in appdb_hostnames] + return len(appdb_automation_agents) == 0 + + KubernetesTester.wait_until(no_automation_agents_have_registered, timeout=600, sleep_time=5) + + def assert_monitoring_data_exists( + self, database_name: str = "admin", period: str = "P1DT12H", timeout: int = 120, all_hosts: bool = True + ): + """ + Asserts the existence of monitoring measurements in this Ops Manager instance. + """ + appdb_hosts = self.get_appdb_hosts() + host_ids = [host["id"] for host in appdb_hosts] + project_id = [host["groupId"] for host in appdb_hosts][0] + tester = self.get_om_tester() + + def agent_is_showing_metrics(): + for host_id in host_ids: + measurements = tester.api_read_monitoring_measurements( + host_id, + database_name=database_name, + project_id=project_id, + period=period, + ) + if measurements is None and all_hosts: + return False + elif measurements is None and not all_hosts: + continue + + found = False + for measurement in measurements: + if len(measurement["dataPoints"]) > 0: + found = True + break + + if all_hosts and not found: + return False + elif not all_hosts and found: + return True + + if all_hosts: + return True + return False + + KubernetesTester.wait_until( + agent_is_showing_metrics, + timeout=timeout, + ) + + def get_appdb_resource(self) -> MongoDB: + mdb = MongoDB(name=self.app_db_name(), namespace=self.namespace) + # We "artificially" add SCRAM authentication to make syntax match the normal MongoDB - + # this will let the mongo_uri() method work correctly + # (opsmanager_types.go does the same) + mdb["spec"] = self["spec"]["applicationDatabase"] + mdb["spec"]["type"] = MongoDB.Types.REPLICA_SET + mdb["spec"]["security"] = {"authentication": {"modes": ["SCRAM"]}} + return mdb + + def services(self, member_cluster_name: Optional[str] = None) -> List[Optional[kubernetes.client.V1Service]]: + """Returns a two element list with internal and external Services. + + Any of them might be None if the Service is not found. + """ + services = [] + service_names = ( + self.svc_name(member_cluster_name), + self.external_svc_name(member_cluster_name), + ) + + for name in service_names: + try: + svc = kubernetes.client.CoreV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_service(name, self.namespace) + services.append(svc) + except ApiException: + services.append(None) + + return [services[0], services[1]] + + def read_statefulset(self, member_cluster_name: str = None) -> kubernetes.client.V1StatefulSet: + if member_cluster_name is None: + member_cluster_name = self.pick_one_om_member_cluster_name() + + return kubernetes.client.AppsV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_stateful_set(self.om_sts_name(member_cluster_name), self.namespace) + + def pick_one_appdb_member_cluster_name(self) -> Optional[str]: + if self.is_appdb_multi_cluster(): + return self.get_appdb_indexed_cluster_spec_items()[0][1]["clusterName"] + else: + return None + + def pick_one_om_member_cluster_name(self) -> Optional[str]: + if self.is_om_multi_cluster(): + return self.get_om_indexed_cluster_spec_items()[0][1]["clusterName"] + else: + return None + + def read_appdb_statefulset(self, member_cluster_name: Optional[str] = None) -> kubernetes.client.V1StatefulSet: + if member_cluster_name is None: + member_cluster_name = self.pick_one_appdb_member_cluster_name() + return kubernetes.client.AppsV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_stateful_set(self.app_db_sts_name(member_cluster_name), self.namespace) + + def read_backup_statefulset(self, member_cluster_name: Optional[str] = None) -> kubernetes.client.V1StatefulSet: + if member_cluster_name is None: + member_cluster_name = self.pick_one_om_member_cluster_name() + + return kubernetes.client.AppsV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_stateful_set(self.backup_daemon_sts_name(member_cluster_name), self.namespace) + + def read_om_pods(self) -> list[tuple[kubernetes.client.ApiClient, kubernetes.client.V1Pod]]: + if self.is_om_multi_cluster(): + om_pod_names = self.get_om_pod_names_in_member_clusters() + member_cluster_client_map = get_member_cluster_client_map() + list_of_pods = [] + for cluster_name, om_pod_name in om_pod_names: + member_cluster_client = member_cluster_client_map[cluster_name].api_client + api_client = kubernetes.client.CoreV1Api(api_client=member_cluster_client) + list_of_pods.append( + ( + member_cluster_client, + api_client.read_namespaced_pod(om_pod_name, self.namespace), + ) + ) + return list_of_pods + else: + api_client = kubernetes.client.ApiClient() + return [ + ( + api_client, + kubernetes.client.CoreV1Api(api_client=api_client).read_namespaced_pod(podname, self.namespace), + ) + for podname in get_pods(self.name + "-{}", self.get_total_number_of_om_replicas()) + ] + + def get_om_pod_names_in_member_clusters(self) -> list[tuple[str, str]]: + """Returns list of tuples (cluster_name, pod_name) ordered by cluster index. + Pod names are generated according to member count in spec.clusterSpecList. + Clusters are ordered by cluster indexes in -cluster-mapping config map. + """ + pod_names_per_cluster = [] + for cluster_idx, cluster_spec_item in self.get_om_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + if is_member_cluster(cluster_name): + pod_names = multi_cluster_pod_names(self.name, [(cluster_idx, int(cluster_spec_item["members"]))]) + else: + pod_names = [ + self.om_pod_name(cluster_name, pod_idx) + for pod_idx in range(0, self.get_om_replicas_in_member_cluster(cluster_name)) + ] + + pod_names_per_cluster.extend([(cluster_name, pod_name) for pod_name in pod_names]) + + return pod_names_per_cluster + + def get_om_cluster_spec_item(self, member_cluster_name: str) -> dict[str, any]: + cluster_spec_items = [ + cluster_spec_item + for idx, cluster_spec_item in self.get_om_indexed_cluster_spec_items() + if cluster_spec_item["clusterName"] == member_cluster_name + ] + if len(cluster_spec_items) == 0: + raise Exception(f"{member_cluster_name} not found on OM's cluster_spec_items") + + return cluster_spec_items[0] + + def get_om_sts_names_in_member_clusters(self) -> list[tuple[str, str]]: + """Returns list of tuples (cluster_name, sts_name) ordered by cluster index. + Statefulset names are generated according to member count in spec.clusterSpecList. + Clusters are ordered by cluster indexes in -cluster-mapping config map. + """ + sts_names_per_cluster = [] + for cluster_idx, cluster_spec_item in self.get_om_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + sts_names_per_cluster.append((cluster_name, self.om_sts_name(cluster_name))) + + return sts_names_per_cluster + + def get_appdb_sts_names_in_member_clusters(self) -> list[tuple[str, str]]: + """Returns list of tuples (cluster_name, sts_name) ordered by cluster index. + Statefulset names are generated according to member count in spec.applicationDatabase.clusterSpecList. + Clusters are ordered by cluster indexes in -cluster-mapping config map. + """ + sts_names_per_cluster = [] + for cluster_idx, cluster_spec_item in self.get_appdb_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + sts_names_per_cluster.append((cluster_name, self.app_db_sts_name(cluster_name))) + + return sts_names_per_cluster + + def get_backup_sts_names_in_member_clusters(self) -> list[tuple[str, str]]: + """ """ + sts_names_per_cluster = [] + for cluster_idx, cluster_spec_item in self.get_om_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + sts_names_per_cluster.append((cluster_name, self.backup_daemon_sts_name(cluster_name))) + + return sts_names_per_cluster + + def get_om_member_cluster_names(self) -> list[str]: + """Returns list of OpsManager member cluster names ordered by cluster index.""" + member_cluster_names = [] + for cluster_idx, cluster_spec_item in self.get_om_indexed_cluster_spec_items(): + member_cluster_names.append(cluster_spec_item["clusterName"]) + + return member_cluster_names + + def get_appdb_member_cluster_names(self) -> list[str]: + """Returns list of AppDB member cluster names ordered by cluster index.""" + member_cluster_names = [] + for cluster_idx, cluster_spec_item in self.get_appdb_indexed_cluster_spec_items(): + member_cluster_names.append(cluster_spec_item["clusterName"]) + + return member_cluster_names + + def backup_daemon_pod_names(self, member_cluster_name: Optional[str] = None) -> list[tuple[str, str]]: + """ + Returns list of tuples (cluster_name, pod_name) ordered by cluster index. + Pod names are generated according to member count in spec.clusterSpecList[i].backup.members + """ + pod_names_per_cluster = [] + for _, cluster_spec_item in self.get_om_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + if member_cluster_name is not None and cluster_name != member_cluster_name: + continue + members_in_cluster = cluster_spec_item.get("backup", {}).get( + "members", self.get_backup_members_count(member_cluster_name=cluster_name) + ) + pod_names = [ + f"{self.backup_daemon_sts_name(member_cluster_name=cluster_name)}-{idx}" + for idx in range(int(members_in_cluster)) + ] + + pod_names_per_cluster.extend([(cluster_name, pod_name) for pod_name in pod_names]) + + return pod_names_per_cluster + + def get_appdb_pod_names_in_member_clusters(self) -> list[tuple[str, str]]: + """Returns list of tuples (cluster_name, pod_name) ordered by cluster index. + Pod names are generated according to member count in spec.applicationDatabase.clusterSpecList. + Clusters are ordered by cluster indexes in -cluster-mapping config map. + """ + pod_names_per_cluster = [] + for ( + cluster_index, + cluster_spec_item, + ) in self.get_appdb_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + pod_names = multi_cluster_pod_names( + self.app_db_name(), [(cluster_index, int(cluster_spec_item["members"]))] + ) + pod_names_per_cluster.extend([(cluster_name, pod_name) for pod_name in pod_names]) + + return pod_names_per_cluster + + def get_appdb_process_hostnames_in_member_clusters(self) -> list[tuple[str, str]]: + """Returns list of tuples (cluster_name, service name) ordered by cluster index. + Service names are generated according to member count in spec.applicationDatabase.clusterSpecList. + Clusters are ordered by cluster indexes in -cluster-mapping config map. + """ + service_names_per_cluster = [] + for ( + cluster_index, + cluster_spec_item, + ) in self.get_appdb_indexed_cluster_spec_items(): + cluster_name = cluster_spec_item["clusterName"] + service_names = multi_cluster_service_names( + self.app_db_name(), [(cluster_index, int(cluster_spec_item["members"]))] + ) + service_names_per_cluster.extend([(cluster_name, service_name) for service_name in service_names]) + + return service_names_per_cluster + + def get_appdb_hostnames_for_monitoring(self) -> list[str]: + """ + Returns list of hostnames for appdb members. + In case of multicluster appdb, hostnames are generated from the pod service fqdn. + In case of single cluster, the hostnames use the headless service. + """ + hostnames = [] + appdb_resource = self.get_appdb_resource() + external_domain = appdb_resource["spec"].get("externalAccess", {}).get("externalDomain") + if self.is_appdb_multi_cluster(): + for cluster_index, cluster_spec_item in self.get_appdb_indexed_cluster_spec_items(): + pod_names = multi_cluster_pod_names( + self.app_db_name(), [(cluster_index, int(cluster_spec_item["members"]))] + ) + + cluster_external_domain = cluster_spec_item.get("externalAccess", {}).get( + "externalDomain", external_domain + ) + if cluster_external_domain is not None: + hostnames.extend([f"{pod_name}.{cluster_external_domain}" for pod_name in pod_names]) + else: + hostnames.extend([f"{pod_name}-svc.{self.namespace}.svc.cluster.local" for pod_name in pod_names]) + else: + resource_name = appdb_resource["metadata"]["name"] + service_name = f"{resource_name}-svc" + namespace = appdb_resource["metadata"]["namespace"] + + for index in range(appdb_resource["spec"]["members"]): + if external_domain is not None: + hostnames.append(f"{resource_name}-{index}.{external_domain}") + else: + hostnames.append(f"{resource_name}-{index}.{service_name}.{namespace}.svc.cluster.local") + + return hostnames + + def get_appdb_indexed_cluster_spec_items(self) -> list[tuple[int, dict[str, any]]]: + """Returns ordered list (by cluster index) of tuples (cluster index, clusterSpecItem) from spec.applicationDatabase.clusterSpecList. + Cluster indexes are read from -cluster-mapping config map. + """ + if not self.is_appdb_multi_cluster(): + return self.get_legacy_central_cluster(self.get_appdb_members_count()) + + cluster_index_mapping = read_deployment_state(self.app_db_name(), self.namespace)["clusterMapping"] + result = [] + for cluster_spec_item in self["spec"]["applicationDatabase"].get("clusterSpecList", []): + result.append( + ( + int(cluster_index_mapping[cluster_spec_item["clusterName"]]), + cluster_spec_item, + ) + ) + + return sorted(result, key=lambda x: x[0]) + + def get_om_indexed_cluster_spec_items(self) -> list[tuple[int, dict[str, str]]]: + """Returns an ordered list (by cluster index) of tuples (cluster index, clusterSpecItem) from spec.clusterSpecList. + Cluster indexes are read from -cluster-mapping config map. + """ + if not self.is_om_multi_cluster(): + return self.get_legacy_central_cluster(self.get_total_number_of_om_replicas()) + + cluster_mapping = read_deployment_state(self.name, self.namespace)["clusterMapping"] + result = [ + ( + int(cluster_mapping[cluster_spec_item["clusterName"]]), + cluster_spec_item, + ) + for cluster_spec_item in self["spec"].get("clusterSpecList", []) + ] + return sorted(result, key=lambda x: x[0]) + + def read_deployment_state(self, resource_name: str) -> dict[str, Any]: + deployment_state_cm = read_configmap( + self.namespace, + f"{resource_name}-state", + get_central_cluster_client(), + ) + state = json.loads(deployment_state_cm["state"]) + return state + + @staticmethod + def get_legacy_central_cluster(replicas: int) -> list[tuple[int, dict[str, str]]]: + return [(0, {"clusterName": LEGACY_CENTRAL_CLUSTER_NAME, "members": str(replicas)})] + + def read_appdb_pods(self) -> list[tuple[kubernetes.client.ApiClient, kubernetes.client.V1Pod]]: + """Returns list of tuples[api_client used, pod].""" + if self.is_appdb_multi_cluster(): + appdb_pod_names = self.get_appdb_pod_names_in_member_clusters() + member_cluster_client_map = get_member_cluster_client_map() + list_of_pods = [] + for cluster_name, appdb_pod_name in appdb_pod_names: + member_cluster_client = member_cluster_client_map[cluster_name].api_client + api_client = kubernetes.client.CoreV1Api(api_client=member_cluster_client) + list_of_pods.append( + ( + member_cluster_client, + api_client.read_namespaced_pod(appdb_pod_name, self.namespace), + ) + ) + + return list_of_pods + else: + api_client = kubernetes.client.ApiClient() + return [ + ( + api_client, + kubernetes.client.CoreV1Api(api_client=api_client).read_namespaced_pod(pod_name, self.namespace), + ) + for pod_name in get_pods(self.app_db_name() + "-{}", self.get_appdb_members_count()) + ] + + def read_backup_pods(self) -> list[tuple[kubernetes.client.ApiClient, kubernetes.client.V1Pod]]: + if self.is_om_multi_cluster(): + backup_pod_names = self.backup_daemon_pod_names() + member_cluster_client_map = get_member_cluster_client_map() + list_of_pods = [] + for cluster_name, backup_pod_name in backup_pod_names: + member_cluster_client = member_cluster_client_map[cluster_name].api_client + api_client = kubernetes.client.CoreV1Api(api_client=member_cluster_client) + list_of_pods.append( + ( + member_cluster_client, + api_client.read_namespaced_pod(backup_pod_name, self.namespace), + ) + ) + return list_of_pods + else: + api_client = kubernetes.client.ApiClient() + return [ + ( + api_client, + kubernetes.client.CoreV1Api().read_namespaced_pod(pod_name, self.namespace), + ) + for pod_name in get_pods( + self.backup_daemon_sts_name() + "-{}", + self.get_backup_members_count(member_cluster_name=LEGACY_CENTRAL_CLUSTER_NAME), + ) + ] + + @staticmethod + def get_backup_daemon_container_status( + backup_daemon_pod: kubernetes.client.V1Pod, + ) -> kubernetes.client.V1ContainerStatus: + return next(filter(lambda c: c.name == "mongodb-backup-daemon", backup_daemon_pod.status.container_statuses)) + + def wait_until_backup_pods_become_ready(self, timeout=300): + def backup_daemons_are_ready(): + try: + for _, backup_pod in self.read_backup_pods(): + if not MongoDBOpsManager.get_backup_daemon_container_status(backup_pod).ready: + return False + return True + except Exception as e: + print("Error checking if pod is ready: " + str(e)) + return False + + KubernetesTester.wait_until(backup_daemons_are_ready, timeout=timeout) + + def read_gen_key_secret(self, member_cluster_name: Optional[str] = None) -> kubernetes.client.V1Secret: + return kubernetes.client.CoreV1Api(get_member_cluster_api_client(member_cluster_name)).read_namespaced_secret( + self.name + "-gen-key", self.namespace + ) + + def read_api_key_secret(self, namespace=None) -> kubernetes.client.V1Secret: + """Reads the API key secret for the global admin created by the Operator. Note, that the secret is + located in the Operator namespace - not Ops Manager one, so the 'namespace' parameter must be passed + if the Ops Manager is installed in a separate namespace""" + if namespace is None: + namespace = self.namespace + return kubernetes.client.CoreV1Api().read_namespaced_secret(self.api_key_secret(namespace), namespace) + + def read_appdb_generated_password_secret(self) -> kubernetes.client.V1Secret: + return kubernetes.client.CoreV1Api().read_namespaced_secret(self.app_db_name() + "-om-password", self.namespace) + + def read_appdb_generated_password(self) -> str: + data = self.read_appdb_generated_password_secret().data + return KubernetesTester.decode_secret(data)["password"] + + def read_appdb_agent_password_secret(self) -> kubernetes.client.V1Secret: + return kubernetes.client.CoreV1Api().read_namespaced_secret( + self.app_db_name() + "-agent-password", self.namespace + ) + + def read_appdb_agent_keyfile_secret(self) -> kubernetes.client.V1Secret: + return kubernetes.client.CoreV1Api().read_namespaced_secret(self.app_db_name() + "-keyfile", self.namespace) + + def read_appdb_connection_url(self) -> str: + secret = kubernetes.client.CoreV1Api().read_namespaced_secret( + self.get_appdb_connection_url_secret_name(), self.namespace + ) + return KubernetesTester.decode_secret(secret.data)["connectionString"] + + def read_appdb_members_from_connection_url_secret(self) -> str: + return re.findall(r"[@,]([^@,\/]+)", self.read_appdb_connection_url()) + + def create_admin_secret( + self, + user_name="jane.doe@example.com", + password="Passw0rd.", + first_name="Jane", + last_name="Doe", + api_client: Optional[kubernetes.client.ApiClient] = None, + ): + data = { + "Username": user_name, + "Password": password, + "FirstName": first_name, + "LastName": last_name, + } + create_or_update_secret(self.namespace, self.get_admin_secret_name(), data, api_client=api_client) + + def get_automation_config_tester(self) -> AutomationConfigTester: + api_client = None + if self.is_appdb_multi_cluster(): + cluster_name = self.pick_one_appdb_member_cluster_name() + api_client = get_member_cluster_client_map()[cluster_name].api_client + + secret = ( + kubernetes.client.CoreV1Api(api_client=api_client) + .read_namespaced_secret(self.app_db_name() + "-config", self.namespace) + .data + ) + automation_config_str = b64decode(secret["cluster-config.json"]).decode("utf-8") + return AutomationConfigTester(json.loads(automation_config_str)) + + def get_or_create_mongodb_connection_config_map( + self, + mongodb_name: str, + project_name: str, + namespace=None, + api_client: Optional[kubernetes.client.ApiClient] = None, + ) -> str: + """Creates the configmap containing the information needed to connect to OM""" + config_map_name = f"{mongodb_name}-config" + data = { + "baseUrl": self.om_status().get_url(), + "projectName": project_name, + "orgId": "", + } + + # the namespace can be different from OM one if the MongoDB is created in a separate namespace + if namespace is None: + namespace = self.namespace + + try: + create_configmap(namespace, config_map_name, data, api_client=api_client) + except ApiException as e: + if e.status != 409: + raise + + # If the ConfigMap already exist, it will be updated with + # an updated status_url() + KubernetesTester.update_configmap(namespace, config_map_name, data) + + return config_map_name + + def get_om_tester( + self, + project_name: Optional[str] = None, + base_url: Optional[str] = None, + api_client: Optional[kubernetes.client.ApiClient] = None, + ) -> OMTester: + """Returns the instance of OMTester helping to check the state of Ops Manager deployed in Kubernetes.""" + agent_api_key = self.agent_api_key(api_client) + api_key_secret = read_secret( + KubernetesTester.get_namespace(), + self.api_key_secret(KubernetesTester.get_namespace(), api_client=api_client), + api_client=api_client, + ) + + # Check if it's an old stile secret or a new one + if "publicApiKey" in api_key_secret: + om_context = OMContext( + self.om_status().get_url() if not base_url else base_url, + api_key_secret["user"], + api_key_secret["publicApiKey"], + project_name=project_name, + agent_api_key=agent_api_key, + ) + else: + om_context = OMContext( + self.om_status().get_url() if not base_url else base_url, + api_key_secret["publicKey"], + api_key_secret["privateKey"], + project_name=project_name, + agent_api_key=agent_api_key, + ) + return OMTester(om_context) + + def get_appdb_service_names_in_multi_cluster(self) -> list[str]: + cluster_indexes_with_members = self.get_appdb_member_cluster_indexes_with_member_count() + for _, cluster_spec_item in self.get_appdb_indexed_cluster_spec_items(): + return multi_cluster_service_names(self.app_db_name(), cluster_indexes_with_members) + + def get_appdb_member_cluster_indexes_with_member_count(self) -> list[tuple[int, int]]: + return [ + (cluster_index, int(cluster_spec_item["members"])) + for cluster_index, cluster_spec_item in self.get_appdb_indexed_cluster_spec_items() + ] + + def get_appdb_tester(self, **kwargs) -> MongoTester: + if self.is_appdb_multi_cluster(): + return MultiReplicaSetTester( + service_names=self.get_appdb_service_names_in_multi_cluster(), + port="27017", + namespace=self.namespace, + **kwargs, + ) + else: + return ReplicaSetTester( + self.app_db_name(), + replicas_count=self.appdb_status().get_members(), + **kwargs, + ) + + def pod_urls(self): + """Returns http urls to each pod in the Ops Manager""" + return [ + "http://{}".format(host) + for host in build_list_of_hosts( + self.name, self.namespace, self.get_total_number_of_om_replicas(), port=8080 + ) + ] + + def set_version(self, version: Optional[str]): + """Sets a specific `version` if set. If `version` is None, then skip.""" + if version is not None: + self["spec"]["version"] = version + return self + + def update_key_to_programmatic(self): + """ + Attempts to create a Programmatic API Key to be used after updating to + newer OM5, which don't support old-style API Key. + """ + + url = self.om_status().get_url() + whitelist_endpoint = f"{url}/api/public/v1.0/admin/whitelist" + headers = {"Content-Type": "application/json", "Accept": "application/json"} + whitelist_entries = [ + {"cidrBlock": "0.0.0.0/1", "description": "first block"}, + {"cidrBlock": "128.0.0.0/1", "description": "second block"}, + ] + + secret_name = self.api_key_secret(self.namespace) + current_creds = read_secret(self.namespace, secret_name) + user = current_creds["user"] + password = current_creds["publicApiKey"] + auth = HTTPDigestAuth(user, password) + + for entry in whitelist_entries: + response = requests.post(whitelist_endpoint, json=entry, headers=headers, auth=auth) + assert response.status_code == 200 + + data = { + "desc": "Creating a programmatic API key before updating to 5.0.0", + "roles": ["GLOBAL_OWNER"], + } + + endpoint = f"{url}/api/public/v1.0/admin/apiKeys" + response = requests.post(endpoint, json=data, headers=headers, auth=auth) + response_data = response.json() + if "privateKey" not in response_data: + assert response_data == {} + + new_creds = { + "publicApiKey": response_data["privateKey"], + "user": response_data["publicKey"], + } + + KubernetesTester.update_secret(self.namespace, secret_name, new_creds) + + def allow_mdb_rc_versions(self): + """ + Sets configurations parameters for OM to be able to download RC versions. + """ + + if "configuration" not in self["spec"]: + self["spec"]["configuration"] = {} + + self["spec"]["configuration"]["mms.featureFlag.automation.mongoDevelopmentVersions"] = "enabled" + self["spec"]["configuration"]["mongodb.release.autoDownload.rc"] = "true" + self["spec"]["configuration"]["mongodb.release.autoDownload.development"] = "true" + + def set_appdb_version(self, version: str): + self["spec"]["applicationDatabase"]["version"] = version + + def __repr__(self): + # FIX: this should be __unicode__ + return "MongoDBOpsManager| status:".format(self.get_status()) + + def get_appdb_members_count(self) -> int: + if self.is_appdb_multi_cluster(): + return sum(i[1] for i in self.get_appdb_member_cluster_indexes_with_member_count()) + return self["spec"]["applicationDatabase"]["members"] + + def get_appdb_connection_url_secret_name(self): + return f"{self.app_db_name()}-connection-string" + + def get_total_number_of_om_replicas(self) -> int: + if not self.is_om_multi_cluster(): + return self["spec"]["replicas"] + + return sum([item["members"] for _, item in self.get_om_indexed_cluster_spec_items()]) + + def get_om_replicas_in_member_cluster(self, member_cluster_name: Optional[str] = None) -> int: + if is_member_cluster(member_cluster_name): + return self.get_om_cluster_spec_item(member_cluster_name)["members"] + + return self["spec"]["replicas"] + + def get_backup_members_count(self, member_cluster_name: Optional[str] = None) -> int: + if not self["spec"].get("backup", {}).get("enabled", False): + return 0 + + if is_member_cluster(member_cluster_name): + cluster_spec_item = self.get_om_cluster_spec_item(member_cluster_name) + members = cluster_spec_item.get("backup", {}).get("members", None) + if members is not None: + return members + + return self["spec"]["backup"].get("members", 0) + + def get_admin_secret_name(self) -> str: + return self["spec"]["adminCredentials"] + + def get_version(self) -> str: + return self["spec"]["version"] + + def get_status(self) -> Optional[str]: + if "status" not in self: + return None + return self["status"] + + def api_key_secret(self, namespace: str, api_client: Optional[kubernetes.client.ApiClient] = None) -> str: + old_secret_name = self.name + "-admin-key" + + # try to read the old secret, if it's present return it, else return the new secret name + try: + kubernetes.client.CoreV1Api(api_client=api_client).read_namespaced_secret(old_secret_name, namespace) + except ApiException as e: + if e.status == 404: + return "{}-{}-admin-key".format(self.namespace, self.name) + + return old_secret_name + + def agent_api_key(self, api_client: Optional[kubernetes.client.ApiClient] = None) -> str: + secret_name = None + member_cluster = self.pick_one_appdb_member_cluster_name() + appdb_sts = self.read_appdb_statefulset(member_cluster_name=member_cluster) + + for volume in appdb_sts.spec.template.spec.volumes: + if volume.name == "agent-api-key": + secret_name = volume.secret.secret_name + break + + if secret_name == None: + return None + + return read_secret(self.namespace, secret_name, get_member_cluster_api_client(member_cluster))["agentApiKey"] + + def om_sts_name(self, member_cluster_name: Optional[str] = None) -> str: + if is_member_cluster(member_cluster_name): + cluster_idx = self.get_om_member_cluster_index(member_cluster_name) + return f"{self.name}-{cluster_idx}" + else: + return self.name + + def om_pod_name(self, member_cluster_name: str, pod_idx: int) -> str: + if is_member_cluster(member_cluster_name): + cluster_idx = self.get_om_member_cluster_index(member_cluster_name) + return f"{self.name}-{cluster_idx}-{pod_idx}" + else: + return f"{self.name}-{pod_idx}" + + def app_db_name(self) -> str: + return self.name + "-db" + + def app_db_sts_name(self, member_cluster_name: Optional[str] = None) -> str: + if is_member_cluster(member_cluster_name): + cluster_idx = self.get_appdb_member_cluster_index(member_cluster_name) + return f"{self.name}-db-{cluster_idx}" + else: + return self.name + "-db" + + def get_om_member_cluster_index(self, member_cluster_name: str) -> int: + for cluster_idx, cluster_spec_item in self.get_om_indexed_cluster_spec_items(): + if cluster_spec_item["clusterName"] == member_cluster_name: + return cluster_idx + raise Exception(f"member cluster {member_cluster_name} not found in OM cluster spec items") + + def get_appdb_member_cluster_index(self, member_cluster_name: str) -> int: + for ( + cluster_idx, + cluster_spec_item, + ) in self.get_appdb_indexed_cluster_spec_items(): + if cluster_spec_item["clusterName"] == member_cluster_name: + return cluster_idx + + raise Exception(f"member cluster {member_cluster_name} not found in AppDB cluster spec items") + + def app_db_password_secret_name(self) -> str: + return self.app_db_name() + "-om-user-password" + + def backup_daemon_sts_name(self, member_cluster_name: Optional[str] = None) -> str: + return self.om_sts_name(member_cluster_name) + "-backup-daemon" + + def backup_daemon_pods_headless_fqdns(self) -> list[str]: + fqdns = [] + for member_cluster_name in self.get_om_member_cluster_names(): + member_fqdns = [ + f"{pod_name}.{self.backup_daemon_sts_name(member_cluster_name)}-svc.{self.namespace}.svc.cluster.local" + for _, pod_name in self.backup_daemon_pod_names(member_cluster_name=member_cluster_name) + ] + fqdns.extend(member_fqdns) + + return fqdns + + def svc_name(self, member_cluster_name: Optional[str] = None) -> str: + return self.name + "-svc" + + def external_svc_name(self, member_cluster_name: Optional[str] = None) -> str: + return self.name + "-svc-ext" + + def download_mongodb_binaries(self, version: str): + """Downloads mongodb binary in each OM pod, optional downloads MongoDB Tools""" + distros = [ + f"mongodb-linux-x86_64-rhel80-{version}.tgz", + f"mongodb-linux-x86_64-rhel8-{version}.tgz", + f"mongodb-linux-x86_64-ubuntu1604-{version}.tgz", + f"mongodb-linux-x86_64-ubuntu1804-{version}.tgz", + ] + + for api_client, pod in self.read_om_pods(): + for distro in distros: + cmd = [ + "curl", + "-L", + f"https://fastdl.mongodb.org/linux/{distro}", + "-o", + f"/mongodb-ops-manager/mongodb-releases/{distro}", + ] + + KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + self.namespace, + cmd, + container="mongodb-ops-manager", + api_client=api_client, + ) + + def is_appdb_multi_cluster(self): + return self["spec"].get("applicationDatabase", {}).get("topology", "") == "MultiCluster" + + def is_om_multi_cluster(self): + return self["spec"].get("topology", "") == "MultiCluster" + + class StatusCommon: + def assert_reaches_phase( + self, + phase: Phase, + msg_regexp=None, + timeout=None, + ignore_errors=False, + ): + intermediate_events = ( + # This can be an intermediate error, right before we check for this secret we create it. + # The cluster might just be slow + "failed to locate the api key secret", + ) + + start_time = time.time() + self.ops_manager.wait_for( + lambda s: in_desired_state( + current_state=self.get_phase(), + desired_state=phase, + current_generation=self.ops_manager.get_generation(), + observed_generation=self.get_observed_generation(), + current_message=self.get_message(), + msg_regexp=msg_regexp, + ignore_errors=ignore_errors, + intermediate_events=intermediate_events, + ), + timeout, + should_raise=True, + ) + end_time = time.time() + span = trace.get_current_span() + span.set_attribute("meko_resource", self.__class__.__name__) + span.set_attribute("meko_action", "assert_phase") + span.set_attribute("meko_desired_phase", phase.name) + span.set_attribute("meko_time_needed", end_time - start_time) + logger.debug( + f"Reaching phase {phase.name} for resource {self.__class__.__name__} took {end_time - start_time}s" + ) + + def assert_abandons_phase(self, phase: Phase, timeout=None): + return self.ops_manager.wait_for(lambda s: self.get_phase() != phase, timeout, should_raise=True) + + def assert_status_resource_not_ready(self, name: str, kind: str = "StatefulSet", msg_regexp=None, idx=0): + """Checks the element in 'resources_not_ready' field by index 'idx'""" + assert self.get_resources_not_ready()[idx]["kind"] == kind + assert self.get_resources_not_ready()[idx]["name"] == name + assert re.search(msg_regexp, self.get_resources_not_ready()[idx]["message"]) is not None + + def assert_empty_status_resources_not_ready(self): + assert self.get_resources_not_ready() is None + + class BackupStatus(StatusCommon): + def __init__(self, ops_manager: MongoDBOpsManager): + self.ops_manager = ops_manager + + def assert_abandons_phase(self, phase: Phase, timeout=400): + super().assert_abandons_phase(phase, timeout) + + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=800, ignore_errors=True): + super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors) + + def get_phase(self) -> Optional[Phase]: + try: + return Phase[self.ops_manager.get_status()["backup"]["phase"]] + except (KeyError, TypeError): + return None + + def get_message(self) -> Optional[str]: + try: + return self.ops_manager.get_status()["backup"]["message"] + except (KeyError, TypeError): + return None + + def get_observed_generation(self) -> Optional[int]: + try: + return self.ops_manager.get_status()["backup"]["observedGeneration"] + except (KeyError, TypeError): + return None + + def get_resources_not_ready(self) -> Optional[List[Dict]]: + try: + return self.ops_manager.get_status()["backup"]["resourcesNotReady"] + except (KeyError, TypeError): + return None + + class AppDbStatus(StatusCommon): + def __init__(self, ops_manager: MongoDBOpsManager): + self.ops_manager = ops_manager + + def assert_abandons_phase(self, phase: Phase, timeout=400): + super().assert_abandons_phase(phase, timeout) + + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=1000, ignore_errors=False): + super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors) + + def get_phase(self) -> Optional[Phase]: + try: + return Phase[self.ops_manager.get_status()["applicationDatabase"]["phase"]] + except (KeyError, TypeError): + return None + + def get_message(self) -> Optional[str]: + try: + return self.ops_manager.get_status()["applicationDatabase"]["message"] + except (KeyError, TypeError): + return None + + def get_observed_generation(self) -> Optional[int]: + try: + return self.ops_manager.get_status()["applicationDatabase"]["observedGeneration"] + except (KeyError, TypeError): + return None + + def get_version(self) -> Optional[str]: + try: + return self.ops_manager.get_status()["applicationDatabase"]["version"] + except (KeyError, TypeError): + return None + + def get_members(self) -> Optional[int]: + try: + return self.ops_manager.get_status()["applicationDatabase"]["members"] + except (KeyError, TypeError): + return None + + def get_resources_not_ready(self) -> Optional[List[Dict]]: + try: + return self.ops_manager.get_status()["applicationDatabase"]["resourcesNotReady"] + except (KeyError, TypeError): + return None + + class OmStatus(StatusCommon): + def __init__(self, ops_manager: MongoDBOpsManager): + self.ops_manager = ops_manager + + def assert_abandons_phase(self, phase: Phase, timeout=400): + super().assert_abandons_phase(phase, timeout) + + def assert_reaches_phase(self, phase: Phase, msg_regexp=None, timeout=1200, ignore_errors=False): + super().assert_reaches_phase(phase, msg_regexp, timeout, ignore_errors) + + def get_phase(self) -> Optional[Phase]: + try: + return Phase[self.ops_manager.get_status()["opsManager"]["phase"]] + except (KeyError, TypeError): + return None + + def get_message(self) -> Optional[str]: + try: + return self.ops_manager.get_status()["opsManager"]["message"] + except (KeyError, TypeError): + return None + + def get_observed_generation(self) -> Optional[int]: + try: + return self.ops_manager.get_status()["opsManager"]["observedGeneration"] + except (KeyError, TypeError): + return None + + def get_last_transition(self) -> Optional[int]: + try: + return self.ops_manager.get_status()["opsManager"]["lastTransition"] + except (KeyError, TypeError): + return None + + def get_url(self) -> Optional[str]: + try: + return self.ops_manager.get_status()["opsManager"]["url"] + except (KeyError, TypeError): + return None + + def get_replicas(self) -> Optional[int]: + try: + return self.ops_manager.get_status()["opsManager"]["replicas"] + except (KeyError, TypeError): + return None + + def get_resources_not_ready(self) -> Optional[List[Dict]]: + try: + return self.ops_manager.get_status()["opsManager"]["resourcesNotReady"] + except (KeyError, TypeError): + return None diff --git a/docker/mongodb-enterprise-tests/kubetester/security_context.py b/docker/mongodb-enterprise-tests/kubetester/security_context.py new file mode 100644 index 000000000..b44930f11 --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/security_context.py @@ -0,0 +1,31 @@ +from kubernetes import client + + +def assert_pod_security_context(pod: client.V1Pod, managed: bool): + sc = pod.spec.security_context + + if managed: + # Note, that this code is a bit fragile as may depend on the version of Openshift. + assert sc.fs_group != 2000 + # se_linux_options is set on Openshift + assert sc.se_linux_options.level is not None + assert sc.se_linux_options.role is None + assert sc.se_linux_options.type is None + assert sc.run_as_non_root + assert sc.run_as_user == 2000 + else: + assert sc.fs_group == 2000 + # In Kops and Kind se_linux_options is set to None + assert sc.se_linux_options is None + + assert sc.run_as_group is None + + +def assert_pod_container_security_context(container: client.V1Container, managed: bool): + sc = container.security_context + + if managed: + assert sc is None + else: + assert sc.read_only_root_filesystem + assert sc.allow_privilege_escalation is False diff --git a/docker/mongodb-enterprise-tests/kubetester/test_identifiers.py b/docker/mongodb-enterprise-tests/kubetester/test_identifiers.py new file mode 100644 index 000000000..8ac13c9cd --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/test_identifiers.py @@ -0,0 +1,53 @@ +import os +from typing import Any + +import yaml +from kubetester.kubetester import running_locally + +test_identifiers: dict = None + + +def set_test_identifier(identifier: str, value: Any) -> Any: + """ + Persists random test identifier for subsequent local runs. + + Useful for creating resources with random names, e.g., S3 buckets. + When the identifier exists, it disregards passed value and returns saved one. + Appends identifier to .test_identifier file in working directory. + """ + global test_identifiers + + if not running_locally(): + return value + + # this check is for in-memory cache, if the value already exists, we're trying to set value again for the same key + if test_identifiers is not None and identifier in test_identifiers: + raise Exception( + f"cannot override {identifier} test identifier, existing value: {test_identifiers[identifier]}, new value: {value}." + f"There is a high chance this function is called multiple times " + f"in the same test case with the same bucket-name/identifier." + f"The solution is to find the method calls and give each bucket a unique name." + ) + + # test_identifiers is an in-memory cache/global that makes + # sure we don't generate multiple bucket names for the same test bucket + if test_identifiers is None: + test_identifiers = dict() + + test_identifiers_local = dict() + test_identifiers_file = ".test_identifiers" + if os.path.exists(test_identifiers_file): + with open("%s" % test_identifiers_file) as f: + test_identifiers_local = yaml.safe_load(f) + + # We have found the bucket in the cache. Let's re-use it and save it to the in-memory cache. + if identifier in test_identifiers_local: + test_identifiers[identifier] = test_identifiers_local[identifier] + else: + # The bucket is not in the cache. Let's save it in-memory and to the file cache. + test_identifiers[identifier] = value + test_identifiers_local[identifier] = value + with open("%s" % test_identifiers_file, "w") as f: + yaml.dump(test_identifiers_local, f) + + return test_identifiers[identifier] diff --git a/scripts/git-hooks/applypatch-msg b/docker/mongodb-enterprise-tests/kubetester/tests/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/applypatch-msg rename to docker/mongodb-enterprise-tests/kubetester/tests/__init__.py diff --git a/docker/mongodb-enterprise-tests/kubetester/tests/test___init__.py b/docker/mongodb-enterprise-tests/kubetester/tests/test___init__.py new file mode 100644 index 000000000..9504556dd --- /dev/null +++ b/docker/mongodb-enterprise-tests/kubetester/tests/test___init__.py @@ -0,0 +1,159 @@ +import unittest +from unittest.mock import MagicMock + +import kubernetes.client +from kubeobject import CustomObject + + +class TestCreateOrUpdate(ctx, unittest.TestCase): + def test_create_or_update_is_not_bound(self): + api_client = MagicMock() + custom_object = CustomObject( + api_client=api_client, + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + custom_object.bound = False + custom_object.create = MagicMock() + + custom_object.update() + + custom_object.create.assert_called_once() + + def test_create_or_update_is_not_bound_exists_update(self): + api_client = MagicMock() + custom_object = CustomObject( + api_client=api_client, + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + custom_object.bound = False + custom_object.create = MagicMock() + custom_object.update = MagicMock() + + custom_object.create.side_effect = kubernetes.client.ApiException(status=409) + custom_object.update() + + custom_object.update.assert_called_once() + custom_object.create.assert_called_once() + + def test_create_or_update_is_bound_update(self): + api_client = MagicMock() + custom_object = CustomObject( + api_client=api_client, + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + custom_object.bound = True + custom_object.update = MagicMock() + custom_object.load = MagicMock() + + custom_object.update() + custom_object.update.assert_called_once() + custom_object.load.assert_not_called() + + def test_create_or_update_is_bound_update_409_10_times(self): + custom_object = CustomObject( + api_client=MagicMock(), + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + loaded_object = CustomObject( + api_client=MagicMock(), + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + loaded_object["spec"] = {"my": "test"} + + custom_object.bound = True + custom_object.update = MagicMock() + custom_object.api.get_namespaced_custom_object = MagicMock() + custom_object.api.get_namespaced_custom_object.return_value = loaded_object + + exception_count = 0 + + def raise_exception(): + nonlocal exception_count + exception_count += 1 + raise kubernetes.client.ApiException(status=409) + + custom_object.update.side_effect = raise_exception + + with self.assertRaises(Exception) as context: + custom_object.update() + self.assertTrue("Tried client side merge" in str(context.exception)) + + custom_object.update.assert_called() + assert exception_count == 10 + + def test_create_or_update_is_bound_update_409_few_times(self): + api_client = MagicMock() + object_to_api_server = CustomObject( + api_client=api_client, + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + object_from_api_server = CustomObject( + api_client=api_client, + name="mock", + namespace="mock", + plural="mock", + kind="mock", + group="mock", + version="v1", + ) + + object_to_api_server["spec"] = {"override": "test"} + object_to_api_server["status"] = {"status": "pending"} + + object_from_api_server["spec"] = {"my": "test"} + object_from_api_server["status"] = {"status": "running"} + + object_to_api_server.bound = True + object_to_api_server.update = MagicMock() + object_to_api_server.api.get_namespaced_custom_object = MagicMock() + object_to_api_server.api.get_namespaced_custom_object.return_value = object_from_api_server + + exception_count = 0 + + def raise_exception(): + nonlocal exception_count + exception_count += 1 + if exception_count < 3: + raise kubernetes.client.ApiException(status=409) + + object_to_api_server.update.side_effect = raise_exception + + object_to_api_server.update() + object_to_api_server.update.assert_called() + object_to_api_server.api.get_namespaced_custom_object.assert_called() + + # assert specs were taken from object_to_api_server + assert object_to_api_server["spec"] == {"override": "test"} + + # assert status is taken from object_from_api_server + assert object_to_api_server["status"] == {"status": "running"} diff --git a/scripts/git-hooks/commit-msg b/docker/mongodb-enterprise-tests/kubetester/vault.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/commit-msg rename to docker/mongodb-enterprise-tests/kubetester/vault.py diff --git a/docker/mongodb-enterprise-tests/pytest.ini b/docker/mongodb-enterprise-tests/pytest.ini new file mode 100644 index 000000000..90d20706f --- /dev/null +++ b/docker/mongodb-enterprise-tests/pytest.ini @@ -0,0 +1,32 @@ +[pytest] + +# Run tests under the tests directory +testpaths = tests + +# Only files matching the following globs will be read as part of the test suite. +# The files under `mixed` match the *_test.py because they are generic in nature. +python_files = *.py + +# -s -- disable stdout capture: should be added to `PYTEST_ADDOPTS` if needed +# -x -- stop after first failure +# -v -- increase verbosity +# -rA -- show a summary at the end. That means all prints will be shown at the end of the test run, more here: https://docs.pytest.org/en/7.1.x/how-to/output.html#producing-a-detailed-summary-report +addopts = -x -rA -v --color=yes --setup-show --junitxml=/tmp/results/myreport.xml + +# Logging configuration +# log_cli = 1 # Set this to have logs printed right away instead of captured. +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +log_cli_date_format=%Y-%m-%d %H:%M:%S + +# by default, marking a test with xfail will not cause a test suite failure. +# setting it to true ensures a failed test suite on an unexpected pass. +xfail_strict = true + +# With newer 6.x version of pytest, the markers need to be defined in this file +# in a 'markers' list. +# To avoid this warning we can add the following filter, or add each custom mark. +# TODO: Start using filenames to invoke the tests, and not the markers; we have +# abused the concept too much and pytest 6.0 is complaining about it. +filterwarnings = + ignore::UserWarning diff --git a/scripts/git-hooks/fsmonitor-watchman b/docker/mongodb-enterprise-tests/tests/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/fsmonitor-watchman rename to docker/mongodb-enterprise-tests/tests/__init__.py diff --git a/scripts/git-hooks/post-update b/docker/mongodb-enterprise-tests/tests/authentication/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/post-update rename to docker/mongodb-enterprise-tests/tests/authentication/__init__.py diff --git a/docker/mongodb-enterprise-tests/tests/authentication/conftest.py b/docker/mongodb-enterprise-tests/tests/authentication/conftest.py new file mode 100644 index 000000000..f7bc5a63d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/conftest.py @@ -0,0 +1,240 @@ +import os +from typing import Dict, Generator, List, Optional + +from kubernetes import client +from kubetester import get_pod_when_ready, read_secret +from kubetester.certs import generate_cert +from kubetester.helm import helm_uninstall, helm_upgrade +from kubetester.ldap import ( + LDAPUser, + OpenLDAP, + add_user_to_group, + create_user, + ensure_group, + ensure_organization, + ensure_organizational_unit, + ldap_initialize, +) +from pytest import fixture +from tests.conftest import is_member_cluster + +LDAP_PASSWORD = "LDAPPassword." +LDAP_NAME = "openldap" +LDAP_POD_LABEL = "app=openldap" +LDAP_PORT_PLAIN = 389 +LDAP_PORT_TLS = 636 +LDAP_PROTO_PLAIN = "ldap" +LDAP_PROTO_TLS = "ldaps" + +AUTOMATION_AGENT_NAME = "mms-automation-agent" + + +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if "default_operator" not in item.fixturenames: + item.fixturenames.insert(0, "default_operator") + + +def openldap_install( + namespace: str, + name: str = LDAP_NAME, + cluster_client: Optional[client.ApiClient] = None, + cluster_name: Optional[str] = None, + helm_args: Dict[str, str] = None, + tls: bool = False, +) -> OpenLDAP: + if is_member_cluster(cluster_name): + os.environ["HELM_KUBECONTEXT"] = cluster_name + + if helm_args is None: + helm_args = {} + helm_args.update({"namespace": namespace, "fullnameOverride": name, "nameOverride": name}) + + # check if the openldap pod exists, if not do a helm upgrade + pods = client.CoreV1Api(api_client=cluster_client).list_namespaced_pod(namespace, label_selector=f"app={name}") + if not pods.items: + print(f"performing helm upgrade of openldap") + + helm_upgrade( + name=name, + namespace=namespace, + helm_args=helm_args, + helm_chart_path="vendor/openldap", + helm_override_path=True, + ) + get_pod_when_ready(namespace, f"app={name}", api_client=cluster_client) + if tls: + return OpenLDAP( + ldap_url(namespace, name, LDAP_PROTO_TLS, LDAP_PORT_TLS), + ldap_admin_password(namespace, name, api_client=cluster_client), + ) + + return OpenLDAP( + ldap_url(namespace, name), + ldap_admin_password(namespace, name, api_client=cluster_client), + ) + + +@fixture(scope="module") +def openldap_tls( + namespace: str, + openldap_cert: str, + ca_path: str, +) -> OpenLDAP: + """Installs an OpenLDAP server with TLS configured and returns a reference to it. + + In order to do it, this fixture will install the vendored openldap Helm chart + located in `vendor/openldap` directory inside the `tests` container image. + """ + + helm_args = { + "tls.enabled": "true", + "tls.secret": openldap_cert, + # Do not require client certificates + "env.LDAP_TLS_VERIFY_CLIENT": "never", + "namespace": namespace, + } + server = openldap_install(namespace, name=LDAP_NAME, helm_args=helm_args, tls=True) + # When creating a new OpenLDAP container with TLS enabled, the container is ready, but the server is not accepting + # requests, as it's generating DH parameters for the TLS config. Only using retries!=0 for ldap_initialize when creating + # the OpenLDAP server. + ldap_initialize(server, ca_path=ca_path, retries=10) + return server + + +@fixture(scope="module") +def openldap(namespace: str) -> OpenLDAP: + """Installs a OpenLDAP server and returns a reference to it. + + In order to do it, this fixture will install the vendored openldap Helm chart + located in `vendor/openldap` directory inside the `tests` container image. + """ + ref = openldap_install(namespace, LDAP_NAME) + print(f"Returning OpenLDAP=: {ref}") + return ref + + +@fixture(scope="module") +def secondary_openldap(namespace: str) -> OpenLDAP: + return openldap_install(namespace, f"{LDAP_NAME}secondary") + + +@fixture(scope="module") +def openldap_cert(namespace: str, issuer: str) -> str: + """Returns a new secret to be used to enable TLS on LDAP.""" + host = ldap_host(namespace, LDAP_NAME) + return generate_cert(namespace, "openldap", host, issuer) + + +@fixture(scope="module") +def ldap_mongodb_user_tls(openldap_tls: OpenLDAP, ca_path: str) -> LDAPUser: + user = LDAPUser("mdb0", LDAP_PASSWORD) + create_user(openldap_tls, user, ca_path=ca_path) + return user + + +@fixture(scope="module") +def ldap_mongodb_x509_agent_user(openldap: OpenLDAP, namespace: str, ca_path: str) -> LDAPUser: + organization_name = "cluster.local-agent" + user = LDAPUser( + AUTOMATION_AGENT_NAME, + LDAP_PASSWORD, + ) + + ensure_organization(openldap, organization_name, ca_path=ca_path) + + ensure_organizational_unit(openldap, namespace, o=organization_name, ca_path=ca_path) + create_user(openldap, user, ou=namespace, o=organization_name) + + ensure_group( + openldap, + cn=AUTOMATION_AGENT_NAME, + ou=namespace, + o=organization_name, + ca_path=ca_path, + ) + + add_user_to_group( + openldap, + user=AUTOMATION_AGENT_NAME, + group_cn=AUTOMATION_AGENT_NAME, + ou=namespace, + o=organization_name, + ) + return user + + +@fixture(scope="module") +def ldap_mongodb_agent_user(openldap: OpenLDAP) -> LDAPUser: + user = LDAPUser(AUTOMATION_AGENT_NAME, LDAP_PASSWORD) + + ensure_organizational_unit(openldap, "groups") + create_user(openldap, user, ou="groups") + + ensure_group(openldap, cn="agents", ou="groups") + add_user_to_group(openldap, user=AUTOMATION_AGENT_NAME, group_cn="agents", ou="groups") + + return user + + +@fixture(scope="module") +def secondary_ldap_mongodb_agent_user(secondary_openldap: OpenLDAP) -> LDAPUser: + user = LDAPUser(AUTOMATION_AGENT_NAME, LDAP_PASSWORD) + + ensure_organizational_unit(secondary_openldap, "groups") + create_user(secondary_openldap, user, ou="groups") + + ensure_group(secondary_openldap, cn="agents", ou="groups") + add_user_to_group(secondary_openldap, user=AUTOMATION_AGENT_NAME, group_cn="agents", ou="groups") + + return user + + +@fixture(scope="module") +def ldap_mongodb_user(openldap: OpenLDAP) -> LDAPUser: + print(f"Creating LDAP user {openldap}") + user = LDAPUser("mdb0", LDAP_PASSWORD) + + ensure_organizational_unit(openldap, "groups") + create_user(openldap, user, ou="groups") + + ensure_group(openldap, cn="users", ou="groups") + add_user_to_group(openldap, user="mdb0", group_cn="users", ou="groups") + + return user + + +@fixture(scope="module") +def ldap_mongodb_users(openldap: OpenLDAP) -> List[LDAPUser]: + user_list = [LDAPUser("mdb0", LDAP_PASSWORD)] + for user in user_list: + create_user(openldap, user) + + return user_list + + +@fixture(scope="module") +def secondary_ldap_mongodb_users(secondary_openldap: OpenLDAP) -> List[LDAPUser]: + user_list = [LDAPUser("mdb0", LDAP_PASSWORD)] + for user in user_list: + create_user(secondary_openldap, user) + + return user_list + + +def ldap_host(namespace: str, name: str) -> str: + return "{}.{}.svc.cluster.local".format(name, namespace) + + +def ldap_url( + namespace: str, + name: str, + proto: str = LDAP_PROTO_PLAIN, + port: int = LDAP_PORT_PLAIN, +) -> str: + host = ldap_host(namespace, name) + return "{}://{}:{}".format(proto, host, port) + + +def ldap_admin_password(namespace: str, name: str, api_client: Optional[client.ApiClient] = None) -> str: + return read_secret(namespace, name, api_client=api_client)["LDAP_ADMIN_PASSWORD"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-agent-auth.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-agent-auth.yaml new file mode 100644 index 000000000..de23b5f2e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-agent-auth.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: ldap-replica-set +spec: + type: ReplicaSet + members: 3 + version: 4.4.4-ent + + # make persistent by default to speed up the test + + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + + agents: + mode: "" + + modes: ["LDAP", "SCRAM"] + ldap: + bindQueryUser: "" + servers: "" + bindQueryPasswordSecretRef: + name: "" + transportSecurity: "" diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-replica-set-roles.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-replica-set-roles.yaml new file mode 100644 index 000000000..242b831cc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-replica-set-roles.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: ldap-replica-set +spec: + type: ReplicaSet + members: 3 + version: 4.4.0-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + agents: + mode: "SCRAM" + enabled: true + modes: ["LDAP", "SCRAM"] + ldap: + bindQueryUser: "" + servers: "" + bindQueryPasswordSecretRef: + name: "" + transportSecurity: "" + roles: + - role: "cn=users,ou=groups,dc=example,dc=org" + db: admin + privileges: + - actions: + - insert + resource: + collection: foo + db: foo + - actions: + - insert + - find + resource: + collection: "" + db: admin + diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-replica-set.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-replica-set.yaml new file mode 100644 index 000000000..725d8c1d6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-replica-set.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: ldap-replica-set +spec: + type: ReplicaSet + members: 3 + version: 4.4.0-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + agents: + mode: "SCRAM" + enabled: true + # Enabled LDAP and SCRAM Authentication Mode + modes: ["LDAP", "SCRAM"] + ldap: + servers: "" + transportSecurity: "" + bindQueryUser: "" + bindQueryPasswordSecretRef: + name: "" + \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-sharded-cluster-user.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-sharded-cluster-user.yaml new file mode 100644 index 000000000..23cd96f21 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-sharded-cluster-user.yaml @@ -0,0 +1,17 @@ +--- +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: ldap-user-0 + spec: + username: "uid=mdb0,dc=example,dc=org" + db: "$external" + mongodbResourceRef: + name: ldap-sharded-cluster + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "readWriteAnyDatabase" + - db: "admin" + name: "dbAdminAnyDatabase" diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-sharded-cluster.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-sharded-cluster.yaml new file mode 100644 index 000000000..261822776 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-sharded-cluster.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: ldap-sharded-cluster +spec: + type: ShardedCluster + + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 1 + configServerCount: 3 + + version: 4.4.0-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + # Enabled LDAP Authentication Mode + modes: ["LDAP"] + ldap: + servers: "" + transportSecurity: "tls" + # Specify the LDAP Distinguished Name to which + # MongoDB binds when connecting to the LDAP server + bindQueryUser: "cn=admin,dc=example,dc=org" + bindQueryPasswordSecretRef: + name: "" diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-user.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-user.yaml new file mode 100644 index 000000000..1a885070f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/ldap/ldap-user.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: ldap-user-0 +spec: + username: "uid=mdb0,dc=example,dc=org" + db: "$external" + mongodbResourceRef: + name: ldap-replica-set + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "readWriteAnyDatabase" + - db: "admin" + name: "dbAdminAnyDatabase" diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-basic.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-basic.yaml new file mode 100644 index 000000000..23a147fc0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-basic.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: replica-set +spec: + members: 3 + version: 4.4.22 + type: ReplicaSet + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-explicit-scram-sha-1.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-explicit-scram-sha-1.yaml new file mode 100644 index 000000000..6f675c15b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-explicit-scram-sha-1.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-scram-sha-1 +spec: + members: 3 + version: 5.0.5 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + authentication: + agents: + # This may look weird, but without it we'll get this from OpsManager: + # Cannot configure SCRAM-SHA-1 without using MONGODB-CR in te Agent Mode","reason":"Cannot configure SCRAM-SHA-1 without using MONGODB-CR in te Agent Mode + mode: MONGODB-CR + enabled: true + modes: ["SCRAM-SHA-1", "MONGODB-CR"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram-sha-256-x509-internal-cluster.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram-sha-256-x509-internal-cluster.yaml new file mode 100644 index 000000000..035eb9818 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram-sha-256-x509-internal-cluster.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + tls: + enabled: true + authentication: + agents: + mode: SCRAM + enabled: true + modes: ["SCRAM", "X509"] + internalCluster: X509 diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram-sha-256.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram-sha-256.yaml new file mode 100644 index 000000000..e659aa4d4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram-sha-256.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + authentication: + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram.yaml new file mode 100644 index 000000000..780f7aba5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-scram.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-scram +spec: + members: 3 + version: 4.4.2 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + authentication: + agents: + mode: SCRAM + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-tls-scram-sha-256.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-tls-scram-sha-256.yaml new file mode 100644 index 000000000..17bba38ca --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-tls-scram-sha-256.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: replica-set-scram-256-and-x509 +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + tls: + enabled: true + authentication: + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-x509-to-scram-256.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-x509-to-scram-256.yaml new file mode 100644 index 000000000..d6390ea17 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/replica-set-x509-to-scram-256.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: replica-set-x509-to-scram-256 +spec: + members: 3 + version: 4.4.0-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + tls: + enabled: true + authentication: + agents: + mode: X509 + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/scram-sha-user.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/scram-sha-user.yaml new file mode 100644 index 000000000..8796bdbac --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/scram-sha-user.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mms-user-1 +spec: + passwordSecretKeyRef: + name: mms-user-1-password + key: password + username: "mms-user-1" + db: "admin" + mongodbResourceRef: + name: "my-replica-set" + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "userAdminAnyDatabase" + - db: "admin" + name: "readWrite" diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-explicit-scram-sha-1.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-explicit-scram-sha-1.yaml new file mode 100644 index 000000000..f4070ad60 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-explicit-scram-sha-1.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster-scram-sha-1 +spec: + shardCount: 1 + type: ShardedCluster + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 5.0.5 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: true + security: + authentication: + agents: + # This may look weird, but without it we'll get this from OpsManager: + # Cannot configure SCRAM-SHA-1 without using MONGODB-CR in te Agent Mode","reason":"Cannot configure SCRAM-SHA-1 without using MONGODB-CR in te Agent Mode + mode: MONGODB-CR + enabled: true + modes: ["SCRAM-SHA-1", "MONGODB-CR"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-1.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-1.yaml new file mode 100644 index 000000000..d557ae680 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-1.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster-scram-sha-1 +spec: + shardCount: 1 + type: ShardedCluster + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.2 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: WARN + persistent: true + security: + authentication: + agents: + mode: SCRAM + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-256-x509-internal-cluster.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-256-x509-internal-cluster.yaml new file mode 100644 index 000000000..25d338970 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-256-x509-internal-cluster.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sharded-cluster-scram-sha-256 +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + security: + tls: + enabled: true + authentication: + agents: + mode: SCRAM + enabled: true + modes: [ "SCRAM", "X509" ] + internalCluster: X509 diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-256.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-256.yaml new file mode 100644 index 000000000..2b58bbde5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-scram-sha-256.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sharded-cluster-scram-sha-256 +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + security: + authentication: + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-tls-scram-sha-256.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-tls-scram-sha-256.yaml new file mode 100644 index 000000000..ee6febab9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-tls-scram-sha-256.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sharded-cluster-tls-scram-sha-256 +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + security: + tls: + enabled: true + authentication: + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-x509-internal-cluster-auth-transition.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-x509-internal-cluster-auth-transition.yaml new file mode 100644 index 000000000..d051ba898 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-x509-internal-cluster-auth-transition.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sc-internal-cluster-auth-transition +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 1 + configServerCount: 1 + version: 4.4.0-ent + type: ShardedCluster + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + security: + tls: + enabled: true + authentication: + agents: + mode: X509 + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-x509-to-scram-256.yaml b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-x509-to-scram-256.yaml new file mode 100644 index 000000000..2921de354 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/fixtures/sharded-cluster-x509-to-scram-256.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sharded-cluster-x509-to-scram-256 +spec: + shardCount: 1 + mongodsPerShardCount: 2 + mongosCount: 1 + configServerCount: 2 + version: 4.4.0-ent + type: ShardedCluster + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + security: + tls: + enabled: true + authentication: + agents: + mode: X509 + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_agent_ldap.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_agent_ldap.py new file mode 100644 index 000000000..32584f773 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_agent_ldap.py @@ -0,0 +1,174 @@ +from kubetester import create_secret, find_fixture, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.ldap import LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from pytest import fixture, mark + +USER_NAME = "mms-user-1" +PASSWORD = "my-password" + + +@fixture(scope="module") +def replica_set(openldap: OpenLDAP, namespace: str, custom_mdb_prev_version: str) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-agent-auth.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + ac_secret_name = "automation-config-password" + create_secret(namespace, ac_secret_name, {"automationConfigPassword": "LDAPPassword."}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + } + resource["spec"]["security"]["authentication"]["agents"] = { + "mode": "LDAP", + "automationPasswordSecretRef": { + "name": ac_secret_name, + "key": "automationConfigPassword", + }, + "automationUserName": "mms-automation-agent", + } + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + + try_load(resource) + + return resource + + +@fixture(scope="module") +def ldap_user_mongodb(replica_set: MongoDB, namespace: str, ldap_mongodb_user: LDAPUser) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + user.add_roles( + [ + # In order to be able to write to custom db/collections during the tests + Role(db="admin", role="readWriteAnyDatabase"), + ] + ) + + return user.create() + + +@mark.e2e_replica_set_ldap_agent_auth +@mark.usefixtures("ldap_mongodb_agent_user", "ldap_user_mongodb") +def test_replica_set(replica_set: MongoDB): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_ldap_user_mongodb_reaches_updated_phase(ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated, timeout=150) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_new_ldap_users_can_authenticate(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="customDb", + collection="customColl", + attempts=10, + ) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_deployment_is_reachable_with_ldap_agent(replica_set: MongoDB): + tester = replica_set.tester() + # Due to what we found out in + # https://jira.mongodb.org/browse/CLOUDP-68873 + # the agents might report being in goal state, the MDB resource + # would report no errors but the deployment would be unreachable + # See the comment inside the function for further details + tester.assert_deployment_reachable() + + +@mark.e2e_replica_set_ldap_agent_auth +def test_scale_replica_test(replica_set: MongoDB): + replica_set.reload() + replica_set["spec"]["members"] = 5 + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_new_ldap_users_can_authenticate_after_scaling(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="customDb", + collection="customColl", + attempts=10, + ) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_disable_agent_auth(replica_set: MongoDB): + replica_set.reload() + replica_set["spec"]["security"]["authentication"]["enabled"] = False + replica_set["spec"]["security"]["authentication"]["agents"]["enabled"] = False + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_replica_set_connectivity_with_no_auth(replica_set: MongoDB): + tester = replica_set.tester() + tester.assert_connectivity() + + +@mark.e2e_replica_set_ldap_agent_auth +def test_deployment_is_reachable_with_no_auth(replica_set: MongoDB): + tester = replica_set.tester() + tester.assert_deployment_reachable() + + +@mark.e2e_replica_set_ldap_agent_auth +def test_enable_SCRAM_auth(replica_set: MongoDB): + replica_set["spec"]["security"]["authentication"]["agents"]["enabled"] = True + replica_set["spec"]["security"]["authentication"]["agents"]["mode"] = "SCRAM" + replica_set["spec"]["security"]["authentication"]["enabled"] = True + replica_set["spec"]["security"]["authentication"]["mode"] = "SCRAM" + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_replica_set_connectivity_with_SCRAM_auth(replica_set: MongoDB): + tester = replica_set.tester() + tester.assert_connectivity() + + +@mark.e2e_replica_set_ldap_agent_auth +def test_change_version_to_latest(replica_set: MongoDB, custom_mdb_version: str): + replica_set.reload() + replica_set.set_version(ensure_ent_version(custom_mdb_version)) + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_replica_set_ldap_agent_auth +def test_replica_set_connectivity_after_version_change_SCRAM(replica_set: MongoDB): + tester = replica_set.tester() + tester.assert_connectivity() + + +@mark.e2e_replica_set_ldap_agent_auth +def test_deployment_is_reachable_after_version_change_SCRAM(replica_set: MongoDB): + tester = replica_set.tester() + tester.assert_deployment_reachable() diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_custom_roles.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_custom_roles.py new file mode 100644 index 000000000..d1e976a21 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_custom_roles.py @@ -0,0 +1,125 @@ +from kubetester import create_secret, find_fixture +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, generic_user +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer_ca_configmap: str, + namespace: str, + ldap_mongodb_user: LDAPUser, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-replica-set-roles.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + "authzQueryTemplate": "{USER}?memberOf?base", + } + + resource.update() + return resource + + +@fixture(scope="module") +def ldap_user_mongodb( + replica_set: MongoDB, + namespace: str, + ldap_mongodb_user: LDAPUser, + openldap: OpenLDAP, +) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + + user.update() + return user + + +@mark.e2e_replica_set_custom_roles +def test_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_custom_roles +def test_create_ldap_user(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated) + + ac = replica_set.get_automation_config_tester() + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=False) + ac.assert_expected_users(1) + + +@mark.e2e_replica_set_custom_roles +def test_new_ldap_users_can_write_to_database(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="foo", + collection="foo", + attempts=10, + ) + + +@mark.e2e_replica_set_custom_roles +@mark.xfail(reason="The user should not be able to write to a database/collection it is not authorized to write on") +def test_new_ldap_users_can_write_to_other_collection(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="foo", + collection="foo2", + attempts=10, + ) + + +@mark.e2e_replica_set_custom_roles +@mark.xfail(reason="The user should not be able to write to a database/collection it is not authorized to write on") +def test_new_ldap_users_can_write_to_other_database(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="foo2", + collection="foo", + attempts=10, + ) + + +@mark.e2e_replica_set_custom_roles +def test_automation_config_has_roles(replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + + tester.assert_has_expected_number_of_roles(expected_roles=1) + role = { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [ + {"actions": ["insert"], "resource": {"collection": "foo", "db": "foo"}}, + { + "actions": ["insert", "find"], + "resource": {"collection": "", "db": "admin"}, + }, + ], + "authenticationRestrictions": [], + } + tester.assert_expected_role(role_index=0, expected_value=role) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_feature_controls.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_feature_controls.py new file mode 100644 index 000000000..45570f818 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_feature_controls.py @@ -0,0 +1,152 @@ +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="function") +def replicaset(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-basic.yaml"), + namespace=namespace, + ) + if try_load(resource): + return resource + + resource.update() + return resource + + +@mark.e2e_feature_controls_authentication +def test_replicaset_reaches_running_phase(replicaset: MongoDB): + replicaset.assert_reaches_phase(Phase.Running, ignore_errors=True) + + +@mark.e2e_feature_controls_authentication +def test_authentication_is_owned_by_opsmanager(replicaset: MongoDB): + """ + There is no authentication, so feature controls API should allow + authentication changes from Ops Manager UI. + """ + fc = replicaset.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + assert len(fc["policies"]) == 2 + policies = [p["policy"] for p in fc["policies"]] + assert "EXTERNALLY_MANAGED_LOCK" in policies + assert "DISABLE_SET_MONGOD_VERSION" in policies + + for p in fc["policies"]: + if p["policy"] == "EXTERNALLY_MANAGED_LOCK": + assert p["disabledParams"] == [] + + +@mark.e2e_feature_controls_authentication +def test_authentication_disabled_is_owned_by_operator(replicaset: MongoDB): + """ + Authentication has been added to the Spec, on "disabled" mode, + this makes the Operator to *own* authentication and thus + making Feature controls API to restrict any + """ + replicaset["spec"]["security"] = {"authentication": {"enabled": False}} + replicaset.update() + + replicaset.assert_reaches_phase(Phase.Running, timeout=600) + + fc = replicaset.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert len(fc["policies"]) == 3 + + assert policies[0]["disabledParams"] == [] + assert policies[2]["disabledParams"] == [] + + policies = [p["policy"] for p in fc["policies"]] + assert "EXTERNALLY_MANAGED_LOCK" in policies + assert "DISABLE_AUTHENTICATION_MECHANISMS" in policies + assert "DISABLE_SET_MONGOD_VERSION" in policies + + +@mark.e2e_feature_controls_authentication +def test_authentication_enabled_is_owned_by_operator(replicaset: MongoDB): + """ + Authentication has been enabled on the Operator. Authentication is still + owned by the operator so feature controls should be kept the same. + """ + replicaset["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + replicaset.update() + + replicaset.assert_reaches_phase(Phase.Running, timeout=600) + + fc = replicaset.get_om_tester().get_feature_controls() + + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + assert len(fc["policies"]) == 3 + # sort the policies to have pre-determined order + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert policies[0]["disabledParams"] == [] + assert policies[2]["disabledParams"] == [] + + policies = [p["policy"] for p in fc["policies"]] + assert "EXTERNALLY_MANAGED_LOCK" in policies + assert "DISABLE_AUTHENTICATION_MECHANISMS" in policies + assert "DISABLE_SET_MONGOD_VERSION" in policies + + +@mark.e2e_feature_controls_authentication +def test_authentication_disabled_owned_by_opsmanager(replicaset: MongoDB): + """ + Authentication has been disabled (removed) on the Operator. Authentication + is now "owned" by Ops Manager. + """ + last_transition = replicaset.get_status_last_transition_time() + replicaset["spec"]["security"] = None + replicaset.update() + + replicaset.assert_state_transition_happens(last_transition) + replicaset.assert_reaches_phase(Phase.Running, timeout=600) + + fc = replicaset.get_om_tester().get_feature_controls() + + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + # sort the policies to have pre-determined order + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert len(fc["policies"]) == 2 + assert policies[0]["policy"] == "DISABLE_SET_MONGOD_VERSION" + assert policies[1]["policy"] == "EXTERNALLY_MANAGED_LOCK" + assert policies[1]["disabledParams"] == [] + + +@mark.e2e_om_feature_controls_authentication +def test_feature_controls_cleared_on_replica_set_deletion(replica_set: MongoDB): + """ + Replica set was deleted from the cluster. Policies are removed from the OpsManager group. + """ + replica_set.delete() + + def replica_set_deleted() -> bool: + k8s_resource_deleted = None + try: + replica_set.load() + k8s_resource_deleted = False + except kubernetes.client.ApiException: + k8s_resource_deleted = True + automation_config_deleted = None + tester = replica_set.get_automation_config_tester() + try: + tester.assert_empty() + automation_config_deleted = True + except AssertionError: + automation_config_deleted = False + return k8s_resource_deleted and automation_config_deleted + + wait_until(replica_set_deleted, timeout=60) + + fc = replica_set.get_om_tester().get_feature_controls() + + # after deleting the replicaset the policies in the feature control are removed + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + assert len(fc["policies"]) == 0 diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ignore_unkown_users.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ignore_unkown_users.py new file mode 100644 index 000000000..2ba63dbb6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ignore_unkown_users.py @@ -0,0 +1,53 @@ +from kubetester import find_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set( + namespace: str, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("replica-set-scram-sha-256.yaml"), namespace=namespace) + + resource["spec"]["security"]["authentication"]["ignoreUnknownUsers"] = True + + return resource.create() + + +@mark.e2e_replica_set_ignore_unknown_users +def test_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ignore_unknown_users +def test_authoritative_set_false(replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + tester.assert_authoritative_set(False) + + +@mark.e2e_replica_set_ignore_unknown_users +def test_set_ignore_unknown_users_false(replica_set: MongoDB): + replica_set.reload() + replica_set["spec"]["security"]["authentication"]["ignoreUnknownUsers"] = False + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ignore_unknown_users +def test_authoritative_set_true(replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + tester.assert_authoritative_set(True) + + +@mark.e2e_replica_set_ignore_unknown_users +def test_set_ignore_unknown_users_true(replica_set: MongoDB): + replica_set.reload() + replica_set["spec"]["security"]["authentication"]["ignoreUnknownUsers"] = True + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ignore_unknown_users +def test_authoritative_set_false_again(replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + tester.assert_authoritative_set(False) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap.py new file mode 100644 index 000000000..8e2adf7a2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap.py @@ -0,0 +1,299 @@ +import tempfile +from typing import List + +from kubetester import create_secret, find_fixture +from kubetester.certs import create_mongodb_tls_certs, create_x509_user_cert +from kubetester.kubetester import KubernetesTester +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from pytest import fixture, mark + +USER_NAME = "mms-user-1" +PASSWORD = "my-password" +MDB_RESOURCE = "ldap-replica-set" + + +@fixture(scope="module") +def server_certs(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, "ldap-replica-set", "certs-ldap-replica-set-cert") + return "certs" + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer_ca_configmap: str, + ldap_mongodb_agent_user: LDAPUser, + server_certs: str, + namespace: str, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-replica-set.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + ac_secret_name = "automation-config-password" + create_secret( + namespace, + ac_secret_name, + {"automationConfigPassword": ldap_mongodb_agent_user.password}, + ) + + resource["spec"]["security"] = { + "tls": { + "enabled": True, + "ca": issuer_ca_configmap, + }, + "certsSecretPrefix": server_certs, + "authentication": { + "enabled": True, + "modes": ["LDAP", "SCRAM", "X509"], + "ldap": { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + }, + "agents": { + "mode": "LDAP", + "automationPasswordSecretRef": { + "name": ac_secret_name, + "key": "automationConfigPassword", + }, + "automationUserName": ldap_mongodb_agent_user.uid, + }, + }, + } + + return resource.create() + + +@fixture(scope="module") +def user_ldap(replica_set: MongoDB, namespace: str, ldap_mongodb_users: List[LDAPUser]) -> MongoDBUser: + mongodb_user = ldap_mongodb_users[0] + user = generic_user( + namespace, + username=mongodb_user.username, + db="$external", + password=mongodb_user.password, + mongodb_resource=replica_set, + ) + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + + return user.create() + + +@fixture(scope="module") +def user_scram(replica_set: MongoDB, namespace: str) -> MongoDBUser: + user = generic_user( + namespace, + username="mms-user-1", + db="admin", + mongodb_resource=replica_set, + ) + secret_name = "user-password" + secret_key = "password" + create_secret(namespace, secret_name, {secret_key: "my-password"}) + user["spec"]["passwordSecretKeyRef"] = { + "name": secret_name, + "key": secret_key, + } + + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + + return user.create() + + +@mark.e2e_replica_set_ldap +def test_replica_set(replica_set: MongoDB, ldap_mongodb_users: List[LDAPUser]): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ldap +def test_create_ldap_user(replica_set: MongoDB, user_ldap: MongoDBUser): + user_ldap.assert_reaches_phase(Phase.Updated) + + ac = replica_set.get_automation_config_tester() + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=True) + ac.assert_expected_users(1) + + +@mark.e2e_replica_set_ldap +def test_new_mdb_users_are_created_and_can_authenticate(replica_set: MongoDB, user_ldap: MongoDBUser, ca_path: str): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=user_ldap["spec"]["username"], + password=user_ldap.password, + tls_ca_file=ca_path, + attempts=10, + ) + + +@mark.e2e_replica_set_ldap +def test_create_scram_user(replica_set: MongoDB, user_scram: MongoDBUser): + user_scram.assert_reaches_phase(Phase.Updated) + + ac = replica_set.get_automation_config_tester() + ac.assert_authentication_mechanism_enabled("SCRAM-SHA-256", active_auth_mechanism=False) + ac.assert_expected_users(2) + + +@mark.e2e_replica_set_ldap +def test_replica_set_connectivity(replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(ca_path=ca_path) + tester.assert_connectivity() + + +@mark.e2e_replica_set_ldap +def test_ops_manager_state_correctly_updated(replica_set: MongoDB, user_ldap: MongoDBUser): + expected_roles = { + ("admin", "clusterAdmin"), + ("admin", "readWriteAnyDatabase"), + ("admin", "dbAdminAnyDatabase"), + } + + tester = replica_set.get_automation_config_tester() + tester.assert_expected_users(2) + tester.assert_has_user(user_ldap["spec"]["username"]) + tester.assert_user_has_roles(user_ldap["spec"]["username"], expected_roles) + + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", active_auth_mechanism=False) + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=3) + + +@mark.e2e_replica_set_ldap +def test_user_cannot_authenticate_with_incorrect_password(replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(ca_path=ca_path) + + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ssl=True, + tlsCAFile=ca_path, + ) + + +@mark.e2e_replica_set_ldap +def test_user_can_authenticate_with_correct_password(replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(ca_path=ca_path) + tester.assert_scram_sha_authentication( + password=PASSWORD, + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ssl=True, + tlsCAFile=ca_path, + ) + + +@fixture(scope="module") +def user_x509(replica_set: MongoDB, namespace: str) -> MongoDBUser: + user = generic_user( + namespace, + username="CN=x509-testing-user", + db="$external", + mongodb_resource=replica_set, + ) + + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + + return user.create() + + +@mark.e2e_replica_set_ldap +def test_x509_user_created(replica_set: MongoDB, user_x509: MongoDBUser): + user_x509.assert_reaches_phase(Phase.Updated) + + expected_roles = { + ("admin", "clusterAdmin"), + ("admin", "readWriteAnyDatabase"), + ("admin", "dbAdminAnyDatabase"), + } + + tester = replica_set.get_automation_config_tester() + tester.assert_expected_users(3) + tester.assert_has_user(user_x509["spec"]["username"]) + tester.assert_user_has_roles(user_x509["spec"]["username"], expected_roles) + + tester.assert_authentication_mechanism_enabled("MONGODB-X509", active_auth_mechanism=False) + + +@mark.e2e_replica_set_ldap +def test_x509_user_connectivity( + namespace: str, + ca_path: str, + issuer: str, + replica_set: MongoDB, + user_x509: MongoDBUser, +): + cert_file = tempfile.NamedTemporaryFile(delete=False, mode="w") + create_x509_user_cert(issuer, namespace, path=cert_file.name) + + tester = replica_set.tester() + tester.assert_x509_authentication(cert_file_name=cert_file.name, tlsCAFile=ca_path) + + +@mark.e2e_replica_set_ldap +def test_change_ldap_servers( + namespace: str, + replica_set: MongoDB, + secondary_openldap: OpenLDAP, + secondary_ldap_mongodb_users: List[LDAPUser], + secondary_ldap_mongodb_agent_user, +): + secret_name = "bind-query-password-secondary" + create_secret(namespace, secret_name, {"password": secondary_openldap.admin_password}) + ac_secret_name = "automation-config-password-secondary" + create_secret( + namespace, + ac_secret_name, + {"automationConfigPassword": secondary_ldap_mongodb_agent_user.password}, + ) + replica_set.load() + replica_set["spec"]["security"]["authentication"]["ldap"]["servers"] = [secondary_openldap.servers] + replica_set["spec"]["security"]["authentication"]["ldap"]["bindQueryPasswordSecretRef"] = {"name": secret_name} + replica_set["spec"]["security"]["authentication"]["agents"] = { + "mode": "LDAP", + "automationPasswordSecretRef": { + "name": ac_secret_name, + "key": "automationConfigPassword", + }, + "automationUserName": secondary_ldap_mongodb_agent_user.uid, + } + + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replica_set_ldap +def test_replica_set_ldap_settings_are_updated(replica_set: MongoDB, ldap_mongodb_users: List[LDAPUser]): + replica_set.reload() + replica_set["spec"]["security"]["authentication"]["ldap"]["timeoutMS"] = 12345 + replica_set["spec"]["security"]["authentication"]["ldap"]["userCacheInvalidationInterval"] = 60 + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + ac = replica_set.get_automation_config_tester() + assert "timeoutMS" in ac.automation_config["ldap"] + assert "userCacheInvalidationInterval" in ac.automation_config["ldap"] + assert ac.automation_config["ldap"]["timeoutMS"] == 12345 + assert ac.automation_config["ldap"]["userCacheInvalidationInterval"] == 60 diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_agent_client_certs.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_agent_client_certs.py new file mode 100644 index 000000000..fbe511bab --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_agent_client_certs.py @@ -0,0 +1,217 @@ +import tempfile + +from kubetester import create_secret, delete_secret, find_fixture, read_secret +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs, generate_cert +from kubetester.ldap import LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from pytest import fixture, mark + +USER_NAME = "mms-user-1" +PASSWORD = "my-password" + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + resource_name = "ldap-replica-set" + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + resource_name, + f"{resource_name}-cert", + replicas=5, + ) + + +@fixture(scope="module") +def client_cert_path(issuer: str, namespace: str): + spec = { + "commonName": "client-cert", + "subject": {"organizationalUnits": ["mongodb.com"]}, + } + + client_secret = generate_cert( + namespace, + "client-cert", + "client-cert", + issuer, + spec=spec, + ) + client_cert = read_secret(namespace, client_secret) + cert_file = tempfile.NamedTemporaryFile() + with open(cert_file.name, "w") as f: + f.write(client_cert["tls.key"] + client_cert["tls.crt"]) + + yield cert_file.name + + cert_file.close() + + +@fixture(scope="module") +def agent_client_cert(issuer: str, namespace: str) -> str: + spec = { + "commonName": "mms-automation-client-cert", + "subject": {"organizationalUnits": ["mongodb.com"]}, + } + + client_certificate_secret = generate_cert( + namespace, + "mongodb-mms-automation", + "mongodb-mms-automation", + issuer, + spec=spec, + ) + automation_agent_cert = read_secret(namespace, client_certificate_secret) + data = {} + data["tls.crt"], data["tls.key"] = ( + automation_agent_cert["tls.crt"], + automation_agent_cert["tls.key"], + ) + # creates a secret that combines key and crt + create_secret(namespace, "agent-client-cert", data, type="kubernetes.io/tls") + + yield "agent-client-cert" + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer: str, + issuer_ca_configmap: str, + server_certs: str, + agent_client_cert: str, + namespace: str, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-agent-auth.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + ac_secret_name = "automation-config-password" + create_secret(namespace, ac_secret_name, {"automationConfigPassword": "LDAPPassword."}) + + resource["spec"]["security"]["tls"] = { + "enabled": True, + "ca": issuer_ca_configmap, + } + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + } + resource["spec"]["security"]["authentication"]["agents"] = { + "mode": "LDAP", + "automationPasswordSecretRef": { + "name": ac_secret_name, + "key": "automationConfigPassword", + }, + "clientCertificateSecretRef": {"name": agent_client_cert}, + "automationUserName": "mms-automation-agent", + } + + return resource.update() + + +@fixture(scope="module") +def ldap_user_mongodb(replica_set: MongoDB, namespace: str, ldap_mongodb_user: LDAPUser) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + user.add_roles( + [ + # In order to be able to write to custom db/collections during the tests + Role(db="admin", role="readWriteAnyDatabase"), + ] + ) + + return user.update() + + +@mark.e2e_replica_set_ldap_agent_client_certs +@mark.usefixtures("ldap_mongodb_agent_user", "ldap_user_mongodb") +def test_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400, ignore_errors=True) + + +@mark.e2e_replica_set_ldap_agent_client_certs +def test_ldap_user_mongodb_reaches_updated_phase(ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated, timeout=150) + + +@mark.e2e_replica_set_ldap_agent_client_certs +def test_new_ldap_users_can_authenticate(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser, ca_path: str): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="customDb", + collection="customColl", + tls_ca_file=ca_path, + attempts=10, + ) + + +@mark.e2e_replica_set_ldap_agent_client_certs +def test_client_requires_certs(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["authentication"]["requireClientTLSAuthentication"] = True + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + ac_tester = replica_set.get_automation_config_tester() + ac_tester.assert_tls_client_certificate_mode("REQUIRE") + + +@mark.e2e_replica_set_ldap_agent_client_certs +def test_client_can_auth_with_client_certs_provided( + replica_set: MongoDB, + ldap_user_mongodb: MongoDBUser, + ca_path: str, + client_cert_path: str, +): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="customDb", + collection="customColl", + tls_ca_file=ca_path, + ssl_certfile=client_cert_path, + attempts=10, + ) + + +@mark.e2e_replica_set_ldap_agent_client_certs +def test_client_certs_made_optional(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["authentication"]["requireClientTLSAuthentication"] = False + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + ac_tester = replica_set.get_automation_config_tester() + ac_tester.assert_tls_client_certificate_mode("OPTIONAL") + + +@mark.e2e_replica_set_ldap_agent_client_certs +def test_client_can_auth_again_with_no_client_certs(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser, ca_path: str): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="customDb", + collection="customColl", + tls_ca_file=ca_path, + attempts=10, + ) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_group_dn.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_group_dn.py new file mode 100644 index 000000000..8905d58ab --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_group_dn.py @@ -0,0 +1,102 @@ +from kubetester import create_secret, find_fixture +from kubetester.ldap import LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, generic_user +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer_ca_configmap: str, + namespace: str, + ldap_mongodb_user: LDAPUser, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-agent-auth.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + "authzQueryTemplate": "{USER}?memberOf?base", + } + + ac_secret_name = "automation-config-password" + create_secret(namespace, ac_secret_name, {"automationConfigPassword": "LDAPPassword."}) + resource["spec"]["security"]["roles"] = [ + { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [ + {"actions": ["insert"], "resource": {"db": "foo", "collection": "foo"}}, + ], + }, + ] + resource["spec"]["security"]["authentication"]["agents"] = { + "mode": "LDAP", + "automationPasswordSecretRef": { + "name": ac_secret_name, + "key": "automationConfigPassword", + }, + "automationUserName": "mms-automation-agent", + "automationLdapGroupDN": "cn=agents,ou=groups,dc=example,dc=org", + } + return resource.create() + + +@fixture(scope="module") +def ldap_user_mongodb(replica_set: MongoDB, namespace: str, ldap_mongodb_user: LDAPUser) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + + return user.create() + + +@mark.e2e_replica_set_ldap_group_dn +def test_replica_set( + replica_set: MongoDB, + ldap_mongodb_agent_user: LDAPUser, + ldap_user_mongodb: MongoDBUser, +): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ldap_group_dn +def test_ldap_user_mongodb_reaches_updated_phase(ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated, timeout=150) + + +@mark.e2e_replica_set_ldap_group_dn +def test_new_ldap_users_can_authenticate(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="foo", + collection="foo", + attempts=10, + ) + + +@mark.e2e_replica_set_ldap_group_dn +def test_deployment_is_reachable_with_ldap_agent(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + # Due to what we found out in + # https://jira.mongodb.org/browse/CLOUDP-68873 + # the agents might report being in goal state, the MDB resource + # would report no errors but the deployment would be unreachable + # See the comment inside the function for further details + tester.assert_deployment_reachable() diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_group_dn_with_x509_agent.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_group_dn_with_x509_agent.py new file mode 100644 index 000000000..6ac8abd06 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_group_dn_with_x509_agent.py @@ -0,0 +1,118 @@ +import random +import time +from datetime import datetime, timezone + +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import create_secret, find_fixture, kubetester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_x509_agent_tls_certs, + create_x509_mongodb_tls_certs, +) +from kubetester.ldap import LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, generic_user +from pytest import fixture, mark + +MDB_RESOURCE = "ldap-replica-set" + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer_ca_configmap: str, + server_certs: str, + agent_certs: str, + namespace: str, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-agent-auth.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "CN=mms-automation-agent,(.+),L=NY,ST=NY,C=US", substitution: "uid=mms-automation-agent,{0},dc=example,dc=org"}, {match: "(.+)", substitution:"uid={0},ou=groups,dc=example,dc=org"}]', + "authzQueryTemplate": "{USER}?memberOf?base", + } + + resource["spec"]["security"]["tls"] = {"enabled": True, "ca": issuer_ca_configmap} + resource["spec"]["security"]["roles"] = [ + { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [ + {"actions": ["insert"], "resource": {"db": "foo", "collection": "foo"}}, + ], + }, + ] + resource["spec"]["security"]["authentication"]["modes"] = ["LDAP", "SCRAM", "X509"] + resource["spec"]["security"]["authentication"]["agents"] = { + "mode": "X509", + "automationLdapGroupDN": f"cn=mms-automation-agent,ou={namespace},o=cluster.local-agent,dc=example,dc=org", + } + return resource.create() + + +@fixture(scope="module") +def ldap_user_mongodb(replica_set: MongoDB, namespace: str, ldap_mongodb_user: LDAPUser) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + + return user.create() + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_x509_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@mark.e2e_replica_set_ldap_group_dn_with_x509_agent +def test_replica_set(replica_set: MongoDB, ldap_mongodb_x509_agent_user: LDAPUser, namespace: str): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ldap_group_dn_with_x509_agent +def test_ldap_user_mongodb_reaches_updated_phase(ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated, timeout=150) + + +@mark.e2e_replica_set_ldap_group_dn_with_x509_agent +def test_new_ldap_users_can_authenticate(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser, ca_path: str): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="foo", + collection="foo", + attempts=10, + tls_ca_file=ca_path, + ) + + +@mark.e2e_replica_set_ldap_group_dn_with_x509_agent +def test_deployment_is_reachable_with_ldap_agent(replica_set: MongoDB): + tester = replica_set.tester() + # Due to what we found out in + # https://jira.mongodb.org/browse/CLOUDP-68873 + # the agents might report being in goal state, the MDB resource + # would report no errors but the deployment would be unreachable + # See the comment inside the function for further details + tester.assert_deployment_reachable() diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_tls.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_tls.py new file mode 100644 index 000000000..f6bf8f3d2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_tls.py @@ -0,0 +1,148 @@ +import time +from typing import Dict + +from kubetester import create_or_update_secret, find_fixture, wait_until +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from pytest import fixture, mark + + +@fixture(scope="module") +def operator_installation_config_quick_recovery(operator_installation_config: Dict[str, str]) -> Dict[str, str]: + """ + This sets the recovery backoff time to 10s for the replicaset reconciliation. It seems like setting it higher + ensures that the automatic recovery doesn't get triggered at all since the `lastTransitionTime` in the status gets updated + too often. + TODO: investigate why and when this mechanism was broken. + """ + operator_installation_config["customEnvVars"] = ( + operator_installation_config["customEnvVars"] + "\&MDB_AUTOMATIC_RECOVERY_BACKOFF_TIME_S=10" + ) + return operator_installation_config + + +@fixture(scope="module") +def operator_installation_config(operator_installation_config_quick_recovery: Dict[str, str]) -> Dict[str, str]: + return operator_installation_config_quick_recovery + + +@fixture(scope="module") +def replica_set( + openldap_tls: OpenLDAP, + issuer_ca_configmap: str, + namespace: str, +) -> MongoDB: + """ + This function sets up ReplicaSet resource for testing. The testing procedure includes CLOUDP-189433, that requires + putting the resource into "Pending" state and the automatically recovering it. + """ + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-replica-set.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_or_update_secret(namespace, secret_name, {"password": openldap_tls.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap_tls.servers], + "bindQueryPasswordSecretRef": {"name": secret_name}, + "transportSecurity": "none", # For testing CLOUDP-189433 + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + } + + resource.update() + return resource + + +@fixture(scope="module") +def ldap_user_mongodb(replica_set: MongoDB, namespace: str, ldap_mongodb_user_tls: LDAPUser) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user_tls.username, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user_tls.password, + ) + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + + return user.create() + + +@mark.e2e_replica_set_ldap_tls +def test_replica_set_pending_CLOUDP_189433(replica_set: MongoDB): + """ + This function tests CLOUDP-189433. The resource needs to enter the "Pending" state and without the automatic + recovery, it would stay like this forever (since we wouldn't push the new AC with a fix). + """ + replica_set.assert_reaches_phase(Phase.Pending, timeout=100) + + +@mark.e2e_replica_set_ldap_tls +def test_turn_tls_on_CLOUDP_189433(replica_set: MongoDB): + """ + This function tests CLOUDP-189433. The user attempts to fix the AutomationConfig. + Before updating the AutomationConfig, we need to ensure the operator pushed the wrong one to Ops Manager. + """ + + def wait_for_ac_pushed() -> bool: + ac = replica_set.get_automation_config_tester().automation_config + try: + transport_security = ac["ldap"]["transportSecurity"] + version = ac["version"] + if version < 4: + return False + if transport_security != "none": + return False + return True + except KeyError: + return False + + wait_until(wait_for_ac_pushed, timeout=200) + + resource = replica_set.load() + resource["spec"]["security"]["authentication"]["ldap"]["transportSecurity"] = "tls" + resource.update() + + +@mark.e2e_replica_set_ldap_tls +def test_replica_set_CLOUDP_189433(replica_set: MongoDB): + """ + This function tests CLOUDP-189433. The recovery mechanism kicks in and pushes Automation Config. The ReplicaSet + goes into running state. + """ + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replica_set_ldap_tls +def test_create_ldap_user(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated) + + ac = replica_set.get_automation_config_tester() + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=False) + ac.assert_expected_users(1) + + +@mark.e2e_replica_set_ldap_tls +def test_new_ldap_users_can_authenticate(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication(ldap_user_mongodb["spec"]["username"], ldap_user_mongodb.password, attempts=10) + + +@mark.e2e_replica_set_ldap_tls +def test_remove_ldap_settings(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + replica_set.load() + replica_set["spec"]["security"]["authentication"]["ldap"] = None + replica_set["spec"]["security"]["authentication"]["modes"] = ["SCRAM"] + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=400) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_user_to_dn_mapping.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_user_to_dn_mapping.py new file mode 100644 index 000000000..b3163a071 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_ldap_user_to_dn_mapping.py @@ -0,0 +1,76 @@ +from kubetester import create_secret, find_fixture +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer_ca_configmap: str, + namespace: str, + ldap_mongodb_user: LDAPUser, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-replica-set.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + } + + return resource.create() + + +@fixture(scope="module") +def ldap_user_mongodb( + replica_set: MongoDB, + namespace: str, + ldap_mongodb_user: LDAPUser, + openldap: OpenLDAP, +) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + + return user.create() + + +@mark.e2e_replica_set_ldap_user_to_dn_mapping +def test_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_ldap_user_to_dn_mapping +def test_create_ldap_user(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated) + + ac = replica_set.get_automation_config_tester() + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=False) + ac.assert_expected_users(1) + + +@mark.e2e_replica_set_ldap_user_to_dn_mapping +def test_new_ldap_users_can_authenticate(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication(ldap_user_mongodb["spec"]["username"], ldap_user_mongodb.password, attempts=10) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_1_connectivity.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_1_connectivity.py new file mode 100644 index 000000000..18200f10a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_1_connectivity.py @@ -0,0 +1,19 @@ +import pytest +from kubetester.mongotester import ReplicaSetTester +from pytest import fixture +from tests.authentication.sha1_connectivity_tests import SHA1ConnectivityTests + + +@pytest.mark.e2e_replica_set_scram_sha_1_user_connectivity +class TestReplicaSetSHA1Connectivity(SHA1ConnectivityTests): + @fixture + def yaml_file(self): + return "replica-set-explicit-scram-sha-1.yaml" + + @fixture + def mdb_resource_name(self): + return "replica-set-scram-sha-1" + + @fixture + def mongo_tester(self, mdb_resource_name: str): + return ReplicaSetTester(mdb_resource_name, 3) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_256_connectivity.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_256_connectivity.py new file mode 100644 index 000000000..051df5c4c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_256_connectivity.py @@ -0,0 +1,227 @@ +from typing import Dict + +from kubetester import ( + create_or_update_secret, + create_secret, + find_fixture, + read_secret, + update_secret, + wait_until, +) +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from pytest import fixture, mark + +MDB_RESOURCE = "my-replica-set" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +CONNECTION_STRING_SECRET_NAME = "my-replica-set-connection-string" +USER_PASSWORD = "my-password" +USER_DATABASE = "admin" + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("replica-set-scram-sha-256.yaml"), + namespace=namespace, + name=MDB_RESOURCE, + ) + resource.set_version(custom_mdb_version) + + resource["spec"]["security"]["authentication"] = { + "ignoreUnknownUsers": True, + "enabled": True, + "modes": ["SCRAM"], + } + + return resource.update() + + +@fixture(scope="module") +def scram_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(find_fixture("scram-sha-user.yaml"), namespace=namespace) + + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + {"password": USER_PASSWORD}, + ) + + return resource.update() + + +@fixture(scope="module") +def standard_secret(replica_set: MongoDB): + secret_name = "{}-{}-{}".format(replica_set.name, USER_NAME, USER_DATABASE) + return read_secret(replica_set.namespace, secret_name) + + +@fixture(scope="module") +def connection_string_secret(replica_set: MongoDB): + return read_secret(replica_set.namespace, CONNECTION_STRING_SECRET_NAME) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +class TestReplicaSetCreation(KubernetesTester): + def test_replica_set_created(self, replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + def test_replica_set_connectivity(self, replica_set: MongoDB): + replica_set.assert_connectivity() + + def test_ops_manager_state_correctly_updated(self, replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + tester.assert_expected_users(0) + tester.assert_authoritative_set(False) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_create_user(scram_user: MongoDBUser): + scram_user.assert_reaches_phase(Phase.Updated) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +class TestReplicaSetIsUpdatedWithNewUser(KubernetesTester): + def test_replica_set_connectivity(self, replica_set: MongoDB): + replica_set.assert_connectivity() + + def test_ops_manager_state_correctly_updated(self, replica_set: MongoDB): + expected_roles = { + (USER_DATABASE, "clusterAdmin"), + (USER_DATABASE, "userAdminAnyDatabase"), + (USER_DATABASE, "readWrite"), + (USER_DATABASE, "userAdminAnyDatabase"), + } + + tester = replica_set.get_automation_config_tester() + tester.assert_has_user(USER_NAME) + tester.assert_user_has_roles(USER_NAME, expected_roles) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + tester.assert_expected_users(1) + tester.assert_authoritative_set(False) + + def test_user_can_authenticate_with_correct_password(self, replica_set: MongoDB): + replica_set.tester().assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) + + def test_user_cannot_authenticate_with_incorrect_password(self, replica_set: MongoDB): + replica_set.tester().assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +class TestCanChangePassword(KubernetesTester): + def test_user_can_authenticate_with_new_password(self, namespace: str, replica_set: MongoDB): + update_secret(namespace, PASSWORD_SECRET_NAME, {"password": "my-new-password7"}) + replica_set.tester().assert_scram_sha_authentication( + password="my-new-password7", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) + + def test_user_cannot_authenticate_with_old_password(self, replica_set: MongoDB): + replica_set.tester().assert_scram_sha_authentication_fails( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_credentials_secret_is_created(replica_set: MongoDB, standard_secret: Dict[str, str]): + assert "username" in standard_secret + assert "password" in standard_secret + assert "connectionString.standard" in standard_secret + assert "connectionString.standardSrv" in standard_secret + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_credentials_can_connect_to_db(replica_set: MongoDB, standard_secret: Dict[str, str]): + print("Connecting with {}".format(standard_secret["connectionString.standard"])) + replica_set.assert_connectivity_from_connection_string(standard_secret["connectionString.standard"], tls=False) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_credentials_can_connect_to_db_with_srv(replica_set: MongoDB, standard_secret: Dict[str, str]): + print("Connecting with {}".format(standard_secret["connectionString.standardSrv"])) + replica_set.assert_connectivity_from_connection_string(standard_secret["connectionString.standardSrv"], tls=False) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_update_user_with_connection_string_secret(scram_user: MongoDBUser): + scram_user.load() + scram_user["spec"]["connectionStringSecretName"] = CONNECTION_STRING_SECRET_NAME + scram_user.update() + + scram_user.assert_reaches_phase(Phase.Updated) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_credentials_can_connect_to_db_with_connection_string_secret( + replica_set: MongoDB, connection_string_secret: Dict[str, str] +): + print("Connecting with {}".format(connection_string_secret["connectionString.standard"])) + replica_set.assert_connectivity_from_connection_string( + connection_string_secret["connectionString.standard"], tls=False + ) + + print("Connecting with {}".format(connection_string_secret["connectionString.standardSrv"])) + replica_set.assert_connectivity_from_connection_string( + connection_string_secret["connectionString.standardSrv"], tls=False + ) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_authentication_is_still_configured_after_remove_authentication(namespace: str, replica_set: MongoDB): + replica_set["spec"]["security"]["authentication"] = None + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + def ac_updated() -> bool: + tester = replica_set.get_automation_config_tester() + # authentication remains enabled as the operator is not configuring it when + # spec.security.authentication is not configured + try: + tester.assert_has_user(USER_NAME) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + tester.assert_expected_users(1) + tester.assert_authoritative_set(False) + return True + except AssertionError: + return False + + wait_until(ac_updated, timeout=600) + + +@mark.e2e_replica_set_scram_sha_256_user_connectivity +def test_authentication_can_be_disabled_without_modes(namespace: str, replica_set: MongoDB): + replica_set["spec"]["security"]["authentication"] = { + "enabled": False, + } + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + def auth_disabled() -> bool: + tester = replica_set.get_automation_config_tester() + # we have explicitly set authentication to be disabled + try: + tester.assert_has_user(USER_NAME) + tester.assert_authentication_disabled(remaining_users=1) + return True + except AssertionError: + return False + + wait_until(auth_disabled, timeout=600) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_256_user_first.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_256_user_first.py new file mode 100644 index 000000000..e2fd1394d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_256_user_first.py @@ -0,0 +1,78 @@ +import pytest +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from pytest import fixture + +MDB_RESOURCE = "my-replica-set" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@fixture(scope="module") +def scram_user(namespace) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user.yaml"), namespace=namespace) + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + KubernetesTester.create_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + return resource + + +@fixture(scope="module") +def replica_set(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-basic.yaml"), + namespace=namespace, + name="my-replica-set", + ) + + return resource + + +@pytest.mark.e2e_replica_set_scram_sha_256_user_first +def test_replica_set_created(replica_set: MongoDB): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_replica_set_scram_sha_256_user_first +def test_user_pending(scram_user: MongoDBUser): + """pending phase as auth has not yet been enabled""" + scram_user.update() + scram_user.assert_reaches_phase(Phase.Pending, timeout=50) + + +@pytest.mark.e2e_replica_set_scram_sha_256_user_first +def test_replica_set_auth_enabled(replica_set: MongoDB): + replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_replica_set_scram_sha_256_user_first +def test_user_created(scram_user: MongoDBUser): + scram_user.assert_reaches_phase(Phase.Updated, timeout=50) + + +@pytest.mark.e2e_replica_set_scram_sha_256_user_first +def test_replica_set_connectivity(replica_set: MongoDB): + replica_set.assert_connectivity() + + +@pytest.mark.e2e_replica_set_scram_sha_256_user_first +def test_ops_manager_state_correctly_updated(replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + tester.assert_expected_users(1) + tester.assert_authoritative_set(True) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_and_x509.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_and_x509.py new file mode 100644 index 000000000..cc69b878e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_and_x509.py @@ -0,0 +1,169 @@ +import tempfile + +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_mongodb_tls_certs, + create_x509_user_cert, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester + +MDB_RESOURCE = "replica-set-scram-256-and-x509" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@pytest.fixture(scope="module") +def replica_set(namespace: str, issuer_ca_configmap: str, server_certs: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("replica-set-tls-scram-sha-256.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.mark.e2e_replica_set_scram_sha_and_x509 +class TestReplicaSetCreation(KubernetesTester): + def test_replica_set_running(self, replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + def test_replica_set_connectivity(self, replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + def test_ops_manager_state_correctly_updated(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + + +@pytest.mark.e2e_replica_set_scram_sha_and_x509 +class TestCreateMongoDBUser(KubernetesTester): + """ + description: | + Creates a MongoDBUser + create: + file: scram-sha-user.yaml + patch: '[{"op":"replace","path":"/spec/mongodbResourceRef/name","value": "replica-set-scram-256-and-x509" }]' + wait_until: in_updated_state + timeout: 150 + """ + + @classmethod + def setup_class(cls): + print(f"creating password for MongoDBUser {USER_NAME} in secret/{PASSWORD_SECRET_NAME} ") + KubernetesTester.create_secret( + KubernetesTester.get_namespace(), + PASSWORD_SECRET_NAME, + { + "password": USER_PASSWORD, + }, + ) + super().setup_class() + + def test_create_user(self): + pass + + +@pytest.mark.e2e_replica_set_scram_sha_and_x509 +class TestScramUserCanAuthenticate(KubernetesTester): + def test_user_cannot_authenticate_with_incorrect_password(self, ca_path: str): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + ) + + def test_user_can_authenticate_with_correct_password(self, ca_path): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + ) + + def test_enable_x509(self, replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["authentication"]["modes"].append("X509") + replica_set["spec"]["security"]["authentication"]["agents"] = {"mode": "SCRAM"} + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + def test_automation_config_was_updated(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + # when both agents.mode is set to SCRAM, X509 should not be used as agent auth + tester.assert_authentication_mechanism_enabled("MONGODB-X509", active_auth_mechanism=False) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + + tester.assert_expected_users(1) + + +@pytest.mark.e2e_replica_set_scram_sha_and_x509 +class TestAddMongoDBUser(KubernetesTester): + """ + create: + file: test-x509-user.yaml + patch: '[{"op":"replace","path":"/spec/mongodbResourceRef/name","value": "replica-set-scram-256-and-x509" }]' + wait_until: user_exists + """ + + def test_add_user(self): + assert True + + @staticmethod + def user_exists(): + ac = KubernetesTester.get_automation_config() + users = ac["auth"]["usersWanted"] + return "CN=x509-testing-user" in [user["user"] for user in users] + + +@pytest.mark.e2e_replica_set_scram_sha_and_x509 +class TestX509CertCreationAndApproval(KubernetesTester): + def setup_method(self): + super().setup_method() + self.cert_file = tempfile.NamedTemporaryFile(delete=False, mode="w") + + def test_create_user_and_authenticate(self, issuer: str, namespace: str, ca_path: str): + create_x509_user_cert(issuer, namespace, path=self.cert_file.name) + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_x509_authentication(cert_file_name=self.cert_file.name, tlsCAFile=ca_path) + + def teardown(self): + self.cert_file.close() + + +@pytest.mark.e2e_replica_set_scram_sha_and_x509 +class TestCanStillAuthAsScramUsers(KubernetesTester): + def test_user_cannot_authenticate_with_incorrect_password(self, ca_path: str): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + ) + + def test_user_can_authenticate_with_correct_password(self, ca_path: str): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + ) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_upgrade.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_upgrade.py new file mode 100644 index 000000000..092703641 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_sha_upgrade.py @@ -0,0 +1,53 @@ +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester + +MDB_RESOURCE = "my-replica-set-scram" + + +@pytest.mark.e2e_replica_set_scram_sha_1_upgrade +class TestCreateScramSha1ReplicaSet(KubernetesTester): + + def test_create_replicaset(self, custom_mdb_version: str): + resource = MongoDB.from_yaml(load_fixture("replica-set-scram.yaml"), namespace=self.namespace) + resource.set_version(custom_mdb_version) + resource.update() + + resource.assert_reaches_phase(Phase.Running) + + def test_assert_connectivity(self): + ReplicaSetTester(MDB_RESOURCE, 3).assert_connectivity() + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + + tester.assert_expected_users(0) + tester.assert_authoritative_set(True) + + +@pytest.mark.e2e_replica_set_scram_sha_1_upgrade +class TestReplicaSetDeleted(KubernetesTester): + """ + description: | + Deletes the Replica Set. + delete: + file: replica-set-scram.yaml + wait_until: mongo_resource_deleted + timeout: 120 + """ + + def test_authentication_was_disabled(self): + def authentication_was_disabled(): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + try: + tester.assert_authentication_disabled() + return True + except AssertionError: + return False + + KubernetesTester.wait_until(authentication_was_disabled, timeout=10, sleep_time=1) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_x509_ic_manual_certs.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_x509_ic_manual_certs.py new file mode 100644 index 000000000..8f1d57370 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_x509_ic_manual_certs.py @@ -0,0 +1,80 @@ +from kubetester.certs import SetProperties, create_mongodb_tls_certs +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + +MDB_RESOURCE = "my-replica-set" +SUBJECT = {"organizations": ["MDB Tests"], "organizationalUnits": ["Servers"]} +SERVER_SET = SetProperties(MDB_RESOURCE, MDB_RESOURCE + "-svc", 3) + + +@fixture(scope="module") +def all_certs(issuer, namespace) -> None: + """Generates TLS Certificates for the servers.""" + spec_server = { + "subject": SUBJECT, + "usages": ["server auth"], + } + + spec_client = { + "subject": SUBJECT, + "usages": ["client auth"], + } + + server_set = SERVER_SET + create_mongodb_tls_certs( + issuer, + namespace, + server_set.name, + server_set.name + "-cert", + server_set.replicas, + service_name=server_set.service, + spec=spec_server, + ) + create_mongodb_tls_certs( + issuer, + namespace, + server_set.name, + server_set.name + "-clusterfile", + server_set.replicas, + service_name=server_set.service, + spec=spec_client, + ) + + +@fixture(scope="module") +def replica_set( + namespace: str, + all_certs, + issuer_ca_configmap: str, +) -> MongoDB: + _ = all_certs + mdb: MongoDB = MongoDB.from_yaml( + _fixture("replica-set-scram-sha-256-x509-internal-cluster.yaml"), + namespace=namespace, + ) + mdb["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return mdb.create() + + +@mark.e2e_replica_set_scram_x509_ic_manual_certs +def test_create_replica_set_with_x509_internal_cluster(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) + + +@mark.e2e_replica_set_scram_x509_ic_manual_certs +def test_create_replica_can_connect(replica_set: MongoDB, ca_path: str): + # The ca_path fixture indicates a relative path in the testing Pod to the + # CA file we can use to validate against the certificates generated + # by cert-manager. + replica_set.assert_connectivity(ca_path=ca_path) + + +@mark.e2e_replica_set_scram_x509_ic_manual_certs +def test_ops_manager_state_was_updated_correctly(replica_set: MongoDB): + ac_tester = replica_set.get_automation_config_tester() + ac_tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + ac_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + ac_tester.assert_expected_users(0) + ac_tester.assert_authoritative_set(True) + ac_tester.assert_internal_cluster_authentication_enabled() diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_x509_internal_cluster.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_x509_internal_cluster.py new file mode 100644 index 000000000..62b9b4b0f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_scram_x509_internal_cluster.py @@ -0,0 +1,51 @@ +from kubetester import create_or_update_secret, read_secret +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_x509_agent_tls_certs, + create_x509_mongodb_tls_certs, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + +MDB_RESOURCE = "my-replica-set" + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + create_x509_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + secret_name = f"{MDB_RESOURCE}-cert" + data = read_secret(namespace, secret_name) + secret_type = "kubernetes.io/tls" + create_or_update_secret(namespace, f"{MDB_RESOURCE}-clusterfile", data, type=secret_type) + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@fixture(scope="module") +def mdb(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml( + load_fixture("replica-set-scram-sha-256-x509-internal-cluster.yaml"), + namespace=namespace, + ) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.update() + + +@mark.e2e_replica_set_scram_x509_internal_cluster +class TestReplicaSetScramX509Internal(KubernetesTester): + def test_mdb_is_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=600) + + def test_ops_manager_state_was_updated_correctly(self): + ac_tester = AutomationConfigTester(self.get_automation_config()) + ac_tester.assert_authentication_enabled() + ac_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + ac_tester.assert_expected_users(0) + ac_tester.assert_authoritative_set(True) + ac_tester.assert_internal_cluster_authentication_enabled() diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_update_roles_no_privileges.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_update_roles_no_privileges.py new file mode 100644 index 000000000..a9f05c0c0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_update_roles_no_privileges.py @@ -0,0 +1,128 @@ +from kubetester import create_secret, find_fixture, wait_until +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, generic_user +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set( + openldap: OpenLDAP, + issuer_ca_configmap: str, + namespace: str, + ldap_mongodb_user: LDAPUser, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("ldap/ldap-replica-set-roles.yaml"), namespace=namespace) + + secret_name = "bind-query-password" + create_secret(namespace, secret_name, {"password": openldap.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + "authzQueryTemplate": "{USER}?memberOf?base", + } + + yield resource.create() + resource.delete() + + +@fixture(scope="module") +def ldap_user_mongodb( + replica_set: MongoDB, + namespace: str, + ldap_mongodb_user: LDAPUser, + openldap: OpenLDAP, +) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user.uid, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user.password, + ) + + yield user.create() + user.delete() + + +@mark.e2e_replica_set_update_roles_no_privileges +def test_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_update_roles_no_privileges +def test_create_ldap_user(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + ldap_user_mongodb.assert_reaches_phase(Phase.Updated) + + ac = replica_set.get_automation_config_tester() + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=False) + ac.assert_expected_users(1) + + +@mark.e2e_replica_set_update_roles_no_privileges +def test_new_ldap_users_can_write_to_database(replica_set: MongoDB, ldap_user_mongodb: MongoDBUser): + tester = replica_set.tester() + + tester.assert_ldap_authentication( + username=ldap_user_mongodb["spec"]["username"], + password=ldap_user_mongodb.password, + db="foo", + collection="foo", + attempts=10, + ) + + +@mark.e2e_replica_set_update_roles_no_privileges +def test_automation_config_has_roles(replica_set: MongoDB): + tester = replica_set.get_automation_config_tester() + + tester.assert_has_expected_number_of_roles(expected_roles=1) + role = { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [ + {"actions": ["insert"], "resource": {"collection": "foo", "db": "foo"}}, + { + "actions": ["insert", "find"], + "resource": {"collection": "", "db": "admin"}, + }, + ], + "authenticationRestrictions": [], + } + tester.assert_expected_role(role_index=0, expected_value=role) + + +@mark.e2e_replica_set_update_roles_no_privileges +def test_update_role(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["roles"] = [ + { + "db": "admin", + "role": "cn=users,ou=groups,dc=example,dc=org", + "roles": [{"db": "admin", "role": "readWriteAnyDatabase"}], + } + ] + replica_set.update() + + +@mark.e2e_replica_set_update_roles_no_privileges +def test_automation_config_has_new_roles(replica_set: MongoDB): + role = { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [], + "roles": [{"db": "admin", "role": "readWriteAnyDatabase"}], + "authenticationRestrictions": [], + } + + def has_role() -> bool: + tester = replica_set.get_automation_config_tester() + return tester.get_role_at_index(0) == role + + wait_until(has_role, timeout=90, sleep_time=5) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/replica_set_x509_to_scram_transition.py b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_x509_to_scram_transition.py new file mode 100644 index 000000000..ea23e7cb8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/replica_set_x509_to_scram_transition.py @@ -0,0 +1,187 @@ +import time + +import pytest +from kubetester import try_load +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_agent_tls_certs, + create_mongodb_tls_certs, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester +from kubetester.omtester import get_rs_cert_names +from pytest import fixture + +MDB_RESOURCE = "replica-set-x509-to-scram-256" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@fixture(scope="module") +def replica_set(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("replica-set-x509-to-scram-256.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + try_load(res) + return res + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +class TestEnableX509ForReplicaSet(KubernetesTester): + def test_replica_set_running(self, replica_set: MongoDB): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-X509") + tester.assert_authoritative_set(True) + tester.assert_authentication_enabled() + tester.assert_expected_users(0) + + def test_deployment_is_reachable(self, replica_set: MongoDB): + tester = replica_set.tester() + # Due to what we found out in + # https://jira.mongodb.org/browse/CLOUDP-68873 + # the agents might report being in goal state, the MDB resource + # would report no errors but the deployment would be unreachable + # See the comment inside the function for further details + time.sleep(20) + tester.assert_deployment_reachable() + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +def test_enable_scram_and_x509(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["authentication"]["modes"] = ["X509", "SCRAM"] + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +def test_x509_is_still_configured(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=300) + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-X509") + tester.assert_authoritative_set(True) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", active_auth_mechanism=False) + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + tester.assert_expected_users(0) + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +class TestReplicaSetDisableAuthentication(KubernetesTester): + def test_disable_auth(self, replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["authentication"]["enabled"] = False + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + def test_assert_connectivity(self, replica_set: MongoDB, ca_path: str): + replica_set.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_disabled("MONGODB-X509") + tester.assert_authentication_mechanism_disabled("SCRAM-SHA-256") + tester.assert_authentication_disabled() + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +class TestCanEnableScramSha256: + def test_can_enable_scram_sha_256(self, replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["authentication"]["enabled"] = True + replica_set["spec"]["security"]["authentication"]["modes"] = ["SCRAM"] + replica_set["spec"]["security"]["authentication"]["agents"]["mode"] = "SCRAM" + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + def test_assert_connectivity(self, replica_set: MongoDB, ca_path: str): + replica_set.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_disabled("MONGODB-X509") + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_expected_users(0) + tester.assert_authentication_enabled() + tester.assert_authoritative_set(True) + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +class TestCreateScramSha256User(KubernetesTester): + """ + description: | + Creates a SCRAM-SHA-256 user + create: + file: scram-sha-user.yaml + patch: '[{"op":"replace","path":"/spec/mongodbResourceRef/name","value": "replica-set-x509-to-scram-256" }]' + wait_until: in_updated_state + timeout: 150 + """ + + @classmethod + def setup_class(cls): + print(f"creating password for MongoDBUser {USER_NAME} in secret/{PASSWORD_SECRET_NAME} ") + KubernetesTester.create_secret( + KubernetesTester.get_namespace(), + PASSWORD_SECRET_NAME, + { + "password": USER_PASSWORD, + }, + ) + super().setup_class() + + def test_user_can_authenticate_with_correct_password(self, ca_path: str): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ssl=True, + tlsCAFile=ca_path, + # As of today, user CRs don't have the status/phase fields. So there's no other way + # to verify that they were created other than just spinning and checking. + # See https://jira.mongodb.org/browse/CLOUDP-150729 + # 120 * 5s ~= 600s - the usual timeout we use + attempts=120, + ) + + def test_user_cannot_authenticate_with_incorrect_password(self, ca_path: str): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ssl=True, + tlsCAFile=ca_path, + ) + + +@pytest.mark.e2e_replica_set_x509_to_scram_transition +class TestReplicaSetDeleted(KubernetesTester): + """ + description: | + Deletes the Replica Set + delete: + file: replica-set-x509-to-scram-256.yaml + wait_until: mongo_resource_deleted + timeout: 240 + """ + + def test_noop(self): + pass diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sha1_connectivity_tests.py b/docker/mongodb-enterprise-tests/tests/authentication/sha1_connectivity_tests.py new file mode 100644 index 000000000..d9bd2b264 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sha1_connectivity_tests.py @@ -0,0 +1,150 @@ +import kubernetes +from kubetester import MongoDB, create_or_update_secret, try_load +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.mongotester import MongoTester +from pytest import fixture + + +class SHA1ConnectivityTests: + PASSWORD_SECRET_NAME = "mms-user-1-password" + USER_PASSWORD = "my-password" + USER_NAME = "mms-user-1" + + @fixture + def yaml_file(self): + raise Exception("Not implemented, should be defined in a subclass") + + @fixture + def mdb_resource_name(self): + raise Exception("Not implemented, should be defined in a subclass") + + @fixture + def mongo_tester(self, mdb_resource_name: str): + raise Exception("Not implemented, should be defined in a subclass") + + @fixture + def mdb(self, namespace, mdb_resource_name, yaml_file, custom_mdb_version: str): + mdb = MongoDB.from_yaml( + yaml_fixture(yaml_file), + namespace=namespace, + name=mdb_resource_name, + ) + mdb["spec"]["version"] = custom_mdb_version + + try_load(mdb) + return mdb + + def test_create_cluster(self, mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running) + + def test_cluster_connectivity(self, mongo_tester: MongoTester): + mongo_tester.assert_connectivity() + + def test_ops_manager_state_correctly_updated(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-CR") + tester.assert_authoritative_set(True) + tester.assert_authentication_enabled(2) + tester.assert_expected_users(0) + + # CreateMongoDBUser + + def test_create_secret(self): + print(f"creating password for MongoDBUser {self.USER_NAME} in secret/{self.PASSWORD_SECRET_NAME} ") + + create_or_update_secret( + KubernetesTester.get_namespace(), + self.PASSWORD_SECRET_NAME, + { + "password": self.USER_PASSWORD, + }, + ) + + def test_create_user(self, namespace: str, mdb_resource_name: str): + mdb = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user.yaml"), + namespace=namespace, + ) + mdb["spec"]["mongodbResourceRef"]["name"] = mdb_resource_name + + mdb.update() + mdb.assert_reaches_phase(Phase.Updated, timeout=150) + + # ClusterIsUpdatedWithNewUser + + def test_ops_manager_state_with_users_correctly_updated(self): + expected_roles = { + ("admin", "clusterAdmin"), + ("admin", "userAdminAnyDatabase"), + ("admin", "readWrite"), + ("admin", "userAdminAnyDatabase"), + } + + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_has_user(self.USER_NAME) + tester.assert_user_has_roles(self.USER_NAME, expected_roles) + tester.assert_expected_users(1) + + def test_user_cannot_authenticate_with_incorrect_password(self, mongo_tester: MongoTester): + mongo_tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-1", + ) + + def test_user_can_authenticate_with_correct_password(self, mongo_tester: MongoTester): + mongo_tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-1", + attempts=20, + ) + + # CanChangePassword + + def test_update_secret(self, mdb: MongoDB): + print(f"updating password for MongoDBUser {self.USER_NAME} in secret/{self.PASSWORD_SECRET_NAME}") + KubernetesTester.update_secret( + KubernetesTester.get_namespace(), + self.PASSWORD_SECRET_NAME, + {"password": "my-new-password"}, + ) + + def test_user_can_authenticate_with_new_password(self, mongo_tester: MongoTester): + mongo_tester.assert_scram_sha_authentication( + password="my-new-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-1", + attempts=20, + ) + + def test_user_cannot_authenticate_with_old_password(self, mongo_tester: MongoTester): + mongo_tester.assert_scram_sha_authentication_fails( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-1", + ) + + def test_authentication_is_disabled_once_resource_is_deleted(namespace: str, mdb: MongoDB): + mdb.delete() + + def resource_is_deleted() -> bool: + try: + mdb.load() + return False + except kubernetes.client.ApiException as e: + return e.status == 404 + + # wait until the resource is deleted + run_periodically(resource_is_deleted, timeout=300) + + def authentication_was_disabled() -> bool: + return KubernetesTester.get_automation_config()["auth"]["disabled"] + + run_periodically(authentication_was_disabled, timeout=60) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_ldap.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_ldap.py new file mode 100644 index 000000000..a6bf37e30 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_ldap.py @@ -0,0 +1,149 @@ +import time +from typing import Dict, List + +from kubetester import wait_until +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.ldap import LDAPUser, OpenLDAP +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from kubetester.mongotester import ShardedClusterTester +from pytest import fixture, mark + + +@fixture(scope="module") +def operator_installation_config(operator_installation_config_quick_recovery: Dict[str, str]) -> Dict[str, str]: + return operator_installation_config_quick_recovery + + +@fixture(scope="module") +def sharded_cluster( + openldap_tls: OpenLDAP, + namespace: str, + issuer_ca_configmap: str, +) -> MongoDB: + bind_query_password_secret = "bind-query-password" + resource = MongoDB.from_yaml(yaml_fixture("ldap/ldap-sharded-cluster.yaml"), namespace=namespace) + + KubernetesTester.create_secret(namespace, bind_query_password_secret, {"password": openldap_tls.admin_password}) + + resource["spec"]["security"]["authentication"]["ldap"] = { + "servers": [openldap_tls.servers], + "bindQueryPasswordSecretRef": {"name": bind_query_password_secret}, + "transportSecurity": "none", # For testing CLOUDP-229222 + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + } + resource["spec"]["security"]["authentication"]["agents"] = {"mode": "SCRAM"} + resource["spec"]["security"]["authentication"]["modes"] = ["LDAP", "SCRAM"] + + return resource.create() + + +@fixture(scope="module") +def ldap_user_mongodb(replica_set: MongoDB, namespace: str, ldap_mongodb_user_tls: LDAPUser) -> MongoDBUser: + """Returns a list of MongoDBUsers (already created) and their corresponding passwords.""" + user = generic_user( + namespace, + username=ldap_mongodb_user_tls.username, + db="$external", + mongodb_resource=replica_set, + password=ldap_mongodb_user_tls.password, + ) + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + + return user.create() + + +@mark.e2e_sharded_cluster_ldap +def test_sharded_cluster_pending_CLOUDP_229222(sharded_cluster: MongoDB): + """ + This function tests CLOUDP-229222. The resource needs to enter the "Pending" state and without the automatic + recovery, it would stay like this forever (since we wouldn't push the new AC with a fix). + """ + sharded_cluster.assert_reaches_phase(Phase.Pending, timeout=100) + + +@mark.e2e_sharded_cluster_ldap +def test_sharded_cluster_turn_tls_on_CLOUDP_229222(sharded_cluster: MongoDB): + """ + This function tests CLOUDP-229222. The user attempts to fix the AutomationConfig. + Before updating the AutomationConfig, we need to ensure the operator pushed the wrong one to Ops Manager. + """ + + def wait_for_ac_exists() -> bool: + ac = sharded_cluster.get_automation_config_tester().automation_config + try: + _ = ac["ldap"]["transportSecurity"] + _ = ac["version"] + return True + except KeyError: + return False + + wait_until(wait_for_ac_exists, timeout=800) + current_version = sharded_cluster.get_automation_config_tester().automation_config["version"] + + def wait_for_ac_pushed() -> bool: + ac = sharded_cluster.get_automation_config_tester().automation_config + try: + transport_security = ac["ldap"]["transportSecurity"] + new_version = ac["version"] + if transport_security != "none": + return False + if new_version <= current_version: + return False + return True + except KeyError: + return False + + wait_until(wait_for_ac_pushed, timeout=800) + + resource = sharded_cluster.load() + resource["spec"]["security"]["authentication"]["ldap"]["transportSecurity"] = "tls" + resource.update() + + +@mark.e2e_sharded_cluster_ldap +def test_sharded_cluster_CLOUDP_229222(sharded_cluster: MongoDB): + """ + This function tests CLOUDP-229222. The recovery mechanism kicks in and pushes Automation Config. The ReplicaSet + goes into running state. + """ + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) + + +# TODO: Move to a MongoDBUsers (based on KubeObject) for this user creation. +@mark.e2e_sharded_cluster_ldap +class TestAddLDAPUsers(KubernetesTester): + """ + name: Create LDAP Users + create_many: + file: ldap/ldap-sharded-cluster-user.yaml + wait_until: all_users_ready + """ + + def test_users_ready(self): + pass + + @staticmethod + def all_users_ready(): + ac = KubernetesTester.get_automation_config() + automation_config_users = 0 + for user in ac["auth"]["usersWanted"]: + if user["user"] != "mms-backup-agent" and user["user"] != "mms-monitoring-agent": + automation_config_users += 1 + + return automation_config_users == 1 + + +@mark.e2e_sharded_cluster_ldap +def test_new_mdb_users_are_created(sharded_cluster: MongoDB, ldap_mongodb_users: List[LDAPUser]): + tester = ShardedClusterTester(sharded_cluster.name, 1) + + for ldap_user in ldap_mongodb_users: + tester.assert_ldap_authentication(ldap_user.username, ldap_user.password, attempts=10) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_1_connectivity.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_1_connectivity.py new file mode 100644 index 000000000..3c72324f2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_1_connectivity.py @@ -0,0 +1,19 @@ +import pytest +from kubetester.mongotester import ShardedClusterTester +from pytest import fixture +from tests.authentication.sha1_connectivity_tests import SHA1ConnectivityTests + + +@pytest.mark.e2e_sharded_cluster_scram_sha_1_user_connectivity +class TestShardedClusterSHA1Connectivity(SHA1ConnectivityTests): + @fixture + def yaml_file(self): + return "sharded-cluster-explicit-scram-sha-1.yaml" + + @fixture + def mdb_resource_name(self): + return "my-sharded-cluster-scram-sha-1" + + @fixture + def mongo_tester(self, mdb_resource_name: str): + return ShardedClusterTester(mdb_resource_name, 2) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_256_connectivity.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_256_connectivity.py new file mode 100644 index 000000000..029e91fb2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_256_connectivity.py @@ -0,0 +1,128 @@ +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ShardedClusterTester + +MDB_RESOURCE = "sharded-cluster-scram-sha-256" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@pytest.mark.e2e_sharded_cluster_scram_sha_256_user_connectivity +class TestShardedClusterCreation(KubernetesTester): + """ + description: | + Creates a Sharded Cluster and checks everything is created as expected. + """ + + def test_create_sharded_cluster(self, custom_mdb_version: str): + resource = MongoDB.from_yaml(load_fixture("sharded-cluster-scram-sha-256.yaml"), namespace=self.namespace) + resource.set_version(custom_mdb_version) + resource.update() + + resource.assert_reaches_phase(Phase.Running) + + def test_sharded_cluster_connectivity(self): + ShardedClusterTester(MDB_RESOURCE, 2).assert_connectivity() + + def test_ops_manager_state_correctly_updated(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + + +@pytest.mark.e2e_sharded_cluster_scram_sha_256_user_connectivity +class TestCreateMongoDBUser(KubernetesTester): + """ + description: | + Creates a MongoDBUser + create: + file: scram-sha-user.yaml + patch: '[{"op":"replace","path":"/spec/mongodbResourceRef/name","value": "sharded-cluster-scram-sha-256" }]' + wait_until: in_updated_state + timeout: 150 + """ + + @classmethod + def setup_class(cls): + print(f"creating password for MongoDBUser {USER_NAME} in secret/{PASSWORD_SECRET_NAME} ") + KubernetesTester.create_secret( + KubernetesTester.get_namespace(), + PASSWORD_SECRET_NAME, + { + "password": USER_PASSWORD, + }, + ) + super().setup_class() + + def test_create_user(self): + pass + + +@pytest.mark.e2e_sharded_cluster_scram_sha_256_user_connectivity +class TestShardedClusterIsUpdatedWithNewUser(KubernetesTester): + def test_sharded_cluster_connectivity(self): + ShardedClusterTester(MDB_RESOURCE, 2).assert_connectivity() + + def test_ops_manager_state_correctly_updated(self): + expected_roles = { + ("admin", "clusterAdmin"), + ("admin", "userAdminAnyDatabase"), + ("admin", "readWrite"), + ("admin", "userAdminAnyDatabase"), + } + + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_has_user(USER_NAME) + tester.assert_user_has_roles(USER_NAME, expected_roles) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + tester.assert_expected_users(1) + + def test_user_cannot_authenticate_with_incorrect_password(self): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) + + def test_user_can_authenticate_with_correct_password(self): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_256_user_connectivity +class TestCanChangePassword(KubernetesTester): + @classmethod + def setup_env(cls): + print(f"updating password for MongoDBUser {USER_NAME} in secret/{PASSWORD_SECRET_NAME}") + KubernetesTester.update_secret( + KubernetesTester.get_namespace(), + PASSWORD_SECRET_NAME, + {"password": "my-new-password"}, + ) + + def test_user_can_authenticate_with_new_password(self): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication( + password="my-new-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + attempts=20, + ) + + def test_user_cannot_authenticate_with_old_password(self): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication_fails( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_and_x509.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_and_x509.py new file mode 100644 index 000000000..c8885aeb0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_and_x509.py @@ -0,0 +1,196 @@ +import tempfile + +import pytest +from kubetester import create_secret, try_load +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + create_sharded_cluster_certs, + create_x509_agent_tls_certs, + create_x509_user_cert, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.mongotester import ShardedClusterTester + +MDB_RESOURCE = "sharded-cluster-tls-scram-sha-256" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + ) + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.fixture(scope="module") +def sharded_cluster(namespace: str, server_certs, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("sharded-cluster-tls-scram-sha-256.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + + try_load(res) + return res + + +@pytest.fixture(scope="module") +def mongodb_user_password_secret(namespace: str) -> str: + create_secret( + namespace=namespace, + name=PASSWORD_SECRET_NAME, + data={ + "password": USER_PASSWORD, + }, + ) + return PASSWORD_SECRET_NAME + + +@pytest.fixture(scope="module") +def scram_user(sharded_cluster: MongoDB, mongodb_user_password_secret: str, namespace: str) -> MongoDBUser: + user = MongoDBUser.from_yaml(load_fixture("scram-sha-user.yaml"), namespace=namespace) + user["spec"]["mongodbResourceRef"]["name"] = sharded_cluster.name + user["spec"]["passwordSecretKeyRef"]["name"] = mongodb_user_password_secret + return user.create() + + +@pytest.fixture(scope="module") +def x509_user(sharded_cluster: MongoDB, namespace: str) -> MongoDBUser: + user = MongoDBUser.from_yaml(load_fixture("test-x509-user.yaml"), namespace=namespace) + user["spec"]["mongodbResourceRef"]["name"] = sharded_cluster.name + return user.create() + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_sharded_cluster_running(sharded_cluster: MongoDB): + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_sharded_cluster_connectivity(sharded_cluster: MongoDB, ca_path: str): + tester = sharded_cluster.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_ops_manager_state_correctly_updated_sha(): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_user_reaches_updated_phase(scram_user: MongoDBUser): + scram_user.assert_reaches_phase(Phase.Updated, timeout=150) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_user_can_authenticate_with_correct_password(ca_path: str): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + # As of today, user CRs don't have the status/phase fields. So there's no other way + # to verify that they were created other than just spinning and checking. + # See https://jira.mongodb.org/browse/CLOUDP-150729 + # 120 * 5s ~= 600s - the usual timeout we use + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + attempts=120, + ) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_user_cannot_authenticate_with_incorrect_password(ca_path: str): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + ) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_enable_x509(sharded_cluster: MongoDB): + sharded_cluster.load() + sharded_cluster["spec"]["security"]["authentication"]["modes"].append("X509") + sharded_cluster["spec"]["security"]["authentication"]["agents"] = {"mode": "SCRAM"} + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=900) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_ops_manager_state_correctly_updated_sha_and_x509(): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-X509", active_auth_mechanism=False) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + tester.assert_expected_users(1) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_x509_user_reaches_updated_phase(x509_user: MongoDBUser): + x509_user.assert_reaches_phase(Phase.Updated, timeout=150) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +def test_x509_user_exists_in_automation_config(x509_user: MongoDBUser): + ac = KubernetesTester.get_automation_config() + users = ac["auth"]["usersWanted"] + return x509_user["spec"]["username"] in (user["user"] for user in users) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +class TestX509CertCreationAndApproval(KubernetesTester): + def setup_method(self): + super().setup_method() + self.cert_file = tempfile.NamedTemporaryFile(delete=False, mode="w") + + def test_create_user_and_authenticate(self, issuer: str, namespace: str, ca_path: str): + create_x509_user_cert(issuer, namespace, path=self.cert_file.name) + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_x509_authentication(cert_file_name=self.cert_file.name, tlsCAFile=ca_path) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_and_x509 +class TestCanStillAuthAsScramUsers(KubernetesTester): + def test_user_can_authenticate_with_correct_password(self, ca_path: str): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + # As of today, user CRs don't have the status/phase fields. So there's no other way + # to verify that they were created other than just spinning and checking. + # See https://jira.mongodb.org/browse/CLOUDP-150729 + # 120 * 5s ~= 600s - the usual timeout we use + attempts=120, + ) + + def test_user_cannot_authenticate_with_incorrect_password(self, ca_path: str): + tester = ShardedClusterTester(MDB_RESOURCE, 2) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + ssl=True, + auth_mechanism="SCRAM-SHA-256", + tlsCAFile=ca_path, + ) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_upgrade.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_upgrade.py new file mode 100644 index 000000000..516bae9e5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_sha_upgrade.py @@ -0,0 +1,49 @@ +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ShardedClusterTester + +MDB_RESOURCE = "my-sharded-cluster-scram-sha-1" + + +@pytest.mark.e2e_sharded_cluster_scram_sha_1_upgrade +class TestCreateScramSha1ShardedCluster(KubernetesTester): + """ + description: | + Creates a ShardedCluster with SCRAM-SHA-1 authentication + """ + + def test_create_sharded_cluster(self, custom_mdb_version: str): + resource = MongoDB.from_yaml(load_fixture("sharded-cluster-scram-sha-1.yaml"), namespace=self.namespace) + resource.set_version(custom_mdb_version) + resource.update() + + resource.assert_reaches_phase(Phase.Running) + + def test_assert_connectivity(self): + ShardedClusterTester(MDB_RESOURCE, 2).assert_connectivity() + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + + tester.assert_expected_users(0) + tester.assert_authoritative_set(True) + + +@pytest.mark.e2e_sharded_cluster_scram_sha_1_upgrade +class TestShardedClusterDeleted(KubernetesTester): + """ + description: | + Deletes the Sharded Cluster + delete: + file: sharded-cluster-scram-sha-1.yaml + wait_until: mongo_resource_deleted + timeout: 240 + """ + + def test_noop(self): + pass diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_x509_ic_manual_certs.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_x509_ic_manual_certs.py new file mode 100644 index 000000000..88a6a6e64 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_x509_ic_manual_certs.py @@ -0,0 +1,82 @@ +from kubetester.certs import SetProperties, create_mongodb_tls_certs +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + +MDB_RESOURCE = "sharded-cluster-scram-sha-256" +SUBJECT = {"organizations": ["MDB Tests"], "organizationalUnits": ["Servers"]} +SERVER_SETS = frozenset( + [ + SetProperties(MDB_RESOURCE + "-0", MDB_RESOURCE + "-sh", 3), + SetProperties(MDB_RESOURCE + "-config", MDB_RESOURCE + "-cs", 3), + SetProperties(MDB_RESOURCE + "-mongos", MDB_RESOURCE + "-svc", 2), + ] +) + + +@fixture(scope="module") +def all_certs(issuer, namespace) -> None: + """Generates all required TLS certificates: Servers and Client/Member.""" + spec_server = { + "subject": SUBJECT, + "usages": ["server auth"], + } + spec_client = { + "subject": SUBJECT, + "usages": ["client auth"], + } + + for server_set in SERVER_SETS: + create_mongodb_tls_certs( + issuer, + namespace, + server_set.name, + server_set.name + "-cert", + server_set.replicas, + service_name=server_set.service, + spec=spec_server, + ) + create_mongodb_tls_certs( + issuer, + namespace, + server_set.name, + server_set.name + "-clusterfile", + server_set.replicas, + service_name=server_set.service, + spec=spec_client, + ) + + +@fixture(scope="module") +def sharded_cluster( + namespace: str, + all_certs, + issuer_ca_configmap: str, +) -> MongoDB: + mdb: MongoDB = MongoDB.from_yaml( + _fixture("sharded-cluster-scram-sha-256-x509-internal-cluster.yaml"), + namespace=namespace, + ) + mdb["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return mdb.create() + + +@mark.e2e_sharded_cluster_scram_x509_ic_manual_certs +def test_create_replica_set_with_x509_internal_cluster(sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_sharded_cluster_scram_x509_ic_manual_certs +def test_create_replica_can_connect(sharded_cluster: MongoDB, ca_path: str): + sharded_cluster.assert_connectivity(ca_path=ca_path) + + +@mark.e2e_sharded_cluster_scram_x509_ic_manual_certs +def test_ops_manager_state_was_updated_correctly(sharded_cluster: MongoDB): + ac_tester = sharded_cluster.get_automation_config_tester() + ac_tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + ac_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + ac_tester.assert_internal_cluster_authentication_enabled() + + ac_tester.assert_expected_users(0) + ac_tester.assert_authoritative_set(True) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_x509_internal_cluster.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_x509_internal_cluster.py new file mode 100644 index 000000000..f555073b0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_scram_x509_internal_cluster.py @@ -0,0 +1,58 @@ +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_sharded_cluster_certs, + create_x509_agent_tls_certs, + create_x509_mongodb_tls_certs, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as find_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.omtester import get_sc_cert_names +from pytest import fixture, mark + +MDB_RESOURCE_NAME = "sharded-cluster-scram-sha-256" + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE_NAME) + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE_NAME, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + internal_auth=True, + x509_certs=True, + ) + + +@fixture(scope="module") +def sharded_cluster(namespace: str, server_certs, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-scram-sha-256-x509-internal-cluster.yaml"), + namespace=namespace, + ) + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + yield resource.create() + + +@mark.e2e_sharded_cluster_scram_x509_internal_cluster +class TestReplicaSetScramX509Internal(KubernetesTester): + def test_create_resource(self, sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_ops_manager_state_was_updated_correctly(self): + ac_tester = AutomationConfigTester(self.get_automation_config()) + ac_tester.assert_authentication_enabled() + ac_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + ac_tester.assert_internal_cluster_authentication_enabled() + + ac_tester.assert_expected_users(0) + ac_tester.assert_authoritative_set(True) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_internal_cluster_transition.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_internal_cluster_transition.py new file mode 100644 index 000000000..957be146d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_internal_cluster_transition.py @@ -0,0 +1,55 @@ +from kubetester import find_fixture +from kubetester.certs import create_sharded_cluster_certs, create_x509_agent_tls_certs +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + +MDB_RESOURCE_NAME = "sc-internal-cluster-auth-transition" + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE_NAME) + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE_NAME, + shards=2, + mongod_per_shard=3, + config_servers=1, + mongos=1, + internal_auth=True, + x509_certs=True, + ) + + +@fixture(scope="module") +def sc(namespace: str, server_certs, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-x509-internal-cluster-auth-transition.yaml"), + namespace=namespace, + ) + resource["spec"]["security"] = { + "tls": { + "enabled": True, + "ca": issuer_ca_configmap, + }, + "authentication": {"enabled": True, "modes": ["X509"]}, + } + yield resource.update() + + +@mark.e2e_sharded_cluster_internal_cluster_transition +def test_create_resource(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_sharded_cluster_internal_cluster_transition +def test_enable_internal_cluster_authentication(sc: MongoDB): + sc.load() + sc["spec"]["security"]["authentication"]["internalCluster"] = "X509" + sc.update() + + sc.assert_reaches_phase(Phase.Running, timeout=2400) diff --git a/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py new file mode 100644 index 000000000..682539339 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/authentication/sharded_cluster_x509_to_scram_transition.py @@ -0,0 +1,172 @@ +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_mongodb_tls_certs, + create_sharded_cluster_certs, + create_x509_agent_tls_certs, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ShardedClusterTester +from kubetester.omtester import get_sc_cert_names +from pytest import fixture + +MDB_RESOURCE = "sharded-cluster-x509-to-scram-256" +USER_NAME = "mms-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE, + shards=2, + mongod_per_shard=3, + config_servers=2, + mongos=1, + ) + + +@fixture(scope="module") +def sharded_cluster(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + load_fixture("sharded-cluster-x509-to-scram-256.yaml"), + namespace=namespace, + ) + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + yield resource.create() + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +class TestEnableX509ForShardedCluster(KubernetesTester): + def test_create_resource(self, sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-X509") + tester.assert_authentication_enabled() + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +def test_enable_scram_and_x509(sharded_cluster: MongoDB): + sharded_cluster.load() + sharded_cluster["spec"]["security"]["authentication"]["modes"] = ["X509", "SCRAM"] + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=900) + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +def test_x509_is_still_configured(): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-X509") + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", active_auth_mechanism=False) + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +class TestShardedClusterDisableAuthentication(KubernetesTester): + def test_disable_auth(self, sharded_cluster: MongoDB): + sharded_cluster.load() + sharded_cluster["spec"]["security"]["authentication"]["enabled"] = False + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1500) + + def test_assert_connectivity(self, ca_path: str): + ShardedClusterTester(MDB_RESOURCE, 1, ssl=True, ca_path=ca_path).assert_connectivity() + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_disabled("MONGODB-X509") + tester.assert_authentication_disabled() + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +class TestCanEnableScramSha256: + def test_can_enable_scram_sha_256(self, sharded_cluster: MongoDB): + sharded_cluster.load() + sharded_cluster["spec"]["security"]["authentication"]["enabled"] = True + sharded_cluster["spec"]["security"]["authentication"]["modes"] = [ + "SCRAM", + ] + sharded_cluster["spec"]["security"]["authentication"]["agents"]["mode"] = "SCRAM" + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_assert_connectivity(self, ca_path: str): + ShardedClusterTester(MDB_RESOURCE, 1, ssl=True, ca_path=ca_path).assert_connectivity(attempts=25) + + def test_ops_manager_state_updated_correctly(self): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_disabled("MONGODB-X509") + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled() + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +class TestCreateScramSha256User(KubernetesTester): + """ + description: | + Creates a SCRAM-SHA-256 user + create: + file: scram-sha-user.yaml + patch: '[{"op":"replace","path":"/spec/mongodbResourceRef/name","value": "sharded-cluster-x509-to-scram-256" }]' + wait_until: in_updated_state + timeout: 150 + """ + + @classmethod + def setup_class(cls): + print(f"creating password for MongoDBUser {USER_NAME} in secret/{PASSWORD_SECRET_NAME} ") + KubernetesTester.create_secret( + KubernetesTester.get_namespace(), + PASSWORD_SECRET_NAME, + { + "password": USER_PASSWORD, + }, + ) + super().setup_class() + + def test_user_can_authenticate_with_incorrect_password(self, ca_path: str): + tester = ShardedClusterTester(MDB_RESOURCE, 1) + tester.assert_scram_sha_authentication_fails( + password="invalid-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ssl=True, + tlsCAFile=ca_path, + ) + + def test_user_can_authenticate_with_correct_password(self, ca_path: str): + tester = ShardedClusterTester(MDB_RESOURCE, 1) + tester.assert_scram_sha_authentication( + password="my-password", + username="mms-user-1", + auth_mechanism="SCRAM-SHA-256", + ssl=True, + tlsCAFile=ca_path, + ) + + +@pytest.mark.e2e_sharded_cluster_x509_to_scram_transition +class TestShardedClusterDeleted(KubernetesTester): + """ + description: | + Deletes the Sharded Cluster + delete: + file: sharded-cluster-x509-to-scram-256.yaml + wait_until: mongo_resource_deleted + timeout: 240 + """ + + def test_noop(self): + pass diff --git a/scripts/git-hooks/pre-applypatch b/docker/mongodb-enterprise-tests/tests/clusterwideoperator/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/pre-applypatch rename to docker/mongodb-enterprise-tests/tests/clusterwideoperator/__init__.py diff --git a/scripts/git-hooks/pre-merge-commit b/docker/mongodb-enterprise-tests/tests/clusterwideoperator/conftest.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/pre-merge-commit rename to docker/mongodb-enterprise-tests/tests/clusterwideoperator/conftest.py diff --git a/docker/mongodb-enterprise-tests/tests/clusterwideoperator/om_multiple.py b/docker/mongodb-enterprise-tests/tests/clusterwideoperator/om_multiple.py new file mode 100644 index 000000000..8759899e7 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/clusterwideoperator/om_multiple.py @@ -0,0 +1,181 @@ +from typing import Dict + +import kubernetes +from kubetester import create_or_update_secret, read_secret +from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml +from kubetester.helm import helm_template +from kubetester.kubetester import create_testing_namespace +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + get_central_cluster_client, + get_evergreen_task_id, + get_member_cluster_api_client, + get_member_cluster_clients, + get_multi_cluster_operator_clustermode, + get_multi_cluster_operator_installation_config, + get_operator_clusterwide, + get_operator_installation_config, + get_version_id, + is_multi_cluster, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +def _prepare_om_namespace(ops_manager_namespace: str, operator_installation_config: dict[str, str]): + """create a new namespace and configures all necessary service accounts there""" + install_database_roles( + ops_manager_namespace, + operator_installation_config, + api_client=get_central_cluster_client(), + ) + + +def install_database_roles( + namespace: str, + operator_installation_config: dict[str, str], + api_client: kubernetes.client.ApiClient, +): + try: + yaml_file = helm_template( + helm_args={ + "registry.imagePullSecrets": operator_installation_config["registry.imagePullSecrets"], + }, + templates="templates/database-roles.yaml", + helm_options=[f"--namespace {namespace}"], + ) + create_or_replace_from_yaml(api_client, yaml_file) + except Exception as e: + print(f"Caught exception while installing database roles: {e}") + raise e + + +def create_om_admin_secret(ops_manager_namespace: str, api_client: kubernetes.client.ApiClient = None): + data = dict( + Username="test-user", + Password="@Sihjifutestpass21nnH", + FirstName="foo", + LastName="bar", + ) + create_or_update_secret( + namespace=ops_manager_namespace, + name="ops-manager-admin-secret", + data=data, + api_client=api_client, + ), + + +def prepare_multi_cluster_namespace(namespace: str, new_namespace: str): + operator_installation_config = get_multi_cluster_operator_installation_config(namespace) + image_pull_secret_name = operator_installation_config["registry.imagePullSecrets"] + image_pull_secret_data = read_secret(namespace, image_pull_secret_name, api_client=get_central_cluster_client()) + for member_cluster_client in get_member_cluster_clients(): + install_database_roles( + new_namespace, + operator_installation_config, + member_cluster_client.api_client, + ) + create_testing_namespace( + get_evergreen_task_id(), + new_namespace, + member_cluster_client.api_client, + True, + ) + create_or_update_secret( + new_namespace, + image_pull_secret_name, + image_pull_secret_data, + type="kubernetes.io/dockerconfigjson", + api_client=member_cluster_client.api_client, + ) + + +def ops_manager(namespace: str, custom_version: str, custom_appdb_version: str) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource + + +def prepare_namespace(namespace: str, new_namespace: str, operator_installation_config): + if is_multi_cluster(): + prepare_multi_cluster_namespace(namespace, new_namespace) + else: + _prepare_om_namespace(new_namespace, operator_installation_config) + create_om_admin_secret(new_namespace, api_client=get_central_cluster_client()) + + +@fixture(scope="module") +def om1( + namespace: str, + custom_version: str, + custom_appdb_version: str, + operator_installation_config: Dict[str, str], +) -> MongoDBOpsManager: + prepare_namespace(namespace, "om-1", operator_installation_config) + om = ops_manager("om-1", custom_version, custom_appdb_version) + om.update() + return om + + +@fixture(scope="module") +def om2( + namespace: str, + custom_version: str, + custom_appdb_version: str, + operator_installation_config: Dict[str, str], +) -> MongoDBOpsManager: + prepare_namespace(namespace, "om-2", operator_installation_config) + om = ops_manager("om-2", custom_version, custom_appdb_version) + om.update() + return om + + +@fixture(scope="module") +def om_operator_clusterwide(namespace: str): + if is_multi_cluster(): + return get_multi_cluster_operator_clustermode(namespace) + else: + return get_operator_clusterwide(namespace, get_operator_installation_config(namespace)) + + +@mark.e2e_om_multiple +def test_install_operator(om_operator_clusterwide: Operator): + om_operator_clusterwide.assert_is_running() + + +@mark.e2e_om_multiple +def test_create_namespaces(evergreen_task_id: str): + create_testing_namespace(evergreen_task_id, "om-1") + create_testing_namespace(evergreen_task_id, "om-2") + + +@mark.e2e_om_multiple +def test_multiple_om_create(om1: MongoDBOpsManager, om2: MongoDBOpsManager): + om1.om_status().assert_reaches_phase(Phase.Running, timeout=1100) + om2.om_status().assert_reaches_phase(Phase.Running, timeout=1100) + + +@mark.e2e_om_multiple +def test_image_pull_secret_om_created_1(namespace: str, operator_installation_config: Dict[str, str]): + """check if imagePullSecrets was cloned in the OM namespace""" + secret_name = operator_installation_config["registry.imagePullSecrets"] + secretDataInOperatorNs = read_secret(namespace, secret_name) + secretDataInOmNs = read_secret("om-1", secret_name) + assert secretDataInOperatorNs == secretDataInOmNs + + +@mark.e2e_om_multiple +def test_image_pull_secret_om_created_2(namespace: str, operator_installation_config: Dict[str, str]): + """check if imagePullSecrets was cloned in the OM namespace""" + secret_name = operator_installation_config["registry.imagePullSecrets"] + secretDataInOperatorNs = read_secret(namespace, secret_name) + secretDataInOmNs = read_secret("om-2", secret_name) + assert secretDataInOperatorNs == secretDataInOmNs diff --git a/scripts/git-hooks/pre-push b/docker/mongodb-enterprise-tests/tests/common/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/pre-push rename to docker/mongodb-enterprise-tests/tests/common/__init__.py diff --git a/docker/mongodb-enterprise-tests/tests/common/constants.py b/docker/mongodb-enterprise-tests/tests/common/constants.py new file mode 100644 index 000000000..70861838e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/common/constants.py @@ -0,0 +1,10 @@ +MEMBER_CLUSTER_1 = "kind-e2e-cluster-1" +MEMBER_CLUSTER_2 = "kind-e2e-cluster-2" +MEMBER_CLUSTER_3 = "kind-e2e-cluster-3" + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +MONGODB_PORT = 30000 + +S3_OPLOG_NAME = "s3-oplog" +S3_BLOCKSTORE_NAME = "s3-blockstore" diff --git a/scripts/git-hooks/pre-rebase b/docker/mongodb-enterprise-tests/tests/common/ops_manager/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/pre-rebase rename to docker/mongodb-enterprise-tests/tests/common/ops_manager/__init__.py diff --git a/docker/mongodb-enterprise-tests/tests/common/ops_manager/multi_cluster.py b/docker/mongodb-enterprise-tests/tests/common/ops_manager/multi_cluster.py new file mode 100644 index 000000000..bbaae3e7e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/common/ops_manager/multi_cluster.py @@ -0,0 +1,54 @@ +import kubernetes +from kubetester.awss3client import s3_endpoint +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.opsmanager import MongoDBOpsManager +from tests.common.constants import S3_BLOCKSTORE_NAME, S3_OPLOG_NAME +from tests.conftest import AWS_REGION + + +def ops_manager_multi_cluster_with_tls_s3_backups( + namespace: str, + name: str, + central_cluster_client: kubernetes.client.ApiClient, + custom_appdb_version: str, + s3_bucket_blockstore: str, + s3_bucket_oplog: str, +): + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_tls_s3.yaml"), name=name, namespace=namespace + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.allow_mdb_rc_versions() + resource.set_appdb_version(custom_appdb_version) + + # configure S3 Blockstore + resource["spec"]["backup"]["s3Stores"][0]["name"] = S3_BLOCKSTORE_NAME + resource["spec"]["backup"]["s3Stores"][0]["s3SecretRef"]["name"] = S3_BLOCKSTORE_NAME + "-secret" + resource["spec"]["backup"]["s3Stores"][0]["s3BucketEndpoint"] = s3_endpoint(AWS_REGION) + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket_blockstore + resource["spec"]["backup"]["s3Stores"][0]["s3RegionOverride"] = AWS_REGION + # configure S3 Oplog + resource["spec"]["backup"]["s3OpLogStores"][0]["name"] = S3_OPLOG_NAME + resource["spec"]["backup"]["s3OpLogStores"][0]["s3SecretRef"]["name"] = S3_OPLOG_NAME + "-secret" + resource["spec"]["backup"]["s3OpLogStores"][0]["s3BucketEndpoint"] = s3_endpoint(AWS_REGION) + resource["spec"]["backup"]["s3OpLogStores"][0]["s3BucketName"] = s3_bucket_oplog + resource["spec"]["backup"]["s3OpLogStores"][0]["s3RegionOverride"] = AWS_REGION + + # configure memory overrides so OM doesn't crash + resource["spec"]["statefulSet"] = { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "mongodb-ops-manager", + "resources": {"requests": {"memory": "15G"}, "limits": {"memory": "15G"}}, + }, + ] + } + } + } + } + resource.create_admin_secret(api_client=central_cluster_client) + + return resource diff --git a/scripts/git-hooks/pre-receive b/docker/mongodb-enterprise-tests/tests/common/placeholders/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/pre-receive rename to docker/mongodb-enterprise-tests/tests/common/placeholders/__init__.py diff --git a/docker/mongodb-enterprise-tests/tests/common/placeholders/placeholders.py b/docker/mongodb-enterprise-tests/tests/common/placeholders/placeholders.py new file mode 100644 index 000000000..c28fe718d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/common/placeholders/placeholders.py @@ -0,0 +1,111 @@ +def get_annotations_with_placeholders_for_single_cluster(): + return { + "podIndex": "value={podIndex}", + "namespace": "value={namespace}", + "resourceName": "value={resourceName}", + "podName": "value={podName}", + "statefulSetName": "value={statefulSetName}", + "externalServiceName": "value={externalServiceName}", + "mongodProcessDomain": "value={mongodProcessDomain}", + "mongodProcessFQDN": "value={mongodProcessFQDN}", + } + + +def get_annotations_with_placeholders_for_multi_cluster(prefix: str = ""): + return { + "podIndex": prefix + "value={podIndex}", + "namespace": prefix + "value={namespace}", + "resourceName": prefix + "value={resourceName}", + "podName": prefix + "value={podName}", + "statefulSetName": prefix + "value={statefulSetName}", + "externalServiceName": prefix + "value={externalServiceName}", + "mongodProcessDomain": prefix + "value={mongodProcessDomain}", + "mongodProcessFQDN": prefix + "value={mongodProcessFQDN}", + "clusterName": prefix + "value={clusterName}", + "clusterIndex": prefix + "value={clusterIndex}", + } + + +def get_expected_annotations_single_cluster(name: str, namespace: str, pod_idx: int): + """Returns annotations with resolved placeholder in the context of + running single-cluster deployment without using external domains, so + with FQDNs from headless services""" + return { + "podIndex": f"value={pod_idx}", + "namespace": f"value={namespace}", + "resourceName": f"value={name}", + "podName": f"value={name}-{pod_idx}", + "statefulSetName": f"value={name}", + "externalServiceName": f"value={name}-{pod_idx}-svc-external", + "mongodProcessDomain": f"value={name}-svc.{namespace}.svc.cluster.local", + "mongodProcessFQDN": f"value={name}-{pod_idx}.{name}-svc.{namespace}.svc.cluster.local", # headless pod fqdn + } + + +def get_expected_annotations_single_cluster_with_external_domain( + name: str, namespace: str, pod_idx: int, external_domain: str +): + """Returns annotations with resolved placeholder in the context of running + single-cluster deployment with external domains, so with FQDNs from external domains""" + pod_name = f"{name}-{pod_idx}" + return { + "podIndex": f"value={pod_idx}", + "namespace": f"value={namespace}", + "resourceName": f"value={name}", + "podName": f"value={pod_name}", + "statefulSetName": f"value={name}", + "externalServiceName": f"value={pod_name}-svc-external", + "mongodProcessDomain": f"value={external_domain}", + "mongodProcessFQDN": f"value={pod_name}.{external_domain}", + } + + +def get_expected_annotations_multi_cluster( + name: str, namespace: str, pod_idx: int, cluster_name: str, cluster_index: int, prefix: str = "" +): + """Returns annotations with resolved placeholders in the context of running + multi-cluster deployment without external domains, so with FQDNs from pod services. + """ + statefulset_name = f"{name}-{cluster_index}" + pod_name = f"{statefulset_name}-{pod_idx}" + + return { + "podIndex": f"{prefix}value={pod_idx}", + "namespace": f"{prefix}value={namespace}", + "resourceName": f"{prefix}value={name}", + "podName": f"{prefix}value={pod_name}", + "statefulSetName": f"{prefix}value={statefulset_name}", + "externalServiceName": f"{prefix}value={pod_name}-svc-external", + "mongodProcessDomain": f"{prefix}value={namespace}.svc.cluster.local", + "mongodProcessFQDN": f"{prefix}value={pod_name}-svc.{namespace}.svc.cluster.local", + "clusterName": f"{prefix}value={cluster_name}", + "clusterIndex": f"{prefix}value={cluster_index}", + } + + +def get_expected_annotations_multi_cluster_no_mesh( + name: str, + namespace: str, + pod_idx: int, + external_domain: str, + cluster_name: str, + cluster_index: int, + prefix: str = "", +): + """Returns annotations with resolved placeholder in the context of running + multi-cluster deployment without a service mesh so with FQDNs from external domains. + """ + statefulset_name = f"{name}-{cluster_index}" + pod_name = f"{statefulset_name}-{pod_idx}" + return { + "podIndex": f"{prefix}value={pod_idx}", + "namespace": f"{prefix}value={namespace}", + "resourceName": f"{prefix}value={name}", + "podName": f"{prefix}value={pod_name}", + "statefulSetName": f"{prefix}value={statefulset_name}", + "externalServiceName": f"{prefix}value={pod_name}-svc-external", + "mongodProcessDomain": f"{prefix}value={external_domain}", + "mongodProcessFQDN": f"{prefix}value={pod_name}.{external_domain}", + "clusterName": f"{prefix}value={cluster_name}", + "clusterIndex": f"{prefix}value={cluster_index}", + } diff --git a/docker/mongodb-enterprise-tests/tests/conftest.py b/docker/mongodb-enterprise-tests/tests/conftest.py new file mode 100644 index 000000000..f570f865b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/conftest.py @@ -0,0 +1,1623 @@ +import json +import logging +import os +import subprocess +import tempfile +import time +from typing import Any, Callable, Dict, List, Optional + +import kubernetes +import requests +from kubernetes import client +from kubernetes.client import ApiextensionsV1Api +from kubetester import ( + create_or_update_configmap, + get_deployments, + get_pod_when_ready, + is_pod_ready, + read_secret, + update_configmap, +) +from kubetester.awss3client import AwsS3Client +from kubetester.certs import ( + Certificate, + ClusterIssuer, + Issuer, + create_mongodb_tls_certs, + create_multi_cluster_mongodb_tls_certs, +) +from kubetester.helm import helm_install_from_chart +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as _fixture +from kubetester.kubetester import running_locally +from kubetester.mongodb_multi import MultiClusterClient +from kubetester.omtester import OMContext, OMTester +from kubetester.operator import Operator +from pymongo.errors import ServerSelectionTimeoutError +from pytest import fixture +from tests import test_logger +from tests.multicluster import prepare_multi_cluster_namespaces + +try: + kubernetes.config.load_kube_config() +except Exception: + kubernetes.config.load_incluster_config() + +AWS_REGION = "us-east-1" + +KUBECONFIG_FILEPATH = "/etc/config/kubeconfig" +MULTI_CLUSTER_CONFIG_DIR = "/etc/multicluster" +# AppDB monitoring is disabled by default for e2e tests. +# If monitoring is needed use monitored_appdb_operator_installation_config / operator_with_monitored_appdb +MONITOR_APPDB_E2E_DEFAULT = "true" +MULTI_CLUSTER_OPERATOR_NAME = "mongodb-enterprise-operator-multi-cluster" +CLUSTER_HOST_MAPPING = { + "us-central1-c_central": "https://35.232.85.244", + "us-east1-b_member-1a": "https://35.243.222.230", + "us-east1-c_member-2a": "https://34.75.94.207", + "us-west1-a_member-3a": "https://35.230.121.15", +} + +LEGACY_CENTRAL_CLUSTER_NAME: str = "__default" +LEGACY_DEPLOYMENT_STATE_VERSION: str = "1.27.0" + +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def namespace() -> str: + return get_namespace() + + +def get_namespace() -> str: + return os.environ["NAMESPACE"] + + +@fixture(scope="module") +def version_id() -> str: + return get_version_id() + + +def get_version_id(): + """ + Returns VERSION_ID if it has been defined, or "latest" otherwise. + """ + if "OVERRIDE_VERSION_ID" in os.environ: + return os.environ["OVERRIDE_VERSION_ID"] + return os.environ.get("VERSION_ID", "latest") + + +@fixture(scope="module") +def operator_installation_config(namespace: str) -> Dict[str, str]: + return get_operator_installation_config(namespace) + + +def get_operator_installation_config(namespace): + """Returns the ConfigMap containing configuration data for the Operator to be created. + Created in the single_e2e.sh""" + config = KubernetesTester.read_configmap(namespace, "operator-installation-config") + config["customEnvVars"] = f"OPS_MANAGER_MONITOR_APPDB={MONITOR_APPDB_E2E_DEFAULT}" + if os.getenv("OM_DEBUG_HTTP") == "true": + print("Adding OM_DEBUG_HTTP=true to operator_installation_config") + config["customEnvVars"] += "\&OM_DEBUG_HTTP=true" + + if local_operator(): + config["operator.replicas"] = 0 + return config + + +@fixture(scope="module") +def monitored_appdb_operator_installation_config(operator_installation_config: Dict[str, str]) -> Dict[str, str]: + """Returns the ConfigMap containing configuration data for the Operator to be created + and for the AppDB to be monitored. + Created in the single_e2e.sh""" + config = operator_installation_config + config["customEnvVars"] = "OPS_MANAGER_MONITOR_APPDB=true" + return config + + +def get_multi_cluster_operator_installation_config(namespace: str) -> Dict[str, str]: + """Returns the ConfigMap containing configuration data for the Operator to be created. + Created in the single_e2e.sh""" + config = KubernetesTester.read_configmap( + namespace, + "operator-installation-config", + api_client=get_central_cluster_client(), + ) + config["customEnvVars"] = f"OPS_MANAGER_MONITOR_APPDB={MONITOR_APPDB_E2E_DEFAULT}" + return config + + +@fixture(scope="module") +def multi_cluster_operator_installation_config( + central_cluster_client: kubernetes.client.ApiClient, namespace: str +) -> Dict[str, str]: + return get_multi_cluster_operator_installation_config(namespace) + + +@fixture(scope="module") +def multi_cluster_monitored_appdb_operator_installation_config( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + multi_cluster_operator_installation_config: dict[str, str], +) -> Dict[str, str]: + multi_cluster_operator_installation_config["customEnvVars"] = f"OPS_MANAGER_MONITOR_APPDB=true" + return multi_cluster_operator_installation_config + + +@fixture(scope="module") +def operator_clusterwide( + namespace: str, + operator_installation_config: Dict[str, str], +) -> Operator: + return get_operator_clusterwide(namespace, operator_installation_config) + + +def get_operator_clusterwide(namespace, operator_installation_config): + helm_args = operator_installation_config.copy() + helm_args["operator.watchNamespace"] = "*" + return Operator(namespace=namespace, helm_args=helm_args).install() + + +@fixture(scope="module") +def operator_vault_secret_backend( + namespace: str, + monitored_appdb_operator_installation_config: Dict[str, str], +) -> Operator: + helm_args = monitored_appdb_operator_installation_config.copy() + helm_args["operator.vaultSecretBackend.enabled"] = "true" + return Operator(namespace=namespace, helm_args=helm_args).install() + + +@fixture(scope="module") +def operator_vault_secret_backend_tls( + namespace: str, + monitored_appdb_operator_installation_config: Dict[str, str], +) -> Operator: + helm_args = monitored_appdb_operator_installation_config.copy() + helm_args["operator.vaultSecretBackend.enabled"] = "true" + helm_args["operator.vaultSecretBackend.tlsSecretRef"] = "vault-tls" + return Operator(namespace=namespace, helm_args=helm_args).install() + + +@fixture(scope="module") +def operator_installation_config_quick_recovery(operator_installation_config: Dict[str, str]) -> Dict[str, str]: + """ + This functions appends automatic recovery settings for CLOUDP-189433. In order to make the test runnable in + reasonable time, we override the Recovery back off to 120 seconds. This gives enough time for the initial + automation config to be published and statefulsets to be created before forcing the recovery. + """ + operator_installation_config["customEnvVars"] = ( + operator_installation_config["customEnvVars"] + "\&MDB_AUTOMATIC_RECOVERY_BACKOFF_TIME_S=120" + ) + return operator_installation_config + + +@fixture(scope="module") +def evergreen_task_id() -> str: + return get_evergreen_task_id() + + +def get_evergreen_task_id(): + taskId = os.environ.get("TASK_ID", "") + return taskId + + +@fixture(scope="module") +def managed_security_context() -> str: + return os.environ.get("MANAGED_SECURITY_CONTEXT", "False") + + +@fixture(scope="module") +def aws_s3_client(evergreen_task_id: str) -> AwsS3Client: + return get_aws_s3_client(evergreen_task_id) + + +def get_aws_s3_client(evergreen_task_id: str = ""): + tags = {"environment": "mongodb-enterprise-operator-tests"} + + if evergreen_task_id != "": + tags["evg_task"] = evergreen_task_id + + return AwsS3Client("us-east-1", **tags) + + +@fixture(scope="session") +def crd_api(): + return ApiextensionsV1Api() + + +@fixture(scope="module") +def cert_manager() -> str: + result = install_cert_manager( + cluster_client=get_central_cluster_client(), + cluster_name=get_central_cluster_name(), + ) + wait_for_cert_manager_ready(cluster_client=get_central_cluster_client()) + return result + + +@fixture(scope="module") +def issuer(cert_manager: str, namespace: str) -> str: + return create_issuer(namespace=namespace, api_client=get_central_cluster_client()) + + +@fixture(scope="module") +def intermediate_issuer(cert_manager: str, issuer: str, namespace: str) -> str: + """ + This fixture creates an intermediate "Issuer" in the testing namespace + """ + # Create the Certificate for the intermediate CA based on the issuer fixture + intermediate_ca_cert = Certificate(namespace=namespace, name="intermediate-ca-issuer") + intermediate_ca_cert["spec"] = { + "isCA": True, + "commonName": "intermediate-ca-issuer", + "secretName": "intermediate-ca-secret", + "issuerRef": {"name": issuer}, + "dnsNames": ["intermediate-ca.example.com"], + } + intermediate_ca_cert.create().block_until_ready() + + # Create the intermediate issuer + issuer = Issuer(name="intermediate-ca-issuer", namespace=namespace) + issuer["spec"] = {"ca": {"secretName": "intermediate-ca-secret"}} + issuer.create().block_until_ready() + + return "intermediate-ca-issuer" + + +@fixture(scope="module") +def multi_cluster_issuer( + cert_manager: str, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> str: + return create_issuer(namespace, central_cluster_client) + + +@fixture(scope="module") +def multi_cluster_clusterissuer( + cert_manager: str, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> str: + return create_issuer(namespace, central_cluster_client, clusterwide=True) + + +@fixture(scope="module") +def issuer_ca_filepath(): + return get_issuer_ca_filepath() + + +def get_issuer_ca_filepath(): + return _fixture("ca-tls-full-chain.crt") + + +@fixture(scope="module") +def custom_logback_file_path(): + return _fixture("custom_logback.xml") + + +@fixture(scope="module") +def amazon_ca_1_filepath(): + return _fixture("amazon-ca-1.pem") + + +@fixture(scope="module") +def amazon_ca_2_filepath(): + return _fixture("amazon-ca-2.pem") + + +@fixture(scope="module") +def multi_cluster_issuer_ca_configmap( + issuer_ca_filepath: str, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> str: + return create_issuer_ca_configmap(issuer_ca_filepath, namespace, api_client=central_cluster_client) + + +def create_issuer_ca_configmap( + issuer_ca_filepath: str, namespace: str, name: str = "issuer-ca", api_client: kubernetes.client.ApiClient = None +): + """This is the CA file which verifies the certificates signed by it.""" + ca = open(issuer_ca_filepath).read() + # The operator expects the CA that validates Ops Manager is contained in + # an entry with a name of "mms-ca.crt" + data = {"ca-pem": ca, "mms-ca.crt": ca} + create_or_update_configmap(namespace, name, data, api_client=api_client) + return name + + +@fixture(scope="module") +def issuer_ca_configmap(issuer_ca_filepath: str, namespace: str) -> str: + """This is the CA file which verifies the certificates signed by it.""" + ca = open(issuer_ca_filepath).read() + + # The operator expects the CA that validates Ops Manager is contained in + # an entry with a name of "mms-ca.crt" + data = {"ca-pem": ca, "mms-ca.crt": ca} + + name = "issuer-ca" + create_or_update_configmap(namespace, name, data) + return name + + +@fixture(scope="module") +def ops_manager_issuer_ca_configmap(issuer_ca_filepath: str, namespace: str) -> str: + """ + This is the CA file which verifies the certificates signed by it. + This CA is used to community with Ops Manager. This is needed by the database pods + which talk to OM. + """ + ca = open(issuer_ca_filepath).read() + + # The operator expects the CA that validates Ops Manager is contained in + # an entry with a name of "mms-ca.crt" + data = {"mms-ca.crt": ca} + + name = "ops-manager-issuer-ca" + create_or_update_configmap(namespace, name, data) + return name + + +@fixture(scope="module") +def app_db_issuer_ca_configmap(issuer_ca_filepath: str, namespace: str) -> str: + """ + This is the custom ca used with the AppDB hosts. This can be the same as the one used + for OM but does not need to be the same. + """ + ca = open(issuer_ca_filepath).read() + + name = "app-db-issuer-ca" + create_or_update_configmap(namespace, name, {"ca-pem": ca}) + return name + + +@fixture(scope="module") +def issuer_ca_plus(issuer_ca_filepath: str, namespace: str) -> str: + """Returns the name of a ConfigMap which includes a custom CA and the full + certificate chain for downloads.mongodb.com, fastdl.mongodb.org, + downloads.mongodb.org. This allows for the use of a custom CA while still + allowing the agent to download from MongoDB servers. + + """ + ca = open(issuer_ca_filepath).read() + plus_ca = open(_fixture("downloads.mongodb.com.chained+root.crt")).read() + + # The operator expects the CA that validates Ops Manager is contained in + # an entry with a name of "mms-ca.crt" + data = {"ca-pem": ca + plus_ca, "mms-ca.crt": ca + plus_ca} + + name = "issuer-plus-ca" + create_or_update_configmap(namespace, name, data) + yield name + + +@fixture(scope="module") +def ca_path() -> str: + """Returns a relative path to a file containing the CA. + This is required to test TLS enabled connections to MongoDB like: + + def test_connect(replica_set: MongoDB, ca_path: str) + replica_set.assert_connectivity(ca_path=ca_path) + """ + return _fixture("ca-tls.crt") + + +@fixture(scope="module") +def custom_mdb_version() -> str: + """Returns a CUSTOM_MDB_VERSION for Mongodb to be created/upgraded to for testing. + Defaults to 5.0.14 (simplifies testing locally)""" + return get_custom_mdb_version() + + +@fixture(scope="module") +def cluster_domain() -> str: + return get_cluster_domain() + + +def get_custom_mdb_version(): + return os.getenv("CUSTOM_MDB_VERSION", "6.0.7") + + +def get_cluster_domain(): + return os.getenv("CLUSTER_DOMAIN", "cluster.local") + + +@fixture(scope="module") +def custom_mdb_prev_version() -> str: + """Returns a CUSTOM_MDB_PREV_VERSION for Mongodb to be created/upgraded to for testing. + Defaults to 5.0.1 (simplifies testing locally)""" + return os.getenv("CUSTOM_MDB_PREV_VERSION", "5.0.1") + + +@fixture(scope="module") +def custom_appdb_version(custom_mdb_version: str) -> str: + """Returns a CUSTOM_APPDB_VERSION for AppDB to be created/upgraded to for testing, + defaults to custom_mdb_version() (in most cases we need to use the same version for MongoDB as for AppDB) + """ + + return get_custom_appdb_version(custom_mdb_version) + + +def get_custom_appdb_version(custom_mdb_version: str = get_custom_mdb_version()): + return os.getenv("CUSTOM_APPDB_VERSION", f"{custom_mdb_version}-ent") + + +@fixture(scope="module") +def custom_version() -> str: + """Returns a CUSTOM_OM_VERSION for OM. + Defaults to 5.0+ (for development)""" + # The variable is set in context files with one of the values ops_manager_60_latest or ops_manager_70_latest + # in .evergreen.yml + return get_custom_om_version() + + +def get_custom_om_version(): + return os.getenv("CUSTOM_OM_VERSION", "6.0.22") + + +@fixture(scope="module") +def default_operator( + namespace: str, + operator_installation_config: Dict[str, str], + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +) -> Operator: + if is_multi_cluster(): + return get_multi_cluster_operator( + namespace, + central_cluster_name, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + ) + return get_default_operator(namespace, operator_installation_config) + + +def get_default_operator( + namespace: str, + operator_installation_config: Dict[str, str], +) -> Operator: + """Installs/upgrades a default Operator used by any test not interested in some custom Operator setting. + TODO we use the helm template | kubectl apply -f process so far as Helm install/upgrade needs more refactoring in + the shared environment""" + operator = Operator( + namespace=namespace, + helm_args=operator_installation_config, + ).upgrade() + + return operator + + +@fixture(scope="module") +def operator_with_monitored_appdb( + namespace: str, + monitored_appdb_operator_installation_config: Dict[str, str], +) -> Operator: + """Installs/upgrades a default Operator used by any test that needs the AppDB monitoring enabled.""" + return Operator( + namespace=namespace, + helm_args=monitored_appdb_operator_installation_config, + ).upgrade() + + +def get_central_cluster_name(): + central_cluster = LEGACY_CENTRAL_CLUSTER_NAME + if is_multi_cluster(): + central_cluster = os.environ.get("CENTRAL_CLUSTER") + if not central_cluster: + raise ValueError("No central cluster specified in environment variable CENTRAL_CLUSTER!") + return central_cluster + + +def is_multi_cluster(): + return len(os.getenv("MEMBER_CLUSTERS", "")) > 0 + + +@fixture(scope="module") +def central_cluster_name() -> str: + return get_central_cluster_name() + + +def get_central_cluster_client() -> kubernetes.client.ApiClient: + if is_multi_cluster(): + return get_cluster_clients()[get_central_cluster_name()] + else: + return kubernetes.client.ApiClient() + + +@fixture(scope="module") +def central_cluster_client() -> kubernetes.client.ApiClient: + return get_central_cluster_client() + + +def get_member_cluster_names() -> List[str]: + if is_multi_cluster(): + member_clusters = os.environ.get("MEMBER_CLUSTERS") + if not member_clusters: + raise ValueError("No member clusters specified in environment variable MEMBER_CLUSTERS!") + return sorted(member_clusters.split()) + else: + return [] + + +@fixture(scope="module") +def member_cluster_names() -> List[str]: + return get_member_cluster_names() + + +def get_member_cluster_clients(cluster_mapping: dict[str, int] = None) -> List[MultiClusterClient]: + if not is_multi_cluster(): + return [MultiClusterClient(kubernetes.client.ApiClient(), LEGACY_CENTRAL_CLUSTER_NAME)] + + member_cluster_clients = [] + for i, cluster_name in enumerate(sorted(get_member_cluster_names())): + cluster_idx = i + + if cluster_mapping: + cluster_idx = cluster_mapping[cluster_name] + + member_cluster_clients.append( + MultiClusterClient(get_cluster_clients()[cluster_name], cluster_name, cluster_idx) + ) + + return member_cluster_clients + + +def get_member_cluster_client_map(deployment_state: dict[str, Any] = None) -> dict[str, MultiClusterClient]: + return { + multi_cluster_client.cluster_name: multi_cluster_client + for multi_cluster_client in get_member_cluster_clients(deployment_state) + } + + +def get_member_cluster_api_client( + member_cluster_name: Optional[str], +) -> kubernetes.client.ApiClient: + if is_member_cluster(member_cluster_name): + return get_cluster_clients()[member_cluster_name] + else: + return kubernetes.client.ApiClient() + + +@fixture(scope="module") +def disable_istio( + namespace: str, + member_cluster_clients: List[MultiClusterClient], +): + for mcc in member_cluster_clients: + api = client.CoreV1Api(api_client=mcc.api_client) + labels = {"istio-injection": "disabled"} + ns = api.read_namespace(name=namespace) + ns.metadata.labels.update(labels) + api.replace_namespace(name=namespace, body=ns) + return None + + +@fixture(scope="module") +def member_cluster_clients() -> List[MultiClusterClient]: + return get_member_cluster_clients() + + +@fixture(scope="module") +def multi_cluster_operator( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +) -> Operator: + return get_multi_cluster_operator( + namespace, + central_cluster_name, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + ) + + +def get_multi_cluster_operator( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +) -> Operator: + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + + # when running with the local operator, this is executed by scripts/dev/prepare_local_e2e_run.sh + if not local_operator(): + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names) + return _install_multi_cluster_operator( + namespace, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + # override the serviceAccountName for the operator deployment + "operator.createOperatorServiceAccount": "false", + }, + central_cluster_name, + ) + + +@fixture(scope="module") +def multi_cluster_operator_with_monitored_appdb( + namespace: str, + central_cluster_name: str, + multi_cluster_monitored_appdb_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +) -> Operator: + print(f"\nSetting HELM_KUBECONTEXT to {central_cluster_name}") + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + + # when running with the local operator, this is executed by scripts/dev/prepare_local_e2e_run.sh + if not local_operator(): + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names) + return _install_multi_cluster_operator( + namespace, + multi_cluster_monitored_appdb_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + # override the serviceAccountName for the operator deployment + "operator.createOperatorServiceAccount": "false", + }, + central_cluster_name, + ) + + +@fixture(scope="module") +def multi_cluster_operator_manual_remediation( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +) -> Operator: + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + if not local_operator(): + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names) + return _install_multi_cluster_operator( + namespace, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + # override the serviceAccountName for the operator deployment + "operator.createOperatorServiceAccount": "false", + "multiCluster.performFailOver": "false", + }, + central_cluster_name, + ) + + +def get_multi_cluster_operator_clustermode(namespace: str) -> Operator: + os.environ["HELM_KUBECONTEXT"] = get_central_cluster_name() + run_kube_config_creation_tool( + get_member_cluster_names(), + namespace, + namespace, + get_member_cluster_names(), + True, + ) + return _install_multi_cluster_operator( + namespace, + get_multi_cluster_operator_installation_config(namespace), + get_central_cluster_client(), + get_member_cluster_clients(), + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + # override the serviceAccountName for the operator deployment + "operator.createOperatorServiceAccount": "false", + "operator.watchNamespace": "*", + }, + get_central_cluster_name(), + ) + + +@fixture(scope="module") +def multi_cluster_operator_clustermode( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], + cluster_clients: Dict[str, kubernetes.client.ApiClient], +) -> Operator: + return get_multi_cluster_operator_clustermode(namespace) + + +@fixture(scope="module") +def install_multi_cluster_operator_set_members_fn( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], +) -> Callable[[List[str]], Operator]: + def _fn(member_cluster_names: List[str]) -> Operator: + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + mcn = ",".join(member_cluster_names) + return _install_multi_cluster_operator( + namespace, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + # override the serviceAccountName for the operator deployment + "operator.createOperatorServiceAccount": "false", + "multiCluster.clusters": "{" + mcn + "}", + }, + central_cluster_name, + ) + + return _fn + + +def _install_multi_cluster_operator( + namespace: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + helm_opts: Dict[str, str], + central_cluster_name: str, + operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME, + helm_chart_path: Optional[str] = "helm_chart", + custom_operator_version: Optional[str] = None, +) -> Operator: + prepare_multi_cluster_namespaces( + namespace, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + skip_central_cluster=True, + ) + multi_cluster_operator_installation_config.update(helm_opts) + + operator = Operator( + name=operator_name, + namespace=namespace, + helm_args=multi_cluster_operator_installation_config, + api_client=central_cluster_client, + helm_chart_path=helm_chart_path, + ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version) + + # If we're running locally, then immediately after installing the deployment, we scale it to zero. + # This way operator in POD is not interfering with locally running one. + if local_operator(): + client.AppsV1Api(api_client=central_cluster_client).patch_namespaced_deployment_scale( + namespace=namespace, + name=operator.name, + body={"spec": {"replicas": 0}}, + ) + + return operator + + +@fixture(scope="module") +def official_operator( + namespace: str, + managed_security_context: str, + operator_installation_config: Dict[str, str], + central_cluster_name: str, + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +) -> Operator: + return install_official_operator( + namespace, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + None, + ) + + +def install_official_operator( + namespace: str, + managed_security_context: str, + operator_installation_config: Dict[str, str], + central_cluster_name: Optional[str], + central_cluster_client: Optional[client.ApiClient], + member_cluster_clients: Optional[List[MultiClusterClient]], + member_cluster_names: Optional[List[str]], + custom_operator_version: Optional[str] = None, +) -> Operator: + """ + Installs the Operator from the official Helm Chart. + + The version installed is always the latest version published as a Helm Chart. + """ + logger.debug( + f"Installing latest released {'multi' if is_multi_cluster() else 'single'} cluster operator from helm charts" + ) + + # When running in Openshift "managedSecurityContext" will be true. + # When running in kind "managedSecurityContext" will be false, but still use the ubi images. + helm_args = { + "registry.imagePullSecrets": operator_installation_config["registry.imagePullSecrets"], + "managedSecurityContext": managed_security_context, + "operator.mdbDefaultArchitecture": operator_installation_config["operator.mdbDefaultArchitecture"], + } + name = "mongodb-enterprise-operator" + + # Note, that we don't intend to install the official Operator to standalone clusters (kops/openshift) as we want to + # avoid damaged CRDs. But we may need to install the "openshift like" environment to Kind instead of the "ubi" + # images are used for installing the dev Operator + helm_args["operator.operator_image_name"] = "{}-ubi".format(name) + + # Note: + # We might want in the future to install CRDs when performing upgrade/downgrade tests, the helm install only takes + # care of the operator deployment. + # A solution is to clone and checkout our helm_charts repository, and apply the CRDs from the right branch + # Leaving below a code snippet to check out the right branch + """ + # Version stored in env variable has format "1.27.0", tag name has format "enterprise-operator-1.27.0" + if custom_operator_version: + checkout_branch = f"enterprise-operator-{custom_operator_version}" + else: + checkout_branch = "main" + + temp_dir = tempfile.mkdtemp() + # Values files are now located in `helm-charts` repo. + clone_and_checkout( + "https://github.com/mongodb/helm-charts", + temp_dir, + checkout_branch, # branch or tag to check out from helm-charts. + ) + """ + + if is_multi_cluster(): + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + # when running with the local operator, this is executed by scripts/dev/prepare_local_e2e_run.sh + if not local_operator(): + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names) + helm_args.update( + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + # override the serviceAccountName for the operator deployment + "operator.createOperatorServiceAccount": "false", + "multiCluster.clusters": operator_installation_config["multiCluster.clusters"], + } + ) + # The "official" Operator will be installed, from the Helm Repo ("mongodb/enterprise-operator") + # We pass helm_args as operator installation config below instead of the full configmap data, otherwise + # it overwrites registries and image versions, and we wouldn't use the official images but the dev ones + return _install_multi_cluster_operator( + namespace, + helm_args, + central_cluster_client, + member_cluster_clients, + helm_opts=helm_args, + central_cluster_name=get_central_cluster_name(), + helm_chart_path="mongodb/enterprise-operator", + custom_operator_version=custom_operator_version, + ) + else: + # When testing the UBI image type we need to assume a few things + # The "official" Operator will be installed, from the Helm Repo ("mongodb/enterprise-operator") + return Operator( + namespace=namespace, + helm_args=helm_args, + helm_chart_path="mongodb/enterprise-operator", + name=name, + ).install(custom_operator_version=custom_operator_version) + + +# Function dumping the list of deployments and all their container images in logs. +# This is useful for example to ensure we are installing the correct operator version. +def log_deployments_info(namespace: str): + logger.debug(f"Dumping deployments list and container images in namespace {namespace}:") + logger.debug(log_deployment_and_images(get_deployments(namespace))) + + +def log_deployment_and_images(deployments): + images, deployment_names = extract_container_images_and_deployments(deployments) + for deployment in deployment_names: + logger.debug(f"Deployment {deployment} contains images {images.get(deployment, 'error_getting_key')}") + + +# Extract container images and deployments names from the nested dict returned by kubetester +# Handles any missing key gracefully +def extract_container_images_and_deployments(deployments) -> (Dict[str, str], List[str]): + deployment_images = {} + deployment_names = [] + deployments = deployments.to_dict() + + if "items" not in deployments: + logger.debug("Error: 'items' field not found in the response.") + return deployment_images + + for deployment in deployments.get("items", []): + try: + deployment_name = deployment["metadata"].get("name", "Unknown") + deployment_names.append(deployment_name) + containers = deployment["spec"]["template"]["spec"].get("containers", []) + + # Extract images used by each container in the deployment + images = [container.get("image", "No Image Specified") for container in containers] + + # Store it in a dictionary, to be logged outside of this function + deployment_images[deployment_name] = images + + except KeyError as e: + logger.debug( + f"KeyError: Missing expected key in deployment {deployment.get('metadata', {}).get('name', 'Unknown')} - {e}" + ) + except Exception as e: + logger.debug( + f"Error: An unexpected error occurred for deployment {deployment.get('metadata', {}).get('name', 'Unknown')} - {e}" + ) + + return deployment_images, deployment_names + + +def setup_agent_config(agent, with_process_support): + log_rotate_config_for_process = { + "sizeThresholdMB": "100", + "percentOfDiskspace": "0.4", + "numTotal": 10, + "timeThresholdHrs": 1, + "numUncompressed": 2, + } + + log_rotate_for_backup_monitoring = {"sizeThresholdMB": 100, "timeThresholdHrs": 10} + + agent["backupAgent"] = {} + agent["monitoringAgent"] = {} + + if with_process_support: + agent["mongod"] = {} + agent["mongod"]["logRotate"] = log_rotate_config_for_process + agent["mongod"]["auditlogRotate"] = log_rotate_config_for_process + + agent["backupAgent"]["logRotate"] = log_rotate_for_backup_monitoring + agent["monitoringAgent"]["logRotate"] = log_rotate_for_backup_monitoring + + +def setup_log_rotate_for_agents(resource, with_process_support=True): + if "agent" not in resource["spec"] or resource["spec"]["agent"] is None: + resource["spec"]["agent"] = {} + setup_agent_config(resource["spec"]["agent"], with_process_support) + + +def assert_log_rotation_process(process, with_process_support=True): + if with_process_support: + _assert_log_rotation_process(process, "logRotate") + _assert_log_rotation_process(process, "auditLogRotate") + + +def _assert_log_rotation_process(process, key): + assert process[key]["sizeThresholdMB"] == 100 + assert process[key]["timeThresholdHrs"] == 1 + assert process[key]["percentOfDiskspace"] == 0.4 + assert process[key]["numTotal"] == 10 + assert process[key]["numUncompressed"] == 2 + + +def assert_log_rotation_backup_monitoring(agent_config): + assert agent_config["logRotate"]["sizeThresholdMB"] == 100 + assert agent_config["logRotate"]["timeThresholdHrs"] == 10 + + +def _read_multi_cluster_config_value(value: str) -> str: + multi_cluster_config_dir = os.environ.get("MULTI_CLUSTER_CONFIG_DIR", MULTI_CLUSTER_CONFIG_DIR) + filepath = f"{multi_cluster_config_dir}/{value}".rstrip() + if not os.path.isfile(filepath): + raise ValueError(f"{filepath} does not exist!") + with open(filepath, "r") as f: + return f.read().strip() + + +def _get_client_for_cluster( + cluster_name: str, +) -> kubernetes.client.api_client.ApiClient: + token = _read_multi_cluster_config_value(cluster_name) + + if not token: + raise ValueError(f"No token found for cluster {cluster_name}") + + configuration = kubernetes.client.Configuration() + kubernetes.config.load_kube_config( + context=cluster_name, + config_file=os.environ.get("KUBECONFIG", KUBECONFIG_FILEPATH), + client_configuration=configuration, + ) + configuration.host = CLUSTER_HOST_MAPPING.get(cluster_name, configuration.host) + + configuration.verify_ssl = False + configuration.api_key = {"authorization": f"Bearer {token}"} + return kubernetes.client.api_client.ApiClient(configuration=configuration) + + +def install_cert_manager( + cluster_client: Optional[client.ApiClient] = None, + cluster_name: Optional[str] = None, + name="cert-manager", + version="v1.5.4", +) -> str: + if is_member_cluster(cluster_name): + # ensure we cert-manager in the member clusters. + os.environ["HELM_KUBECONTEXT"] = cluster_name + + install_required = True + + if running_locally(): + webhook_ready = is_pod_ready( + name, + f"app.kubernetes.io/instance={name},app.kubernetes.io/component=webhook", + api_client=cluster_client, + ) + controller_ready = is_pod_ready( + name, + f"app.kubernetes.io/instance={name},app.kubernetes.io/component=controller", + api_client=cluster_client, + ) + if webhook_ready is not None and controller_ready is not None: + print("Cert manager already installed, skipping helm install") + install_required = False + + if install_required: + helm_install_from_chart( + name, # cert-manager is installed on a specific namespace + name, + f"jetstack/{name}", + version=version, + custom_repo=("jetstack", "https://charts.jetstack.io"), + helm_args={"installCRDs": "true"}, + ) + + return name + + +def wait_for_cert_manager_ready( + cluster_client: Optional[client.ApiClient] = None, + namespace="cert-manager", +): + # waits until the cert-manager webhook and controller are Ready, otherwise creating + # Certificate Custom Resources will fail. + get_pod_when_ready( + namespace, + f"app.kubernetes.io/instance={namespace},app.kubernetes.io/component=webhook", + api_client=cluster_client, + ) + get_pod_when_ready( + namespace, + f"app.kubernetes.io/instance={namespace},app.kubernetes.io/component=controller", + api_client=cluster_client, + ) + + +def get_cluster_clients() -> dict[str, kubernetes.client.api_client.ApiClient]: + if not is_multi_cluster(): + return { + LEGACY_CENTRAL_CLUSTER_NAME: kubernetes.client.ApiClient(), + } + + member_clusters = [ + _read_multi_cluster_config_value("member_cluster_1"), + _read_multi_cluster_config_value("member_cluster_2"), + ] + + if len(get_member_cluster_names()) == 3: + member_clusters.append(_read_multi_cluster_config_value("member_cluster_3")) + return get_clients_for_clusters(member_clusters) + + +@fixture(scope="module") +def cluster_clients() -> dict[str, kubernetes.client.api_client.ApiClient]: + return get_cluster_clients() + + +def get_clients_for_clusters( + member_cluster_names: List[str], +) -> dict[str, kubernetes.client.ApiClient]: + if not is_multi_cluster(): + return { + LEGACY_CENTRAL_CLUSTER_NAME: kubernetes.client.ApiClient(), + } + + central_cluster = _read_multi_cluster_config_value("central_cluster") + + return {c: _get_client_for_cluster(c) for c in ([central_cluster] + member_cluster_names)} + + +def get_api_servers_from_pod_kubeconfig(kubeconfig: str, cluster_clients: Dict[str, kubernetes.client.ApiClient]): + api_servers = dict() + fd, kubeconfig_tmp_path = tempfile.mkstemp() + with os.fdopen(fd, "w") as fp: + fp.write(kubeconfig) + + for cluster_name, cluster_client in cluster_clients.items(): + configuration = kubernetes.client.Configuration() + kubernetes.config.load_kube_config( + context=cluster_name, + config_file=kubeconfig_tmp_path, + client_configuration=configuration, + ) + api_servers[cluster_name] = configuration.host + + return api_servers + + +def run_kube_config_creation_tool( + member_clusters: List[str], + central_namespace: str, + member_namespace: str, + member_cluster_names: List[str], + cluster_scoped: Optional[bool] = False, + service_account_name: Optional[str] = "mongodb-enterprise-operator-multi-cluster", +): + central_cluster = _read_multi_cluster_config_value("central_cluster") + member_clusters_str = ",".join(member_clusters) + args = [ + os.getenv( + "MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH", + "multi-cluster-kube-config-creator", + ), + "multicluster", + "setup", + "--member-clusters", + member_clusters_str, + "--central-cluster", + central_cluster, + "--member-cluster-namespace", + member_namespace, + "--central-cluster-namespace", + central_namespace, + "--service-account", + service_account_name, + ] + + if os.getenv("MULTI_CLUSTER_CREATE_SERVICE_ACCOUNT_TOKEN_SECRETS") == "true": + args.append("--create-service-account-secrets") + + if not local_operator(): + api_servers = get_api_servers_from_test_pod_kubeconfig(member_namespace, member_cluster_names) + + if len(api_servers) > 0: + args.append("--member-clusters-api-servers") + args.append(",".join([api_servers[member_cluster] for member_cluster in member_clusters])) + + if cluster_scoped: + args.append("--cluster-scoped") + + try: + print(f"Running multi-cluster cli setup tool: {' '.join(args)}") + subprocess.check_output(args, stderr=subprocess.STDOUT) + print("Finished running multi-cluster cli setup tool") + except subprocess.CalledProcessError as exc: + print(f"Status: FAIL Reason: {exc.output}") + raise exc + + +def get_api_servers_from_kubeconfig_secret( + namespace: str, + secret_name: str, + secret_cluster_client: kubernetes.client.ApiClient, + cluster_clients: Dict[str, kubernetes.client.ApiClient], +): + kubeconfig_secret = read_secret(namespace, secret_name, api_client=secret_cluster_client) + return get_api_servers_from_pod_kubeconfig(kubeconfig_secret["kubeconfig"], cluster_clients) + + +def get_api_servers_from_test_pod_kubeconfig(namespace: str, member_cluster_names: List[str]) -> Dict[str, str]: + test_pod_cluster = get_test_pod_cluster_name() + cluster_clients = get_clients_for_clusters(member_cluster_names) + + return get_api_servers_from_kubeconfig_secret( + namespace, + "test-pod-kubeconfig", + cluster_clients[test_pod_cluster], + cluster_clients, + ) + + +def get_test_pod_cluster_name(): + return os.environ["TEST_POD_CLUSTER"] + + +def run_multi_cluster_recovery_tool( + member_clusters: List[str], + central_namespace: str, + member_namespace: str, + cluster_scoped: Optional[bool] = False, +) -> int: + central_cluster = _read_multi_cluster_config_value("central_cluster") + member_clusters_str = ",".join(member_clusters) + args = [ + os.getenv( + "MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH", + "multi-cluster-kube-config-creator", + ), + "multicluster", + "recover", + "--member-clusters", + member_clusters_str, + "--central-cluster", + central_cluster, + "--member-cluster-namespace", + member_namespace, + "--central-cluster-namespace", + central_namespace, + "--operator-name", + MULTI_CLUSTER_OPERATOR_NAME, + "--source-cluster", + member_clusters[0], + ] + if os.getenv("MULTI_CLUSTER_CREATE_SERVICE_ACCOUNT_TOKEN_SECRETS") == "true": + args.append("--create-service-account-secrets") + + if cluster_scoped: + args.extend(["--cluster-scoped", "true"]) + + try: + print(f"Running multi-cluster cli recovery tool: {' '.join(args)}") + subprocess.check_output(args, stderr=subprocess.PIPE) + print("Finished running multi-cluster cli recovery tool") + except subprocess.CalledProcessError as exc: + print("Status: FAIL", exc.returncode, exc.output) + return exc.returncode + return 0 + + +def create_issuer( + namespace: str, + api_client: Optional[client.ApiClient] = None, + clusterwide: bool = False, +): + """ + This fixture creates an "Issuer" in the testing namespace. This requires cert-manager to be installed + in the cluster. The ca-tls.key and ca-tls.crt are the private key and certificates used to generate + certificates. This is based on a Cert-Manager CA Issuer. + More info here: https://cert-manager.io/docs/configuration/ca/ + + Please note, this cert will expire on Dec 8 07:53:14 2023 GMT. + """ + issuer_data = { + "tls.key": open(_fixture("ca-tls.key")).read(), + "tls.crt": open(_fixture("ca-tls.crt")).read(), + } + secret = client.V1Secret( + metadata=client.V1ObjectMeta(name="ca-key-pair"), + string_data=issuer_data, + ) + + try: + if clusterwide: + client.CoreV1Api(api_client=api_client).create_namespaced_secret("cert-manager", secret) + else: + client.CoreV1Api(api_client=api_client).create_namespaced_secret(namespace, secret) + except client.rest.ApiException as e: + if e.status == 409: + print("ca-key-pair already exists") + else: + raise e + + # And then creates the Issuer + if clusterwide: + issuer = ClusterIssuer(name="ca-issuer", namespace="") + else: + issuer = Issuer(name="ca-issuer", namespace=namespace) + + issuer["spec"] = {"ca": {"secretName": "ca-key-pair"}} + issuer.api = kubernetes.client.CustomObjectsApi(api_client=api_client) + + try: + issuer.create().block_until_ready() + except client.rest.ApiException as e: + if e.status == 409: + print("issuer already exists") + else: + raise e + + return "ca-issuer" + + +def local_operator(): + """Checks if the current test run should assume that the operator is running locally, i.e. not in a pod.""" + return os.getenv("LOCAL_OPERATOR", "") == "true" + + +def pod_names(replica_set_name: str, replica_set_members: int) -> list[str]: + """List of pod names for given replica set name.""" + return [f"{replica_set_name}-{i}" for i in range(0, replica_set_members)] + + +def multi_cluster_pod_names(replica_set_name: str, cluster_index_with_members: list[tuple[int, int]]) -> list[str]: + """List of multi-cluster pod names for given replica set name and a list of member counts in member clusters.""" + result_list = [] + for cluster_index, members in cluster_index_with_members: + result_list.extend([f"{replica_set_name}-{cluster_index}-{pod_idx}" for pod_idx in range(0, members)]) + + return result_list + + +def multi_cluster_service_names(replica_set_name: str, cluster_index_with_members: list[tuple[int, int]]) -> list[str]: + """List of multi-cluster service names for given replica set name and a list of member counts in member clusters.""" + return [f"{pod_name}-svc" for pod_name in multi_cluster_pod_names(replica_set_name, cluster_index_with_members)] + + +def is_member_cluster(cluster_name: Optional[str] = None) -> bool: + if cluster_name is not None and cluster_name != LEGACY_CENTRAL_CLUSTER_NAME: + return True + return False + + +def default_external_domain() -> str: + """Default external domain used for testing LoadBalancers on Kind.""" + return "mongodb.interconnected" + + +def external_domain_fqdns( + replica_set_name: str, + replica_set_members: int, + external_domain: str = default_external_domain(), +) -> list[str]: + """Builds list of hostnames for given replica set when connecting to it using external domain.""" + return [f"{pod_name}.{external_domain}" for pod_name in pod_names(replica_set_name, replica_set_members)] + + +def update_coredns_hosts( + host_mappings: list[tuple[str, str]], + cluster_name: Optional[str] = None, + api_client: Optional[kubernetes.client.ApiClient] = None, + additional_rules: list[str] = None, +): + """Updates kube-system/coredns config map with given host_mappings.""" + + mapping_indent = " " * 7 + mapping_string = "\n".join( + [f"{mapping_indent}{host_mapping[0]} {host_mapping[1]}" for host_mapping in host_mappings] + ) + + additional_rules_string = None + if additional_rules is not None: + additional_rules_indent = " " * 4 + additional_rules_string = "\n" + "\n".join( + [f"{additional_rules_indent}{additional_rule}" for additional_rule in additional_rules] + ) + + config_data = {"Corefile": coredns_config("interconnected", mapping_string, additional_rules_string)} + + if cluster_name is None: + cluster_name = LEGACY_CENTRAL_CLUSTER_NAME + + print(f"Updating coredns for cluster: {cluster_name} with the following hosts list: {host_mappings}") + update_configmap("kube-system", "coredns", config_data, api_client=api_client) + + +def coredns_config(tld: str, mappings: str, additional_rules: str = None): + """Returns coredns config map data with mappings inserted.""" + return f""" +.:53 {{ + errors + health {{ + lameduck 5s + }} + ready + kubernetes cluster.local in-addr.arpa ip6.arpa {{ + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + }} + prometheus :9153 + forward . /etc/resolv.conf {{ + max_concurrent 1000 + }} + cache 30 + loop + reload + loadbalance + debug{additional_rules or ""} + hosts /etc/coredns/customdomains.db {tld} {{ +{mappings} + ttl 10 + reload 1m + fallthrough + }} +}} +""" + + +def create_appdb_certs( + namespace: str, + issuer: str, + appdb_name: str, + cluster_index_with_members: list[tuple[int, int]] = None, + cert_prefix="appdb", + clusterwide: bool = False, + additional_domains: Optional[List[str]] = None, +) -> str: + if cluster_index_with_members is None: + cluster_index_with_members = [(0, 1), (1, 2)] + + appdb_cert_name = f"{cert_prefix}-{appdb_name}-cert" + + if is_multi_cluster(): + service_fqdns = [ + f"{svc}.{namespace}.svc.cluster.local" + for svc in multi_cluster_service_names(appdb_name, cluster_index_with_members) + ] + create_multi_cluster_mongodb_tls_certs( + issuer, + appdb_cert_name, + get_member_cluster_clients(), + get_central_cluster_client(), + service_fqdns=service_fqdns, + namespace=namespace, + clusterwide=clusterwide, + additional_domains=additional_domains, + ) + else: + create_mongodb_tls_certs( + issuer, + namespace, + appdb_name, + appdb_cert_name, + clusterwide=clusterwide, + additional_domains=additional_domains, + ) + + return cert_prefix + + +def pytest_sessionfinish(session, exitstatus): + project_id = os.environ.get("OM_PROJECT_ID", "") + if project_id: + base_url = os.environ.get("OM_HOST") + user = os.environ.get("OM_USER") + key = os.environ.get("OM_API_KEY") + ids = project_id.split(",") + for project_id in ids: + try: + tester = OMTester( + OMContext( + base_url=base_url, + public_key=key, + project_id=project_id, + user=user, + ) + ) + + # let's only access om if its healthy and around. + status_code, _ = tester.request_health(base_url) + if status_code == requests.status_codes.codes.OK: + ev = tester.get_project_events().json()["results"] + with open(f"/tmp/diagnostics/{project_id}-events.json", "w", encoding="utf-8") as f: + json.dump(ev, f, ensure_ascii=False, indent=4) + else: + logging.info("om is not healthy - not collecting events information") + + except Exception as e: + continue + + +def install_multi_cluster_operator_cluster_scoped( + watch_namespaces: list[str], + namespace: str = get_namespace(), + central_cluster_name: str = get_central_cluster_name(), + central_cluster_client: client.ApiClient = get_central_cluster_client(), + multi_cluster_operator_installation_config: dict[str, str] = None, + member_cluster_clients: list[kubernetes.client.ApiClient] = None, + cluster_clients: dict[str, kubernetes.client.ApiClient] = None, + member_cluster_names: list[str] = None, +) -> Operator: + if multi_cluster_operator_installation_config is None: + multi_cluster_operator_installation_config = get_multi_cluster_operator_installation_config(namespace).copy() + if member_cluster_clients is None: + member_cluster_clients = get_member_cluster_clients().copy() + if cluster_clients is None: + cluster_clients = get_cluster_clients().copy() + if member_cluster_names is None: + member_cluster_names = get_member_cluster_names().copy() + + print( + f"Installing multi cluster operator in context: {central_cluster_name} and with watched namespaces: {watch_namespaces}" + ) + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + member_cluster_namespaces = ",".join(watch_namespaces) + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names, True) + + return _install_multi_cluster_operator( + namespace, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + "operator.createOperatorServiceAccount": "false", + "operator.watchNamespace": member_cluster_namespaces, + }, + central_cluster_name, + ) + + +def assert_data_got_restored(test_data, collection1, collection2=None, timeout=300): + """The data in the db has been restored to the initial state. Note, that this happens eventually - so + we need to loop for some time (usually takes 60 seconds max). This is different from restoring from a + specific snapshot (see the previous class) where the FINISHED restore job means the data has been restored. + For PIT restores FINISHED just means the job has been created and the agents will perform restore eventually + """ + print("\nWaiting until the db data is restored") + start_time = time.time() + last_error = None + + while True: + elapsed_time = time.time() - start_time + if elapsed_time > timeout: + logger.debug("\nExisting data in MDB: {}".format(list(collection1.find()))) + if collection2 is not None: + logger.debug("\nExisting data in MDB: {}".format(list(collection2.find()))) + raise AssertionError( + f"The data hasn't been restored in {timeout // 60} minutes! Last assertion error was: {last_error}" + ) + + try: + records = list(collection1.find()) + assert records == [test_data] + + if collection2 is not None: + records = list(collection2.find()) + assert records == [test_data] + + return + except AssertionError as e: + logger.debug(f"assertionError while asserting data got restored: {e}") + last_error = e + pass + except ServerSelectionTimeoutError: + # The mongodb driver complains with `No replica set members + # match selector "Primary()",` This could be related with DNS + # not being functional, or the database going through a + # re-election process. Let's give it another chance to succeed. + logger.debug(f"ServerSelectionTimeoutError, are we going through a re-election?") + pass + except Exception as e: + # We ignore Exception as there is usually a blip in connection (backup restore + # results in reelection or whatever) + # "Connection reset by peer" or "not master and slaveOk=false" + logger.error("Exception happened while waiting for db data restore: ", e) + # this is definitely the sign of a problem - no need continuing as each connection times out + # after many minutes + if "Connection refused" in str(e): + raise e + + time.sleep(1) # Sleep for a short duration before the next check + + +def verify_pvc_expanded( + first_data_pvc_name, + first_journal_pvc_name, + first_logs_pvc_name, + namespace, + resized_storage_size, + initial_storage_size, +): + data_pvc = client.CoreV1Api().read_namespaced_persistent_volume_claim(first_data_pvc_name, namespace) + assert data_pvc.status.capacity["storage"] == resized_storage_size + journal_pvc = client.CoreV1Api().read_namespaced_persistent_volume_claim(first_journal_pvc_name, namespace) + assert journal_pvc.status.capacity["storage"] == resized_storage_size + logs_pvc = client.CoreV1Api().read_namespaced_persistent_volume_claim(first_logs_pvc_name, namespace) + assert logs_pvc.status.capacity["storage"] == initial_storage_size diff --git a/scripts/git-hooks/prepare-commit-msg b/docker/mongodb-enterprise-tests/tests/mixed/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/prepare-commit-msg rename to docker/mongodb-enterprise-tests/tests/mixed/__init__.py diff --git a/docker/mongodb-enterprise-tests/tests/mixed/all_mongodb_resources_parallel_test.py b/docker/mongodb-enterprise-tests/tests/mixed/all_mongodb_resources_parallel_test.py new file mode 100644 index 000000000..786e2d54d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/mixed/all_mongodb_resources_parallel_test.py @@ -0,0 +1,68 @@ +import threading +import time + +import pytest +from kubetester.kubetester import KubernetesTester, fixture, run_periodically + +mdb_resources = { + "my-standalone": fixture("standalone.yaml"), + "sh001-single": fixture("sharded-cluster-single.yaml"), + "my-replica-set-single": fixture("replica-set-single.yaml"), +} + + +@pytest.mark.e2e_all_mongodb_resources_parallel +class TestRaceConditions(KubernetesTester): + """ + name: Test for no race conditions during creation of three mongodb resources in parallel. + description: | + Makes sure no duplicated organizations/groups are created, while 3 mongodb resources + are created in parallel. Also the automation config doesn't miss entries. + """ + + random_storage_name = None + + @classmethod + def setup_env(cls): + threads = [] + for filename in mdb_resources.values(): + args = (cls.get_namespace(), filename) + threads.append(threading.Thread(target=cls.create_custom_resource_from_file, args=args)) + + [t.start() for t in threads] + [t.join() for t in threads] + + print("Waiting until any of the resources gets to 'Running' state..") + run_periodically(TestRaceConditions.any_resource_created, timeout=360) + + def test_one_resource_created_only(self): + mongodbs = [ + KubernetesTester.get_namespaced_custom_object(KubernetesTester.get_namespace(), resource, "MongoDB") + for resource in mdb_resources.keys() + ] + assert len([m for m in mongodbs if m["status"]["phase"] == "Running"]) == 1 + + # No duplicated organizations were created and automation config is consistent + organizations = KubernetesTester.find_organizations(KubernetesTester.get_om_group_name()) + assert len(organizations) == 1 + groups = KubernetesTester.find_groups_in_organization(organizations[0], KubernetesTester.get_om_group_name()) + assert len(groups) == 1 + + config = KubernetesTester.get_automation_config() + + # making sure that only one single mdb resource was created + replica_set_created = len(config["replicaSets"]) == 1 and len(config["processes"]) == 1 + sharded_cluster_created = len(config["replicaSets"]) == 2 and len(config["processes"]) == 3 + standalone_created = len(config["replicaSets"]) == 0 and len(config["processes"]) == 1 + + assert replica_set_created + sharded_cluster_created + standalone_created == 1 + + @staticmethod + def any_resource_created(): + namespace = KubernetesTester.get_namespace() + results = [ + KubernetesTester.check_phase(namespace, "MongoDB", resource, "Running") for resource in mdb_resources.keys() + ] + + print("Standalone ready: {}, sharded cluster ready: {}, replica set ready: {}".format(*results)) + return any(results) diff --git a/docker/mongodb-enterprise-tests/tests/mixed/conftest.py b/docker/mongodb-enterprise-tests/tests/mixed/conftest.py new file mode 100644 index 000000000..cb0664057 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/mixed/conftest.py @@ -0,0 +1,4 @@ +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if "default_operator" not in item.fixturenames: + item.fixturenames.insert(0, "default_operator") diff --git a/docker/mongodb-enterprise-tests/tests/mixed/crd_validation.py b/docker/mongodb-enterprise-tests/tests/mixed/crd_validation.py new file mode 100644 index 000000000..9eceef7a5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/mixed/crd_validation.py @@ -0,0 +1,39 @@ +""" +Checks that the CRD conform to Structural Schema: +https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#specifying-a-structural-schema +""" + +from kubernetes.client import ApiextensionsV1Api, V1CustomResourceDefinition +from pytest import mark + + +def crd_has_expected_conditions(resource: V1CustomResourceDefinition) -> bool: + for condition in resource.status.conditions: + if condition.type == "NonStructuralSchema": + return False + + return True + + +@mark.e2e_crd_validation +def test_mongodb_crd_is_valid(crd_api: ApiextensionsV1Api): + resource = crd_api.read_custom_resource_definition("mongodb.mongodb.com") + assert crd_has_expected_conditions(resource) + + +@mark.e2e_crd_validation +def test_mongodb_users_crd_is_valid(crd_api: ApiextensionsV1Api): + resource = crd_api.read_custom_resource_definition("mongodbusers.mongodb.com") + assert crd_has_expected_conditions(resource) + + +@mark.e2e_crd_validation +def test_opsmanagers_crd_is_valid(crd_api: ApiextensionsV1Api): + resource = crd_api.read_custom_resource_definition("opsmanagers.mongodb.com") + assert crd_has_expected_conditions(resource) + + +@mark.e2e_crd_validation +def test_mongodbmulti_crd_is_valid(crd_api: ApiextensionsV1Api): + resource = crd_api.read_custom_resource_definition("mongodbmulticluster.mongodb.com") + assert crd_has_expected_conditions(resource) diff --git a/docker/mongodb-enterprise-tests/tests/mixed/failures_on_multi_clusters.py b/docker/mongodb-enterprise-tests/tests/mixed/failures_on_multi_clusters.py new file mode 100644 index 000000000..0a30fad3d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/mixed/failures_on_multi_clusters.py @@ -0,0 +1,132 @@ +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="class") +def replica_set(namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + yield resource.create() + + resource.delete() + + +@fixture(scope="class") +def replica_set_single(namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml(yaml_fixture("replica-set-single.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + yield resource.create() + + resource.delete() + + +@fixture(scope="class") +def sharded_cluster(namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml(yaml_fixture("sharded-cluster.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + yield resource.create() + + resource.delete() + + +@fixture(scope="class") +def sharded_cluster_single(namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml(yaml_fixture("sharded-cluster-single.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + yield resource.create() + + resource.delete() + + +@mark.e2e_multiple_cluster_failures +class TestNoTwoReplicaSetsCanBeCreatedOnTheSameProject: + def test_replica_set_get_to_running_state(self, replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) + + assert "warnings" not in replica_set["status"] + + def test_no_warnings_when_scaling(self, replica_set: MongoDB): + replica_set["spec"]["members"] = 5 + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=500) + assert "warnings" not in replica_set["status"] + + def test_second_mdb_resource_fails(self, replica_set_single: MongoDB): + replica_set_single.assert_reaches_phase(Phase.Pending) + + assert ( + replica_set_single["status"]["message"] + == "Cannot have more than 1 MongoDB Cluster per project (see https://docs.mongodb.com/kubernetes-operator/stable/tutorial/migrate-to-single-resource/)" + ) + + assert "warnings" not in replica_set_single["status"] + + # pylint: disable=unused-argument + def test_automation_config_is_correct(self, replica_set, replica_set_single): + config = KubernetesTester.get_automation_config() + + assert len(config["processes"]) == 5 + for process in config["processes"]: + assert not process["name"].startswith("my-replica-set-single") + + assert len(config["sharding"]) == 0 + assert len(config["replicaSets"]) == 1 + + +@mark.e2e_multiple_cluster_failures +class TestNoTwoClustersCanBeCreatedOnTheSameProject: + def test_sharded_cluster_reaches_running_phase(self, sharded_cluster: MongoDB): + # Unfortunately, Sharded cluster takes a long time to even start. + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=600) + + assert "warnings" not in sharded_cluster["status"] + + def test_second_mdb_sharded_cluster_fails(self, sharded_cluster_single: MongoDB): + sharded_cluster_single.assert_reaches_phase(Phase.Pending) + + assert "warnings" not in sharded_cluster_single["status"] + + # pylint: disable=unused-argument + def test_sharded_cluster_automation_config_is_correct(self, sharded_cluster, sharded_cluster_single): + config = KubernetesTester.get_automation_config() + + for process in config["processes"]: + assert not process["name"].startswith("sh001-single") + + # Creating a Sharded Cluster will result in 1 entry added to `sharding` and ... + assert len(config["sharding"]) == 1 + # ... and also 2 entries into `replicaSets` (1 shard and 1 config replica set, in this case) + assert len(config["replicaSets"]) == 2 + + +@mark.e2e_multiple_cluster_failures +class TestNoTwoDifferentTypeOfResourceCanBeCreatedOnTheSameProject: + def test_multiple_test_different_type_fails(self, replica_set_single: MongoDB): + replica_set_single.assert_reaches_phase(Phase.Running) + + def test_adding_sharded_cluster_fails(self, sharded_cluster_single: MongoDB): + sharded_cluster_single.assert_reaches_phase(Phase.Pending) + + status = sharded_cluster_single["status"] + assert status["phase"] == "Pending" + assert ( + status["message"] + == "Cannot have more than 1 MongoDB Cluster per project (see https://docs.mongodb.com/kubernetes-operator/stable/tutorial/migrate-to-single-resource/)" + ) + + assert "warnings" not in status + + # pylint: disable=unused-argument + def test_automation_config_contains_one_cluster(self, replica_set_single, sharded_cluster_single): + config = KubernetesTester.get_automation_config() + + assert len(config["processes"]) == 1 + + for process in config["processes"]: + assert not process["name"].startswith("sh001-single") + + assert len(config["sharding"]) == 0 + assert len(config["replicaSets"]) == 1 diff --git a/docker/mongodb-enterprise-tests/tests/mixed/fixtures/sample-mdb-object-to-test.yaml b/docker/mongodb-enterprise-tests/tests/mixed/fixtures/sample-mdb-object-to-test.yaml new file mode 100644 index 000000000..6fc9f05bc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/mixed/fixtures/sample-mdb-object-to-test.yaml @@ -0,0 +1,12 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-mdb-object-to-test +spec: + version: 5.0.15 + type: Standalone + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/__init__.py b/docker/mongodb-enterprise-tests/tests/multicluster/__init__.py new file mode 100644 index 000000000..5499e39d6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/__init__.py @@ -0,0 +1,29 @@ +import os +from typing import Dict, List + +from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml +from kubetester.helm import helm_template +from kubetester.mongodb_multi import MultiClusterClient + + +def prepare_multi_cluster_namespaces( + namespace: str, + multi_cluster_operator_installation_config: Dict[str, str], + member_cluster_clients: List[MultiClusterClient], + central_cluster_name: str, + skip_central_cluster: bool = True, +): + """create a new namespace and configures all necessary service accounts there""" + + helm_args = multi_cluster_operator_installation_config + yaml_file = helm_template( + helm_args=helm_args, + templates="templates/database-roles.yaml", + helm_options=[f"--namespace {namespace}"], + ) + # create database roles in member clusters. + for mcc in member_cluster_clients: + if skip_central_cluster and mcc.cluster_name == central_cluster_name: + continue + create_or_replace_from_yaml(mcc.api_client, yaml_file) + os.remove(yaml_file) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/conftest.py b/docker/mongodb-enterprise-tests/tests/multicluster/conftest.py new file mode 100644 index 000000000..51d1a0123 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/conftest.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +import ipaddress +import urllib +from typing import Dict, Generator, List, Optional +from urllib import parse + +import kubernetes +from kubeobject import CustomObject +from kubetester import create_or_update_namespace, create_or_update_secret +from kubetester.certs import generate_cert +from kubetester.kubetester import create_testing_namespace +from kubetester.ldap import ( + LDAPUser, + OpenLDAP, + add_user_to_group, + create_user, + ensure_group, + ensure_organizational_unit, + ldap_initialize, +) +from kubetester.mongodb_multi import MultiClusterClient +from pytest import fixture +from tests.authentication.conftest import ( + AUTOMATION_AGENT_NAME, + LDAP_NAME, + LDAP_PASSWORD, + ldap_host, + openldap_install, +) +from tests.conftest import ( + create_issuer, + get_api_servers_from_test_pod_kubeconfig, + install_cert_manager, + wait_for_cert_manager_ready, +) + + +@fixture(scope="module") +def member_cluster_cert_manager(member_cluster_clients: List[MultiClusterClient]) -> str: + member_cluster_one = member_cluster_clients[0] + result = install_cert_manager( + cluster_client=member_cluster_one.api_client, + cluster_name=member_cluster_one.cluster_name, + ) + wait_for_cert_manager_ready(cluster_client=member_cluster_one.api_client) + return result + + +@fixture(scope="module") +def multi_cluster_ldap_issuer( + member_cluster_cert_manager: str, + member_cluster_clients: List[MultiClusterClient], +): + member_cluster_one = member_cluster_clients[0] + # create openldap namespace if it doesn't exist + create_or_update_namespace( + "openldap", + {"istio-injection": "enabled"}, + api_client=member_cluster_one.api_client, + ) + + return create_issuer("openldap", member_cluster_one.api_client) + + +@fixture(scope="module") +def multicluster_openldap_cert(member_cluster_clients: List[MultiClusterClient], multi_cluster_ldap_issuer: str) -> str: + """Returns a new secret to be used to enable TLS on LDAP.""" + + member_cluster_one = member_cluster_clients[0] + + host = ldap_host("openldap", LDAP_NAME) + return generate_cert( + "openldap", + "openldap", + host, + multi_cluster_ldap_issuer, + api_client=member_cluster_one.api_client, + ) + + +@fixture(scope="module") +def multicluster_openldap_tls( + member_cluster_clients: List[MultiClusterClient], + multicluster_openldap_cert: str, + ca_path: str, +) -> Generator[OpenLDAP, None, None]: + member_cluster_one = member_cluster_clients[0] + helm_args = { + "tls.enabled": "true", + "tls.secret": multicluster_openldap_cert, + # Do not require client certificates + "env.LDAP_TLS_VERIFY_CLIENT": "never", + } + server = openldap_install( + "openldap", + LDAP_NAME, + helm_args=helm_args, + cluster_client=member_cluster_one.api_client, + cluster_name=member_cluster_one.cluster_name, + tls=True, + ) + # When creating a new OpenLDAP container with TLS enabled, the container is ready, but the server is not accepting + # requests, as it's generating DH parameters for the TLS config. Only using retries!=0 for ldap_initialize when creating + # the OpenLDAP server. + ldap_initialize(server, ca_path=ca_path, retries=10) + return server + + +@fixture(scope="module") +def ldap_mongodb_user(multicluster_openldap_tls: OpenLDAP, ca_path: str) -> LDAPUser: + user = LDAPUser("mdb0", LDAP_PASSWORD) + + ensure_organizational_unit(multicluster_openldap_tls, "groups", ca_path=ca_path) + create_user(multicluster_openldap_tls, user, ou="groups", ca_path=ca_path) + + ensure_group(multicluster_openldap_tls, cn="users", ou="groups", ca_path=ca_path) + add_user_to_group( + multicluster_openldap_tls, + user="mdb0", + group_cn="users", + ou="groups", + ca_path=ca_path, + ) + + return user + + +@fixture(scope="module") +def ldap_mongodb_users(multicluster_openldap_tls: OpenLDAP, ca_path: str) -> List[LDAPUser]: + user_list = [LDAPUser("mdb0", LDAP_PASSWORD)] + for user in user_list: + create_user(multicluster_openldap_tls, user, ou="groups", ca_path=ca_path) + return user_list + + +@fixture(scope="module") +def ldap_mongodb_agent_user(multicluster_openldap_tls: OpenLDAP, ca_path: str) -> LDAPUser: + user = LDAPUser(AUTOMATION_AGENT_NAME, LDAP_PASSWORD) + + ensure_organizational_unit(multicluster_openldap_tls, "groups", ca_path=ca_path) + create_user(multicluster_openldap_tls, user, ou="groups", ca_path=ca_path) + + ensure_group(multicluster_openldap_tls, cn="agents", ou="groups", ca_path=ca_path) + add_user_to_group( + multicluster_openldap_tls, + user=AUTOMATION_AGENT_NAME, + group_cn="agents", + ou="groups", + ca_path=ca_path, + ) + + return user + + +# more details https://istio.io/latest/docs/tasks/traffic-management/egress/egress-control/ +@fixture(scope="module") +def service_entries( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], +) -> List[CustomObject]: + return create_service_entries_objects(namespace, central_cluster_client, member_cluster_names) + + +@fixture(scope="module") +def test_patch_central_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient) -> str: + corev1 = kubernetes.client.CoreV1Api(api_client=central_cluster_client) + ns = corev1.read_namespace(namespace) + ns.metadata.labels["istio-injection"] = "enabled" + corev1.patch_namespace(namespace, ns) + + +def check_valid_ip(ip_str: str) -> bool: + try: + ipaddress.ip_address(ip_str) + return True + except ValueError: + return False + + +def create_service_entries_objects( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], +) -> List[CustomObject]: + service_entries = [] + + allowed_hosts_service_entry = CustomObject( + name="allowed-hosts", + namespace=namespace, + kind="ServiceEntry", + plural="serviceentries", + group="networking.istio.io", + version="v1beta1", + api_client=central_cluster_client, + ) + + allowed_addresses_service_entry = CustomObject( + name="allowed-addresses", + namespace=namespace, + kind="ServiceEntry", + plural="serviceentries", + group="networking.istio.io", + version="v1beta1", + api_client=central_cluster_client, + ) + + api_servers = get_api_servers_from_test_pod_kubeconfig(namespace, member_cluster_names) + + host_parse_results = [urllib.parse.urlparse(api_servers[member_cluster]) for member_cluster in member_cluster_names] + + hosts = set(["cloud-qa.mongodb.com"]) + addresses = set() + ports = [{"name": "https", "number": 443, "protocol": "HTTPS"}] + + for host_parse_result in host_parse_results: + if host_parse_result.port is not None and host_parse_result.port != "": + ports.append( + { + "name": f"https-{host_parse_result.port}", + "number": host_parse_result.port, + "protocol": "HTTPS", + } + ) + if check_valid_ip(host_parse_result.hostname): + addresses.add(host_parse_result.hostname) + else: + hosts.add(host_parse_result.hostname) + + allowed_hosts_service_entry["spec"] = { + # by default the access mode is set to "REGISTRY_ONLY" which means only the hosts specified + # here would be accessible from the operator pod + "hosts": list(hosts), + "exportTo": ["."], + "location": "MESH_EXTERNAL", + "ports": ports, + "resolution": "DNS", + } + service_entries.append(allowed_hosts_service_entry) + + if len(addresses) > 0: + allowed_addresses_service_entry["spec"] = { + "hosts": ["kubernetes", "kubernetes-master", "kube-apiserver"], + "addresses": list(addresses), + "exportTo": ["."], + "location": "MESH_EXTERNAL", + "ports": ports, + # when allowing by IP address we do not want to resolve IP using HTTP host field + "resolution": "NONE", + } + service_entries.append(allowed_addresses_service_entry) + + return service_entries + + +def cluster_spec_list( + member_cluster_names: List[str], + members: List[int], + member_configs: Optional[List[List[Dict]]] = None, + backup_configs: Optional[List[Dict]] = None, +): + + if member_configs is None and backup_configs is None: + result = [] + for name, members in zip(member_cluster_names, members): + if members is not None: + result.append({"clusterName": name, "members": members}) + return result + elif member_configs is not None: + result = [] + for name, members, memberConfig in zip(member_cluster_names, members, member_configs): + if members is not None: + result.append({"clusterName": name, "members": members, "memberConfig": memberConfig}) + return result + elif backup_configs is not None: + result = [] + for name, members, backupConfig in zip(member_cluster_names, members, backup_configs): + if members is not None: + result.append({"clusterName": name, "members": members, "backup": backupConfig}) + return result + + +def create_namespace( + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + task_id: str, + namespace: str, + image_pull_secret_name: str, + image_pull_secret_data: Dict[str, str], +) -> str: + for member_cluster_client in member_cluster_clients: + create_testing_namespace(task_id, namespace, member_cluster_client.api_client, True) + create_or_update_secret( + namespace, + image_pull_secret_name, + image_pull_secret_data, + type="kubernetes.io/dockerconfigjson", + api_client=member_cluster_client.api_client, + ) + + create_testing_namespace(task_id, namespace, central_cluster_client) + create_or_update_secret( + namespace, + image_pull_secret_name, + image_pull_secret_data, + type="kubernetes.io/dockerconfigjson", + api_client=central_cluster_client, + ) + + return namespace diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-central-sts-override.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-central-sts-override.yaml new file mode 100644 index 000000000..d8434d35f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-central-sts-override.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + duplicateServiceObjects: false + credentials: my-credentials + statefulSet: + spec: + template: + spec: + # FIXME workaround for sleep infinity hanging + shareProcessNamespace: true + containers: + - name: sidecar1 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 2 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-cluster.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-cluster.yaml new file mode 100644 index 000000000..2e142bcb8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-cluster.yaml @@ -0,0 +1,20 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + persistent: false + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 2 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-dr.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-dr.yaml new file mode 100644 index 000000000..a44e78f3b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-dr.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + persistent: false + credentials: my-credentials + duplicateServiceObjects: true + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: gke_k8s-rdas_us-east1-b_member-1a + members: 2 + statefulSet: + spec: + template: + spec: + securityContext: + fsGroup: 2000 + - clusterName: gke_k8s-rdas_us-east1-c_member-2a + members: 1 + statefulSet: + spec: + template: + spec: + securityContext: + fsGroup: 2000 + - clusterName: gke_k8s-rdas_us-west1-a_member-3a + members: 2 + statefulSet: + spec: + template: + spec: + securityContext: + fsGroup: 2000 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-pvc-resize.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-pvc-resize.yaml new file mode 100644 index 000000000..124cc8001 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-pvc-resize.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 2 + persistent: true + statefulSet: + spec: + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 1Gi + storageClassName: csi-hostpath-sc diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-split-horizon.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-split-horizon.yaml new file mode 100644 index 000000000..07c138a6c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-split-horizon.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + connectivity: + replicaSetHorizons: + - "test-horizon": "ec2-52-56-69-123.eu-west-2.compute.amazonaws.com:30100" + - "test-horizon": "ec2-3-9-165-220.eu-west-2.compute.amazonaws.com:30100" + - "test-horizon": "ec2-3-10-22-163.eu-west-2.compute.amazonaws.com:30100" + + + version: 4.4.0-ent + type: ReplicaSet + persistent: true + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 1 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-sts-override.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-sts-override.yaml new file mode 100644 index 000000000..b0c36bbea --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi-sts-override.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + statefulSet: + spec: + selector: + matchLabels: + app: "multi-replica-set" + template: + metadata: + labels: + app: "multi-replica-set" + spec: + containers: + - name: sidecar1 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + - clusterName: kind-e2e-cluster-2 + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar2 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + - clusterName: kind-e2e-cluster-3 + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar3 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi.yaml new file mode 100644 index 000000000..376caa441 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-multi.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 2 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-user.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-user.yaml new file mode 100644 index 000000000..bda215813 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-user.yaml @@ -0,0 +1,20 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mms-user-1 +spec: + passwordSecretKeyRef: + name: mms-user-1-password + key: password + username: "mms-user-1" + db: "admin" + mongodbResourceRef: + name: "multi-replica-set" + namespace: + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "userAdminAnyDatabase" + - db: "admin" + name: "readWrite" diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-x509-user.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-x509-user.yaml new file mode 100644 index 000000000..3e7f09b9e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/mongodb-x509-user.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: test-x509-user +spec: + username: 'CN=x509-testing-user' + db: '$external' + mongodbResourceRef: + name: "multi-replica-set" + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "userAdminAnyDatabase" + - db: "admin" + name: "readWrite" diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/split-horizon-node-port.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/split-horizon-node-port.yaml new file mode 100644 index 000000000..0fa05bc18 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/split-horizon-node-port.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: my-service + labels: + controller: mongodb-enterprise-operator +spec: + type: NodePort + selector: + controller: mongodb-enterprise-operator + ports: + - port: 27017 + targetPort: 27017 + nodePort: 30007 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/split-horizon-node-ports/split-horizon-node-port.yaml b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/split-horizon-node-ports/split-horizon-node-port.yaml new file mode 100644 index 000000000..8f9c1b6f5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/fixtures/split-horizon-node-ports/split-horizon-node-port.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: my-service + labels: + controller: mongodb-enterprise-operator + mongodbmulticluster: -multi-cluster-replica-set + statefulset.kubernetes.io/pod-name: multi-cluster-replica-set-0-0 +spec: + type: NodePort + selector: + controller: mongodb-enterprise-operator + statefulset.kubernetes.io/pod-name: multi-cluster-replica-set-0-0 + ports: + - port: 30100 + targetPort: 27017 + nodePort: 30100 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/manual_multi_cluster_tls_no_mesh_2_clusters_eks_gke.py b/docker/mongodb-enterprise-tests/tests/multicluster/manual_multi_cluster_tls_no_mesh_2_clusters_eks_gke.py new file mode 100644 index 000000000..0569e9608 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/manual_multi_cluster_tls_no_mesh_2_clusters_eks_gke.py @@ -0,0 +1,143 @@ +# This is a manual test deploying MongoDBMultiCluster on 2 clusters in GKE and EKS. +# Steps to execute: +# 1. After the test is executed it will create external services of type LoadBalancer. All pods will be not ready +# due to lack of external connectivity. +# 2. Go to mc.mongokubernetes.com Route53 hosted zone: https://us-east-1.console.aws.amazon.com/route53/v2/hostedzones#ListRecordSets/Z04069951X9SBFR8OQUFM +# 3. Copy provisioned hostnames from external services in EKS and update CNAME records for: +# * multi-cluster-rs-0-0.aws-member-cluster.eks.mc.mongokubernetes.com +# * multi-cluster-rs-0-1.aws-member-cluster.eks.mc.mongokubernetes.com +# 4. Copy IP addresses of external services in GKE and update A record for: +# * multi-cluster-rs-1-0.gke-member-cluster.gke.mc.mongokubernetes.com +# * multi-cluster-rs-1-1.gke-member-cluster.gke.mc.mongokubernetes.com +# 5. After few minutes everything should be ready. + +from typing import List + +import kubernetes +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture +from tests.multicluster.conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-rs" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +BUNDLE_PEM_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert-pem" + + +@fixture(scope="module") +def cert_additional_domains() -> list[str]: + return [ + "*.gke-member-cluster.gke.mc.mongokubernetes.com", + "*.aws-member-cluster.eks.mc.mongokubernetes.com", + ] + + +@fixture(scope="module") +def mongodb_multi_unmarshalled(namespace: str, member_cluster_names: List[str]) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource["spec"]["persistent"] = False + # These domains map 1:1 to the CoreDNS file. Please be mindful when updating them. + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1]) + + resource["spec"]["externalAccess"] = {} + resource["spec"]["clusterSpecList"][0]["externalAccess"] = { + "externalDomain": "aws-member-cluster.eks.mc.mongokubernetes.com", + "externalService": { + "annotations": {"cloud.google.com/l4-rbs": "enabled"}, + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": True, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + ], + }, + }, + } + resource["spec"]["clusterSpecList"][1]["externalAccess"] = { + "externalDomain": "gke-member-cluster.gke.mc.mongokubernetes.com", + "externalService": { + "annotations": { + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "instance", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", + }, + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": True, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + ], + }, + }, + } + + return resource + + +@fixture(scope="function") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + mongodb_multi_unmarshalled: MongoDBMulti, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBMulti: + mongodb_multi_unmarshalled["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + mongodb_multi_unmarshalled.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return mongodb_multi_unmarshalled.update() + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: list[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, + cert_additional_domains: list[str], +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + additional_domains=cert_additional_domains, + ) + + +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +def test_create_mongodb_multi( + mongodb_multi: MongoDBMulti, + namespace: str, + server_certs: str, + multi_cluster_issuer_ca_configmap: str, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=2400) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_2_cluster_clusterwide_replicaset.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_2_cluster_clusterwide_replicaset.py new file mode 100644 index 000000000..3af1723d9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_2_cluster_clusterwide_replicaset.py @@ -0,0 +1,273 @@ +from typing import Dict, List + +import kubernetes +import pytest +from kubetester import ( + create_or_update_configmap, + create_or_update_secret, + read_configmap, + read_secret, +) +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator + +from . import prepare_multi_cluster_namespaces +from .conftest import cluster_spec_list, create_namespace + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" + + +@pytest.fixture(scope="module") +def mdba_ns(namespace: str): + return "{}-mdb-ns-a".format(namespace) + + +@pytest.fixture(scope="module") +def mdbb_ns(namespace: str): + return "{}-mdb-ns-b".format(namespace) + + +@pytest.fixture(scope="module") +def mongodb_multi_a_unmarshalled( + central_cluster_client: kubernetes.client.ApiClient, + mdba_ns: str, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", mdba_ns) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1]) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@pytest.fixture(scope="module") +def mongodb_multi_b_unmarshalled( + central_cluster_client: kubernetes.client.ApiClient, + mdbb_ns: str, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", mdbb_ns) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1]) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@pytest.fixture(scope="module") +def server_certs_a( + multi_cluster_clusterissuer: str, + mongodb_multi_a_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_clusterissuer, + f"{CERT_SECRET_PREFIX}-{mongodb_multi_a_unmarshalled.name}-cert", + member_cluster_clients, + central_cluster_client, + mongodb_multi_a_unmarshalled, + clusterwide=True, + ) + + +@pytest.fixture(scope="module") +def server_certs_b( + multi_cluster_clusterissuer: str, + mongodb_multi_b_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_clusterissuer, + f"{CERT_SECRET_PREFIX}-{mongodb_multi_b_unmarshalled.name}-cert", + member_cluster_clients, + central_cluster_client, + mongodb_multi_b_unmarshalled, + clusterwide=True, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi_a( + central_cluster_client: kubernetes.client.ApiClient, + mdba_ns: str, + server_certs_a: str, + mongodb_multi_a_unmarshalled: MongoDBMulti, + issuer_ca_filepath: str, +) -> MongoDBMulti: + ca = open(issuer_ca_filepath).read() + + # The operator expects the CA that validates Ops Manager is contained in + # an entry with a name of "mms-ca.crt" + data = {"ca-pem": ca, "mms-ca.crt": ca} + name = "issuer-ca" + + create_or_update_configmap(mdba_ns, name, data, api_client=central_cluster_client) + + resource = mongodb_multi_a_unmarshalled + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": name, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@pytest.fixture(scope="module") +def mongodb_multi_b( + central_cluster_client: kubernetes.client.ApiClient, + mdbb_ns: str, + server_certs_b: str, + mongodb_multi_b_unmarshalled: MongoDBMulti, + issuer_ca_filepath: str, +) -> MongoDBMulti: + ca = open(issuer_ca_filepath).read() + + # The operator expects the CA that validates Ops Manager is contained in + # an entry with a name of "mms-ca.crt" + data = {"ca-pem": ca, "mms-ca.crt": ca} + name = "issuer-ca" + + create_or_update_configmap(mdbb_ns, name, data, api_client=central_cluster_client) + + resource = mongodb_multi_b_unmarshalled + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": name, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_create_kube_config_file(cluster_clients: Dict, member_cluster_names: List[str]): + clients = cluster_clients + + assert len(clients) == 2 + assert member_cluster_names[0] in clients + assert member_cluster_names[1] in clients + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_create_namespaces( + namespace: str, + mdba_ns: str, + mdbb_ns: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + evergreen_task_id: str, + multi_cluster_operator_installation_config: Dict[str, str], +): + image_pull_secret_name = multi_cluster_operator_installation_config["registry.imagePullSecrets"] + image_pull_secret_data = read_secret(namespace, image_pull_secret_name, api_client=central_cluster_client) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + mdba_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + mdbb_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_deploy_operator(multi_cluster_operator_clustermode: Operator): + multi_cluster_operator_clustermode.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_prepare_namespace( + multi_cluster_operator_installation_config: Dict[str, str], + member_cluster_clients: List[MultiClusterClient], + central_cluster_name: str, + mdba_ns: str, + mdbb_ns: str, +): + prepare_multi_cluster_namespaces( + mdba_ns, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + skip_central_cluster=False, + ) + + prepare_multi_cluster_namespaces( + mdbb_ns, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + skip_central_cluster=False, + ) + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_copy_configmap_and_secret_across_ns( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + multi_cluster_operator_installation_config: Dict[str, str], + mdba_ns: str, + mdbb_ns: str, +): + data = read_configmap(namespace, "my-project", api_client=central_cluster_client) + data["projectName"] = mdba_ns + create_or_update_configmap(mdba_ns, "my-project", data, api_client=central_cluster_client) + + data["projectName"] = mdbb_ns + create_or_update_configmap(mdbb_ns, "my-project", data, api_client=central_cluster_client) + + data = read_secret(namespace, "my-credentials", api_client=central_cluster_client) + create_or_update_secret(mdba_ns, "my-credentials", data, api_client=central_cluster_client) + create_or_update_secret(mdbb_ns, "my-credentials", data, api_client=central_cluster_client) + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_create_mongodb_multi_nsa(mongodb_multi_a: MongoDBMulti): + mongodb_multi_a.assert_reaches_phase(Phase.Running, timeout=800) + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_enable_mongodb_multi_nsa_auth(mongodb_multi_a: MongoDBMulti): + mongodb_multi_a.reload() + mongodb_multi_a["spec"]["authentication"] = ( + { + "agents": {"mode": "SCRAM"}, + "enabled": True, + "modes": ["SCRAM"], + }, + ) + + +@pytest.mark.e2e_multi_cluster_2_clusters_clusterwide +def test_create_mongodb_multi_nsb(mongodb_multi_b: MongoDBMulti): + mongodb_multi_b.assert_reaches_phase(Phase.Running, timeout=800) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_2_cluster_replicaset.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_2_cluster_replicaset.py new file mode 100644 index 000000000..cc10e299c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_2_cluster_replicaset.py @@ -0,0 +1,106 @@ +from typing import Dict, List + +import kubernetes +import pytest +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator + +from .conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, member_cluster_names: List[str], custom_mdb_version: str +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1]) + resource.set_version(ensure_ent_version(custom_mdb_version)) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + server_certs: str, + mongodb_multi_unmarshalled: MongoDBMulti, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBMulti: + resource = mongodb_multi_unmarshalled + + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return resource.create() + + +@pytest.mark.e2e_multi_cluster_2_clusters_replica_set +def test_create_kube_config_file(cluster_clients: Dict, member_cluster_names: List[str]): + clients = cluster_clients + + assert len(clients) == 2 + assert member_cluster_names[0] in clients + assert member_cluster_names[1] in clients + + +@pytest.mark.e2e_multi_cluster_2_clusters_replica_set +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_2_clusters_replica_set +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_multi_cluster_2_clusters_replica_set +def test_statefulset_is_created_across_multiple_clusters( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_2_clusters_replica_set +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_agent_flags.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_agent_flags.py new file mode 100644 index 000000000..4438fc333 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_agent_flags.py @@ -0,0 +1,96 @@ +from typing import List + +import kubernetes +from kubetester import client +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.common.placeholders import placeholders +from tests.multicluster.conftest import cluster_spec_list + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi-cluster.yaml"), "multi-replica-set", namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + # override agent startup flags + resource["spec"]["agent"] = {"startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFile"}} + resource["spec"]["agent"]["logLevel"] = "DEBUG" + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource.update() + + +@mark.e2e_multi_cluster_agent_flags +def test_create_mongodb_multi(multi_cluster_operator: Operator, mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@mark.e2e_multi_cluster_agent_flags +def test_multi_replicaset_has_agent_flags( + namespace: str, + member_cluster_clients: List[MultiClusterClient], +): + cluster_1_client = member_cluster_clients[0] + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFile* | wc -l", + ] + result = KubernetesTester.run_command_in_pod_container( + "multi-replica-set-0-0", + namespace, + cmd, + container="mongodb-enterprise-database", + api_client=cluster_1_client.api_client, + ) + assert result != "0" + + +@mark.e2e_multi_cluster_agent_flags +def test_placeholders_in_external_services( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + for cluster_spec_item in mongodb_multi["spec"]["clusterSpecList"]: + annotations = placeholders.get_annotations_with_placeholders_for_multi_cluster( + prefix=f'{cluster_spec_item["clusterName"]},' + ) + external_access = cluster_spec_item.get("externalAccess", {}) + external_service = external_access.get("externalService", {}) + external_service["annotations"] = annotations + external_access["externalService"] = external_service + cluster_spec_item["externalAccess"] = external_access + + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=300) + + name = mongodb_multi["metadata"]["name"] + for _, member_cluster_client in enumerate(member_cluster_clients): + members = mongodb_multi.get_item_spec(member_cluster_client.cluster_name)["members"] + for pod_idx in range(0, members): + cluster_idx = member_cluster_client.cluster_index + service = client.CoreV1Api(api_client=member_cluster_client.api_client).read_namespaced_service( + f"{name}-{cluster_idx}-{pod_idx}-svc-external", namespace + ) + cluster_name = member_cluster_client.cluster_name + expected_annotations = placeholders.get_expected_annotations_multi_cluster( + name=name, + namespace=namespace, + pod_idx=pod_idx, + cluster_index=cluster_idx, + cluster_name=cluster_name, + prefix=f"{cluster_name},", + ) + assert service.metadata.annotations == expected_annotations diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_automated_disaster_recovery.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_automated_disaster_recovery.py new file mode 100644 index 000000000..bb48b422c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_automated_disaster_recovery.py @@ -0,0 +1,160 @@ +from typing import List, Optional + +import kubernetes +from kubeobject import CustomObject +from kubernetes import client +from kubetester import delete_statefulset, statefulset_is_deleted +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import get_member_cluster_api_client + +from .conftest import cluster_spec_list, create_service_entries_objects + +FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3" + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["persistent"] = False + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return resource + + +@mark.e2e_multi_cluster_disaster_recovery +def test_label_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient): + api = client.CoreV1Api(api_client=central_cluster_client) + + labels = {"istio-injection": "enabled"} + ns = api.read_namespace(name=namespace) + + ns.metadata.labels.update(labels) + api.replace_namespace(name=namespace, body=ns) + + +@mark.e2e_multi_cluster_disaster_recovery +def test_create_service_entry(service_entries: List[CustomObject]): + for service_entry in service_entries: + service_entry.update() + + +@mark.e2e_multi_cluster_disaster_recovery +@mark.e2e_multi_cluster_multi_disaster_recovery +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_disaster_recovery +@mark.e2e_multi_cluster_multi_disaster_recovery +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_multi_cluster_disaster_recovery +def test_update_service_entry_block_failed_cluster_traffic( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], +): + healthy_cluster_names = [ + cluster_name for cluster_name in member_cluster_names if cluster_name != FAILED_MEMBER_CLUSTER_NAME + ] + service_entries = create_service_entries_objects(namespace, central_cluster_client, healthy_cluster_names) + for service_entry in service_entries: + print(f"service_entry={service_entries}") + service_entry.update() + + +@mark.e2e_multi_cluster_disaster_recovery +def test_mongodb_multi_leaves_running_state( + mongodb_multi: MongoDBMulti, +): + mongodb_multi.load() + mongodb_multi.assert_abandons_phase(Phase.Running, timeout=300) + + +@mark.e2e_multi_cluster_disaster_recovery +def test_delete_database_statefulset_in_failed_cluster(mongodb_multi: MongoDBMulti, member_cluster_names: list[str]): + failed_cluster_idx = member_cluster_names.index(FAILED_MEMBER_CLUSTER_NAME) + sts_name = f"{mongodb_multi.name}-{failed_cluster_idx}" + try: + delete_statefulset( + mongodb_multi.namespace, + sts_name, + propagation_policy="Background", + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + run_periodically( + lambda: statefulset_is_deleted( + mongodb_multi.namespace, + sts_name, + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + + +@mark.e2e_multi_cluster_disaster_recovery +@mark.e2e_multi_cluster_multi_disaster_recovery +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity() + + +@mark.e2e_multi_cluster_disaster_recovery +def test_replica_reaches_running(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@mark.e2e_multi_cluster_disaster_recovery +@mark.e2e_multi_cluster_multi_disaster_recovery +def test_number_numbers_in_ac(mongodb_multi: MongoDBMulti): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + desiredmembers = 0 + for c in mongodb_multi["spec"]["clusterSpecList"]: + desiredmembers += c["members"] + + processes = tester.get_replica_set_processes(mongodb_multi.name) + assert len(processes) == desiredmembers + + +@mark.e2e_multi_cluster_disaster_recovery +def test_sts_count_in_member_cluster( + mongodb_multi: MongoDBMulti, + member_cluster_names: list[str], + member_cluster_clients: List[MultiClusterClient], +): + failed_cluster_idx = member_cluster_names.index(FAILED_MEMBER_CLUSTER_NAME) + clients = member_cluster_clients[:] + clients.pop(failed_cluster_idx) + # assert the distribution of member cluster3 nodes. + + statefulsets = mongodb_multi.read_statefulsets(clients) + cluster_one_client = clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 3 + + cluster_two_client = clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 2 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_backup_restore.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_backup_restore.py new file mode 100644 index 000000000..f16e717d9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_backup_restore.py @@ -0,0 +1,507 @@ +import datetime +import time +from typing import Dict, List, Optional + +import kubernetes +import kubernetes.client +import pymongo +from kubernetes import client +from kubetester import ( + create_or_update_configmap, + create_or_update_secret, + get_default_storage_class, + read_service, + try_load, +) +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser +from kubetester.omtester import OMTester +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pymongo.errors import ServerSelectionTimeoutError +from pytest import fixture, mark +from tests.conftest import assert_data_got_restored, update_coredns_hosts + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +MONGODB_PORT = 30000 + + +HEAD_PATH = "/head/" +OPLOG_RS_NAME = "my-mongodb-oplog" +BLOCKSTORE_RS_NAME = "my-mongodb-blockstore" +USER_PASSWORD = "/qwerty@!#:" + + +@fixture(scope="module") +def ops_manager_certs( + namespace: str, + multi_cluster_issuer: str, + central_cluster_client: kubernetes.client.ApiClient, +): + return create_ops_manager_tls_certs( + multi_cluster_issuer, + namespace, + "om-backup", + secret_name="mdb-om-backup-cert", + # We need the interconnected certificate since we update coreDNS later with that ip -> domain + # because our central cluster is not part of the mesh, but we can access the pods via external IPs. + # Since we are using TLS we need a certificate for a hostname, an IP does not work, hence + # f"om-backup.{namespace}.interconnected" -> IP setup below + additional_domains=[ + "fastdl.mongodb.org", + f"om-backup.{namespace}.interconnected", + ], + api_client=central_cluster_client, + ) + + +def create_project_config_map(om: MongoDBOpsManager, mdb_name, project_name, client, custom_ca): + name = f"{mdb_name}-config" + data = { + "baseUrl": om.om_status().get_url(), + "projectName": project_name, + "sslMMSCAConfigMap": custom_ca, + "orgId": "", + } + + create_or_update_configmap(om.namespace, name, data, client) + + +def new_om_data_store( + mdb: MongoDB, + id: str, + assignment_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "id": id, + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "ssl": mdb.is_tls_enabled(), + "assignmentEnabled": assignment_enabled, + } + + +@fixture(scope="module") +def ops_manager( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + custom_version: Optional[str], + custom_appdb_version: str, + ops_manager_certs: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup.yaml"), namespace=namespace + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["externalConnectivity"] = {"type": "LoadBalancer"} + resource["spec"]["security"] = { + "certsSecretPrefix": "mdb", + "tls": {"ca": multi_cluster_issuer_ca_configmap}, + } + # remove s3 config + del resource["spec"]["backup"]["s3Stores"] + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource.create_admin_secret(api_client=central_cluster_client) + + try_load(resource) + + return resource + + +@fixture(scope="module") +def oplog_replica_set( + ops_manager, + namespace, + custom_mdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ) + + create_project_config_map( + om=ops_manager, + project_name="development", + mdb_name=OPLOG_RS_NAME, + client=central_cluster_client, + custom_ca=multi_cluster_issuer_ca_configmap, + ) + + resource.configure(ops_manager, "development") + + resource["spec"]["opsManager"]["configMapRef"]["name"] = OPLOG_RS_NAME + "-config" + resource.set_version(custom_mdb_version) + + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def blockstore_replica_set( + ops_manager, + namespace, + custom_mdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=BLOCKSTORE_RS_NAME, + ) + + create_project_config_map( + om=ops_manager, + project_name="blockstore", + mdb_name=BLOCKSTORE_RS_NAME, + client=central_cluster_client, + custom_ca=multi_cluster_issuer_ca_configmap, + ) + + resource.configure(ops_manager, "blockstore") + + resource.set_version(custom_mdb_version) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def blockstore_user( + namespace, + blockstore_replica_set: MongoDB, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user-backing-db.yaml"), namespace=namespace) + resource["spec"]["mongodbResourceRef"]["name"] = blockstore_replica_set.name + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + api_client=central_cluster_client, + ) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def oplog_user( + namespace, + oplog_replica_set: MongoDB, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user-backing-db.yaml"), + namespace=namespace, + name="mms-user-2", + ) + resource["spec"]["mongodbResourceRef"]["name"] = oplog_replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "mms-user-2-password" + resource["spec"]["username"] = "mms-user-2" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + api_client=central_cluster_client, + ) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@mark.e2e_multi_cluster_backup_restore +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_backup_restore +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. The OM is expected to get to 'Pending' state + eventually as it will wait for oplog db to be created + """ + + def test_create_om( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + ops_manager["spec"]["backup"]["members"] = 1 + + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=1800, + ) + + def test_daemon_statefulset( + self, + ops_manager: MongoDBOpsManager, + ): + def stateful_set_becomes_ready(): + stateful_set = ops_manager.read_backup_statefulset() + return stateful_set.status.ready_replicas == 1 and stateful_set.status.current_replicas == 1 + + KubernetesTester.wait_until(stateful_set_becomes_ready, timeout=300) + + stateful_set = ops_manager.read_backup_statefulset() + # pod template has volume mount request + assert (HEAD_PATH, "head") in ( + (mount.mount_path, mount.name) for mount in stateful_set.spec.template.spec.containers[0].volume_mounts + ) + + def test_backup_daemon_services_created( + self, + namespace, + central_cluster_client: kubernetes.client.ApiClient, + ): + """Backup creates two additional services for queryable backup""" + services = client.CoreV1Api(api_client=central_cluster_client).list_namespaced_service(namespace).items + + backup_services = [s for s in services if s.metadata.name.startswith("om-backup")] + + assert len(backup_services) >= 3 + + +@mark.e2e_multi_cluster_backup_restore +class TestBackupDatabasesAdded: + """name: Creates mongodb resources for oplog and blockstore and waits until OM resource gets to + running state""" + + def test_backup_mdbs_created( + self, + oplog_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + ): + """Creates mongodb databases all at once""" + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_oplog_user_created(self, oplog_user: MongoDBUser): + oplog_user.assert_reaches_phase(Phase.Updated) + + def test_om_failed_oplog_no_user_ref(self, ops_manager: MongoDBOpsManager): + """Waits until Backup is in failed state as blockstore doesn't have reference to the user""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + msg_regexp=".*is configured to use SCRAM-SHA authentication mode, the user " + "must be specified using 'mongodbUserRef'", + ) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, oplog_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + + assert ops_manager.backup_status().get_message() is None + + +class TestBackupForMongodb: + @fixture(scope="module") + def base_url( + self, + ops_manager: MongoDBOpsManager, + ) -> str: + """ + The base_url makes OM accessible from member clusters via a special interconnected dns address. + This address only works for member clusters. + """ + interconnected_field = f"https://om-backup.{ops_manager.namespace}.interconnected" + new_address = f"{interconnected_field}:8443" + + return new_address + + @fixture(scope="module") + def project_one( + self, + ops_manager: MongoDBOpsManager, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + base_url: str, + ) -> OMTester: + return ops_manager.get_om_tester( + project_name=f"{namespace}-project-one", + api_client=central_cluster_client, + base_url=base_url, + ) + + @fixture(scope="function") + def mongodb_multi_one_collection(self, mongodb_multi_one: MongoDBMulti): + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient( + mongodb_multi_one.tester(port=MONGODB_PORT).cnx_string, + **mongodb_multi_one.tester(port=MONGODB_PORT).default_opts, + )["testdb"] + return collection["testcollection"] + + @fixture(scope="module") + def mongodb_multi_one( + self, + ops_manager: MongoDBOpsManager, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: List[str], + base_url, + custom_mdb_version: str, + ) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi.yaml"), + "multi-replica-set-one", + namespace, + # the project configmap should be created in the central cluster. + ).configure(ops_manager, f"{namespace}-project-one", api_client=central_cluster_client) + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["clusterSpecList"] = [ + {"clusterName": member_cluster_names[0], "members": 2}, + {"clusterName": member_cluster_names[1], "members": 1}, + {"clusterName": member_cluster_names[2], "members": 2}, + ] + + # creating a cluster with backup should work with custom ports + resource["spec"].update({"additionalMongodConfig": {"net": {"port": MONGODB_PORT}}}) + + resource.configure_backup(mode="enabled") + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + data = KubernetesTester.read_configmap( + namespace, "multi-replica-set-one-config", api_client=central_cluster_client + ) + KubernetesTester.delete_configmap(namespace, "multi-replica-set-one-config", api_client=central_cluster_client) + data["baseUrl"] = base_url + data["sslMMSCAConfigMap"] = multi_cluster_issuer_ca_configmap + create_or_update_configmap( + namespace, + "multi-replica-set-one-config", + data, + api_client=central_cluster_client, + ) + + return resource.update() + + @mark.e2e_multi_cluster_backup_restore + def test_setup_om_connection( + self, + ops_manager: MongoDBOpsManager, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + ): + """ + The base_url makes OM accessible from member clusters via a special interconnected dns address. + """ + ops_manager.load() + external_svc_name = ops_manager.external_svc_name() + svc = read_service(ops_manager.namespace, external_svc_name, api_client=central_cluster_client) + # we have no hostName, but the ip is resolvable. + ip = svc.status.load_balancer.ingress[0].ip + + interconnected_field = f"om-backup.{ops_manager.namespace}.interconnected" + + # let's make sure that every client can connect to OM. + for c in member_cluster_clients: + update_coredns_hosts( + host_mappings=[(ip, interconnected_field)], + api_client=c.api_client, + cluster_name=c.cluster_name, + ) + + # let's make sure that the operator can connect to OM via that given address. + update_coredns_hosts( + host_mappings=[(ip, interconnected_field)], + api_client=central_cluster_client, + cluster_name="central-cluster", + ) + + new_address = f"https://{interconnected_field}:8443" + # updating the central url app setting to point at the external address, + # this allows agents in other clusters to communicate correctly with this OM instance. + ops_manager["spec"]["configuration"]["mms.centralUrl"] = new_address + ops_manager.update() + + @mark.e2e_multi_cluster_backup_restore + def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): + # we might fail connection in the beginning since we set a custom dns in coredns + mongodb_multi_one.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=1200) + + @skip_if_local + @mark.e2e_multi_cluster_backup_restore + def test_add_test_data(self, mongodb_multi_one_collection): + max_attempts = 100 + while max_attempts > 0: + try: + mongodb_multi_one_collection.insert_one(TEST_DATA) + return + except Exception as e: + print(e) + max_attempts -= 1 + time.sleep(6) + + @mark.e2e_multi_cluster_backup_restore + def test_mdb_backed_up(self, project_one: OMTester): + project_one.wait_until_backup_snapshots_are_ready(expected_count=1) + + @mark.e2e_multi_cluster_backup_restore + def test_change_mdb_data(self, mongodb_multi_one_collection): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + time.sleep(30) + mongodb_multi_one_collection.insert_one({"foo": "bar"}) + + @mark.e2e_multi_cluster_backup_restore + def test_pit_restore(self, project_one: OMTester): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + + pit_datetme = datetime.datetime.now() - datetime.timedelta(seconds=15) + pit_millis = time_to_millis(pit_datetme) + print("Restoring back to the moment 15 seconds ago (millis): {}".format(pit_millis)) + + project_one.create_restore_job_pit(pit_millis) + + @mark.e2e_multi_cluster_backup_restore + def test_data_got_restored(self, mongodb_multi_one_collection): + assert_data_got_restored(TEST_DATA, mongodb_multi_one_collection, timeout=1200) + + +def time_to_millis(date_time) -> int: + """https://stackoverflow.com/a/11111177/614239""" + epoch = datetime.datetime.utcfromtimestamp(0) + pit_millis = (date_time - epoch).total_seconds() * 1000 + return pit_millis diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_backup_restore_no_mesh.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_backup_restore_no_mesh.py new file mode 100644 index 000000000..809b4b030 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_backup_restore_no_mesh.py @@ -0,0 +1,667 @@ +# This test sets up ops manager in a multicluster "no-mesh" environment. +# It tests the back-up functionality with a multi-cluster replica-set when the replica-set is deployed outside of a service-mesh context. + +import datetime +import time +from typing import List, Optional, Tuple + +import kubernetes +import kubernetes.client +import pymongo +from kubernetes import client +from kubetester import ( + create_or_update_configmap, + create_or_update_secret, + get_default_storage_class, + read_service, + try_load, +) +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser +from kubetester.omtester import OMTester +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import assert_data_got_restored, update_coredns_hosts + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +MONGODB_PORT = 30000 +HEAD_PATH = "/head/" +OPLOG_RS_NAME = "my-mongodb-oplog" +BLOCKSTORE_RS_NAME = "my-mongodb-blockstore" +USER_PASSWORD = "/qwerty@!#:" + + +@fixture(scope="module") +def ops_manager_certs( + namespace: str, + multi_cluster_issuer: str, + central_cluster_client: kubernetes.client.ApiClient, +): + return create_ops_manager_tls_certs( + multi_cluster_issuer, + namespace, + "om-backup", + secret_name="mdb-om-backup-cert", + # We need the interconnected certificate since we update coreDNS later with that ip -> domain + # because our central cluster is not part of the mesh, but we can access the pods via external IPs. + # Since we are using TLS we need a certificate for a hostname, an IP does not work, hence + # f"om-backup.{namespace}.interconnected" -> IP setup below + additional_domains=[ + "fastdl.mongodb.org", + f"om-backup.{namespace}.interconnected", + ], + api_client=central_cluster_client, + ) + + +def create_project_config_map(om: MongoDBOpsManager, mdb_name, project_name, client, custom_ca): + name = f"{mdb_name}-config" + data = { + "baseUrl": om.om_status().get_url(), + "projectName": project_name, + "sslMMSCAConfigMap": custom_ca, + "orgId": "", + } + + create_or_update_configmap(om.namespace, name, data, client) + + +def new_om_data_store( + mdb: MongoDB, + id: str, + assignment_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> dict: + return { + "id": id, + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "ssl": mdb.is_tls_enabled(), + "assignmentEnabled": assignment_enabled, + } + + +@fixture(scope="module") +def ops_manager( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + custom_version: str, + custom_appdb_version: str, + ops_manager_certs: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup.yaml"), namespace=namespace + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["externalConnectivity"] = {"type": "LoadBalancer"} + resource["spec"]["security"] = { + "certsSecretPrefix": "mdb", + "tls": {"ca": multi_cluster_issuer_ca_configmap}, + } + # remove s3 config + del resource["spec"]["backup"]["s3Stores"] + resource.set_version(custom_version) + resource.set_appdb_version(ensure_ent_version(custom_appdb_version)) + + resource.allow_mdb_rc_versions() + resource.create_admin_secret(api_client=central_cluster_client) + + try_load(resource) + + return resource + + +@fixture(scope="module") +def oplog_replica_set( + ops_manager, + namespace, + custom_mdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ) + + create_project_config_map( + om=ops_manager, + project_name="development", + mdb_name=OPLOG_RS_NAME, + client=central_cluster_client, + custom_ca=multi_cluster_issuer_ca_configmap, + ) + + resource.configure(ops_manager, "development") + + resource["spec"]["opsManager"]["configMapRef"]["name"] = OPLOG_RS_NAME + "-config" + resource.set_version(custom_mdb_version) + + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def blockstore_replica_set( + ops_manager, + namespace, + custom_mdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=BLOCKSTORE_RS_NAME, + ) + + create_project_config_map( + om=ops_manager, + project_name="blockstore", + mdb_name=BLOCKSTORE_RS_NAME, + client=central_cluster_client, + custom_ca=multi_cluster_issuer_ca_configmap, + ) + + resource.configure(ops_manager, "blockstore") + + resource.set_version(custom_mdb_version) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def blockstore_user( + namespace, + blockstore_replica_set: MongoDB, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user-backing-db.yaml"), namespace=namespace) + resource["spec"]["mongodbResourceRef"]["name"] = blockstore_replica_set.name + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + api_client=central_cluster_client, + ) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def oplog_user( + namespace, + oplog_replica_set: MongoDB, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user-backing-db.yaml"), + namespace=namespace, + name="mms-user-2", + ) + resource["spec"]["mongodbResourceRef"]["name"] = oplog_replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "mms-user-2-password" + resource["spec"]["username"] = "mms-user-2" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + api_client=central_cluster_client, + ) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def replica_set_external_hosts() -> List[Tuple[str, str]]: + return [ + ("172.18.255.211", "test.kind-e2e-cluster-1.interconnected"), + ( + "172.18.255.211", + "multi-replica-set-one-0-0.kind-e2e-cluster-1.interconnected", + ), + ( + "172.18.255.212", + "multi-replica-set-one-0-1.kind-e2e-cluster-1.interconnected", + ), + ( + "172.18.255.213", + "multi-replica-set-one-0-2.kind-e2e-cluster-1.interconnected", + ), + ("172.18.255.221", "test.kind-e2e-cluster-2.interconnected"), + ( + "172.18.255.221", + "multi-replica-set-one-1-0.kind-e2e-cluster-2.interconnected", + ), + ( + "172.18.255.222", + "multi-replica-set-one-1-1.kind-e2e-cluster-2.interconnected", + ), + ( + "172.18.255.223", + "multi-replica-set-one-1-2.kind-e2e-cluster-2.interconnected", + ), + ("172.18.255.231", "test.kind-e2e-cluster-3.interconnected"), + ( + "172.18.255.231", + "multi-replica-set-one-2-0.kind-e2e-cluster-3.interconnected", + ), + ( + "172.18.255.232", + "multi-replica-set-one-2-1.kind-e2e-cluster-3.interconnected", + ), + ( + "172.18.255.233", + "multi-replica-set-one-2-2.kind-e2e-cluster-3.interconnected", + ), + ] + + +@fixture(scope="module") +def disable_istio( + multi_cluster_operator: Operator, + namespace: str, + member_cluster_clients: List[MultiClusterClient], +): + for mcc in member_cluster_clients: + api = client.CoreV1Api(api_client=mcc.api_client) + labels = {"istio-injection": "disabled"} + ns = api.read_namespace(name=namespace) + ns.metadata.labels.update(labels) + api.replace_namespace(name=namespace, body=ns) + return None + + +@mark.e2e_multi_cluster_backup_restore_no_mesh +def test_update_coredns( + replica_set_external_hosts: List[Tuple[str, str]], + cluster_clients: dict[str, kubernetes.client.ApiClient], +): + """ + This test updates the coredns config in the member clusters to allow connecting to the other replica set members + through an external address. + """ + for cluster_name, cluster_api in cluster_clients.items(): + update_coredns_hosts(replica_set_external_hosts, cluster_name, api_client=cluster_api) + + +@mark.e2e_multi_cluster_backup_restore_no_mesh +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_backup_restore_no_mesh +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. The OM is expected to get to 'Pending' state + eventually as it will wait for oplog db to be created + """ + + def test_create_om( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + ops_manager["spec"]["backup"]["members"] = 1 + + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=1800, + ) + + def test_daemon_statefulset( + self, + ops_manager: MongoDBOpsManager, + ): + def stateful_set_becomes_ready(): + stateful_set = ops_manager.read_backup_statefulset() + return stateful_set.status.ready_replicas == 1 and stateful_set.status.current_replicas == 1 + + KubernetesTester.wait_until(stateful_set_becomes_ready, timeout=300) + + stateful_set = ops_manager.read_backup_statefulset() + # pod template has volume mount request + assert (HEAD_PATH, "head") in ( + (mount.mount_path, mount.name) for mount in stateful_set.spec.template.spec.containers[0].volume_mounts + ) + + def test_backup_daemon_services_created( + self, + namespace, + central_cluster_client: kubernetes.client.ApiClient, + ): + """Backup creates two additional services for queryable backup""" + services = client.CoreV1Api(api_client=central_cluster_client).list_namespaced_service(namespace).items + + backup_services = [s for s in services if s.metadata.name.startswith("om-backup")] + + assert len(backup_services) >= 3 + + +@mark.e2e_multi_cluster_backup_restore_no_mesh +class TestBackupDatabasesAdded: + """name: Creates mongodb resources for oplog and blockstore and waits until OM resource gets to + running state""" + + def test_backup_mdbs_created( + self, + oplog_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + ): + """Creates mongodb databases all at once""" + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_oplog_user_created(self, oplog_user: MongoDBUser): + oplog_user.assert_reaches_phase(Phase.Updated) + + def test_om_failed_oplog_no_user_ref(self, ops_manager: MongoDBOpsManager): + """Waits until Backup is in failed state as blockstore doesn't have reference to the user""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + msg_regexp=".*is configured to use SCRAM-SHA authentication mode, the user " + "must be specified using 'mongodbUserRef'", + ) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, oplog_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + + assert ops_manager.backup_status().get_message() is None + + +class TestBackupForMongodb: + @fixture(scope="module") + def base_url( + self, + ops_manager: MongoDBOpsManager, + ) -> str: + """ + The base_url makes OM accessible from member clusters via a special interconnected dns address. + We also use this address for the operator to connect to ops manager, + because the operator and the replica-sets rely on the same project configmap. + """ + interconnected_field = f"https://om-backup.{ops_manager.namespace}.interconnected" + new_address = f"{interconnected_field}:8443" + + return new_address + + @fixture(scope="module") + def project_one( + self, + ops_manager: MongoDBOpsManager, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + base_url: str, + ) -> OMTester: + return ops_manager.get_om_tester( + project_name=f"{namespace}-project-one", + api_client=central_cluster_client, + base_url=base_url, + ) + + @fixture(scope="function") + def mongodb_multi_one_collection(self, mongodb_multi_one: MongoDBMulti): + + tester = mongodb_multi_one.tester( + port=MONGODB_PORT, + service_names=[ + "multi-replica-set-one-0-0.kind-e2e-cluster-1.interconnected", + "multi-replica-set-one-0-1.kind-e2e-cluster-1.interconnected", + "multi-replica-set-one-1-0.kind-e2e-cluster-2.interconnected", + "multi-replica-set-one-2-0.kind-e2e-cluster-3.interconnected", + "multi-replica-set-one-2-1.kind-e2e-cluster-3.interconnected", + ], + external=True, + ) + + collection = pymongo.MongoClient(tester.cnx_string, **tester.default_opts)["testdb"] + + return collection["testcollection"] + + @fixture(scope="module") + def mongodb_multi_one( + self, + ops_manager: MongoDBOpsManager, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + disable_istio, + namespace: str, + member_cluster_names: List[str], + base_url, + custom_mdb_version: str, + ) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi.yaml"), + "multi-replica-set-one", + namespace, + # the project configmap should be created in the central cluster. + ).configure(ops_manager, f"{namespace}-project-one", api_client=central_cluster_client) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource["spec"]["clusterSpecList"] = [ + {"clusterName": member_cluster_names[0], "members": 2}, + {"clusterName": member_cluster_names[1], "members": 1}, + {"clusterName": member_cluster_names[2], "members": 2}, + ] + + resource["spec"]["externalAccess"] = {} + resource["spec"]["clusterSpecList"][0]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-1.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": MONGODB_PORT, + }, + { + "name": "backup", + "port": MONGODB_PORT + 1, + }, + { + "name": "testing0", + "port": MONGODB_PORT + 2, + }, + ], + } + }, + } + resource["spec"]["clusterSpecList"][1]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-2.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": MONGODB_PORT, + }, + { + "name": "backup", + "port": MONGODB_PORT + 1, + }, + { + "name": "testing1", + "port": MONGODB_PORT + 2, + }, + ], + } + }, + } + resource["spec"]["clusterSpecList"][2]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-3.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": MONGODB_PORT, + }, + { + "name": "backup", + "port": MONGODB_PORT + 1, + }, + { + "name": "testing2", + "port": MONGODB_PORT + 2, + }, + ], + } + }, + } + + # creating a cluster with backup should work with custom ports + resource["spec"].update({"additionalMongodConfig": {"net": {"port": MONGODB_PORT}}}) + + resource.configure_backup(mode="enabled") + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + data = KubernetesTester.read_configmap( + namespace, "multi-replica-set-one-config", api_client=central_cluster_client + ) + KubernetesTester.delete_configmap(namespace, "multi-replica-set-one-config", api_client=central_cluster_client) + data["baseUrl"] = base_url + data["sslMMSCAConfigMap"] = multi_cluster_issuer_ca_configmap + create_or_update_configmap( + namespace, + "multi-replica-set-one-config", + data, + api_client=central_cluster_client, + ) + + return resource.update() + + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_setup_om_connection( + self, + replica_set_external_hosts: List[Tuple[str, str]], + ops_manager: MongoDBOpsManager, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + ): + """ + test_setup_om_connection makes OM accessible from member clusters via a special interconnected dns address. + """ + ops_manager.load() + external_svc_name = ops_manager.external_svc_name() + svc = read_service(ops_manager.namespace, external_svc_name, api_client=central_cluster_client) + # we have no hostName, but the ip is resolvable. + ip = svc.status.load_balancer.ingress[0].ip + + interconnected_field = f"om-backup.{ops_manager.namespace}.interconnected" + + # let's make sure that every client can connect to OM. + hosts = replica_set_external_hosts[:] + hosts.append((ip, interconnected_field)) + + for c in member_cluster_clients: + update_coredns_hosts( + host_mappings=hosts, + api_client=c.api_client, + cluster_name=c.cluster_name, + ) + + # let's make sure that the operator can connect to OM via that given address. + update_coredns_hosts( + host_mappings=[(ip, interconnected_field)], + api_client=central_cluster_client, + cluster_name="central-cluster", + ) + + new_address = f"https://{interconnected_field}:8443" + # updating the central url app setting to point at the external address, + # this allows agents in other clusters to communicate correctly with this OM instance. + ops_manager["spec"]["configuration"]["mms.centralUrl"] = new_address + ops_manager.update() + + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): + # we might fail connection in the beginning since we set a custom dns in coredns + mongodb_multi_one.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=1500) + + @skip_if_local + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_add_test_data(self, mongodb_multi_one_collection): + max_attempts = 100 + while max_attempts > 0: + try: + mongodb_multi_one_collection.insert_one(TEST_DATA) + return + except Exception as e: + print(e) + max_attempts -= 1 + time.sleep(6) + + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_mdb_backed_up(self, project_one: OMTester): + project_one.wait_until_backup_snapshots_are_ready(expected_count=1) + + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_change_mdb_data(self, mongodb_multi_one_collection): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + time.sleep(30) + mongodb_multi_one_collection.insert_one({"foo": "bar"}) + + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_pit_restore(self, project_one: OMTester): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + + pit_datetme = datetime.datetime.now() - datetime.timedelta(seconds=15) + pit_millis = time_to_millis(pit_datetme) + print("Restoring back to the moment 15 seconds ago (millis): {}".format(pit_millis)) + + project_one.create_restore_job_pit(pit_millis) + + @mark.e2e_multi_cluster_backup_restore_no_mesh + def test_data_got_restored(self, mongodb_multi_one_collection): + assert_data_got_restored(TEST_DATA, mongodb_multi_one_collection) + + +def time_to_millis(date_time) -> int: + """https://stackoverflow.com/a/11111177/614239""" + epoch = datetime.datetime.utcfromtimestamp(0) + pit_millis = (date_time - epoch).total_seconds() * 1000 + return pit_millis diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_cli_recover.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_cli_recover.py new file mode 100644 index 000000000..f4a2b55ed --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_cli_recover.py @@ -0,0 +1,136 @@ +from typing import Callable, List + +import kubernetes +import pytest +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from tests.conftest import ( + MULTI_CLUSTER_OPERATOR_NAME, + run_kube_config_creation_tool, + run_multi_cluster_recovery_tool, +) +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set" +BUNDLE_SECRET_NAME = f"prefix-{RESOURCE_NAME}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + # ensure certs are created for the members during scale up + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi(mongodb_multi_unmarshalled: MongoDBMulti, server_certs: str) -> MongoDBMulti: + mongodb_multi_unmarshalled["spec"]["clusterSpecList"].pop() + mongodb_multi_unmarshalled.update() + return mongodb_multi_unmarshalled + + +@pytest.mark.e2e_multi_cluster_recover +def test_deploy_operator( + install_multi_cluster_operator_set_members_fn: Callable[[List[str]], Operator], + member_cluster_names: List[str], + namespace: str, +): + run_kube_config_creation_tool(member_cluster_names[:-1], namespace, namespace, member_cluster_names) + # deploy the operator without the final cluster + operator = install_multi_cluster_operator_set_members_fn(member_cluster_names[:-1]) + operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_recover +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_multi_cluster_recover +def test_recover_operator_add_cluster( + member_cluster_names: List[str], + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +): + return_code = run_multi_cluster_recovery_tool(member_cluster_names, namespace, namespace) + assert return_code == 0 + operator = Operator( + name=MULTI_CLUSTER_OPERATOR_NAME, + namespace=namespace, + api_client=central_cluster_client, + ) + operator._wait_for_operator_ready() + operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_recover +def test_mongodb_multi_recovers_adding_cluster(mongodb_multi: MongoDBMulti, member_cluster_names: List[str]): + mongodb_multi.load() + + mongodb_multi["spec"]["clusterSpecList"].append({"clusterName": member_cluster_names[-1], "members": 2}) + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_multi_cluster_recover +def test_recover_operator_remove_cluster( + member_cluster_names: List[str], + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +): + return_code = run_multi_cluster_recovery_tool(member_cluster_names[1:], namespace, namespace) + assert return_code == 0 + operator = Operator( + name=MULTI_CLUSTER_OPERATOR_NAME, + namespace=namespace, + api_client=central_cluster_client, + ) + operator._wait_for_operator_ready() + operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_recover +def test_mongodb_multi_recovers_removing_cluster(mongodb_multi: MongoDBMulti, member_cluster_names: List[str]): + mongodb_multi.load() + + last_transition_time = mongodb_multi.get_status_last_transition_time() + + mongodb_multi["spec"]["clusterSpecList"].pop(0) + mongodb_multi.update() + mongodb_multi.assert_state_transition_happens(last_transition_time) + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1500) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_clusterwide.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_clusterwide.py new file mode 100644 index 000000000..60f00caa3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_clusterwide.py @@ -0,0 +1,230 @@ +import os +import time +from typing import Dict, List + +import kubernetes +from kubernetes import client +from kubetester import create_or_update_configmap, create_or_update_secret, read_secret +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + MULTI_CLUSTER_OPERATOR_NAME, + _install_multi_cluster_operator, + run_kube_config_creation_tool, +) + +from . import prepare_multi_cluster_namespaces +from .conftest import cluster_spec_list, create_namespace + + +@fixture(scope="module") +def mdba_ns(namespace: str): + return "{}-mdb-ns-a".format(namespace) + + +@fixture(scope="module") +def mdbb_ns(namespace: str): + return "{}-mdb-ns-b".format(namespace) + + +@fixture(scope="module") +def unmanaged_mdb_ns(namespace: str): + return "{}-mdb-ns-c".format(namespace) + + +@fixture(scope="module") +def mongodb_multi_a( + central_cluster_client: kubernetes.client.ApiClient, + mdba_ns: str, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", mdba_ns) + resource.set_version(custom_mdb_version) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@fixture(scope="module") +def mongodb_multi_b( + central_cluster_client: kubernetes.client.ApiClient, + mdbb_ns: str, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", mdbb_ns) + resource.set_version(custom_mdb_version) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@fixture(scope="module") +def unmanaged_mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + unmanaged_mdb_ns: str, + member_cluster_names: List[str], +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", unmanaged_mdb_ns) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@fixture(scope="module") +def install_operator( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[kubernetes.client.ApiClient], + cluster_clients: Dict[str, kubernetes.client.ApiClient], + member_cluster_names: List[str], + mdba_ns: str, + mdbb_ns: str, +) -> Operator: + print(f"Installing operator in context: {central_cluster_name}") + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + member_cluster_namespaces = mdba_ns + "," + mdbb_ns + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names, True) + + return _install_multi_cluster_operator( + namespace, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + "operator.createOperatorServiceAccount": "false", + "operator.watchNamespace": member_cluster_namespaces, + }, + central_cluster_name, + ) + + +@mark.e2e_multi_cluster_specific_namespaces +def test_create_namespaces( + namespace: str, + mdba_ns: str, + mdbb_ns: str, + unmanaged_mdb_ns: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + evergreen_task_id: str, + multi_cluster_operator_installation_config: Dict[str, str], +): + image_pull_secret_name = multi_cluster_operator_installation_config["registry.imagePullSecrets"] + image_pull_secret_data = read_secret(namespace, image_pull_secret_name) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + mdba_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + mdbb_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + unmanaged_mdb_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + +@mark.e2e_multi_cluster_specific_namespaces +def test_prepare_namespace( + multi_cluster_operator_installation_config: Dict[str, str], + member_cluster_clients: List[MultiClusterClient], + central_cluster_name: str, + mdba_ns: str, + mdbb_ns: str, +): + prepare_multi_cluster_namespaces( + mdba_ns, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + ) + + prepare_multi_cluster_namespaces( + mdbb_ns, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + ) + + +@mark.e2e_multi_cluster_clusterwide +def test_deploy_operator(multi_cluster_operator_clustermode: Operator): + multi_cluster_operator_clustermode.assert_is_running() + + +@mark.e2e_multi_cluster_specific_namespaces +def test_deploy_operator(install_operator: Operator): + install_operator.assert_is_running() + + +@mark.e2e_multi_cluster_specific_namespaces +def test_copy_configmap_and_secret_across_ns( + namespace: str, + central_cluster_client: client.ApiClient, + multi_cluster_operator_installation_config: Dict[str, str], + mdba_ns: str, + mdbb_ns: str, +): + data = KubernetesTester.read_configmap(namespace, "my-project", api_client=central_cluster_client) + data["projectName"] = mdba_ns + create_or_update_configmap(mdba_ns, "my-project", data, api_client=central_cluster_client) + + data["projectName"] = mdbb_ns + create_or_update_configmap(mdbb_ns, "my-project", data, api_client=central_cluster_client) + + data = read_secret(namespace, "my-credentials", api_client=central_cluster_client) + create_or_update_secret(mdba_ns, "my-credentials", data, api_client=central_cluster_client) + create_or_update_secret(mdbb_ns, "my-credentials", data, api_client=central_cluster_client) + + +@mark.e2e_multi_cluster_specific_namespaces +def test_create_mongodb_multi_nsa(mongodb_multi_a: MongoDBMulti): + mongodb_multi_a.assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_multi_cluster_specific_namespaces +def test_create_mongodb_multi_nsb(mongodb_multi_b: MongoDBMulti): + mongodb_multi_b.assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_multi_cluster_specific_namespaces +def test_create_mongodb_multi_unmanaged(unmanaged_mongodb_multi: MongoDBMulti): + """ + For an unmanaged resource, the status should not be updated! + """ + for i in range(10): + time.sleep(5) + + unmanaged_mongodb_multi.reload() + assert "status" not in unmanaged_mongodb_multi diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_dr_connect.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_dr_connect.py new file mode 100644 index 000000000..ed155d6af --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_dr_connect.py @@ -0,0 +1,108 @@ +import subprocess +import time +from typing import Dict + +import kubernetes +import pytest +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.operator import Operator + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +CLUSTER_TO_DELETE = "member-3a" + + +# this test is intended to run locally, using telepresence. Make sure to configure the cluster_context to api-server mapping +# in the "cluster_host_mapping" fixture before running it. It is intented to be run locally with the command: make e2e-telepresence test=e2e_multi_cluster_dr local=true +@pytest.fixture(scope="module") +def mongodb_multi(central_cluster_client: kubernetes.client.ApiClient, namespace: str) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi-dr.yaml"), "multi-replica-set", namespace) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + # return resource.load() + return resource.create() + + +@pytest.fixture(scope="module") +def mongodb_multi_collection(mongodb_multi: MongoDBMulti): + collection = mongodb_multi.tester().client["testdb"] + return collection["testcollection"] + + +@pytest.mark.e2e_multi_cluster_dr +def test_create_kube_config_file(cluster_clients: Dict): + clients = cluster_clients + assert len(clients) == 4 + + +@pytest.mark.e2e_multi_cluster_dr +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_dr +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_multi_cluster_dr +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity() + + +@pytest.mark.e2e_multi_cluster_dr +def test_add_test_data(mongodb_multi_collection): + # TODO: remove this retry mechanism, for some reason the resource exits the running state and then + # enters it later. The subsequent test fails because the resource is not actually + max_attempts = 100 + while max_attempts > 0: + try: + mongodb_multi_collection.insert_one(TEST_DATA) + return + except Exception as e: + print(e) + max_attempts -= 1 + time.sleep(6) + + +@pytest.mark.e2e_multi_cluster_dr +def test_delete_member_3_cluster(): + # delete 3rd cluster with gcloud command + # gcloud container clusters delete member-3a --zone us-west1-a + subprocess.call( + [ + "gcloud", + "container", + "clusters", + "delete", + CLUSTER_TO_DELETE, + "--zone", + "us-west1-a", + "--quiet", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +@pytest.mark.e2e_multi_cluster_dr +def test_replica_set_is_reachable_after_deletetion(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity() + + +@pytest.mark.e2e_multi_cluster_dr +def test_add_test_data_after_deletion(mongodb_multi_collection, capsys): + max_attempts = 100 + while max_attempts > 0: + try: + mongodb_multi_collection.insert_one(TEST_DATA.copy()) + return + except Exception as e: + with capsys.disabled(): + print(e) + max_attempts -= 1 + time.sleep(6) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_enable_tls.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_enable_tls.py new file mode 100644 index 000000000..39c3da752 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_enable_tls.py @@ -0,0 +1,93 @@ +from typing import List + +import kubernetes +from kubetester import read_secret +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +BUNDLE_PEM_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert-pem" +USER_NAME = "my-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@fixture(scope="module") +def mongodb_multi_unmarshalled(namespace: str, member_cluster_names, custom_mdb_version: str) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + return resource + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + mongodb_multi_unmarshalled: MongoDBMulti, +) -> MongoDBMulti: + + resource = mongodb_multi_unmarshalled + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource.create() + + +@mark.e2e_multi_cluster_enable_tls +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_enable_tls +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti, namespace: str): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_multi_cluster_enable_tls +def test_enabled_tls_mongodb_multi( + mongodb_multi: MongoDBMulti, + namespace: str, + server_certs: str, + multi_cluster_issuer_ca_configmap: str, + member_cluster_clients: List[MultiClusterClient], +): + mongodb_multi.load() + mongodb_multi["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1300) + + # assert the presence of the generated pem certificates in each member cluster + for client in member_cluster_clients: + read_secret( + namespace=namespace, + name=BUNDLE_PEM_SECRET_NAME, + api_client=client.api_client, + ) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_ldap.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_ldap.py new file mode 100644 index 000000000..2b30104ea --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_ldap.py @@ -0,0 +1,345 @@ +import time +from typing import Dict, List + +import kubernetes +from kubetester import create_secret, wait_until +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_static_containers +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser, Role, generic_user +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import get_multi_cluster_operator_installation_config +from tests.multicluster.conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-replica-set-ldap" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +USER_NAME = "mms-user-1" +PASSWORD = "my-password" +LDAP_NAME = "openldap" + + +@fixture(scope="module") +def multi_cluster_operator_installation_config(namespace) -> Dict[str, str]: + config = get_multi_cluster_operator_installation_config(namespace=namespace) + config["customEnvVars"] = config["customEnvVars"] + "\&MDB_AUTOMATIC_RECOVERY_BACKOFF_TIME_S=360" + return config + + +@fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + member_cluster_names, + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + # This test has always been tested with 5.0.5-ent. After trying to unify its variant and upgrading it + # to MDB 6 we realized that our EVG hosts contain outdated docker and seccomp libraries in the host which + # cause MDB process to exit. It might be a good idea to try uncommenting it after migrating to newer EVG hosts. + # See https://github.com/docker-library/mongo/issues/606 for more information + # resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource.set_version(ensure_ent_version("5.0.5-ent")) + + # Setting the initial clusterSpecList to more members than we need to generate + # the certificates for all the members once the RS is scaled up. + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + return resource + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def mongodb_multi( + mongodb_multi_unmarshalled: MongoDBMulti, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names, + member_cluster_clients: List[MultiClusterClient], + namespace: str, + multi_cluster_issuer_ca_configmap: str, + server_certs: str, + multicluster_openldap_tls: OpenLDAP, + ldap_mongodb_agent_user: LDAPUser, + issuer_ca_configmap: str, +) -> MongoDBMulti: + resource = mongodb_multi_unmarshalled + + secret_name = "bind-query-password" + create_secret( + namespace, + secret_name, + {"password": multicluster_openldap_tls.admin_password}, + api_client=central_cluster_client, + ) + ac_secret_name = "automation-config-password" + create_secret( + namespace, + ac_secret_name, + {"automationConfigPassword": ldap_mongodb_agent_user.password}, + api_client=central_cluster_client, + ) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [1, 1, 1]) + + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "enabled": True, + "ca": multi_cluster_issuer_ca_configmap, + }, + "authentication": { + "enabled": True, + "modes": ["LDAP", "SCRAM"], # SCRAM for testing CLOUDP-229222 + "ldap": { + "servers": [multicluster_openldap_tls.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "transportSecurity": "none", # For testing CLOUDP-229222 + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + "timeoutMS": 12345, + "userCacheInvalidationInterval": 60, + }, + "agents": { + "mode": "SCRAM", # SCRAM for testing CLOUDP-189433 + "automationPasswordSecretRef": { + "name": ac_secret_name, + "key": "automationConfigPassword", + }, + "automationUserName": ldap_mongodb_agent_user.uid, + "automationLdapGroupDN": "cn=agents,ou=groups,dc=example,dc=org", + }, + }, + } + resource["spec"]["additionalMongodConfig"] = {"net": {"ssl": {"mode": "preferSSL"}}} + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + resource.update() + return resource + + +@fixture(scope="module") +def user_ldap( + mongodb_multi: MongoDBMulti, + namespace: str, + ldap_mongodb_user: LDAPUser, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBUser: + mongodb_user = ldap_mongodb_user + user = generic_user( + namespace, + username=mongodb_user.uid, + db="$external", + password=mongodb_user.password, + mongodb_resource=mongodb_multi, + ) + user.add_roles( + [ + Role(db="admin", role="clusterAdmin"), + Role(db="admin", role="readWriteAnyDatabase"), + Role(db="admin", role="dbAdminAnyDatabase"), + ] + ) + user.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + user.update() + return user + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_mongodb_multi_pending(mongodb_multi: MongoDBMulti): + """ + This function tests CLOUDP-229222. The resource needs to enter the "Pending" state and without the automatic + recovery, it would stay like this forever (since we wouldn't push the new AC with a fix). + """ + mongodb_multi.assert_reaches_phase(Phase.Pending, timeout=100) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_turn_tls_on_CLOUDP_229222(mongodb_multi: MongoDBMulti): + """ + This function tests CLOUDP-229222. The user attempts to fix the AutomationConfig. + Before updating the AutomationConfig, we need to ensure the operator pushed the wrong one to Ops Manager. + """ + + def wait_for_ac_exists() -> bool: + ac = mongodb_multi.get_automation_config_tester().automation_config + try: + _ = ac["ldap"]["transportSecurity"] + _ = ac["version"] + return True + except KeyError: + return False + + wait_until(wait_for_ac_exists, timeout=200) + current_version = mongodb_multi.get_automation_config_tester().automation_config["version"] + + def wait_for_ac_pushed() -> bool: + ac = mongodb_multi.get_automation_config_tester().automation_config + try: + transport_security = ac["ldap"]["transportSecurity"] + new_version = ac["version"] + if transport_security != "none": + return False + if new_version <= current_version: + return False + return True + except KeyError: + return False + + wait_until(wait_for_ac_pushed, timeout=500) + + resource = mongodb_multi.load() + + resource["spec"]["security"]["authentication"]["ldap"]["transportSecurity"] = "tls" + resource.update() + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_multi_replicaset_CLOUDP_229222(mongodb_multi: MongoDBMulti): + """ + This function tests CLOUDP-229222. The recovery mechanism kicks in and pushes Automation Config. The ReplicaSet + goes into running state. + """ + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1900) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_restore_mongodb_multi_ldap_configuration(mongodb_multi: MongoDBMulti): + """ + This function restores the initial desired security configuration to carry on with the next tests normally. + """ + resource = mongodb_multi.load() + + resource["spec"]["security"]["authentication"]["modes"] = ["LDAP"] + resource["spec"]["security"]["authentication"]["ldap"]["transportSecurity"] = "tls" + resource["spec"]["security"]["authentication"]["agents"]["mode"] = "LDAP" + + resource.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=800) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_create_ldap_user(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser): + user_ldap.assert_reaches_phase(Phase.Updated) + ac = AutomationConfigTester(KubernetesTester.get_automation_config()) + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=True) + ac.assert_expected_users(1) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_ldap_user_created_and_can_authenticate(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_ldap_authentication( + username=user_ldap["spec"]["username"], + password=user_ldap.password, + tls_ca_file=ca_path, + attempts=10, + ) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_ops_manager_state_correctly_updated(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser): + expected_roles = { + ("admin", "clusterAdmin"), + ("admin", "readWriteAnyDatabase"), + ("admin", "dbAdminAnyDatabase"), + } + ac = AutomationConfigTester(KubernetesTester.get_automation_config()) + ac.assert_expected_users(1) + ac.assert_has_user(user_ldap["spec"]["username"]) + ac.assert_user_has_roles(user_ldap["spec"]["username"], expected_roles) + ac.assert_authentication_mechanism_enabled("PLAIN", active_auth_mechanism=True) + ac.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=1) + + assert "userCacheInvalidationInterval" in ac.automation_config["ldap"] + assert "timeoutMS" in ac.automation_config["ldap"] + assert ac.automation_config["ldap"]["userCacheInvalidationInterval"] == 60 + assert ac.automation_config["ldap"]["timeoutMS"] == 12345 + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_deployment_is_reachable_with_ldap_agent(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_deployment_reachable() + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_scale_mongodb_multi(mongodb_multi: MongoDBMulti, member_cluster_names): + mongodb_multi.reload() + mongodb_multi["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=800) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_new_ldap_user_can_authenticate_after_scaling( + mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser, ca_path: str +): + tester = mongodb_multi.tester() + tester.assert_ldap_authentication( + username=user_ldap["spec"]["username"], + password=user_ldap.password, + tls_ca_file=ca_path, + attempts=10, + ) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_disable_agent_auth(mongodb_multi: MongoDBMulti): + mongodb_multi.reload() + mongodb_multi["spec"]["security"]["authentication"]["enabled"] = False + mongodb_multi["spec"]["security"]["authentication"]["agents"]["enabled"] = False + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_mongodb_multi_connectivity_with_no_auth(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity() + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap +def test_deployment_is_reachable_with_no_auth(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_deployment_reachable() diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_ldap_custom_roles.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_ldap_custom_roles.py new file mode 100644 index 000000000..8e0184cc2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_ldap_custom_roles.py @@ -0,0 +1,233 @@ +from typing import List + +import kubernetes +from kubetester import create_secret +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_static_containers +from kubetester.ldap import LDAP_AUTHENTICATION_MECHANISM, LDAPUser, OpenLDAP +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser, generic_user +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-replica-set-ldap" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +USER_NAME = "mms-user-1" +PASSWORD = "my-password" +LDAP_NAME = "openldap" + + +@fixture(scope="module") +def mongodb_multi_unmarshalled(namespace: str, member_cluster_names, custom_mdb_version: str) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + + # This test has always been tested with 5.0.5-ent. After trying to unify its variant and upgrading it + # to MDB 6 we realized that our EVG hosts contain outdated docker and seccomp libraries in the host which + # cause MDB process to exit. It might be a good idea to try uncommenting it after migrating to newer EVG hosts. + # See https://github.com/docker-library/mongo/issues/606 for more information + # resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_version("5.0.5-ent") + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + return resource + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def mongodb_multi( + mongodb_multi_unmarshalled: MongoDBMulti, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + namespace: str, + multi_cluster_issuer_ca_configmap: str, + server_certs: str, + multicluster_openldap_tls: OpenLDAP, + ldap_mongodb_agent_user: LDAPUser, + issuer_ca_configmap: str, +) -> MongoDBMulti: + resource = mongodb_multi_unmarshalled + secret_name = "bind-query-password" + create_secret( + namespace, + secret_name, + {"password": multicluster_openldap_tls.admin_password}, + api_client=central_cluster_client, + ) + ac_secret_name = "automation-config-password" + create_secret( + namespace, + ac_secret_name, + {"automationConfigPassword": ldap_mongodb_agent_user.password}, + api_client=central_cluster_client, + ) + + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "enabled": True, + "ca": multi_cluster_issuer_ca_configmap, + }, + "authentication": { + "enabled": True, + "modes": ["LDAP", "SCRAM"], + "ldap": { + "servers": [multicluster_openldap_tls.servers], + "bindQueryUser": "cn=admin,dc=example,dc=org", + "bindQueryPasswordSecretRef": {"name": secret_name}, + "transportSecurity": "tls", + "validateLDAPServerConfig": True, + "caConfigMapRef": {"name": issuer_ca_configmap, "key": "ca-pem"}, + "userToDNMapping": '[{match: "(.+)",substitution: "uid={0},ou=groups,dc=example,dc=org"}]', + "authzQueryTemplate": "{USER}?memberOf?base", + }, + "agents": { + "mode": "SCRAM", + }, + }, + "roles": [ + { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "foo"}, + }, + { + "actions": ["insert", "find"], + "resource": {"collection": "", "db": "admin"}, + }, + ], + }, + ], + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + resource.update() + return resource + + +@fixture(scope="module") +def user_ldap( + mongodb_multi: MongoDBMulti, + namespace: str, + ldap_mongodb_user: LDAPUser, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBUser: + mongodb_user = ldap_mongodb_user + user = generic_user( + namespace, + username=mongodb_user.uid, + db="$external", + password=mongodb_user.password, + mongodb_resource=mongodb_multi, + ) + user.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + user.update() + return user + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +def test_create_mongodb_multi_with_ldap(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +def test_create_ldap_user(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser): + user_ldap.assert_reaches_phase(Phase.Updated) + ac = AutomationConfigTester(KubernetesTester.get_automation_config()) + ac.assert_authentication_mechanism_enabled(LDAP_AUTHENTICATION_MECHANISM, active_auth_mechanism=False) + ac.assert_expected_users(1) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +def test_ldap_user_can_write_to_database(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_ldap_authentication( + username=user_ldap["spec"]["username"], + password=user_ldap.password, + tls_ca_file=ca_path, + db="foo", + collection="foo", + attempts=10, + ) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +@mark.xfail(reason="The user should not be able to write to a database/collection it is not authorized to write on") +def test_ldap_user_can_write_to_other_collection(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_ldap_authentication( + username=user_ldap["spec"]["username"], + password=user_ldap.password, + tls_ca_file=ca_path, + db="foo", + collection="foo2", + attempts=10, + ) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +@mark.xfail(reason="The user should not be able to write to a database/collection it is not authorized to write on") +def test_ldap_user_can_write_to_other_database(mongodb_multi: MongoDBMulti, user_ldap: MongoDBUser, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_ldap_authentication( + username=user_ldap["spec"]["username"], + password=user_ldap.password, + tls_ca_file=ca_path, + db="foo2", + collection="foo", + attempts=10, + ) + + +@skip_if_static_containers +@mark.e2e_multi_cluster_with_ldap_custom_roles +def test_automation_config_has_roles(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.get_automation_config_tester() + role = { + "role": "cn=users,ou=groups,dc=example,dc=org", + "db": "admin", + "privileges": [ + {"actions": ["insert"], "resource": {"collection": "foo", "db": "foo"}}, + { + "actions": ["insert", "find"], + "resource": {"collection": "", "db": "admin"}, + }, + ], + "authenticationRestrictions": [], + } + tester.assert_expected_role(role_index=0, expected_value=role) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_pvc_resize.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_pvc_resize.py new file mode 100644 index 000000000..7db917ecd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_pvc_resize.py @@ -0,0 +1,69 @@ +from typing import List + +import kubernetes +import pytest +from kubernetes import client +from kubetester import get_statefulset, try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set-pvc-resize" +RESIZED_STORAGE_SIZE = "2Gi" + + +@pytest.fixture(scope="module") +def mongodb_multi( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: list[str], +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi-pvc-resize.yaml"), RESOURCE_NAME, namespace) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + try_load(resource) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + return resource + + +@pytest.mark.e2e_multi_cluster_pvc_resize +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_pvc_resize +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=2000) + + +@pytest.mark.e2e_multi_cluster_pvc_resize +def test_mongodb_multi_resize_pvc_state_changes(mongodb_multi: MongoDBMulti): + # Update the resource + mongodb_multi.load() + mongodb_multi["spec"]["statefulSet"]["spec"]["volumeClaimTemplates"][0]["spec"]["resources"]["requests"][ + "storage" + ] = RESIZED_STORAGE_SIZE + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Pending, timeout=400) + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_multi_cluster_pvc_resize +def test_mongodb_multi_resize_finished( + mongodb_multi: MongoDBMulti, namespace: str, member_cluster_clients: List[MultiClusterClient] +): + statefulsets = [] + for i, c in enumerate(member_cluster_clients): + statefulsets.append((get_statefulset(namespace, f"{RESOURCE_NAME}-{i}", c.api_client), c.api_client)) + + for sts, c in statefulsets: + assert sts.spec.volume_claim_templates[0].spec.resources.requests["storage"] == RESIZED_STORAGE_SIZE + first_pvc_name = f"data-{sts.metadata.name}-0" + pvc = client.CoreV1Api(api_client=c).read_namespaced_persistent_volume_claim(first_pvc_name, namespace) + assert pvc.status.capacity["storage"] == RESIZED_STORAGE_SIZE diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_reconcile_races.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_reconcile_races.py new file mode 100644 index 000000000..138e766c6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_reconcile_races.py @@ -0,0 +1,270 @@ +# It's intended to check for reconcile data races. +import json +import time +from typing import Optional + +import kubernetes.client +import pytest +from kubetester import create_or_update_secret, find_fixture, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.mongodb_user import MongoDBUser +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from tests.conftest import ( + get_central_cluster_client, + get_custom_mdb_version, + get_member_cluster_names, +) +from tests.multicluster.conftest import cluster_spec_list + + +@pytest.fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(find_fixture("om_validation.yaml"), namespace=namespace, name="om") + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + try_load(resource) + return resource + + +@pytest.fixture(scope="module") +def ops_manager2( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(find_fixture("om_validation.yaml"), namespace=namespace, name="om2") + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + try_load(resource) + return resource + + +def get_replica_set(ops_manager, namespace: str, idx: int) -> MongoDB: + name = f"mdb-{idx}-rs" + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=name, + ).configure(ops_manager, name, api_client=get_central_cluster_client()) + resource.set_version(get_custom_mdb_version()) + + try_load(resource) + return resource + + +def get_mdbmc(ops_manager, namespace: str, idx: int) -> MongoDBMulti: + name = f"mdb-{idx}-mc" + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi-cluster.yaml"), + namespace=namespace, + name=name, + ).configure(ops_manager, name, api_client=get_central_cluster_client()) + + try_load(resource) + return resource + + +def get_sharded(ops_manager, namespace: str, idx: int) -> MongoDB: + name = f"mdb-{idx}-sh" + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-single.yaml"), + namespace=namespace, + name=name, + ).configure(ops_manager, name, api_client=get_central_cluster_client()) + try_load(resource) + return resource + + +def get_standalone(ops_manager, namespace: str, idx: int) -> MongoDB: + name = f"mdb-{idx}-st" + resource = MongoDB.from_yaml( + yaml_fixture("standalone.yaml"), + namespace=namespace, + name=name, + ).configure(ops_manager, name, api_client=get_central_cluster_client()) + try_load(resource) + return resource + + +def get_user(ops_manager, namespace: str, idx: int, mdb: MongoDB) -> MongoDBUser: + name = f"{mdb.name}-user-{idx}" + resource = MongoDBUser.from_yaml( + yaml_fixture("mongodb-user.yaml"), + namespace=namespace, + name=name, + ) + try_load(resource) + return resource + + +def get_all_sharded(ops_manager, namespace) -> list[MongoDB]: + return [get_sharded(ops_manager, namespace, idx) for idx in range(0, 4)] + + +def get_all_rs(ops_manager, namespace) -> list[MongoDB]: + return [get_replica_set(ops_manager, namespace, idx) for idx in range(0, 5)] + + +def get_all_mdbmc(ops_manager, namespace) -> list[MongoDB]: + return [get_mdbmc(ops_manager, namespace, idx) for idx in range(0, 4)] + + +def get_all_standalone(ops_manager, namespace) -> list[MongoDB]: + return [get_standalone(ops_manager, namespace, idx) for idx in range(0, 5)] + + +def get_all_users(ops_manager, namespace, mdb: MongoDB) -> list[MongoDBUser]: + return [get_user(ops_manager, namespace, idx, mdb) for idx in range(0, 2)] + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_create_om(ops_manager: MongoDBOpsManager, ops_manager2: MongoDBOpsManager): + ops_manager.update() + ops_manager2.update() + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_om_ready(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=1800) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1800) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_om2_ready(ops_manager2: MongoDBOpsManager): + ops_manager2.appdb_status().assert_reaches_phase(Phase.Running, timeout=1800) + ops_manager2.om_status().assert_reaches_phase(Phase.Running, timeout=1800) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_create_mdb(ops_manager: MongoDBOpsManager, namespace: str): + for resource in get_all_rs(ops_manager, namespace): + resource["spec"]["security"] = { + "authentication": {"agents": {"mode": "SCRAM"}, "enabled": True, "modes": ["SCRAM"]} + } + resource.set_version(get_custom_mdb_version()) + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_create_mdbmc(ops_manager: MongoDBOpsManager, namespace: str): + for resource in get_all_mdbmc(ops_manager, namespace): + resource.set_version(get_custom_mdb_version()) + resource["spec"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_create_sharded(ops_manager: MongoDBOpsManager, namespace: str): + for resource in get_all_sharded(ops_manager, namespace): + resource.set_version(get_custom_mdb_version()) + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_create_standalone(ops_manager: MongoDBOpsManager, namespace: str): + for resource in get_all_standalone(ops_manager, namespace): + resource.set_version(get_custom_mdb_version()) + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_create_users(ops_manager: MongoDBOpsManager, namespace: str): + create_or_update_secret( + namespace, + "mdb-user-password", + {"password": "password"}, + ) + for mdb in get_all_rs(ops_manager, namespace): + for resource in get_all_users(ops_manager, namespace, mdb): + resource["spec"]["mongodbResourceRef"] = {"name": mdb.name} + resource["spec"]["passwordSecretKeyRef"] = {"name": "mdb-user-password", "key": "password"} + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_pod_logs_race(multi_cluster_operator: Operator): + pods = multi_cluster_operator.list_operator_pods() + pod_name = pods[0].metadata.name + container_name = "mongodb-enterprise-operator-multi-cluster" + pod_logs_str = KubernetesTester.read_pod_logs( + multi_cluster_operator.namespace, pod_name, container_name, api_client=multi_cluster_operator.api_client + ) + contains_race = "WARNING: DATA RACE" in pod_logs_str + assert not contains_race + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_restart_operator_pod(ops_manager: MongoDBOpsManager, namespace: str, multi_cluster_operator: Operator): + # this enforces a requeue of all existing resources, increasing the chances of races to happen + multi_cluster_operator.restart_operator_deployment() + multi_cluster_operator.assert_is_running() + time.sleep(5) + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_pod_logs_race_after_restart(multi_cluster_operator: Operator): + pods = multi_cluster_operator.list_operator_pods() + pod_name = pods[0].metadata.name + container_name = "mongodb-enterprise-operator-multi-cluster" + pod_logs_str = KubernetesTester.read_pod_logs( + multi_cluster_operator.namespace, pod_name, container_name, api_client=multi_cluster_operator.api_client + ) + contains_race = "WARNING: DATA RACE" in pod_logs_str + assert not contains_race + + +@pytest.mark.e2e_om_reconcile_race_with_telemetry +def test_telemetry_configmap(namespace: str): + telemetry_configmap_name = "mongodb-enterprise-operator-telemetry" + config = KubernetesTester.read_configmap(namespace, telemetry_configmap_name) + for ts_key in ["lastSendTimestampClusters", "lastSendTimestampDeployments", "lastSendTimestampOperators"]: + ts_cm = config.get(ts_key) + assert ts_cm.isdigit() # it should be a timestamp + + for ps_key in ["lastSendPayloadClusters", "lastSendPayloadDeployments", "lastSendPayloadOperators"]: + try: + payload_string = config.get(ps_key) + payload = json.loads(payload_string) + # Perform a rudimentary check + assert isinstance(payload, list), "payload should be a list" + assert len(payload) > 0, "payload should not be empty" + except json.JSONDecodeError: + pytest.fail("payload contains invalid JSON data") diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_recover_clusterwide.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_recover_clusterwide.py new file mode 100644 index 000000000..a4e40477a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_recover_clusterwide.py @@ -0,0 +1,359 @@ +import os +from typing import Dict, List + +import kubernetes +from kubeobject import CustomObject +from kubernetes import client +from kubetester import ( + create_or_update_configmap, + create_or_update_secret, + delete_cluster_role, + delete_cluster_role_binding, + delete_statefulset, + read_secret, + statefulset_is_deleted, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + MULTI_CLUSTER_OPERATOR_NAME, + _install_multi_cluster_operator, + run_kube_config_creation_tool, + run_multi_cluster_recovery_tool, +) + +from . import prepare_multi_cluster_namespaces +from .conftest import cluster_spec_list, create_service_entries_objects +from .multi_cluster_clusterwide import create_namespace + +FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3" + + +@fixture(scope="module") +def mdba_ns(namespace: str): + return "{}-mdb-ns-a".format(namespace) + + +@fixture(scope="module") +def mdbb_ns(namespace: str): + return "{}-mdb-ns-b".format(namespace) + + +@fixture(scope="module") +def mongodb_multi_a( + central_cluster_client: kubernetes.client.ApiClient, + mdba_ns: str, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", mdba_ns) + resource.set_version(custom_mdb_version) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@fixture(scope="module") +def mongodb_multi_b( + central_cluster_client: kubernetes.client.ApiClient, + mdbb_ns: str, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", mdbb_ns) + resource.set_version(custom_mdb_version) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@fixture(scope="module") +def install_operator( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: Dict[str, str], + central_cluster_client: client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], + mdba_ns: str, + mdbb_ns: str, +) -> Operator: + os.environ["HELM_KUBECONTEXT"] = central_cluster_name + member_cluster_namespaces = mdba_ns + "," + mdbb_ns + run_kube_config_creation_tool( + member_cluster_names, + namespace, + namespace, + member_cluster_names, + True, + service_account_name=MULTI_CLUSTER_OPERATOR_NAME, + ) + + return _install_multi_cluster_operator( + namespace, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + { + "operator.deployment_name": MULTI_CLUSTER_OPERATOR_NAME, + "operator.name": MULTI_CLUSTER_OPERATOR_NAME, + "operator.createOperatorServiceAccount": "false", + "operator.watchNamespace": member_cluster_namespaces, + "multiCluster.performFailOver": "false", + }, + central_cluster_name, + operator_name=MULTI_CLUSTER_OPERATOR_NAME, + ) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_label_operator_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient): + api = client.CoreV1Api(api_client=central_cluster_client) + + labels = {"istio-injection": "enabled"} + ns = api.read_namespace(name=namespace) + + ns.metadata.labels.update(labels) + api.replace_namespace(name=namespace, body=ns) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_create_namespaces( + namespace: str, + mdba_ns: str, + mdbb_ns: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], + evergreen_task_id: str, + multi_cluster_operator_installation_config: Dict[str, str], +): + image_pull_secret_name = multi_cluster_operator_installation_config["registry.imagePullSecrets"] + image_pull_secret_data = read_secret(namespace, image_pull_secret_name) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + mdba_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + mdbb_ns, + image_pull_secret_name, + image_pull_secret_data, + ) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_create_service_entry(service_entries: List[CustomObject]): + for service_entry in service_entries: + service_entry.update() + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_delete_cluster_role_and_binding( + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: List[MultiClusterClient], +): + role_names = [ + "mongodb-enterprise-operator-multi-cluster-role", + "mongodb-enterprise-operator-multi-cluster", + "mongodb-enterprise-operator-multi-cluster-role-binding", + ] + + for name in role_names: + delete_cluster_role(name, central_cluster_client) + delete_cluster_role_binding(name, central_cluster_client) + + for name in role_names: + for client in member_cluster_clients: + delete_cluster_role(name, client.api_client) + delete_cluster_role_binding(name, client.api_client) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_deploy_operator(install_operator: Operator): + install_operator.assert_is_running() + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_prepare_namespace( + multi_cluster_operator_installation_config: Dict[str, str], + member_cluster_clients: List[MultiClusterClient], + central_cluster_name: str, + mdba_ns: str, + mdbb_ns: str, +): + prepare_multi_cluster_namespaces( + mdba_ns, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + ) + + prepare_multi_cluster_namespaces( + mdbb_ns, + multi_cluster_operator_installation_config, + member_cluster_clients, + central_cluster_name, + ) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_copy_configmap_and_secret_across_ns( + namespace: str, + central_cluster_client: client.ApiClient, + multi_cluster_operator_installation_config: Dict[str, str], + mdba_ns: str, + mdbb_ns: str, +): + data = KubernetesTester.read_configmap(namespace, "my-project", api_client=central_cluster_client) + data["projectName"] = mdba_ns + create_or_update_configmap(mdba_ns, "my-project", data, api_client=central_cluster_client) + + data["projectName"] = mdbb_ns + create_or_update_configmap(mdbb_ns, "my-project", data, api_client=central_cluster_client) + + data = read_secret(namespace, "my-credentials", api_client=central_cluster_client) + create_or_update_secret(mdba_ns, "my-credentials", data, api_client=central_cluster_client) + create_or_update_secret(mdbb_ns, "my-credentials", data, api_client=central_cluster_client) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_create_mongodb_multi_nsa_nsb(mongodb_multi_a: MongoDBMulti, mongodb_multi_b: MongoDBMulti): + mongodb_multi_a.assert_reaches_phase(Phase.Running, timeout=1500) + mongodb_multi_b.assert_reaches_phase(Phase.Running, timeout=1500) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_update_service_entry_block_failed_cluster_traffic( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], +): + # TODO: add a way to simulate local operator connection cut-off + healthy_cluster_names = [ + cluster_name for cluster_name in member_cluster_names if cluster_name != FAILED_MEMBER_CLUSTER_NAME + ] + + service_entries = create_service_entries_objects(namespace, central_cluster_client, healthy_cluster_names) + for service_entry in service_entries: + print(f"service_entry={service_entries}") + service_entry.update() + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_delete_database_statefulsets_in_failed_cluster( + mongodb_multi_a: MongoDBMulti, + mongodb_multi_b: MongoDBMulti, + mdba_ns: str, + mdbb_ns: str, + member_cluster_names: list[str], + member_cluster_clients: List[MultiClusterClient], +): + failed_cluster_idx = member_cluster_names.index(FAILED_MEMBER_CLUSTER_NAME) + sts_a_name = f"{mongodb_multi_a.name}-{failed_cluster_idx}" + sts_b_name = f"{mongodb_multi_b.name}-{failed_cluster_idx}" + + try: + delete_statefulset( + mdba_ns, + sts_a_name, + propagation_policy="Background", + api_client=member_cluster_clients[2].api_client, + ) + delete_statefulset( + mdbb_ns, + sts_b_name, + propagation_policy="Background", + api_client=member_cluster_clients[2].api_client, + ) + + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + run_periodically( + lambda: statefulset_is_deleted( + mdba_ns, + sts_a_name, + api_client=member_cluster_clients[failed_cluster_idx].api_client, + ), + timeout=120, + ) + run_periodically( + lambda: statefulset_is_deleted( + mdbb_ns, + sts_b_name, + api_client=member_cluster_clients[failed_cluster_idx].api_client, + ), + timeout=120, + ) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_mongodb_multi_nsa_enters_failed_stated(mongodb_multi_a: MongoDBMulti): + mongodb_multi_a.load() + mongodb_multi_a.assert_reaches_phase(Phase.Failed, timeout=100) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_mongodb_multi_nsb_enters_failed_stated(mongodb_multi_b: MongoDBMulti): + mongodb_multi_b.load() + mongodb_multi_b.assert_reaches_phase(Phase.Failed, timeout=100) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_recover_operator_remove_cluster( + member_cluster_names: List[str], + namespace: str, + mdba_ns: str, + mdbb_ns: str, + central_cluster_client: kubernetes.client.ApiClient, +): + return_code = run_multi_cluster_recovery_tool(member_cluster_names[:-1], namespace, namespace, True) + assert return_code == 0 + operator = Operator( + name=MULTI_CLUSTER_OPERATOR_NAME, + namespace=namespace, + api_client=central_cluster_client, + ) + operator._wait_for_operator_ready() + operator.assert_is_running() + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_mongodb_multi_nsa_recovers_removing_cluster(mongodb_multi_a: MongoDBMulti): + mongodb_multi_a.load() + + mongodb_multi_a["metadata"]["annotations"]["failedClusters"] = None + mongodb_multi_a["spec"]["clusterSpecList"].pop() + mongodb_multi_a.update() + + mongodb_multi_a.assert_reaches_phase(Phase.Running, timeout=1500) + + +@mark.e2e_multi_cluster_recover_clusterwide +def test_mongodb_multi_nsb_recovers_removing_cluster(mongodb_multi_b: MongoDBMulti): + mongodb_multi_b.load() + + mongodb_multi_b["metadata"]["annotations"]["failedClusters"] = None + mongodb_multi_b["spec"]["clusterSpecList"].pop() + mongodb_multi_b.update() + + mongodb_multi_b.assert_reaches_phase(Phase.Running, timeout=1500) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_recover_network_partition.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_recover_network_partition.py new file mode 100644 index 000000000..630a48acd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_recover_network_partition.py @@ -0,0 +1,155 @@ +from typing import List + +import kubernetes +from kubeobject import CustomObject +from kubernetes import client +from kubetester import delete_statefulset, statefulset_is_deleted +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + MULTI_CLUSTER_OPERATOR_NAME, + get_member_cluster_api_client, + run_multi_cluster_recovery_tool, +) + +from .conftest import cluster_spec_list, create_service_entries_objects + +FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3" +RESOURCE_NAME = "multi-replica-set" + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["persistent"] = False + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource.api = client.CustomObjectsApi(central_cluster_client) + + return resource + + +@mark.e2e_multi_cluster_recover_network_partition +def test_label_namespace(namespace: str, central_cluster_client: client.ApiClient): + + api = client.CoreV1Api(api_client=central_cluster_client) + + labels = {"istio-injection": "enabled"} + ns = api.read_namespace(name=namespace) + + ns.metadata.labels.update(labels) + api.replace_namespace(name=namespace, body=ns) + + +@mark.e2e_multi_cluster_recover_network_partition +def test_create_service_entry(service_entries: List[CustomObject]): + for service_entry in service_entries: + service_entry.update() + + +@mark.e2e_multi_cluster_recover_network_partition +def test_deploy_operator(multi_cluster_operator_manual_remediation: Operator): + multi_cluster_operator_manual_remediation.assert_is_running() + + +@mark.e2e_multi_cluster_recover_network_partition +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@mark.e2e_multi_cluster_recover_network_partition +def test_update_service_entry_block_failed_cluster_traffic( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], +): + healthy_cluster_names = [ + cluster_name for cluster_name in member_cluster_names if cluster_name != FAILED_MEMBER_CLUSTER_NAME + ] + service_entries = create_service_entries_objects( + namespace, + central_cluster_client, + healthy_cluster_names, + ) + for service_entry in service_entries: + print(f"service_entry={service_entries}") + service_entry.update() + + +@mark.e2e_multi_cluster_recover_network_partition +def test_delete_database_statefulset_in_failed_cluster( + mongodb_multi: MongoDBMulti, + member_cluster_names: list[str], +): + failed_cluster_idx = member_cluster_names.index(FAILED_MEMBER_CLUSTER_NAME) + sts_name = f"{mongodb_multi.name}-{failed_cluster_idx}" + try: + delete_statefulset( + mongodb_multi.namespace, + sts_name, + propagation_policy="Background", + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + run_periodically( + lambda: statefulset_is_deleted( + mongodb_multi.namespace, + sts_name, + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + + +@mark.e2e_multi_cluster_recover_network_partition +def test_mongodb_multi_enters_failed_state( + mongodb_multi: MongoDBMulti, + namespace: str, + central_cluster_client: client.ApiClient, +): + mongodb_multi.load() + mongodb_multi.assert_reaches_phase(Phase.Failed, timeout=100) + + +@mark.e2e_multi_cluster_recover_network_partition +def test_recover_operator_remove_cluster( + member_cluster_names: List[str], + namespace: str, + central_cluster_client: client.ApiClient, +): + return_code = run_multi_cluster_recovery_tool(member_cluster_names[:-1], namespace, namespace) + assert return_code == 0 + operator = Operator( + name=MULTI_CLUSTER_OPERATOR_NAME, + namespace=namespace, + api_client=central_cluster_client, + ) + operator._wait_for_operator_ready() + operator.assert_is_running() + + +@mark.e2e_multi_cluster_recover_network_partition +def test_mongodb_multi_recovers_removing_cluster(mongodb_multi: MongoDBMulti, member_cluster_names: List[str]): + mongodb_multi.load() + + last_transition_time = mongodb_multi.get_status_last_transition_time() + + mongodb_multi["metadata"]["annotations"]["failedClusters"] = None + mongodb_multi["spec"]["clusterSpecList"].pop() + mongodb_multi.update() + mongodb_multi.assert_state_transition_happens(last_transition_time) + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1500) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set.py new file mode 100644 index 000000000..b9c5c8834 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set.py @@ -0,0 +1,240 @@ +from typing import Dict, List + +import kubernetes +import pytest +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import delete_statefulset, get_statefulset, wait_until +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from tests.conftest import ( + assert_log_rotation_process, + member_cluster_clients, + setup_log_rotate_for_agents, +) +from tests.multicluster.conftest import cluster_spec_list + +MONGODB_PORT = 30000 + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names, + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi-central-sts-override.yaml"), + "multi-replica-set", + namespace, + ) + resource.set_version(custom_mdb_version) + resource["spec"]["persistent"] = False + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + additional_mongod_config = { + "systemLog": {"logAppend": True, "verbosity": 4}, + "operationProfiling": {"mode": "slowOp"}, + "net": {"port": MONGODB_PORT}, + } + + resource["spec"]["additionalMongodConfig"] = additional_mongod_config + setup_log_rotate_for_agents(resource) + + # TODO: incorporate this into the base class. + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + resource.set_architecture_annotation() + + resource.update() + return resource + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_create_kube_config_file(cluster_clients: Dict, central_cluster_name: str, member_cluster_names: str): + clients = cluster_clients + + assert len(clients) == 4 + for member_cluster_name in member_cluster_names: + assert member_cluster_name in clients + assert central_cluster_name in clients + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=2000) + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_statefulset_is_created_across_multiple_clusters( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 2 + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_pvc_not_created( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + namespace: str, +): + with pytest.raises(kubernetes.client.exceptions.ApiException) as e: + client.CoreV1Api(api_client=member_cluster_clients[0].api_client).read_namespaced_persistent_volume_claim( + f"data-{mongodb_multi.name}-{0}-{0}", namespace + ) + assert e.value.reason == "Not Found" + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_replica_set +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester(port=MONGODB_PORT) + tester.assert_connectivity() + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_statefulset_overrides(mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient]): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + # assert sts.podspec override in cluster1 + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert_container_in_sts("sidecar1", cluster_one_sts) + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_headless_service_creation( + mongodb_multi: MongoDBMulti, + namespace: str, + member_cluster_clients: List[MultiClusterClient], +): + headless_services = mongodb_multi.read_headless_services(member_cluster_clients) + + cluster_one_client = member_cluster_clients[0] + cluster_one_svc = headless_services[cluster_one_client.cluster_name] + ep_one = client.CoreV1Api(api_client=cluster_one_client.api_client).read_namespaced_endpoints( + cluster_one_svc.metadata.name, namespace + ) + assert len(ep_one.subsets[0].addresses) == mongodb_multi.get_item_spec(cluster_one_client.cluster_name)["members"] + + cluster_two_client = member_cluster_clients[1] + cluster_two_svc = headless_services[cluster_two_client.cluster_name] + ep_two = client.CoreV1Api(api_client=cluster_two_client.api_client).read_namespaced_endpoints( + cluster_two_svc.metadata.name, namespace + ) + assert len(ep_two.subsets[0].addresses) == mongodb_multi.get_item_spec(cluster_two_client.cluster_name)["members"] + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_mongodb_options(mongodb_multi: MongoDBMulti): + automation_config_tester = mongodb_multi.get_automation_config_tester() + for process in automation_config_tester.get_replica_set_processes(mongodb_multi.name): + assert process["args2_6"]["systemLog"]["verbosity"] == 4 + assert process["args2_6"]["systemLog"]["logAppend"] + assert process["args2_6"]["operationProfiling"]["mode"] == "slowOp" + assert process["args2_6"]["net"]["port"] == MONGODB_PORT + assert_log_rotation_process(process) + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_update_additional_options(mongodb_multi: MongoDBMulti, central_cluster_client: kubernetes.client.ApiClient): + mongodb_multi["spec"]["additionalMongodConfig"]["systemLog"]["verbosity"] = 2 + mongodb_multi["spec"]["additionalMongodConfig"]["net"]["maxIncomingConnections"] = 100 + # update uses json merge+patch which means that deleting keys is done by setting them to None + mongodb_multi["spec"]["additionalMongodConfig"]["operationProfiling"] = None + + mongodb_multi.update() + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_mongodb_options_were_updated(mongodb_multi: MongoDBMulti): + automation_config_tester = mongodb_multi.get_automation_config_tester() + for process in automation_config_tester.get_replica_set_processes(mongodb_multi.name): + assert process["args2_6"]["systemLog"]["verbosity"] == 2 + assert process["args2_6"]["systemLog"]["logAppend"] + assert process["args2_6"]["net"]["maxIncomingConnections"] == 100 + assert process["args2_6"]["net"]["port"] == MONGODB_PORT + # the mode setting has been removed + assert "mode" not in process["args2_6"]["operationProfiling"] + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_delete_member_cluster_sts( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + sts_name = "{}-0".format(mongodb_multi.name) + api_client = member_cluster_clients[0].api_client + sts_old = get_statefulset(namespace, sts_name, api_client) + delete_statefulset( + namespace=namespace, + name=sts_name, + api_client=api_client, + ) + + def check_if_sts_was_recreated() -> bool: + try: + sts = get_statefulset(namespace, sts_name, api_client) + return sts.metadata.uid != sts_old.metadata.uid + except ApiException as e: + if e.status == 404: + return False + else: + raise e + + wait_until(check_if_sts_was_recreated, timeout=120) + + # the operator should reconcile and recreate the statefulset + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_multi_cluster_replica_set +def test_cleanup_on_mdbm_delete(mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient]): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + + mongodb_multi.delete() + + def check_sts_not_exist(): + try: + client.AppsV1Api(api_client=cluster_one_client.api_client).read_namespaced_stateful_set( + cluster_one_sts.metadata.name, cluster_one_sts.metadata.namespace + ) + except ApiException as e: + if e.reason == "Not Found": + return True + return False + else: + return False + + KubernetesTester.wait_until(check_sts_not_exist, timeout=200) + + +def assert_container_in_sts(container_name: str, sts: client.V1StatefulSet): + container_names = [c.name for c in sts.spec.template.spec.containers] + assert container_name in container_names diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_deletion.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_deletion.py new file mode 100644 index 000000000..716c90755 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_deletion.py @@ -0,0 +1,114 @@ +from typing import List + +import kubernetes +import pytest +from kubetester import try_load, wait_until +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from tests import test_logger +from tests.multicluster.conftest import cluster_spec_list + +logger = test_logger.get_test_logger(__name__) + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", namespace) + + if try_load(resource): + return resource + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + return resource.update() + + +@pytest.mark.e2e_multi_cluster_replica_set_deletion +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_replica_set_deletion +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@pytest.mark.e2e_multi_cluster_replica_set_deletion +def test_automation_config_has_been_updated(mongodb_multi: MongoDBMulti): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + processes = tester.get_replica_set_processes(mongodb_multi.name) + assert len(processes) == 5 + + +@pytest.mark.e2e_multi_cluster_replica_set_deletion +def test_delete_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.delete() + + def wait_for_deleted() -> bool: + try: + mongodb_multi.load() + return False + except kubernetes.client.ApiException as e: + if e.status == 404: + return True + else: + logger.error(e) + return False + + wait_until(wait_for_deleted, timeout=60) + + +@pytest.mark.e2e_multi_cluster_replica_set_deletion +def test_deployment_has_been_removed_from_automation_config(): + def wait_until_automation_config_is_clean() -> bool: + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + try: + tester.assert_empty() + return True + except AssertionError as e: + logger.error(e) + return False + + wait_until(wait_until_automation_config_is_clean, timeout=60) + + +@pytest.mark.e2e_multi_cluster_replica_set_deletion +def test_kubernetes_resources_have_been_cleaned_up( + mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient] +): + def wait_until_secrets_are_removed() -> bool: + try: + mongodb_multi.read_services(member_cluster_clients) + return False + except kubernetes.client.ApiException as e: + logger.error(e) + return True + + def wait_until_statefulsets_are_removed() -> bool: + try: + mongodb_multi.read_statefulsets(member_cluster_clients) + return False + except kubernetes.client.ApiException as e: + logger.error(e) + return True + + def wait_until_configmaps_are_removed() -> bool: + try: + mongodb_multi.read_configmaps(member_cluster_clients) + return False + except kubernetes.client.ApiException as e: + logger.error(e) + return True + + wait_until(wait_until_secrets_are_removed, timeout=60) + wait_until(wait_until_statefulsets_are_removed, timeout=60) + wait_until(wait_until_configmaps_are_removed, timeout=60) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_ignore_unknown_users.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_ignore_unknown_users.py new file mode 100644 index 000000000..c512c2146 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_ignore_unknown_users.py @@ -0,0 +1,59 @@ +import kubernetes +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi.yaml"), + "multi-replica-set", + namespace, + ) + resource.set_version(custom_mdb_version) + + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + resource["spec"]["security"]["authentication"]["ignoreUnknownUsers"] = True + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return resource.update() + + +@mark.e2e_multi_cluster_replica_set_ignore_unknown_users +def test_replica_set(multi_cluster_operator: Operator, mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_multi_cluster_replica_set_ignore_unknown_users +def test_authoritative_set_false(mongodb_multi: MongoDBMulti): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authoritative_set(False) + + +@mark.e2e_multi_cluster_replica_set_ignore_unknown_users +def test_set_ignore_unknown_users_false(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + mongodb_multi["spec"]["security"]["authentication"]["ignoreUnknownUsers"] = False + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_multi_cluster_replica_set_ignore_unknown_users +def test_authoritative_set_true(mongodb_multi: MongoDBMulti): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authoritative_set(True) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_member_options.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_member_options.py new file mode 100644 index 000000000..3fd5eda88 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_member_options.py @@ -0,0 +1,212 @@ +from typing import Dict + +import kubernetes +import pytest +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names, + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi.yaml"), + "multi-replica-set", + namespace, + ) + resource.set_version(custom_mdb_version) + member_options = [ + [ + { + "votes": 1, + "priority": "0.3", + "tags": { + "cluster": "cluster-1", + "region": "weur", + }, + }, + { + "votes": 1, + "priority": "0.7", + "tags": { + "cluster": "cluster-1", + "region": "eeur", + }, + }, + ], + [ + { + "votes": 1, + "priority": "0.2", + "tags": { + "cluster": "cluster-2", + "region": "apac", + }, + }, + ], + [ + { + "votes": 1, + "priority": "1.3", + "tags": { + "cluster": "cluster-3", + "region": "nwus", + }, + }, + { + "votes": 1, + "priority": "2.7", + "tags": { + "cluster": "cluster-3", + "region": "seus", + }, + }, + ], + ] + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2], member_options) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + resource.update() + return resource + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_create_kube_config_file(cluster_clients: Dict, central_cluster_name: str, member_cluster_names: str): + clients = cluster_clients + + assert len(clients) == 4 + for member_cluster_name in member_cluster_names: + assert member_cluster_name in clients + assert central_cluster_name in clients + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_mongodb_multi_member_options_ac(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + config = mongodb_multi.get_automation_config_tester().automation_config + rs = config["replicaSets"] + member1 = rs[0]["members"][0] + member2 = rs[0]["members"][1] + member3 = rs[0]["members"][2] + member4 = rs[0]["members"][3] + member5 = rs[0]["members"][4] + + assert member1["votes"] == 1 + assert member1["priority"] == 0.3 + assert member1["tags"] == {"cluster": "cluster-1", "region": "weur"} + + assert member2["votes"] == 1 + assert member2["priority"] == 0.7 + assert member2["tags"] == {"cluster": "cluster-1", "region": "eeur"} + + assert member3["votes"] == 1 + assert member3["priority"] == 0.2 + assert member3["tags"] == {"cluster": "cluster-2", "region": "apac"} + + assert member4["votes"] == 1 + assert member4["priority"] == 1.3 + assert member4["tags"] == {"cluster": "cluster-3", "region": "nwus"} + + assert member5["votes"] == 1 + assert member5["priority"] == 2.7 + assert member5["tags"] == {"cluster": "cluster-3", "region": "seus"} + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_mongodb_multi_update_member_options(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + + mongodb_multi["spec"]["clusterSpecList"][0]["memberConfig"][0] = { + "votes": 1, + "priority": "1.3", + "tags": { + "cluster": "cluster-1", + "region": "weur", + "app": "backend", + }, + } + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + config = mongodb_multi.get_automation_config_tester().automation_config + rs = config["replicaSets"] + + updated_member = rs[0]["members"][0] + assert updated_member["votes"] == 1 + assert updated_member["priority"] == 1.3 + assert updated_member["tags"] == { + "cluster": "cluster-1", + "region": "weur", + "app": "backend", + } + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_mongodb_multi_set_member_votes_to_0(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + + mongodb_multi["spec"]["clusterSpecList"][1]["memberConfig"][0]["votes"] = 0 + mongodb_multi["spec"]["clusterSpecList"][1]["memberConfig"][0]["priority"] = "0.0" + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + config = mongodb_multi.get_automation_config_tester().automation_config + rs = config["replicaSets"] + + updated_member = rs[0]["members"][2] + assert updated_member["votes"] == 0 + assert updated_member["priority"] == 0.0 + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_mongodb_multi_set_invalid_votes_and_priority(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + + mongodb_multi["spec"]["clusterSpecList"][1]["memberConfig"][0]["votes"] = 0 + mongodb_multi["spec"]["clusterSpecList"][1]["memberConfig"][0]["priority"] = "0.7" + mongodb_multi.update() + mongodb_multi.assert_reaches_phase( + Phase.Failed, + msg_regexp=".*cannot have 0 votes when priority is greater than 0", + ) + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_mongodb_multi_set_recover_valid_member_options(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + # A member with priority 0.0 could still be a voting member. It cannot become primary and cannot trigger elections. + # https://www.mongodb.com/docs/v5.0/core/replica-set-priority-0-member/#priority-0-replica-set-members + mongodb_multi["spec"]["clusterSpecList"][1]["memberConfig"][0]["votes"] = 1 + mongodb_multi["spec"]["clusterSpecList"][1]["memberConfig"][0]["priority"] = "0.0" + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@pytest.mark.e2e_multi_cluster_replica_set_member_options +def test_mongodb_multi_set_only_one_vote_per_member(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + + mongodb_multi["spec"]["clusterSpecList"][2]["memberConfig"][1]["votes"] = 3 + mongodb_multi["spec"]["clusterSpecList"][2]["memberConfig"][1]["priority"] = "0.1" + mongodb_multi.update() + mongodb_multi.assert_reaches_phase( + Phase.Failed, + msg_regexp=".*cannot have greater than 1 vote", + ) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_migration.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_migration.py new file mode 100644 index 000000000..43c8023c1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_migration.py @@ -0,0 +1,90 @@ +from typing import List + +import kubernetes +import pymongo +import pytest +from kubetester import try_load +from kubetester.kubetester import assert_statefulset_architecture +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import get_default_architecture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import MongoDBBackgroundTester +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +MDBM_RESOURCE = "multi-replica-set-migration" + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version, +) -> MongoDBMulti: + + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDBM_RESOURCE, namespace) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource["spec"]["version"] = custom_mdb_version + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + try_load(resource) + return resource + + +@pytest.fixture(scope="module") +def mdb_health_checker(mongodb_multi: MongoDBMulti) -> MongoDBBackgroundTester: + return MongoDBBackgroundTester( + mongodb_multi.tester(), + allowed_sequential_failures=1, + health_function_params={ + "attempts": 1, + "write_concern": pymongo.WriteConcern(w="majority"), + }, + ) + + +@pytest.mark.e2e_multi_cluster_replica_set_migration +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_replica_set_migration +def test_create_mongodb_multi_running(mongodb_multi: MongoDBMulti): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + +@pytest.mark.e2e_multi_cluster_replica_set_migration +def test_start_background_checker(mdb_health_checker: MongoDBBackgroundTester): + mdb_health_checker.start() + + +@pytest.mark.e2e_multi_cluster_replica_set_migration +def test_migrate_architecture(mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient]): + """ + If the E2E is running with default architecture as non-static, + then the test will migrate to static and vice versa. + """ + original_default_architecture = get_default_architecture() + target_architecture = "non-static" if original_default_architecture == "static" else "static" + + mongodb_multi.trigger_architecture_migration() + + mongodb_multi.load() + assert mongodb_multi["metadata"]["annotations"]["mongodb.com/v1.architecture"] == target_architecture + + mongodb_multi.assert_abandons_phase(Phase.Running, timeout=1000) + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1000) + + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + for statefulset in statefulsets.values(): + assert_statefulset_architecture(statefulset, target_architecture) + + +@pytest.mark.e2e_multi_cluster_replica_set_migration +def test_mdb_healthy_throughout_change_version( + mdb_health_checker: MongoDBBackgroundTester, +): + mdb_health_checker.assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_scale_down.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_scale_down.py new file mode 100644 index 000000000..3c34cb17a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_scale_down.py @@ -0,0 +1,139 @@ +from typing import List + +import kubernetes +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set" +BUNDLE_SECRET_NAME = f"prefix-{RESOURCE_NAME}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + # start at one member in each cluster + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi(mongodb_multi_unmarshalled: MongoDBMulti, server_certs: str) -> MongoDBMulti: + return mongodb_multi_unmarshalled.create() + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_statefulsets_have_been_created_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 2 + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_ops_manager_has_been_updated_correctly_before_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(5) + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_scale_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + mongodb_multi["spec"]["clusterSpecList"][0]["members"] = 1 + mongodb_multi["spec"]["clusterSpecList"][1]["members"] = 1 + mongodb_multi["spec"]["clusterSpecList"][2]["members"] = 1 + mongodb_multi.update() + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1800) + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_statefulsets_have_been_scaled_down_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 1 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 1 + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_ops_manager_has_been_updated_correctly_after_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(3) + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_replica_set_scale_down +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_scale_up.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_scale_up.py new file mode 100644 index 000000000..b1388e270 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_scale_up.py @@ -0,0 +1,142 @@ +from typing import List + +import kubernetes +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set" +BUNDLE_SECRET_NAME = f"prefix-{RESOURCE_NAME}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: List[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi(mongodb_multi_unmarshalled: MongoDBMulti, server_certs: str) -> MongoDBMulti: + # we have created certs for all 5 members, but want to start at only 3. + mongodb_multi_unmarshalled["spec"]["clusterSpecList"][0]["members"] = 1 + mongodb_multi_unmarshalled["spec"]["clusterSpecList"][1]["members"] = 1 + mongodb_multi_unmarshalled["spec"]["clusterSpecList"][2]["members"] = 1 + return mongodb_multi_unmarshalled.create() + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_statefulsets_have_been_created_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 1 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 1 + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_ops_manager_has_been_updated_correctly_before_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(3) + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_scale_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + mongodb_multi["spec"]["clusterSpecList"][0]["members"] = 2 + mongodb_multi["spec"]["clusterSpecList"][1]["members"] = 1 + mongodb_multi["spec"]["clusterSpecList"][2]["members"] = 2 + mongodb_multi.update() + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1800) + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_statefulsets_have_been_scaled_up_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 2 + + +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_ops_manager_has_been_updated_correctly_after_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(5) + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_replica_set_scale_up +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_test_mtls.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_test_mtls.py new file mode 100644 index 000000000..22aacfd13 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_replica_set_test_mtls.py @@ -0,0 +1,228 @@ +from typing import List + +import kubernetes +import pytest +from kubetester import wait_until +from kubetester.kubetester import KubernetesTester, create_testing_namespace +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), "multi-replica-set", namespace) + resource.set_version(custom_mdb_version) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + # TODO: incorporate this into the base class. + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + return resource + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_create_mongo_pod_in_separate_namespace( + member_cluster_clients: List[MultiClusterClient], + evergreen_task_id: str, + namespace: str, +): + cluster_1_client = member_cluster_clients[0] + + # create the namespace to deploy the + create_testing_namespace(evergreen_task_id, f"{namespace}-mongo", api_client=cluster_1_client.api_client) + + corev1 = kubernetes.client.CoreV1Api(api_client=cluster_1_client.api_client) + + # def default_service_account_token_exists() -> bool: + # secrets: kubernetes.client.V1SecretList = corev1.list_namespaced_secret( + # f"{namespace}-mongo" + # ) + # for secret in secrets.items: + # if secret.metadata.name.startswith("default-token"): + # return True + # return False + # + # wait_until(default_service_account_token_exists, timeout=10) + + # create a pod with mongo installed in a separate namespace that does not have istio configured. + corev1.create_namespaced_pod( + f"{namespace}-mongo", + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "mongo", + }, + "spec": { + "containers": [ + { + "image": "mongo", + "name": "mongo", + } + ], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Never", + }, + }, + ) + + def pod_is_ready() -> bool: + try: + pod = corev1.read_namespaced_pod("mongo", f"{namespace}-mongo") + return pod.status.phase == "Running" + except Exception: + return False + + wait_until(pod_is_ready, timeout=60) + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_connectivity_fails_from_second_namespace( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + namespace: str, +): + cluster_1_client = member_cluster_clients[0] + + service_fqdn = f"{mongodb_multi.name}-2-0-svc.{namespace}.svc.cluster.local" + cmd = ["mongosh", "--host", service_fqdn] + + result = KubernetesTester.run_command_in_pod_container( + "mongo", + f"{namespace}-mongo", + cmd, + container="mongo", + api_client=cluster_1_client.api_client, + ) + + failures = [ + "MongoServerSelectionError: connection to", + f"getaddrinfo ENOTFOUND {service_fqdn}", + "HostNotFound", + ] + + assert True in [ + failure in result for failure in failures + ], f"no expected failure messages found in result: {result}" + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_enable_istio_injection( + member_cluster_clients: List[MultiClusterClient], + namespace: str, +): + cluster_1_client = member_cluster_clients[0] + corev1 = kubernetes.client.CoreV1Api(api_client=cluster_1_client.api_client) + ns: kubernetes.client.V1Namespace = corev1.read_namespace(f"{namespace}-mongo") + ns.metadata.labels["istio-injection"] = "enabled" + corev1.patch_namespace(f"{namespace}-mongo", ns) + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_delete_existing_mongo_pod(member_cluster_clients: List[MultiClusterClient], namespace: str): + cluster_1_client = member_cluster_clients[0] + corev1 = kubernetes.client.CoreV1Api(api_client=cluster_1_client.api_client) + corev1.delete_namespaced_pod("mongo", f"{namespace}-mongo") + + def pod_is_deleted() -> bool: + try: + corev1.read_namespaced_pod("mongo", f"{namespace}-mongo") + return False + except kubernetes.client.ApiException: + return True + + wait_until(pod_is_deleted, timeout=120) + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_create_pod_with_istio_sidecar(member_cluster_clients: List[MultiClusterClient], namespace: str): + cluster_1_client = member_cluster_clients[0] + corev1 = kubernetes.client.CoreV1Api(api_client=cluster_1_client.api_client) + # create a pod with mongo installed in a separate namespace that does not have istio configured. + corev1.create_namespaced_pod( + f"{namespace}-mongo", + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "mongo", + }, + "spec": { + "containers": [ + { + "image": "mongo", + "name": "mongo", + } + ], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Never", + }, + }, + ) + + def two_containers_are_present() -> bool: + try: + pod: kubernetes.client.V1Pod = corev1.read_namespaced_pod("mongo", f"{namespace}-mongo") + return len(pod.spec.containers) == 2 and pod.status.phase == "Running" + except Exception: + return False + + # wait for container to back up with sidecar + wait_until(two_containers_are_present, timeout=60) + + +@pytest.mark.e2e_multi_cluster_mtls_test +def test_connectivity_succeeds_from_second_namespace( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + namespace: str, +): + cluster_1_client = member_cluster_clients[0] + cmd = [ + "mongosh", + "--host", + f"{mongodb_multi.name}-0-0-svc.{namespace}.svc.cluster.local", + ] + + def can_connect_to_deployment() -> bool: + result = KubernetesTester.run_command_in_pod_container( + "mongo", + f"{namespace}-mongo", + cmd, + container="mongo", + api_client=cluster_1_client.api_client, + ) + if "Error: network error while attempting to run command 'isMaster' on host" in result: + return False + + if f"getaddrinfo ENOTFOUND" in result: + return False + + if "HostNotFound" in result: + return False + + if f"Connecting to: mongodb://{mongodb_multi.name}-0-0-svc.{namespace}.svc.cluster.local:27017" not in result: + return False + + return True + + wait_until(can_connect_to_deployment, timeout=60) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_down_cluster.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_down_cluster.py new file mode 100644 index 000000000..9eaffe943 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_down_cluster.py @@ -0,0 +1,143 @@ +from typing import List + +import kubernetes +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set" +BUNDLE_SECRET_NAME = f"prefix-{RESOURCE_NAME}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi(mongodb_multi_unmarshalled: MongoDBMulti, server_certs: str) -> MongoDBMulti: + return mongodb_multi_unmarshalled.create() + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_statefulsets_have_been_created_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + + assert len(statefulsets) == 3 + + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 2 + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_ops_manager_has_been_updated_correctly_before_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(5) + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_scale_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.load() + # remove first and last cluster + mongodb_multi["spec"]["clusterSpecList"] = [mongodb_multi["spec"]["clusterSpecList"][1]] + mongodb_multi.update() + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1800, ignore_errors=True) + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_statefulsets_have_been_scaled_down_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets([member_cluster_clients[1]]) + + with pytest.raises(kubernetes.client.exceptions.ApiException) as e: + mongodb_multi.read_statefulsets([member_cluster_clients[0]]) + assert e.value.reason == "Not Found" + + # there should only be one statefulset in the second cluster + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + # there should be no statefulsets in the last cluster + with pytest.raises(kubernetes.client.exceptions.ApiException) as e: + mongodb_multi.read_statefulsets([member_cluster_clients[2]]) + assert e.value.reason == "Not Found" + + +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_ops_manager_has_been_updated_correctly_after_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(1) + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_scale_down_cluster +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti, ca_path: str): + # there should only be one member in cluster 2 so there is just a single service. + tester = mongodb_multi.tester(service_names=[f"{mongodb_multi.name}-1-0-svc"]) + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_up_cluster.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_up_cluster.py new file mode 100644 index 000000000..bef18930b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_up_cluster.py @@ -0,0 +1,140 @@ +from typing import List + +import kubernetes +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set" +BUNDLE_SECRET_NAME = f"prefix-{RESOURCE_NAME}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + # ensure certs are created for the members during scale up + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi(mongodb_multi_unmarshalled: MongoDBMulti, server_certs: str) -> MongoDBMulti: + # remove the last element, we are only starting with 2 clusters we will scale up the 3rd one later. + mongodb_multi_unmarshalled["spec"]["clusterSpecList"].pop() + return mongodb_multi_unmarshalled.update() + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_statefulsets_have_been_created_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + # read all statefulsets except the last one + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients[:-1]) + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_ops_manager_has_been_updated_correctly_before_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(3) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_scale_mongodb_multi(mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient]): + mongodb_multi.load() + mongodb_multi["spec"]["clusterSpecList"].append( + {"members": 2, "clusterName": member_cluster_clients[2].cluster_name} + ) + mongodb_multi.update() + mongodb_multi.assert_abandons_phase(Phase.Running, timeout=120) + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1800) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_statefulsets_have_been_scaled_up_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + + assert len(statefulsets) == 3 + + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = member_cluster_clients[2] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 2 + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_ops_manager_has_been_updated_correctly_after_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(5) + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_scale_up_cluster +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_up_cluster_new_cluster.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_up_cluster_new_cluster.py new file mode 100644 index 000000000..71097ee46 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scale_up_cluster_new_cluster.py @@ -0,0 +1,170 @@ +from typing import Callable, List + +import kubernetes +import pytest +from kubernetes import client +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from tests.conftest import MULTI_CLUSTER_OPERATOR_NAME, run_kube_config_creation_tool +from tests.multicluster.conftest import cluster_spec_list + +RESOURCE_NAME = "multi-replica-set" +BUNDLE_SECRET_NAME = f"prefix-{RESOURCE_NAME}-cert" + + +@pytest.fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + # ensure certs are created for the members during scale up + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource + + +@pytest.fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@pytest.fixture(scope="module") +def mongodb_multi(mongodb_multi_unmarshalled: MongoDBMulti, server_certs: str) -> MongoDBMulti: + mongodb_multi_unmarshalled["spec"]["clusterSpecList"].pop() + return mongodb_multi_unmarshalled.create() + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_deploy_operator( + install_multi_cluster_operator_set_members_fn: Callable[[List[str]], Operator], + member_cluster_names: List[str], + namespace: str, +): + run_kube_config_creation_tool(member_cluster_names[:-1], namespace, namespace, member_cluster_names) + # deploy the operator without the final cluster + operator = install_multi_cluster_operator_set_members_fn(member_cluster_names[:-1]) + operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_statefulsets_have_been_created_correctly( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + clients = {c.cluster_name: c for c in member_cluster_clients} + + # read all statefulsets except the last one + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients[:-1]) + cluster_one_client = clients["kind-e2e-cluster-1"] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = clients["kind-e2e-cluster-2"] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_ops_manager_has_been_updated_correctly_before_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(3) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_delete_deployment(namespace: str, central_cluster_client: kubernetes.client.ApiClient): + client.AppsV1Api(api_client=central_cluster_client).delete_namespaced_deployment( + MULTI_CLUSTER_OPERATOR_NAME, namespace + ) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_re_deploy_operator( + install_multi_cluster_operator_set_members_fn: Callable[[List[str]], Operator], + member_cluster_names: List[str], + namespace: str, +): + run_kube_config_creation_tool(member_cluster_names, namespace, namespace, member_cluster_names) + + # deploy the operator without all clusters + operator = install_multi_cluster_operator_set_members_fn(member_cluster_names) + operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_add_new_cluster_to_mongodb_multi_resource( + mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient] +): + mongodb_multi.load() + mongodb_multi["spec"]["clusterSpecList"].append( + {"members": 2, "clusterName": member_cluster_clients[-1].cluster_name} + ) + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=800) + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_statefulsets_have_been_created_correctly_after_cluster_addition( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + clients = {c.cluster_name: c for c in member_cluster_clients} + # read all statefulsets except the last one + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + cluster_one_client = clients["kind-e2e-cluster-1"] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert cluster_one_sts.status.ready_replicas == 2 + + cluster_two_client = clients["kind-e2e-cluster-2"] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert cluster_two_sts.status.ready_replicas == 1 + + cluster_three_client = clients["kind-e2e-cluster-3"] + cluster_three_sts = statefulsets[cluster_three_client.cluster_name] + assert cluster_three_sts.status.ready_replicas == 2 + + +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_ops_manager_has_been_updated_correctly_after_scaling(): + ac = AutomationConfigTester() + ac.assert_processes_size(5) + + +@skip_if_local +@pytest.mark.e2e_multi_cluster_scale_up_cluster_new_cluster +def test_replica_set_is_reachable(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scram.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scram.py new file mode 100644 index 000000000..d6a24a74c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_scram.py @@ -0,0 +1,215 @@ +from typing import List + +import kubernetes +import pytest +from kubetester import create_or_update_secret, read_secret +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser +from kubetester.mongotester import with_scram +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +MDB_RESOURCE = "multi-replica-set-scram" +USER_NAME = "my-user-1" +USER_RESOURCE = "multi-replica-set-scram-user" +USER_DATABASE = "admin" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" +NEW_USER_PASSWORD = "my-new-password7" + + +@pytest.fixture(scope="function") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names, + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource.set_version(custom_mdb_version) + + resource["spec"]["security"] = { + "authentication": { + "agents": {"mode": "MONGODB-CR"}, + "enabled": True, + "modes": ["SCRAM-SHA-1", "SCRAM-SHA-256", "MONGODB-CR"], + } + } + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return resource + + +@pytest.fixture(scope="function") +def mongodb_user(central_cluster_client: kubernetes.client.ApiClient, namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user.yaml"), USER_RESOURCE, namespace) + + resource["spec"]["username"] = USER_NAME + resource["spec"]["passwordSecretKeyRef"] = { + "name": PASSWORD_SECRET_NAME, + "key": "password", + } + resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return resource + + +@pytest.mark.e2e_multi_cluster_scram +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_scram +def test_create_mongodb_user( + central_cluster_client: kubernetes.client.ApiClient, + mongodb_user: MongoDBUser, + namespace: str, +): + # create user secret first + create_or_update_secret( + namespace=namespace, + name=PASSWORD_SECRET_NAME, + data={"password": USER_PASSWORD}, + api_client=central_cluster_client, + ) + mongodb_user.update() + mongodb_user.assert_reaches_phase(Phase.Pending, timeout=100) + + +@pytest.mark.e2e_multi_cluster_scram +def test_create_mongodb_multi_with_scram(mongodb_multi: MongoDBMulti): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=800) + + +@pytest.mark.e2e_multi_cluster_scram +def test_user_reaches_updated( + central_cluster_client: kubernetes.client.ApiClient, + mongodb_user: MongoDBUser, +): + mongodb_user.assert_reaches_phase(Phase.Updated, timeout=100) + + +@pytest.mark.e2e_multi_cluster_scram +def test_replica_set_connectivity_using_user_password(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity(db="admin", opts=[with_scram(USER_NAME, USER_PASSWORD)]) + + +@pytest.mark.e2e_multi_cluster_scram +def test_change_password_and_check_connectivity( + namespace: str, + mongodb_multi: MongoDBMulti, + central_cluster_client: kubernetes.client.ApiClient, +): + create_or_update_secret( + namespace, + PASSWORD_SECRET_NAME, + {"password": NEW_USER_PASSWORD}, + api_client=central_cluster_client, + ) + tester = mongodb_multi.tester() + tester.assert_scram_sha_authentication( + password=NEW_USER_PASSWORD, + username=USER_NAME, + auth_mechanism="SCRAM-SHA-256", + ) + + +@pytest.mark.e2e_multi_cluster_scram +def test_user_cannot_authenticate_with_old_password(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_scram_sha_authentication_fails( + password=USER_PASSWORD, + username=USER_NAME, + auth_mechanism="SCRAM-SHA-256", + ) + + +@pytest.mark.e2e_multi_cluster_scram +def test_connection_string_secret_was_created( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + for client in member_cluster_clients: + secret_data = read_secret( + namespace, + f"{mongodb_multi.name}-{USER_RESOURCE}-{USER_DATABASE}", + api_client=client.api_client, + ) + assert "username" in secret_data + assert "password" in secret_data + assert "connectionString.standard" in secret_data + assert "connectionString.standardSrv" in secret_data + + +@pytest.mark.e2e_multi_cluster_scram +def test_om_configured_correctly(): + expected_roles = { + ("admin", "clusterAdmin"), + ("admin", "userAdminAnyDatabase"), + ("admin", "readWrite"), + ("admin", "userAdminAnyDatabase"), + } + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_has_user(USER_NAME) + tester.assert_user_has_roles(USER_NAME, expected_roles) + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=3) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", active_auth_mechanism=False) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-1", active_auth_mechanism=False) + tester.assert_authentication_mechanism_enabled("MONGODB-CR", active_auth_mechanism=False) + + +@pytest.mark.e2e_multi_cluster_scram +def test_replica_set_connectivity(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity(db="admin", opts=[with_scram(USER_NAME, NEW_USER_PASSWORD)]) + + +@pytest.mark.e2e_multi_cluster_scram +def test_replica_set_connectivity_from_connection_string_standard( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + secret_data = read_secret( + namespace, + f"{mongodb_multi.name}-{USER_RESOURCE}-{USER_DATABASE}", + api_client=member_cluster_clients[-1].api_client, + ) + tester = mongodb_multi.tester() + tester.cnx_string = secret_data["connectionString.standard"] + tester.assert_connectivity( + db="admin", + opts=[with_scram(USER_NAME, NEW_USER_PASSWORD)], + ) + + +@pytest.mark.e2e_multi_cluster_scram +def test_replica_set_connectivity_from_connection_string_standard_srv( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + secret_data = read_secret( + namespace, + f"{mongodb_multi.name}-{USER_RESOURCE}-{USER_DATABASE}", + api_client=member_cluster_clients[-1].api_client, + ) + tester = mongodb_multi.tester() + tester.cnx_string = secret_data["connectionString.standardSrv"] + tester.assert_connectivity( + db="admin", + opts=[ + with_scram(USER_NAME, NEW_USER_PASSWORD), + ], + ) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_split_horizon.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_split_horizon.py new file mode 100644 index 000000000..39f29d897 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_split_horizon.py @@ -0,0 +1,148 @@ +from typing import List + +import kubernetes +import yaml +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from pytest import fixture, mark + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +USER_NAME = "my-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + +# This test will set up an environment which will configure a resource with split horizon enabled. +# Steps to run this test. + +# 1. Change the nodenames under "additional_domains" +# 2. Run this test with: `make e2e test=e2e_multi_cluster_split_horizon light=true local=true`. +# 3. Wait for the test to pass (this means the environment is set up.) +# 4. Exec into any database pod and note the contents of the files referenced by the fields +# * net.tls.certificateKeyFile +# * net.tlsCAFile +# from the /data/automation-mongod.conf file. + +# 5. Test the connection +# Testing the connection can be done from either the worker node or from your local machine(note accessing traffic from a pod inside the cluster would work irrespective SH is configured correctly or not) +# 1. Acsessing from worker node +# * ssh into any worker node +# * Install the mongo shell +# * Create files from the two files mentioned above. (server.pem and ca.crt) +# * Run "mongo "mongodb://${WORKER_NODE}:30100,${WORKER_NODE}:30101,${WORKER_NODE}:30102/?replicaSet=test-tls-base-rs-external-access" --tls --tlsCertificateKeyFile server.pem --tlsCAFile ca.crt" +# 2. Accessing from local machine +# * Install the mongo shell +# * Create files from the two files mentioned above. (server.pem and ca.crt) +# * Open access to KOPS nodes from your local machine by following these steps(by default KOPS doesn't expose traffic from all ports to the internet) +# : https://stackoverflow.com/questions/45543694/kubernetes-cluster-on-aws-with-kops-nodeport-service-unavailable/45561848#45561848 +# * Run "mongo "mongodb://${WORKER_NODE1}:30100,${WORKER_NODE2}:30101,${WORKER_NODE3}:30102/?replicaSet=test-tls-base-rs-external-access" --tls --tlsCertificateKeyFile server.pem --tlsCAFile ca.crt" +# When split horizon is not configured, specifying the replicaset name should fail. +# When split horizon is configured, it will successfully connect to the primary. + +# Example: mongo "mongodb://ec2-35-178-71-70.eu-west-2.compute.amazonaws.com:30100,ec2-52-56-69-123.eu-west-2.compute.amazonaws.com:30100,ec2-3-10-22-163.eu-west-2.compute.amazonaws.com:30100" --tls --tlsCertificateKeyFile server.pem --tlsCAFile ca-pem + +# 6. Clean the namespace +# * This test creates node ports, which we should delete. + + +@fixture(scope="module") +def mongodb_multi_unmarshalled(namespace: str) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi-split-horizon.yaml"), MDB_RESOURCE, namespace) + return resource + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + additional_domains=[ + "*", + "ec2-35-178-71-70.eu-west-2.compute.amazonaws.com", + "ec2-52-56-69-123.eu-west-2.compute.amazonaws.com", + "ec2-3-10-22-163.eu-west-2.compute.amazonaws.com", + ], + ) + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + server_certs: str, + mongodb_multi_unmarshalled: MongoDBMulti, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBMulti: + + resource = mongodb_multi_unmarshalled + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "enabled": True, + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource.create() + + +@mark.e2e_multi_cluster_split_horizon +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_split_horizon +def test_deploy_mongodb_multi_with_tls( + mongodb_multi: MongoDBMulti, + namespace: str, +): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_multi_cluster_split_horizon +def test_create_node_ports(mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient]): + for mcc in member_cluster_clients: + with open( + yaml_fixture(f"split-horizon-node-ports/split-horizon-node-port.yaml"), + "r", + ) as f: + service_body = yaml.safe_load(f.read()) + + # configure labels and selectors + service_body["metadata"]["labels"][ + "mongodbmulticluster" + ] = f"{mongodb_multi.namespace}-{mongodb_multi.name}" + service_body["metadata"]["labels"][ + "statefulset.kubernetes.io/pod-name" + ] = f"{mongodb_multi.name}-{mcc.cluster_index}-0" + service_body["spec"]["selector"][ + "statefulset.kubernetes.io/pod-name" + ] = f"{mongodb_multi.name}-{mcc.cluster_index}-0" + + KubernetesTester.create_service( + mongodb_multi.namespace, + body=service_body, + api_client=mcc.api_client, + ) + + +@skip_if_local +@mark.e2e_multi_cluster_split_horizon +def test_tls_connectivity(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_sts_override.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_sts_override.py new file mode 100644 index 000000000..270667602 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_sts_override.py @@ -0,0 +1,71 @@ +from typing import List + +import kubernetes +import pytest +from kubernetes import client +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi-sts-override.yaml"), + "multi-replica-set-sts-override", + namespace, + ) + resource.set_version(custom_mdb_version) + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource.update() + + +@pytest.mark.e2e_multi_sts_override +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_sts_override +def test_create_mongodb_multi(mongodb_multi: MongoDBMulti): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_multi_sts_override +def test_statefulset_overrides(mongodb_multi: MongoDBMulti, member_cluster_clients: List[MultiClusterClient]): + statefulsets = mongodb_multi.read_statefulsets(member_cluster_clients) + + # assert sts.podspec override in cluster1 + cluster_one_client = member_cluster_clients[0] + cluster_one_sts = statefulsets[cluster_one_client.cluster_name] + assert_container_in_sts("sidecar1", cluster_one_sts) + assert "multi-replica-set" in cluster_one_sts.spec.template.metadata.labels["app"] + + # assert sts.podspec override in cluster2 + cluster_two_client = member_cluster_clients[1] + cluster_two_sts = statefulsets[cluster_two_client.cluster_name] + assert_container_in_sts("sidecar2", cluster_two_sts) + + +@pytest.mark.e2e_multi_sts_override +def test_access_modes_pvc( + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + namespace: str, +): + pvc = client.CoreV1Api(api_client=member_cluster_clients[0].api_client).read_namespaced_persistent_volume_claim( + f"data-{mongodb_multi.name}-{0}-{0}", namespace + ) + + assert "ReadWriteOnce" in pvc.spec.access_modes + + +def assert_container_in_sts(container_name: str, sts: client.V1StatefulSet): + container_names = [c.name for c in sts.spec.template.spec.containers] + assert container_name in container_names diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_no_mesh.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_no_mesh.py new file mode 100644 index 000000000..62d651c2b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_no_mesh.py @@ -0,0 +1,295 @@ +from typing import List + +import kubernetes +from kubernetes import client +from kubetester import get_service +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.common.placeholders import placeholders +from tests.conftest import update_coredns_hosts +from tests.multicluster.conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +BUNDLE_PEM_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert-pem" + + +@fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, member_cluster_names: List[str], custom_mdb_version: str +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["persistent"] = False + # These domains map 1:1 to the CoreDNS file. Please be mindful when updating them. + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 2, 2]) + + resource["spec"]["externalAccess"] = {} + resource["spec"]["clusterSpecList"][0]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-1.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing0", + "port": 27019, + }, + ], + } + }, + } + resource["spec"]["clusterSpecList"][1]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-2.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing1", + "port": 27019, + }, + ], + } + }, + } + resource["spec"]["clusterSpecList"][2]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-3.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing2", + "port": 27019, + }, + ], + } + }, + } + + return resource + + +@fixture(scope="module") +def disable_istio( + multi_cluster_operator: Operator, + namespace: str, + member_cluster_clients: List[MultiClusterClient], +): + for mcc in member_cluster_clients: + api = client.CoreV1Api(api_client=mcc.api_client) + labels = {"istio-injection": "disabled"} + ns = api.read_namespace(name=namespace) + ns.metadata.labels.update(labels) + api.replace_namespace(name=namespace, body=ns) + return None + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + disable_istio, + namespace: str, + mongodb_multi_unmarshalled: MongoDBMulti, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBMulti: + mongodb_multi_unmarshalled["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + mongodb_multi_unmarshalled.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return mongodb_multi_unmarshalled.update() + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@mark.e2e_multi_cluster_tls_no_mesh +def test_update_coredns(cluster_clients: dict[str, kubernetes.client.ApiClient]): + hosts = [ + ("172.18.255.211", "test.kind-e2e-cluster-1.interconnected"), + ( + "172.18.255.211", + "multi-cluster-replica-set-0-0.kind-e2e-cluster-1.interconnected", + ), + ( + "172.18.255.212", + "multi-cluster-replica-set-0-1.kind-e2e-cluster-1.interconnected", + ), + ( + "172.18.255.213", + "multi-cluster-replica-set-0-2.kind-e2e-cluster-1.interconnected", + ), + ("172.18.255.221", "test.kind-e2e-cluster-2.interconnected"), + ( + "172.18.255.221", + "multi-cluster-replica-set-1-0.kind-e2e-cluster-2.interconnected", + ), + ( + "172.18.255.222", + "multi-cluster-replica-set-1-1.kind-e2e-cluster-2.interconnected", + ), + ( + "172.18.255.223", + "multi-cluster-replica-set-1-2.kind-e2e-cluster-2.interconnected", + ), + ("172.18.255.231", "test.kind-e2e-cluster-3.interconnected"), + ( + "172.18.255.231", + "multi-cluster-replica-set-2-0.kind-e2e-cluster-3.interconnected", + ), + ( + "172.18.255.232", + "multi-cluster-replica-set-2-1.kind-e2e-cluster-3.interconnected", + ), + ( + "172.18.255.233", + "multi-cluster-replica-set-2-2.kind-e2e-cluster-3.interconnected", + ), + ] + + for cluster_name, cluster_api in cluster_clients.items(): + update_coredns_hosts(hosts, cluster_name, api_client=cluster_api) + + +@mark.e2e_multi_cluster_tls_no_mesh +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_tls_no_mesh +def test_create_mongodb_multi( + mongodb_multi: MongoDBMulti, + namespace: str, + server_certs: str, + multi_cluster_issuer_ca_configmap: str, + member_cluster_clients: List[MultiClusterClient], + member_cluster_names: List[str], +): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=2400, ignore_errors=True) + + +@mark.e2e_multi_cluster_tls_no_mesh +def test_service_overrides( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + for cluster_idx, member_cluster_client in enumerate(member_cluster_clients): + for pod_idx in range(0, 2): + external_service_name = f"{mongodb_multi.name}-{cluster_idx}-{pod_idx}-svc-external" + external_service = get_service( + namespace, + external_service_name, + api_client=member_cluster_client.api_client, + ) + + assert external_service is not None + assert external_service.spec.type == "LoadBalancer" + assert external_service.spec.publish_not_ready_addresses + ports = external_service.spec.ports + assert len(ports) == 3 + assert ports[0].name == "mongodb" + assert ports[0].target_port == 27017 + assert ports[0].port == 27017 + assert ports[1].name == "backup" + assert ports[1].target_port == 27018 + assert ports[1].port == 27018 + assert ports[2].name == f"testing{cluster_idx}" + assert ports[2].target_port == 27019 + assert ports[2].port == 27019 + + +@mark.e2e_multi_cluster_tls_no_mesh +def test_placeholders_in_external_services( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], +): + for cluster_spec_item in mongodb_multi["spec"]["clusterSpecList"]: + annotations = placeholders.get_annotations_with_placeholders_for_multi_cluster( + prefix=f'{cluster_spec_item["clusterName"]},' + ) + external_access = cluster_spec_item.get("externalAccess", {}) + external_service = external_access.get("externalService", {}) + external_service["annotations"] = annotations + external_access["externalService"] = external_service + cluster_spec_item["externalAccess"] = external_access + + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=300) + + external_domains = [ + "kind-e2e-cluster-1.interconnected", + "kind-e2e-cluster-2.interconnected", + "kind-e2e-cluster-3.interconnected", + ] + name = mongodb_multi["metadata"]["name"] + for _, member_cluster_client in enumerate(member_cluster_clients): + members = mongodb_multi.get_item_spec(member_cluster_client.cluster_name)["members"] + for pod_idx in range(0, members): + cluster_idx = member_cluster_client.cluster_index + service = client.CoreV1Api(api_client=member_cluster_client.api_client).read_namespaced_service( + f"{name}-{cluster_idx}-{pod_idx}-svc-external", namespace + ) + cluster_name = member_cluster_client.cluster_name + external_domain = external_domains[cluster_idx] + expected_annotations = placeholders.get_expected_annotations_multi_cluster_no_mesh( + name=name, + namespace=namespace, + pod_idx=pod_idx, + external_domain=external_domain, + cluster_index=cluster_idx, + cluster_name=cluster_name, + prefix=f"{cluster_name},", + ) + assert service.metadata.annotations == expected_annotations diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_with_scram.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_with_scram.py new file mode 100644 index 000000000..3b1b5aeae --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_with_scram.py @@ -0,0 +1,229 @@ +from typing import List + +import kubernetes +from kubetester import create_secret, read_secret +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import create_multi_cluster_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser +from kubetester.mongotester import with_scram, with_tls +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +USER_NAME = "my-user-1" +USER_RESOURCE = "multi-replica-set-scram-user" +USER_DATABASE = "admin" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +@fixture(scope="module") +def mongodb_multi_unmarshalled( + namespace: str, + member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["clusterSpecList"] = cluster_spec_list( + member_cluster_names=member_cluster_names, members=[2, 1, 2] + ) + + return resource + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + server_certs: str, + mongodb_multi_unmarshalled: MongoDBMulti, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBMulti: + + resource = mongodb_multi_unmarshalled + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource.create() + + +@fixture(scope="module") +def mongodb_user(central_cluster_client: kubernetes.client.ApiClient, namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodb-user.yaml"), USER_RESOURCE, namespace) + + resource["spec"]["username"] = USER_NAME + resource["spec"]["passwordSecretKeyRef"] = { + "name": PASSWORD_SECRET_NAME, + "key": "password", + } + resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE + resource["spec"]["mongodbResourceRef"]["namespace"] = namespace + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + return resource.create() + + +@mark.e2e_multi_cluster_tls_with_scram +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_tls_with_scram +def test_deploy_mongodb_multi_with_tls( + mongodb_multi: MongoDBMulti, + namespace: str, + member_cluster_clients: List[MultiClusterClient], +): + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_multi_cluster_tls_with_scram +def test_update_mongodb_multi_tls_with_scram( + mongodb_multi: MongoDBMulti, + namespace: str, +): + mongodb_multi.load() + mongodb_multi["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_multi_cluster_tls_with_scram +def test_create_mongodb_user( + central_cluster_client: kubernetes.client.ApiClient, + mongodb_user: MongoDBUser, + namespace: str, +): + # create user secret first + create_secret( + namespace=namespace, + name=PASSWORD_SECRET_NAME, + data={"password": USER_PASSWORD}, + api_client=central_cluster_client, + ) + mongodb_user.assert_reaches_phase(Phase.Updated, timeout=100) + + +@skip_if_local +@mark.e2e_multi_cluster_tls_with_scram +def test_tls_connectivity(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) + + +@skip_if_local +@mark.e2e_multi_cluster_tls_with_scram +def test_replica_set_connectivity_with_scram_and_tls(mongodb_multi: MongoDBMulti, ca_path: str): + tester = mongodb_multi.tester() + tester.assert_connectivity( + db="admin", + opts=[ + with_scram(USER_NAME, USER_PASSWORD), + with_tls(use_tls=True, ca_path=ca_path), + ], + ) + + +@skip_if_local +@mark.e2e_multi_cluster_tls_with_scram +def test_replica_set_connectivity_from_connection_string_standard( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + ca_path: str, +): + secret_data = read_secret( + namespace, + f"{mongodb_multi.name}-{USER_RESOURCE}-{USER_DATABASE}", + api_client=member_cluster_clients[0].api_client, + ) + tester = mongodb_multi.tester() + tester.cnx_string = secret_data["connectionString.standard"] + tester.assert_connectivity( + db="admin", + opts=[ + with_scram(USER_NAME, USER_PASSWORD), + with_tls(use_tls=True, ca_path=ca_path), + ], + ) + + +@skip_if_local +@mark.e2e_multi_cluster_tls_with_scram +def test_replica_set_connectivity_from_connection_string_standard_srv( + namespace: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + ca_path: str, +): + secret_data = read_secret( + namespace, + f"{mongodb_multi.name}-{USER_RESOURCE}-{USER_DATABASE}", + api_client=member_cluster_clients[-1].api_client, + ) + tester = mongodb_multi.tester() + tester.cnx_string = secret_data["connectionString.standardSrv"] + tester.assert_connectivity( + db="admin", + opts=[ + with_scram(USER_NAME, USER_PASSWORD), + with_tls(use_tls=True, ca_path=ca_path), + ], + ) + + +@mark.e2e_multi_cluster_tls_with_scram +def test_mongodb_multi_tls_enable_x509( + mongodb_multi: MongoDBMulti, + namespace: str, +): + mongodb_multi.load() + + mongodb_multi["spec"]["security"]["authentication"]["modes"].append("X509") + mongodb_multi["spec"]["security"]["authentication"]["agents"] = {"mode": "SCRAM"} + mongodb_multi.update() + + # sometimes the agents need more time to register than the time we wait -> + # "Failed to enable Authentication for MongoDB Multi Replicaset" + # after this the agents eventually succeed. + mongodb_multi.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=1000) + + +@mark.e2e_multi_cluster_tls_with_scram +def test_mongodb_multi_tls_automation_config_was_updated( + mongodb_multi: MongoDBMulti, + namespace: str, +): + tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + tester.assert_authentication_mechanism_enabled("MONGODB-X509", active_auth_mechanism=False) + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256") + tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_with_x509.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_with_x509.py new file mode 100644 index 000000000..a68074af2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_tls_with_x509.py @@ -0,0 +1,188 @@ +import tempfile +from typing import List + +import kubernetes +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + create_multi_cluster_mongodb_x509_tls_certs, + create_multi_cluster_x509_agent_certs, + create_multi_cluster_x509_user_cert, +) +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.mongodb_user import MongoDBUser +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + +# TODO This test needs to re-introduce certificate rotation and enabling authentication step by step +# See https://jira.mongodb.org/browse/CLOUDP-311366 + +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" +AGENT_BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-agent-certs" +CLUSTER_BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-clusterfile" + + +@fixture(scope="module") +def mongodb_multi_unmarshalled(namespace: str, member_cluster_names, custom_mdb_version: str) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource["spec"]["additionalMongodConfig"] = {"net": {"tls": {"mode": "requireTLS"}}} + + return resource + + +@fixture(scope="module") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_x509_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def cluster_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + + return create_multi_cluster_mongodb_x509_tls_certs( + multi_cluster_issuer, + CLUSTER_BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def agent_certs( + multi_cluster_issuer: str, + mongodb_multi_unmarshalled: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_x509_agent_certs( + multi_cluster_issuer, + AGENT_BUNDLE_SECRET_NAME, + central_cluster_client, + mongodb_multi_unmarshalled, + ) + + +@fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + server_certs: str, + agent_certs: str, + cluster_certs: str, + mongodb_multi_unmarshalled: MongoDBMulti, + multi_cluster_issuer_ca_configmap: str, + member_cluster_names, +) -> MongoDBMulti: + + resource = mongodb_multi_unmarshalled + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + "authentication": { + "enabled": True, + "modes": ["X509", "SCRAM"], + "agents": {"mode": "X509"}, + "internalCluster": "X509", + }, + } + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource.update() + + return resource + + +@fixture(scope="module") +def mongodb_x509_user(central_cluster_client: kubernetes.client.ApiClient, namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodb-x509-user.yaml"), "multi-replica-set-x509-user", namespace) + resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + resource.update() + + return resource + + +@mark.e2e_multi_cluster_tls_with_x509 +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_tls_with_x509 +def test_deploy_mongodb_multi_with_tls_and_authentication(mongodb_multi: MongoDBMulti, namespace: str): + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_multi_cluster_tls_with_x509 +def test_ops_manager_state_was_updated_correctly(mongodb_multi: MongoDBMulti): + ac_tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + ac_tester.assert_authentication_enabled(expected_num_deployment_auth_mechanisms=2) + ac_tester.assert_authentication_mechanism_enabled("MONGODB-X509") + ac_tester.assert_internal_cluster_authentication_enabled() + + +@mark.e2e_multi_cluster_tls_with_x509 +def test_create_mongodb_x509_user( + central_cluster_client: kubernetes.client.ApiClient, + mongodb_x509_user: MongoDBUser, + namespace: str, +): + mongodb_x509_user.assert_reaches_phase(Phase.Updated, timeout=100) + + +@skip_if_local +@mark.e2e_multi_cluster_tls_with_x509 +def test_x509_user_connectivity( + mongodb_multi: MongoDBMulti, + central_cluster_client: kubernetes.client.ApiClient, + multi_cluster_issuer: str, + namespace: str, + ca_path: str, +): + with tempfile.NamedTemporaryFile(delete=False, mode="w") as cert_file: + create_multi_cluster_x509_user_cert( + multi_cluster_issuer, namespace, central_cluster_client, path=cert_file.name + ) + tester = mongodb_multi.tester() + tester.assert_x509_authentication(cert_file_name=cert_file.name, tlsCAFile=ca_path) + + +# TODO Replace and use this method to check that certificate rotation after enabling TLS and authentication mechanisms +# keeps the resources reachable and in Running state. +def assert_certificate_rotation(central_cluster_client, mongodb_multi, namespace, certificate_name): + cert = Certificate(name=certificate_name, namespace=namespace) + cert.api = kubernetes.client.CustomObjectsApi(api_client=central_cluster_client) + cert.load() + cert["spec"]["dnsNames"].append("foo") # Append DNS to cert to rotate the certificate + cert.update() + # FIXME the assertions below need to be replaced with a robust check that the agents are ready + # and the TLS certificates are rotated. + mongodb_multi.assert_abandons_phase(Phase.Running, timeout=100) + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=1500) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_upgrade_downgrade.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_upgrade_downgrade.py new file mode 100644 index 000000000..253b54bcc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_upgrade_downgrade.py @@ -0,0 +1,101 @@ +import kubernetes +import pymongo +import pytest +from kubetester import try_load +from kubetester.kubetester import ensure_ent_version, fcv_from_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.mongotester import MongoDBBackgroundTester +from kubetester.operator import Operator +from tests.multicluster.conftest import cluster_spec_list + +MDBM_RESOURCE = "multi-replica-set-upgrade" + + +@pytest.fixture(scope="module") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names: list[str], + custom_mdb_prev_version: str, +) -> MongoDBMulti: + + resource = MongoDBMulti.from_yaml(yaml_fixture("mongodb-multi.yaml"), MDBM_RESOURCE, namespace) + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [2, 1, 2]) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + try_load(resource) + return resource + + +@pytest.fixture(scope="module") +def mdb_health_checker(mongodb_multi: MongoDBMulti) -> MongoDBBackgroundTester: + return MongoDBBackgroundTester( + mongodb_multi.tester(), + allowed_sequential_failures=1, + health_function_params={ + "attempts": 1, + "write_concern": pymongo.WriteConcern(w="majority"), + }, + ) + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_create_mongodb_multi_running(mongodb_multi: MongoDBMulti, custom_mdb_prev_version: str): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + mongodb_multi.tester().assert_version(ensure_ent_version(custom_mdb_prev_version)) + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_start_background_checker(mdb_health_checker: MongoDBBackgroundTester): + mdb_health_checker.start() + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_mongodb_multi_upgrade(mongodb_multi: MongoDBMulti, custom_mdb_prev_version: str, custom_mdb_version: str): + mongodb_multi.load() + mongodb_multi["spec"]["version"] = ensure_ent_version(custom_mdb_version) + mongodb_multi["spec"]["featureCompatibilityVersion"] = fcv_from_version(custom_mdb_prev_version) + mongodb_multi.update() + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + + mongodb_multi.tester().assert_version(ensure_ent_version(custom_mdb_version)) + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_upgraded_replica_set_is_reachable(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity() + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_mongodb_multi_downgrade(mongodb_multi: MongoDBMulti, custom_mdb_prev_version: str): + mongodb_multi.load() + mongodb_multi["spec"]["version"] = ensure_ent_version(custom_mdb_prev_version) + mongodb_multi["spec"]["featureCompatibilityVersion"] = fcv_from_version(custom_mdb_prev_version) + mongodb_multi.update() + + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=700) + mongodb_multi.tester().assert_version(ensure_ent_version(custom_mdb_prev_version)) + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_downgraded_replica_set_is_reachable(mongodb_multi: MongoDBMulti): + tester = mongodb_multi.tester() + tester.assert_connectivity() + + +@pytest.mark.e2e_multi_cluster_upgrade_downgrade +def test_mdb_healthy_throughout_change_version( + mdb_health_checker: MongoDBBackgroundTester, +): + mdb_health_checker.assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_validation.py b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_validation.py new file mode 100644 index 000000000..d24fecf3c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster/multi_cluster_validation.py @@ -0,0 +1,45 @@ +import kubernetes +import pytest +import yaml +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.operator import Operator + + +@pytest.mark.e2e_multi_cluster_validation +class TestWebhookValidation(KubernetesTester): + def test_deploy_operator(self, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + def test_unique_cluster_names(self, central_cluster_client: kubernetes.client.ApiClient): + resource = yaml.safe_load(open(yaml_fixture("mongodb-multi-cluster.yaml"))) + resource["spec"]["clusterSpecList"].append({"clusterName": "kind-e2e-cluster-1", "members": 1}) + + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="Multiple clusters with the same name (kind-e2e-cluster-1) are not allowed", + api_client=central_cluster_client, + ) + + def test_only_one_schema(self, central_cluster_client: kubernetes.client.ApiClient): + resource = yaml.safe_load(open(yaml_fixture("mongodb-multi-cluster.yaml"))) + resource["spec"]["cloudManager"] = {"configMapRef": {"name": " my-project"}} + + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="must validate one and only one schema", + api_client=central_cluster_client, + ) + + def test_non_empty_clusterspec_list(self, central_cluster_client: kubernetes.client.ApiClient): + resource = yaml.safe_load(open(yaml_fixture("mongodb-multi-cluster.yaml"))) + resource["spec"]["clusterSpecList"] = [] + + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="ClusterSpecList empty is not allowed, please define at least one cluster", + api_client=central_cluster_client, + ) diff --git a/scripts/git-hooks/update b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/git-hooks/update rename to docker/mongodb-enterprise-tests/tests/multicluster_appdb/__init__.py diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/conftest.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/conftest.py new file mode 100644 index 000000000..dfffaa10c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/conftest.py @@ -0,0 +1,33 @@ +import kubernetes +from kubetester.awss3client import AwsS3Client +from pytest import fixture +from tests.common.constants import S3_BLOCKSTORE_NAME, S3_OPLOG_NAME +from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket + + +@fixture(scope="module") +def s3_bucket_oplog( + aws_s3_client: AwsS3Client, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> str: + yield from create_s3_bucket_oplog(namespace, aws_s3_client, central_cluster_client) + + +def create_s3_bucket_oplog(namespace, aws_s3_client, api_client: kubernetes.client.ApiClient): + create_aws_secret(aws_s3_client, S3_OPLOG_NAME + "-secret", namespace, api_client=api_client) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-oplog") + + +@fixture(scope="module") +def s3_bucket_blockstore( + aws_s3_client: AwsS3Client, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> str: + yield from create_s3_bucket_blockstore(namespace, aws_s3_client, api_client=central_cluster_client) + + +def create_s3_bucket_blockstore(namespace, aws_s3_client, api_client: kubernetes.client.ApiClient): + create_aws_secret(aws_s3_client, S3_BLOCKSTORE_NAME + "-secret", namespace, api_client=api_client) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-blockstore") diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/fixtures/multicluster_appdb_om.yaml b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/fixtures/multicluster_appdb_om.yaml new file mode 100644 index 000000000..d6daf43b0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/fixtures/multicluster_appdb_om.yaml @@ -0,0 +1,29 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup +spec: + replicas: 1 + version: 6.0.13 + adminCredentials: ops-manager-admin-secret + backup: + enabled: false + applicationDatabase: + topology: MultiCluster + members: 3 + version: 6.0.5-ent + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb.py new file mode 100644 index 000000000..2ef31224e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb.py @@ -0,0 +1,163 @@ +import kubernetes +import kubernetes.client +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import create_appdb_certs +from tests.multicluster.conftest import cluster_spec_list + +CERT_PREFIX = "prefix" + + +@fixture(scope="module") +def appdb_member_cluster_names() -> list[str]: + return ["kind-e2e-cluster-2", "kind-e2e-cluster-3"] + + +@fixture(scope="module") +def ops_manager_unmarshalled( + namespace: str, + custom_version: str, + custom_appdb_version: str, + multi_cluster_issuer_ca_configmap: str, + appdb_member_cluster_names: list[str], + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + resource["spec"]["topology"] = "MultiCluster" + resource["spec"]["clusterSpecList"] = cluster_spec_list(["kind-e2e-cluster-2", "kind-e2e-cluster-3"], [2, 2]) + + resource.allow_mdb_rc_versions() + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["backup"] = {"enabled": False} + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list(appdb_member_cluster_names, [2, 3]), + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": CERT_PREFIX, + "tls": {"ca": multi_cluster_issuer_ca_configmap}, + }, + } + + return resource + + +@fixture(scope="module") +def appdb_certs_secret( + namespace: str, + multi_cluster_issuer: str, + ops_manager_unmarshalled: MongoDBOpsManager, +): + return create_appdb_certs( + namespace, + multi_cluster_issuer, + ops_manager_unmarshalled.name + "-db", + cluster_index_with_members=[(0, 5), (1, 5), (2, 5)], + cert_prefix=CERT_PREFIX, + ) + + +@fixture(scope="module") +def ops_manager( + appdb_certs_secret: str, + ops_manager_unmarshalled: MongoDBOpsManager, +) -> MongoDBOpsManager: + resource = ops_manager_unmarshalled.update() + return resource + + +@mark.e2e_multi_cluster_appdb +def test_patch_central_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient): + corev1 = kubernetes.client.CoreV1Api(api_client=central_cluster_client) + ns = corev1.read_namespace(namespace) + ns.metadata.labels["istio-injection"] = "enabled" + corev1.patch_namespace(namespace, ns) + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.assert_appdb_preferred_hostnames_are_added() + ops_manager.assert_appdb_hostnames_are_correct() + + +@mark.e2e_multi_cluster_appdb +def test_scale_up_one_cluster(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + appdb_member_cluster_names, [4, 3] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.assert_appdb_preferred_hostnames_are_added() + ops_manager.assert_appdb_hostnames_are_correct() + + +@mark.e2e_multi_cluster_appdb +def test_scale_down_one_cluster(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + appdb_member_cluster_names, [4, 1] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + # TODO: AppDB does not remove hostnames when scaling down https://jira.mongodb.org/browse/CLOUDP-306333 + # ops_manager.assert_appdb_preferred_hostnames_are_added() + # ops_manager.assert_appdb_hostnames_are_set_correctly() + + +@mark.e2e_multi_cluster_appdb +def test_scale_up_two_clusters(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + appdb_member_cluster_names, [5, 2] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_appdb +def test_scale_down_two_clusters(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + appdb_member_cluster_names, [2, 1] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_appdb +def test_add_cluster_to_cluster_spec(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + cluster_names = ["kind-e2e-cluster-1"] + appdb_member_cluster_names + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list(cluster_names, [2, 2, 1]) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_appdb +def test_remove_cluster_from_cluster_spec(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + cluster_names = ["kind-e2e-cluster-1"] + appdb_member_cluster_names[1:] + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list(cluster_names, [2, 1]) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_appdb +def test_read_cluster_to_cluster_spec(ops_manager: MongoDBOpsManager, appdb_member_cluster_names): + ops_manager.load() + cluster_names = ["kind-e2e-cluster-1"] + appdb_member_cluster_names + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list(cluster_names, [2, 2, 1]) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py new file mode 100644 index 000000000..608febef4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py @@ -0,0 +1,272 @@ +from typing import Optional + +import kubernetes +import kubernetes.client +from kubetester import ( + delete_statefulset, + get_statefulset, + read_configmap, + try_load, + update_configmap, +) +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import create_appdb_certs, get_member_cluster_api_client +from tests.multicluster.conftest import cluster_spec_list + +FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3" +OM_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-1" + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: str, + custom_appdb_version: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + + resource.allow_mdb_rc_versions() + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["backup"] = {"enabled": False} + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list(["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME], [3, 2]), + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": "prefix", + "tls": {"ca": multi_cluster_issuer_ca_configmap}, + }, + } + + return resource + + +@fixture(scope="module") +def appdb_certs_secret( + namespace: str, + multi_cluster_issuer: str, + ops_manager: MongoDBOpsManager, +): + return create_appdb_certs( + namespace, + multi_cluster_issuer, + ops_manager.name + "-db", + cluster_index_with_members=[(0, 5), (1, 5), (2, 5)], + cert_prefix="prefix", + ) + + +@mark.e2e_multi_cluster_appdb_disaster_recovery +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_patch_central_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient): + corev1 = kubernetes.client.CoreV1Api(api_client=central_cluster_client) + ns = corev1.read_namespace(namespace) + ns.metadata.labels["istio-injection"] = "enabled" + corev1.patch_namespace(namespace, ns) + + +@fixture(scope="module") +def config_version(): + class ConfigVersion: + version = 0 + + return ConfigVersion() + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb_disaster_recovery +def test_create_om(ops_manager: MongoDBOpsManager, appdb_certs_secret: str, config_version): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_create_om_majority_down(ops_manager: MongoDBOpsManager, appdb_certs_secret: str, config_version): + # failed cluster has majority members + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME], [2, 3] + ) + + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] + + +@mark.e2e_multi_cluster_appdb_disaster_recovery +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy( + namespace, central_cluster_client: kubernetes.client.ApiClient +): + member_list_cm = read_configmap( + namespace, + "mongodb-enterprise-operator-member-list", + api_client=central_cluster_client, + ) + # this if is only for allowing re-running the test locally + # without it the test function could be executed only once until the map is populated again by running prepare-local-e2e run again + if FAILED_MEMBER_CLUSTER_NAME in member_list_cm: + member_list_cm.pop(FAILED_MEMBER_CLUSTER_NAME) + + # this will trigger operators restart as it panics on changing the configmap + update_configmap( + namespace, + "mongodb-enterprise-operator-member-list", + member_list_cm, + api_client=central_cluster_client, + ) + + +@mark.e2e_multi_cluster_appdb_disaster_recovery +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_delete_om_and_appdb_statefulset_in_failed_cluster( + ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient +): + appdb_sts_name = f"{ops_manager.name}-db-1" + try: + # delete OM to simulate losing Ops Manager application + # this is only for testing unavailability of the OM application, it's not testing losing OM cluster + # we don't delete here any additional resources (secrets, configmaps) that are required for a proper OM recovery testing + delete_statefulset( + ops_manager.namespace, + ops_manager.name, + propagation_policy="Background", + api_client=central_cluster_client, + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + try: + # delete appdb statefulset in failed member cluster to simulate full cluster outage + delete_statefulset( + ops_manager.namespace, + appdb_sts_name, + propagation_policy="Background", + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubernetes.client.ApiClient]): + try: + get_statefulset(namespace, name, api_client=api_client) + return False + except kubernetes.client.ApiException as e: + if e.status == 404: + return True + else: + raise e + + run_periodically( + lambda: statefulset_is_deleted( + ops_manager.namespace, + ops_manager.name, + api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + run_periodically( + lambda: statefulset_is_deleted( + ops_manager.namespace, + appdb_sts_name, + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + + +@mark.e2e_multi_cluster_appdb_disaster_recovery +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_appdb_is_stable_and_om_is_recreated(ops_manager: MongoDBOpsManager, config_version): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + # there shouldn't be any automation config version change when one of the clusters is lost and OM is recreated + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + + +@mark.e2e_multi_cluster_appdb_disaster_recovery +def test_add_appdb_member_to_om_cluster(ops_manager: MongoDBOpsManager, config_version): + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME, OM_MEMBER_CLUSTER_NAME], + [3, 2, 1], + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + # there should be exactly one automation config version change when we add new member + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + 1 + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 2 + 1 + + config_version.version = current_ac_version + + +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_add_appdb_member_to_om_cluster_force_reconfig(ops_manager: MongoDBOpsManager, config_version): + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME, OM_MEMBER_CLUSTER_NAME], + [3, 2, 1], + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Pending) + + ops_manager.reload() + ops_manager["metadata"]["annotations"].update({"mongodb.com/v1.forceReconfigure": "true"}) + ops_manager.update() + + # This can potentially take quite a bit of time. AppDB needs to go up and sync with OM (which will be crashlooping) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 2 + 1 + + config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] + + +@mark.e2e_multi_cluster_appdb_disaster_recovery +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +def test_remove_failed_member_cluster_has_been_scaled_down(ops_manager: MongoDBOpsManager, config_version): + # we remove failed member cluster + # thanks to previous spec stored in the config map, the operator recognizes we need to scale its 2 processes one by one + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", OM_MEMBER_CLUSTER_NAME], [3, 1] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + 2 # two scale downs + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 1 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py new file mode 100644 index 000000000..6d9e1c941 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_s3_based_backup_restore.py @@ -0,0 +1,244 @@ +import datetime +import time + +import kubernetes.client +import pymongo +from kubetester import create_or_update_configmap, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti +from kubetester.omtester import OMTester +from kubetester.opsmanager import MongoDBOpsManager +from pymongo.errors import ServerSelectionTimeoutError +from pytest import fixture, mark +from tests.common.constants import ( + MONGODB_PORT, + S3_BLOCKSTORE_NAME, + S3_OPLOG_NAME, + TEST_DATA, +) +from tests.common.ops_manager.multi_cluster import ( + ops_manager_multi_cluster_with_tls_s3_backups, +) +from tests.conftest import AWS_REGION, assert_data_got_restored +from tests.multicluster.conftest import cluster_spec_list + + +@fixture(scope="module") +def appdb_member_cluster_names() -> list[str]: + return ["kind-e2e-cluster-2", "kind-e2e-cluster-3"] + + +def create_project_config_map(om: MongoDBOpsManager, mdb_name, project_name, client, custom_ca): + name = f"{mdb_name}-config" + data = { + "baseUrl": om.om_status().get_url(), + "projectName": project_name, + "sslMMSCAConfigMap": custom_ca, + "orgId": "", + } + + create_or_update_configmap(om.namespace, name, data, client) + + +@fixture(scope="module") +def multi_cluster_s3_replica_set( + ops_manager, + namespace, + central_cluster_client: kubernetes.client.ApiClient, + appdb_member_cluster_names: list[str], + custom_mdb_version: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi-cluster.yaml"), "multi-replica-set", namespace + ).configure(ops_manager, "s3metadata", api_client=central_cluster_client) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(appdb_member_cluster_names, [1, 2]) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + yield resource.update() + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket_oplog: str, + s3_bucket_blockstore: str, + central_cluster_client: kubernetes.client.ApiClient, + custom_appdb_version: str, + custom_version: str, +) -> MongoDBOpsManager: + resource = ops_manager_multi_cluster_with_tls_s3_backups( + namespace, + "om-backup-tls-s3", + central_cluster_client, + custom_appdb_version, + s3_bucket_blockstore, + s3_bucket_oplog, + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + resource.allow_mdb_rc_versions() + resource.set_version(custom_version) + + del resource["spec"]["security"] + del resource["spec"]["applicationDatabase"]["security"] + + if try_load(resource): + return resource + + return resource + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb_s3_based_backup_restore +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. + """ + + def test_create_om( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager["spec"]["backup"]["members"] = 1 + ops_manager.update() + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + def test_om_is_running( + self, + ops_manager: MongoDBOpsManager, + central_cluster_client: kubernetes.client.ApiClient, + ): + # at this point AppDB is used as the "metadatastore" + ops_manager.backup_status().assert_reaches_phase(Phase.Running, ignore_errors=True) + om_tester = ops_manager.get_om_tester(api_client=central_cluster_client) + om_tester.assert_healthiness() + + def test_add_metadatastore( + self, + multi_cluster_s3_replica_set: MongoDBMulti, + ops_manager: MongoDBOpsManager, + ): + multi_cluster_s3_replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + + # configure metadatastore in om, use dedicate MDB instead of AppDB + ops_manager.load() + ops_manager["spec"]["backup"]["s3Stores"][0]["mongodbResourceRef"] = {"name": multi_cluster_s3_replica_set.name} + ops_manager["spec"]["backup"]["s3OpLogStores"][0]["mongodbResourceRef"] = { + "name": multi_cluster_s3_replica_set.name + } + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running, ignore_errors=True) + + def test_om_s3_stores( + self, + ops_manager: MongoDBOpsManager, + central_cluster_client: kubernetes.client.ApiClient, + ): + om_tester = ops_manager.get_om_tester(api_client=central_cluster_client) + om_tester.assert_s3_stores([{"id": S3_BLOCKSTORE_NAME, "s3RegionOverride": AWS_REGION}]) + om_tester.assert_oplog_s3_stores([{"id": S3_OPLOG_NAME, "s3RegionOverride": AWS_REGION}]) + + +@mark.e2e_multi_cluster_appdb_s3_based_backup_restore +class TestBackupForMongodb: + @fixture(scope="module") + def project_one( + self, + ops_manager: MongoDBOpsManager, + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + ) -> OMTester: + return ops_manager.get_om_tester( + project_name=f"{namespace}-project-one", + api_client=central_cluster_client, + ) + + @fixture(scope="module") + def mongodb_multi_one_collection(self, mongodb_multi_one: MongoDBMulti): + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient( + mongodb_multi_one.tester(port=MONGODB_PORT).cnx_string, + **mongodb_multi_one.tester(port=MONGODB_PORT).default_opts, + )["testdb"] + + return collection["testcollection"] + + @fixture(scope="module") + def mongodb_multi_one( + self, + ops_manager: MongoDBOpsManager, + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + appdb_member_cluster_names: list[str], + custom_mdb_version: str, + ) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml( + yaml_fixture("mongodb-multi.yaml"), + "multi-replica-set-one", + namespace, + # the project configmap should be created in the central cluster. + ).configure(ops_manager, f"{namespace}-project-one", api_client=central_cluster_client) + + resource["spec"]["clusterSpecList"] = cluster_spec_list(appdb_member_cluster_names, [1, 2]) + + # creating a cluster with backup should work with custom ports + resource["spec"].update({"additionalMongodConfig": {"net": {"port": MONGODB_PORT}}}) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource.configure_backup(mode="enabled") + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + + return resource.update() + + def test_mongodb_multi_one_running_state(self, mongodb_multi_one: MongoDBMulti): + # we might fail connection in the beginning since we set a custom dns in coredns + mongodb_multi_one.assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=600) + + def test_add_test_data(self, mongodb_multi_one_collection): + max_attempts = 100 + while max_attempts > 0: + try: + mongodb_multi_one_collection.insert_one(TEST_DATA) + return + except Exception as e: + print(e) + max_attempts -= 1 + time.sleep(6) + + def test_mdb_backed_up(self, project_one: OMTester): + project_one.wait_until_backup_snapshots_are_ready(expected_count=1) + + def test_change_mdb_data(self, mongodb_multi_one_collection): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + time.sleep(30) + mongodb_multi_one_collection.insert_one({"foo": "bar"}) + + def test_pit_restore(self, project_one: OMTester): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + + pit_datetme = datetime.datetime.now() - datetime.timedelta(seconds=15) + pit_millis = time_to_millis(pit_datetme) + print("Restoring back to the moment 15 seconds ago (millis): {}".format(pit_millis)) + + project_one.create_restore_job_pit(pit_millis) + + def test_data_got_restored(self, mongodb_multi_one_collection): + assert_data_got_restored(TEST_DATA, mongodb_multi_one_collection, timeout=1200) + + +def time_to_millis(date_time) -> int: + """https://stackoverflow.com/a/11111177/614239""" + epoch = datetime.datetime.utcfromtimestamp(0) + pit_millis = (date_time - epoch).total_seconds() * 1000 + return pit_millis diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_state_operator_upgrade_downgrade.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_state_operator_upgrade_downgrade.py new file mode 100644 index 000000000..dd4b3bfbe --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_state_operator_upgrade_downgrade.py @@ -0,0 +1,336 @@ +from dataclasses import dataclass +from typing import Dict, List, Optional + +import kubernetes.client +from kubernetes import client +from kubetester import create_or_update_configmap, get_deployments, read_configmap +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import ( + LEGACY_DEPLOYMENT_STATE_VERSION, + create_appdb_certs, + get_central_cluster_name, + get_custom_appdb_version, + install_official_operator, + local_operator, + log_deployments_info, +) +from tests.multicluster.conftest import cluster_spec_list + +CERT_PREFIX = "prefix" +logger = test_logger.get_test_logger(__name__) + +appdb_version = get_custom_appdb_version() + +""" +multicluster_appdb_state_operator_upgrade_downgrade ensures the correctness of the state configmaps of AppDB, when +upgrading/downgrading from/to the legacy state management (versions <= 1.27) and the current operator (from master) +while performing scaling operations accross multiple clusters. +It will always be pinned to version 1.27 (variable LEGACY_DEPLOYMENT_STATE_VERSION) for the initial deployment, so +in the future will test upgrade paths of multiple versions at a time (e.g 1.27 -> currently developed 1.30), even +though we don't officially support these paths. + +The workflow of this test is the following +Install Operator 1.27 -> Deploy OM/AppDB -> Upgrade operator (dev version) -> Scale AppDB +-> Downgrade Operator to 1.27 -> Scale AppDB +At each step, we verify that the state is correct +""" + + +def assert_cm_expected_data( + name: str, namespace: str, expected_data: Optional[Dict], central_cluster_client: kubernetes.client.ApiClient +): + # We try to read the configmap, and catch the exception in case iy doesn't exist + # We later assert this is expected from the test, when expected_data is None + state_configmap_data = None + try: + state_configmap_data = read_configmap(namespace, name, central_cluster_client) + except Exception as e: + logger.error(f"Error when trying to read the configmap {name} in namespace {namespace}: {e}") + + logger.debug(f"Asserting correctness of configmap {name} in namespace {namespace}") + + if state_configmap_data is None: + logger.debug(f"Couldn't find configmap {name} in namespace {namespace}") + assert None == expected_data + else: + logger.debug(f"The configmap {name} in namespace {namespace} contains: {state_configmap_data}") + logger.debug(f"The expected data is: {expected_data}") + assert ( + state_configmap_data == expected_data + ), f"ConfigMap data mismatch, actual: {state_configmap_data} != expected: {expected_data}" + + +# This data class helps to store the different test cases below. +# Each test case defines the AppDB replicas distribution over member clusters, and the expected values of all configmaps +# after reconciliation +@dataclass +class TestCase: + cluster_spec: List[Dict[str, int]] + expected_db_cluster_mapping: Dict[str, str] + expected_db_member_spec: Dict[str, str] + expected_db_state: Optional[Dict[str, str]] + + +# Initial Cluster Spec Test Case +initial_cluster_spec = TestCase( + cluster_spec=cluster_spec_list(["kind-e2e-cluster-2"], [3]), + expected_db_cluster_mapping={ + "kind-e2e-cluster-2": "0", + }, + expected_db_member_spec={ + "kind-e2e-cluster-2": "3", + }, + expected_db_state=None, +) + +# Scale on Upgrade Test Case +scale_on_upgrade = TestCase( + cluster_spec=cluster_spec_list(["kind-e2e-cluster-3", "kind-e2e-cluster-1", "kind-e2e-cluster-2"], [1, 1, 3]), + # Cluster 2 was already in the mapping, it keeps its index. Cluster 3 has index one because it appears before + # cluster 1 in the above spec list + expected_db_cluster_mapping={ + "kind-e2e-cluster-1": "2", + "kind-e2e-cluster-2": "0", + "kind-e2e-cluster-3": "1", + }, + # After full deployment, the LastAppliedMemberSpec Config Map should match the above cluster spec list + expected_db_member_spec={ + "kind-e2e-cluster-1": "1", + "kind-e2e-cluster-2": "3", + "kind-e2e-cluster-3": "1", + }, + # The "state" should contain the same fields as above, but marshalled in a single map + expected_db_state={ + "state": f'{{"clusterMapping":{{"kind-e2e-cluster-1":2,"kind-e2e-cluster-2":0,"kind-e2e-cluster-3":1}},"lastAppliedMemberSpec":{{"kind-e2e-cluster-1":1,"kind-e2e-cluster-2":3,"kind-e2e-cluster-3":1}},"lastAppliedMongoDBVersion":"{appdb_version}"}}' + }, +) + +# Scale on Downgrade Test Case +scale_on_downgrade = TestCase( + cluster_spec=cluster_spec_list(["kind-e2e-cluster-3", "kind-e2e-cluster-1", "kind-e2e-cluster-2"], [1, 2, 0]), + # No new cluster introduced, the mapping stays the same + expected_db_cluster_mapping={ + "kind-e2e-cluster-1": "2", + "kind-e2e-cluster-2": "0", + "kind-e2e-cluster-3": "1", + }, + # Member spec should match the new cluster spec list + expected_db_member_spec={ + "kind-e2e-cluster-1": "2", + "kind-e2e-cluster-2": "0", + "kind-e2e-cluster-3": "1", + }, + # State isn't updated as we downgrade to an operator version that doesn't manage the new state format + expected_db_state=scale_on_upgrade.expected_db_state, +) + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: str, + custom_appdb_version: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + resource["spec"]["topology"] = "MultiCluster" + # OM cluster specs (not rescaled during the test) + resource["spec"]["clusterSpecList"] = cluster_spec_list(["kind-e2e-cluster-1"], [1]) + + resource.allow_mdb_rc_versions() + logger.info(f"Creating admin secret in cluster {get_central_cluster_name()}") + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["backup"] = {"enabled": False} + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": initial_cluster_spec.cluster_spec, + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": CERT_PREFIX, + "tls": {"ca": multi_cluster_issuer_ca_configmap}, + }, + } + + return resource + + +@mark.e2e_multi_cluster_appdb_state_operator_upgrade_downgrade +class TestOpsManagerCreation: + """ + Ensure correct deployment and state of AppDB, with operator version 1.27 installed. + + """ + + # If we want to add CRDs, clone repo at a specific tag and apply CRDs + def test_install_legacy_state_official_operator( + self, + namespace: str, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + ): + logger.info( + f"Installing the official operator from helm charts, with version {LEGACY_DEPLOYMENT_STATE_VERSION}" + ) + operator = install_official_operator( + namespace, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + LEGACY_DEPLOYMENT_STATE_VERSION, + ) + operator.assert_is_running() + # Dumping deployments in logs ensure we are using the correct operator version + log_deployments_info(namespace) + + def test_create_appdb_certs_secret( + self, + namespace: str, + multi_cluster_issuer: str, + ops_manager: MongoDBOpsManager, + ): + create_appdb_certs( + namespace, + multi_cluster_issuer, + ops_manager.name + "-db", + cluster_index_with_members=[(0, 5), (1, 5), (2, 5)], + cert_prefix=CERT_PREFIX, + ) + + def test_create_om( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=700) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + def test_state_correctness( + self, namespace: str, ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient + ): + configmap_name = f"{ops_manager.name}-db-cluster-mapping" + # After deploying the old operator, we expect legacy state in the cluster + expected_data = initial_cluster_spec.expected_db_cluster_mapping + assert_cm_expected_data(configmap_name, namespace, expected_data, central_cluster_client) + + configmap_name = f"{ops_manager.name}-db-member-spec" + expected_data = initial_cluster_spec.expected_db_member_spec + # The expected data is the same for the initial deployment + assert_cm_expected_data(configmap_name, namespace, expected_data, central_cluster_client) + + +@mark.e2e_multi_cluster_appdb_state_operator_upgrade_downgrade +class TestOperatorUpgrade: + """ + Upgrade the operator to latest dev version, scale AppDB, and ensure state correctness. + """ + + def test_install_default_operator(self, namespace: str, multi_cluster_operator: Operator): + logger.info("Installing the operator built from master") + multi_cluster_operator.assert_is_running() + # Dumping deployments in logs ensure we are using the correct operator version + log_deployments_info(namespace) + + def test_scale_appdb(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + # Reordering the clusters triggers a change in the state + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = scale_on_upgrade.cluster_spec + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=500) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=250) + + def test_migrated_state_correctness( + self, namespace: str, ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient + ): + configmap_name = f"{ops_manager.name}-db-state" + # After upgrading the operator, we expect the state to be migrated to the new configmap + assert_cm_expected_data(configmap_name, namespace, scale_on_upgrade.expected_db_state, central_cluster_client) + + def test_old_state_still_exists( + self, namespace: str, ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient + ): + configmap_name = f"{ops_manager.name}-db-cluster-mapping" + assert_cm_expected_data( + configmap_name, namespace, scale_on_upgrade.expected_db_cluster_mapping, central_cluster_client + ) + configmap_name = f"{ops_manager.name}-db-member-spec" + assert_cm_expected_data( + configmap_name, namespace, scale_on_upgrade.expected_db_member_spec, central_cluster_client + ) + + +@mark.e2e_multi_cluster_appdb_state_operator_upgrade_downgrade +class TestOperatorDowngrade: + """ + Downgrade the Operator to 1.27, scale AppDB and ensure state correctness. + """ + + def test_install_legacy_state_official_operator( + self, + namespace: str, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + ): + logger.info(f"Downgrading the operator to version {LEGACY_DEPLOYMENT_STATE_VERSION}, from helm chart release") + operator = install_official_operator( + namespace, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + LEGACY_DEPLOYMENT_STATE_VERSION, + ) + operator.assert_is_running() + # Dumping deployments in logs ensure we are using the correct operator version + log_deployments_info(namespace) + + def test_om_running_after_downgrade(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager.appdb_status().assert_reaches_phase(Phase.Pending, timeout=60) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=350) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=200) + + def test_scale_appdb(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = scale_on_downgrade.cluster_spec + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=200) + + def test_state_correctness_after_downgrade( + self, namespace: str, ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient + ): + configmap_name = f"{ops_manager.name}-db-cluster-mapping" + assert_cm_expected_data( + configmap_name, namespace, scale_on_downgrade.expected_db_cluster_mapping, central_cluster_client + ) + configmap_name = f"{ops_manager.name}-db-member-spec" + assert_cm_expected_data( + configmap_name, namespace, scale_on_downgrade.expected_db_cluster_mapping, central_cluster_client + ) + configmap_name = f"{ops_manager.name}-db-state" + assert_cm_expected_data(configmap_name, namespace, scale_on_downgrade.expected_db_state, central_cluster_client) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_validation.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_validation.py new file mode 100644 index 000000000..bbbbeb313 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_appdb_validation.py @@ -0,0 +1,94 @@ +import kubernetes +import kubernetes.client +import pytest +from kubernetes.client.rest import ApiException +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + + +@fixture(scope="module") +def appdb_member_cluster_names() -> list[str]: + return ["kind-e2e-cluster-2", "kind-e2e-cluster-3"] + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_appdb_version: str, + custom_version: str, + central_cluster_client: kubernetes.client.ApiClient, + appdb_member_cluster_names: list[str], +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + try_load(resource) + + resource["spec"]["version"] = custom_version + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list(appdb_member_cluster_names, [1, 2]), + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + } + + return resource + + +@mark.e2e_multi_cluster_appdb_validation +def test_install_multi_cluster_operator(namespace: str, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb_validation +def test_validate_unique_cluster_name( + ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient +): + ops_manager.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"].append( + {"clusterName": "kind-e2e-cluster-2", "members": 1} + ) + + with pytest.raises( + ApiException, + match=r"Multiple clusters with the same name \(kind-e2e-cluster-2\) are not allowed", + ): + ops_manager.update() + + +@mark.e2e_multi_cluster_appdb_validation +def test_non_empty_clusterspec_list( + ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient +): + ops_manager.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = [] + + with pytest.raises( + ApiException, + match=r"ClusterSpecList empty is not allowed\, please define at least one cluster", + ): + ops_manager.update() + + +@mark.e2e_multi_cluster_appdb_validation +def test_empty_cluster_spec_list_single_cluster( + ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient +): + ops_manager.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"].append( + {"clusterName": "kind-e2e-cluster-4", "members": 1} + ) + ops_manager["spec"]["applicationDatabase"]["topology"] = "SingleCluster" + + with pytest.raises( + ApiException, + match=r"Single cluster AppDB deployment should have empty clusterSpecList", + ): + ops_manager.update() diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_om_networking_clusterwide.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_om_networking_clusterwide.py new file mode 100644 index 000000000..541f709ac --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_om_networking_clusterwide.py @@ -0,0 +1,284 @@ +import kubernetes +from kubetester import read_secret, try_load +from kubetester.awss3client import AwsS3Client +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MultiClusterClient +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + create_appdb_certs, + create_issuer_ca_configmap, + get_central_cluster_client, + get_central_cluster_name, + get_issuer_ca_filepath, + install_multi_cluster_operator_cluster_scoped, +) +from tests.multicluster import prepare_multi_cluster_namespaces +from tests.multicluster.conftest import cluster_spec_list, create_namespace + +from ..common.constants import MEMBER_CLUSTER_1, MEMBER_CLUSTER_2, MEMBER_CLUSTER_3 +from ..common.ops_manager.multi_cluster import ( + ops_manager_multi_cluster_with_tls_s3_backups, +) +from .conftest import create_s3_bucket_blockstore, create_s3_bucket_oplog + +# This test is for checking networking when OM is deployed in a complex multi-cluster scenario involving: +# - OM deployed in different namespace than the operator +# - OM deployed in different clusters than the operator +# - with TLS and custom CAs for AppDB and OM +# - AppDB in MultiCluster mode, but limited to a single member cluster for simplicity +# - S3 backups enabled +# - OM's external connectivity enabled + +OM_NAMESPACE = "mdb-om-mc" +OM_NAME = "om-mc" +OM_CERT_PREFIX = "om-prefix" +APPDB_CERT_PREFIX = "appdb-prefix" + + +@fixture(scope="module") +def om_issuer_ca_configmap() -> str: + return create_issuer_ca_configmap( + get_issuer_ca_filepath(), namespace=OM_NAMESPACE, name="om-issuer-ca", api_client=get_central_cluster_client() + ) + + +@fixture(scope="module") +def appdb_issuer_ca_configmap() -> str: + return create_issuer_ca_configmap( + get_issuer_ca_filepath(), + namespace=OM_NAMESPACE, + name="appdb-issuer-ca", + api_client=get_central_cluster_client(), + ) + + +@fixture(scope="module") +def ops_manager_certs(multi_cluster_clusterissuer: str): + return create_ops_manager_tls_certs( + multi_cluster_clusterissuer, + OM_NAMESPACE, + OM_NAME, + secret_name=f"{OM_CERT_PREFIX}-{OM_NAME}-cert", + clusterwide=True, + ) + + +@fixture(scope="module") +def appdb_certs(multi_cluster_clusterissuer: str): + return create_appdb_certs( + OM_NAMESPACE, + multi_cluster_clusterissuer, + OM_NAME + "-db", + cluster_index_with_members=[(0, 3)], + cert_prefix=APPDB_CERT_PREFIX, + clusterwide=True, + ) + + +@fixture(scope="module") +def s3_bucket_blockstore(aws_s3_client: AwsS3Client) -> str: + return next(create_s3_bucket_blockstore(OM_NAMESPACE, aws_s3_client, api_client=get_central_cluster_client())) + + +@fixture(scope="module") +def s3_bucket_oplog(aws_s3_client: AwsS3Client) -> str: + return next(create_s3_bucket_oplog(OM_NAMESPACE, aws_s3_client, api_client=get_central_cluster_client())) + + +@fixture(scope="function") +def ops_manager( + custom_version: str, + custom_appdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, + ops_manager_certs: str, + appdb_certs: str, + issuer_ca_filepath: str, + s3_bucket_blockstore: str, + s3_bucket_oplog: str, + om_issuer_ca_configmap: str, + appdb_issuer_ca_configmap: str, +) -> MongoDBOpsManager: + resource = ops_manager_multi_cluster_with_tls_s3_backups( + OM_NAMESPACE, OM_NAME, central_cluster_client, custom_appdb_version, s3_bucket_blockstore, s3_bucket_oplog + ) + + if try_load(resource): + return resource + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + resource["spec"]["topology"] = "MultiCluster" + resource["spec"]["clusterSpecList"] = cluster_spec_list([MEMBER_CLUSTER_2], [1], backup_configs=[{"members": 1}]) + resource["spec"]["security"] = { + "certsSecretPrefix": OM_CERT_PREFIX, + "tls": { + "ca": om_issuer_ca_configmap, + }, + } + + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list([MEMBER_CLUSTER_3], [3]), + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": APPDB_CERT_PREFIX, + "tls": { + "ca": appdb_issuer_ca_configmap, + }, + }, + } + + return resource + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_create_namespace( + namespace: str, + central_cluster_client: kubernetes.client.ApiClient, + member_cluster_clients: list[MultiClusterClient], + evergreen_task_id: str, + multi_cluster_operator_installation_config: dict[str, str], +): + image_pull_secret_name = multi_cluster_operator_installation_config["registry.imagePullSecrets"] + image_pull_secret_data = read_secret(namespace, image_pull_secret_name, api_client=central_cluster_client) + + create_namespace( + central_cluster_client, + member_cluster_clients, + evergreen_task_id, + OM_NAMESPACE, + image_pull_secret_name, + image_pull_secret_data, + ) + + prepare_multi_cluster_namespaces( + OM_NAMESPACE, + multi_cluster_operator_installation_config, + member_cluster_clients, + get_central_cluster_name(), + True, + ) + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_deploy_operator(namespace: str): + install_multi_cluster_operator_cluster_scoped(watch_namespaces=[namespace, OM_NAMESPACE]) + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_deploy_ops_manager(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_scale_om_on_different_cluster(ops_manager: MongoDBOpsManager): + ops_manager["spec"]["clusterSpecList"] = [ + { + "clusterName": MEMBER_CLUSTER_2, + "members": 1, + "backup": { + "members": 1, + }, + }, + { + "clusterName": MEMBER_CLUSTER_3, + "members": 2, + }, + ] + + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_scale_backup_daemon_on_different_cluster(ops_manager: MongoDBOpsManager): + ops_manager["spec"]["clusterSpecList"] = [ + { + "clusterName": MEMBER_CLUSTER_2, + "members": 1, + "backup": { + "members": 2, + }, + }, + { + "clusterName": MEMBER_CLUSTER_3, + "members": 2, + "backup": { + "members": 1, + }, + }, + ] + + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_enable_external_connectivity(ops_manager: MongoDBOpsManager): + ops_manager["spec"]["externalConnectivity"] = { + "type": "LoadBalancer", + "port": 9000, + } + # override the default port for the first cluster + ops_manager["spec"]["clusterSpecList"][0]["externalConnectivity"] = { + "type": "LoadBalancer", + "port": 5000, + } + # override the service type for the second cluster + ops_manager["spec"]["clusterSpecList"].append( + { + "clusterName": MEMBER_CLUSTER_1, + "members": 1, + "backup": { + "members": 1, + }, + "externalConnectivity": { + "type": "NodePort", + "port": 30006, + "annotations": { + "test-annotation": "test-value", + }, + }, + } + ) + + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_multi_cluster_om_networking_clusterwide +def test_external_services_are_created(ops_manager: MongoDBOpsManager): + _, external = ops_manager.services(MEMBER_CLUSTER_1) + assert external.spec.type == "NodePort" + assert external.metadata.annotations == {"test-annotation": "test-value"} + assert len(external.spec.ports) == 2 + assert external.spec.ports[0].port == 30006 + assert external.spec.ports[0].target_port == 8443 + + _, external = ops_manager.services(MEMBER_CLUSTER_2) + assert external.spec.type == "LoadBalancer" + assert len(external.spec.ports) == 2 + assert external.spec.ports[0].port == 5000 + assert external.spec.ports[0].target_port == 8443 + + _, external = ops_manager.services(MEMBER_CLUSTER_3) + assert external.spec.type == "LoadBalancer" + assert len(external.spec.ports) == 2 + assert external.spec.ports[0].port == 9000 + assert external.spec.ports[0].target_port == 8443 diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_om_validation.py b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_om_validation.py new file mode 100644 index 000000000..3170c79ad --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_appdb/multicluster_om_validation.py @@ -0,0 +1,74 @@ +import kubernetes +import pytest +from kubernetes.client.rest import ApiException +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.multicluster.conftest import cluster_spec_list + + +@fixture(scope="module") +def appdb_member_cluster_names() -> list[str]: + return ["kind-e2e-cluster-2", "kind-e2e-cluster-3"] + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: str, + custom_appdb_version: str, + appdb_member_cluster_names: list[str], + central_cluster_client: kubernetes.client.ApiClient, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + resource["spec"]["topology"] = "MultiCluster" + resource["spec"]["clusterSpecList"] = cluster_spec_list(["kind-e2e-cluster-2", "kind-e2e-cluster-3"], [2, 2]) + + resource.allow_mdb_rc_versions() + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["backup"] = {"enabled": False} + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list(appdb_member_cluster_names, [2, 3]), + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + } + + return resource + + +@mark.e2e_multi_cluster_om_validation +def test_install_multi_cluster_operator(namespace: str, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_om_validation +def test_topology_is_specified(ops_manager: MongoDBOpsManager): + del ops_manager["spec"]["topology"] + + with pytest.raises( + ApiException, + match=r"Topology 'MultiCluster' must be specified.*", + ): + ops_manager.update() + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_om_validation +def test_validate_cluster_spec_list(ops_manager: MongoDBOpsManager): + ops_manager["spec"]["topology"] = "MultiCluster" + ops_manager["spec"]["clusterSpecList"] = [] + + with pytest.raises( + ApiException, + match=r"At least one ClusterSpecList entry must be specified for MultiCluster mode OM", + ): + ops_manager.update() diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_om/__init__.py b/docker/mongodb-enterprise-tests/tests/multicluster_om/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx-service.yaml b/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx-service.yaml new file mode 100644 index 000000000..0496c3751 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-ext-svc-interconnected +spec: + ports: + - port: 8180 + protocol: TCP + targetPort: 8180 + selector: + run: nginx + type: LoadBalancer diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx.conf b/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx.conf new file mode 100644 index 000000000..b57fd4fd1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx.conf @@ -0,0 +1,13 @@ +events {} +stream { + upstream ops-manager { + server om-mc-1-svc-ext.mongodb-test.interconnected:9000; + server om-mc-2-svc-ext.mongodb-test.interconnected:9000; + server om-mc-3-svc-ext.mongodb-test.interconnected:9000; + } + + server { + listen 8180; + proxy_pass ops-manager; + } +} diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx.yaml b/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx.yaml new file mode 100644 index 000000000..779e82c4c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_om/fixtures/nginx.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx +spec: + replicas: 1 + selector: + matchLabels: + run: nginx + template: + metadata: + labels: + run: nginx + spec: + containers: + - image: macbre/nginx-http3:latest + name: nginx + volumeMounts: + - mountPath: /etc/nginx/nginx.conf + name: nginx-conf + subPath: nginx.conf + volumes: + - configMap: + name: nginx-conf + name: nginx-conf diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py b/docker/mongodb-enterprise-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py new file mode 100644 index 000000000..1f1f3571f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_om/multicluster_om_appdb_no_mesh.py @@ -0,0 +1,636 @@ +import datetime +import json +import time +from typing import List + +import kubernetes +import pymongo +import pytest +import yaml +from kubernetes import client +from kubetester import create_or_update_configmap, create_or_update_service, try_load +from kubetester.awss3client import AwsS3Client +from kubetester.certs import ( + create_multi_cluster_mongodb_tls_certs, + create_ops_manager_tls_certs, +) +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as _fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.mongodb_multi import MongoDBMulti, MultiClusterClient +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + assert_data_got_restored, + create_appdb_certs, + get_central_cluster_client, + get_member_cluster_clients, + update_coredns_hosts, +) +from tests.multicluster_appdb.conftest import ( + create_s3_bucket_blockstore, + create_s3_bucket_oplog, +) + +from .. import test_logger +from ..common.constants import MEMBER_CLUSTER_1, MEMBER_CLUSTER_2, MEMBER_CLUSTER_3 +from ..common.ops_manager.multi_cluster import ( + ops_manager_multi_cluster_with_tls_s3_backups, +) +from ..multicluster.conftest import cluster_spec_list + +# This test is for checking networking when OM is deployed without Service Mesh: +# - with TLS and custom CAs for AppDB and OM +# - AppDB monitoring enabled +# - AppDB in MultiCluster mode, but limited to a single member cluster for simplicity +# - S3 backups enabled +# - OM's external connectivity enabled + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +OM_NAME = "om-mc" +OM_CERT_PREFIX = "om-prefix" +APPDB_CERT_PREFIX = "appdb-prefix" + +# The hostname "nginx-ext-svc-interconnected" is used as the service name, since it does not allow "." in names. +# The "nginx-ext-svc.interconnected" is required since all hostnames needed to be under the "interconnected" TLD in the CoreDNS configuration. +# There is a CoreDNS configuration below that overwrites requests from "nginx-ext-svc-interconnected" to "nginx-ext-svc.interconnected". +NGINX_EXT_SVC_HOSTNAME = "nginx-ext-svc-interconnected" +NGINX_EXT_SVC_COREDNS_HOSTNAME = "nginx-ext-svc.interconnected" + +# The OM external service name are not used by mongodb's or the operator. +# They are only used so that nginx can redirect requests to these hostnames, and we add them in the CoreDNS config. +# This is basically a way for nginx to distribute requests between the 3 load balancer IPs. +OM_1_EXT_SVC_HOSTNAME = "om-mc-1-svc-ext.mongodb-test.interconnected" +OM_2_EXT_SVC_HOSTNAME = "om-mc-2-svc-ext.mongodb-test.interconnected" +OM_3_EXT_SVC_HOSTNAME = "om-mc-3-svc-ext.mongodb-test.interconnected" + +OM_DB_0_0_SVC_HOSTNAME = "om-mc-db-0-0.kind-e2e-cluster-1.interconnected" +OM_DB_1_0_SVC_HOSTNAME = "om-mc-db-1-0.kind-e2e-cluster-2.interconnected" +OM_DB_1_1_SVC_HOSTNAME = "om-mc-db-1-1.kind-e2e-cluster-2.interconnected" +OM_DB_2_0_SVC_HOSTNAME = "om-mc-db-2-0.kind-e2e-cluster-3.interconnected" +OM_DB_2_1_SVC_HOSTNAME = "om-mc-db-2-1.kind-e2e-cluster-3.interconnected" +OM_DB_2_2_SVC_HOSTNAME = "om-mc-db-2-2.kind-e2e-cluster-3.interconnected" +MDB_0_0_SVC_HOSTNAME = "multi-cluster-replica-set-0-0.kind-e2e-cluster-1.interconnected" +MDB_1_0_SVC_HOSTNAME = "multi-cluster-replica-set-1-0.kind-e2e-cluster-2.interconnected" +MDB_2_0_SVC_HOSTNAME = "multi-cluster-replica-set-2-0.kind-e2e-cluster-3.interconnected" +CERT_SECRET_PREFIX = "clustercert" +MDB_RESOURCE = "multi-cluster-replica-set" +BUNDLE_SECRET_NAME = f"{CERT_SECRET_PREFIX}-{MDB_RESOURCE}-cert" + +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, multi_cluster_issuer: str): + additional_domains = [OM_1_EXT_SVC_HOSTNAME, OM_2_EXT_SVC_HOSTNAME, OM_3_EXT_SVC_HOSTNAME, NGINX_EXT_SVC_HOSTNAME] + + return create_ops_manager_tls_certs( + multi_cluster_issuer, + namespace, + OM_NAME, + secret_name=f"{OM_CERT_PREFIX}-{OM_NAME}-cert", + additional_domains=additional_domains, + ) + + +@fixture(scope="module") +def appdb_certs(namespace: str, multi_cluster_issuer: str): + additional_domains = [ + OM_DB_0_0_SVC_HOSTNAME, + OM_DB_1_0_SVC_HOSTNAME, + OM_DB_1_1_SVC_HOSTNAME, + OM_DB_2_0_SVC_HOSTNAME, + OM_DB_2_1_SVC_HOSTNAME, + OM_DB_2_2_SVC_HOSTNAME, + ] + + return create_appdb_certs( + namespace, + multi_cluster_issuer, + OM_NAME + "-db", + cluster_index_with_members=[(0, 1), (1, 2), (2, 3)], + cert_prefix=APPDB_CERT_PREFIX, + additional_domains=additional_domains, + ) + + +@fixture(scope="module") +def s3_bucket_blockstore(namespace: str, aws_s3_client: AwsS3Client) -> str: + return next(create_s3_bucket_blockstore(namespace, aws_s3_client, api_client=get_central_cluster_client())) + + +@fixture(scope="module") +def s3_bucket_oplog(namespace: str, aws_s3_client: AwsS3Client) -> str: + return next(create_s3_bucket_oplog(namespace, aws_s3_client, api_client=get_central_cluster_client())) + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_configure_dns(disable_istio): + host_mappings = [ + ( + "172.18.255.211", + NGINX_EXT_SVC_COREDNS_HOSTNAME, + ), + ( + "172.18.255.212", + OM_DB_0_0_SVC_HOSTNAME, + ), + ( + "172.18.255.213", + OM_1_EXT_SVC_HOSTNAME, + ), + ( + "172.18.255.214", + MDB_0_0_SVC_HOSTNAME, + ), + ( + "172.18.255.221", + OM_DB_1_0_SVC_HOSTNAME, + ), + ( + "172.18.255.222", + OM_DB_1_1_SVC_HOSTNAME, + ), + ( + "172.18.255.223", + OM_2_EXT_SVC_HOSTNAME, + ), + ( + "172.18.255.224", + MDB_1_0_SVC_HOSTNAME, + ), + ( + "172.18.255.231", + OM_DB_2_0_SVC_HOSTNAME, + ), + ( + "172.18.255.232", + OM_DB_2_1_SVC_HOSTNAME, + ), + ( + "172.18.255.233", + OM_DB_2_2_SVC_HOSTNAME, + ), + ( + "172.18.255.234", + OM_3_EXT_SVC_HOSTNAME, + ), + ( + "172.18.255.235", + MDB_2_0_SVC_HOSTNAME, + ), + ] + + # This rule rewrites nginx-ext-svc-interconnected that ends with *-interconnected to *.interconnected + # It's useful when kubefwd propagates LoadBalancer IP by service name nginx-ext-svc-interconnected, + # but for CoreDNS configuration to work it has to end with .interconnected domain (nginx-ext-svc.interconnected) + rewrite_nginx_hostname = f"rewrite name exact {NGINX_EXT_SVC_HOSTNAME} {NGINX_EXT_SVC_COREDNS_HOSTNAME}" + + for c in get_member_cluster_clients(): + update_coredns_hosts( + host_mappings=host_mappings, + api_client=c.api_client, + cluster_name=c.cluster_name, + additional_rules=[rewrite_nginx_hostname], + ) + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_disable_istio(disable_istio): + logger.info("Istio disabled") + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_configure_nginx(namespace: str): + cluster_client = get_central_cluster_client() + + conf = open(_fixture("nginx.conf")).read() + data = {"nginx.conf": conf} + create_or_update_configmap(namespace, "nginx-conf", data, api_client=cluster_client) + + nginx_deployment = yaml.safe_load(open(_fixture("nginx.yaml"))) + apps_api = client.AppsV1Api(api_client=cluster_client) + try: + apps_api.create_namespaced_deployment(namespace, nginx_deployment) + except kubernetes.client.ApiException as e: + if e.status == 409: + apps_api.replace_namespaced_deployment("nginx", namespace, nginx_deployment) + else: + raise Exception(f"failed to create nginx_deployment: {e}") + + nginx_service = yaml.safe_load(open(_fixture("nginx-service.yaml"))) + create_or_update_service(namespace, service=nginx_service) + + +@fixture(scope="function") +def ops_manager( + custom_version: str, + namespace: str, + custom_appdb_version: str, + central_cluster_client: kubernetes.client.ApiClient, + ops_manager_certs: str, + appdb_certs: str, + issuer_ca_filepath: str, + s3_bucket_blockstore: str, + s3_bucket_oplog: str, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBOpsManager: + resource = ops_manager_multi_cluster_with_tls_s3_backups( + namespace, OM_NAME, central_cluster_client, custom_appdb_version, s3_bucket_blockstore, s3_bucket_oplog + ) + + if try_load(resource): + return resource + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + resource["spec"]["topology"] = "MultiCluster" + resource["spec"]["clusterSpecList"] = [ + { + "clusterName": MEMBER_CLUSTER_1, + "members": 1, + "backup": { + "members": 1, + }, + }, + { + "clusterName": MEMBER_CLUSTER_2, + "members": 1, + "backup": { + "members": 1, + }, + }, + { + "clusterName": MEMBER_CLUSTER_3, + "members": 1, + "backup": { + "members": 2, + }, + }, + ] + + resource["spec"]["security"] = { + "certsSecretPrefix": OM_CERT_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + resource["spec"]["externalConnectivity"] = { + "type": "LoadBalancer", + "port": 9000, + } + resource["spec"]["opsManagerURL"] = f"https://{NGINX_EXT_SVC_HOSTNAME}:8180" + + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": APPDB_CERT_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + }, + "externalAccess": {"externalDomain": "some.custom.domain"}, + } + resource["spec"]["applicationDatabase"]["clusterSpecList"] = [ + { + "clusterName": MEMBER_CLUSTER_1, + "members": 1, + "externalAccess": { + "externalDomain": "kind-e2e-cluster-1.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing2", + "port": 27019, + }, + ], + } + }, + }, + }, + { + "clusterName": MEMBER_CLUSTER_2, + "members": 2, + "externalAccess": { + "externalDomain": "kind-e2e-cluster-2.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing2", + "port": 27019, + }, + ], + } + }, + }, + }, + { + "clusterName": MEMBER_CLUSTER_3, + "members": 3, + "externalAccess": { + "externalDomain": "kind-e2e-cluster-3.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing2", + "port": 27019, + }, + ], + } + }, + }, + }, + ] + + return resource + + +@fixture(scope="function") +def server_certs( + multi_cluster_issuer: str, + mongodb_multi: MongoDBMulti, + member_cluster_clients: List[MultiClusterClient], + central_cluster_client: kubernetes.client.ApiClient, +): + return create_multi_cluster_mongodb_tls_certs( + multi_cluster_issuer, + BUNDLE_SECRET_NAME, + member_cluster_clients, + central_cluster_client, + mongodb_multi, + ) + + +@fixture(scope="function") +def mongodb_multi_collection(mongodb_multi: MongoDBMulti, ca_path: str): + + tester = mongodb_multi.tester( + port=27017, + service_names=[MDB_0_0_SVC_HOSTNAME, MDB_1_0_SVC_HOSTNAME, MDB_2_0_SVC_HOSTNAME], + external=True, + use_ssl=True, + ca_path=ca_path, + ) + + collection = pymongo.MongoClient(tester.cnx_string, **tester.default_opts)["testdb"] + + return collection["testcollection"] + + +@fixture(scope="function") +def mongodb_multi( + central_cluster_client: kubernetes.client.ApiClient, + namespace: str, + member_cluster_names, + custom_mdb_version: str, + ops_manager: MongoDBOpsManager, + multi_cluster_issuer_ca_configmap: str, +) -> MongoDBMulti: + resource = MongoDBMulti.from_yaml(_fixture("mongodb-multi.yaml"), MDB_RESOURCE, namespace) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["persistent"] = False + resource["spec"]["clusterSpecList"] = cluster_spec_list(member_cluster_names, [1, 1, 1]) + resource["spec"]["security"] = { + "certsSecretPrefix": CERT_SECRET_PREFIX, + "tls": { + "ca": multi_cluster_issuer_ca_configmap, + }, + } + + resource.configure_backup(mode="enabled") + + resource["spec"]["externalAccess"] = {} + resource["spec"]["clusterSpecList"][0]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-1.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing0", + "port": 27019, + }, + ], + } + }, + } + resource["spec"]["clusterSpecList"][1]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-2.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing1", + "port": 27019, + }, + ], + } + }, + } + resource["spec"]["clusterSpecList"][2]["externalAccess"] = { + "externalDomain": "kind-e2e-cluster-3.interconnected", + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing2", + "port": 27019, + }, + ], + } + }, + } + + create_project_config_map( + om=ops_manager, + project_name="mongodb", + mdb_name=MDB_RESOURCE, + client=central_cluster_client, + custom_ca=multi_cluster_issuer_ca_configmap, + ) + + resource.configure(ops_manager, "mongodb", api_client=get_central_cluster_client()) + + return resource + + +def create_project_config_map(om: MongoDBOpsManager, mdb_name, project_name, client, custom_ca): + name = f"{mdb_name}-config" + data = { + "baseUrl": om.om_status().get_url(), + "projectName": project_name, + "sslMMSCAConfigMap": custom_ca, + "orgId": "", + } + + create_or_update_configmap(om.namespace, name, data, client) + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_deploy_operator(multi_cluster_operator_with_monitored_appdb: Operator): + multi_cluster_operator_with_monitored_appdb.assert_is_running() + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_deploy_ops_manager(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + ops_manager.assert_appdb_monitoring_group_was_created() + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_create_mongodb_multi(server_certs: str, mongodb_multi: MongoDBMulti): + mongodb_multi.update() + mongodb_multi.assert_reaches_phase(Phase.Running, timeout=2400, ignore_errors=True) + + +@skip_if_local +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_add_test_data(mongodb_multi_collection): + max_attempts = 100 + while max_attempts > 0: + try: + mongodb_multi_collection.insert_one(TEST_DATA) + return + except Exception as e: + print(e) + max_attempts -= 1 + time.sleep(6) + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_mdb_backed_up(ops_manager: MongoDBOpsManager): + ops_manager.get_om_tester(project_name="mongodb").wait_until_backup_snapshots_are_ready(expected_count=1) + + +@skip_if_local +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_change_mdb_data(mongodb_multi_collection): + now_millis = time_to_millis(datetime.datetime.now(tz=datetime.UTC)) + print("\nCurrent time (millis): {}".format(now_millis)) + time.sleep(30) + mongodb_multi_collection.insert_one({"foo": "bar"}) + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_pit_restore(ops_manager: MongoDBOpsManager): + now_millis = time_to_millis(datetime.datetime.now(tz=datetime.UTC)) + print("\nCurrent time (millis): {}".format(now_millis)) + + pit_datetime = datetime.datetime.now(tz=datetime.UTC) - datetime.timedelta(seconds=15) + pit_millis = time_to_millis(pit_datetime) + print("Restoring back to the moment 15 seconds ago (millis): {}".format(pit_millis)) + + ops_manager.get_om_tester(project_name="mongodb").create_restore_job_pit(pit_millis) + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_mdb_ready(mongodb_multi: MongoDBMulti): + # Note: that we are not waiting for the restore jobs to get finished as PIT restore jobs get FINISHED status + # right away. + # But the agent might still do work on the cluster, so we need to wait for that to happen. + mongodb_multi.assert_reaches_phase(Phase.Pending) + mongodb_multi.assert_reaches_phase(Phase.Running) + + +@skip_if_local +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_data_got_restored(mongodb_multi_collection): + assert_data_got_restored(TEST_DATA, mongodb_multi_collection, timeout=600) + + +def time_to_millis(date_time) -> int: + epoch = datetime.datetime.fromtimestamp(0, tz=datetime.UTC) + pit_millis = (date_time - epoch).total_seconds() * 1000 + return pit_millis + + +@mark.e2e_multi_cluster_om_appdb_no_mesh +def test_telemetry_configmap(namespace: str): + telemetry_configmap_name = "mongodb-enterprise-operator-telemetry" + config = KubernetesTester.read_configmap(namespace, telemetry_configmap_name) + + try: + payload_string = config.get("lastSendPayloadDeployments") + payload = json.loads(payload_string) + # Perform a rudimentary check + assert isinstance(payload, list), "payload should be a list" + assert len(payload) == 2, "payload should not be empty" + + assert payload[0]["properties"]["type"] == "ReplicaSet" + assert payload[0]["properties"]["externalDomains"] == "ClusterSpecific" + assert payload[1]["properties"]["type"] == "OpsManager" + assert payload[1]["properties"]["externalDomains"] == "Mixed" + except json.JSONDecodeError: + pytest.fail("payload contains invalid JSON data") diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_om/multicluster_om_clusterwide_operator_not_in_mesh_networking.py b/docker/mongodb-enterprise-tests/tests/multicluster_om/multicluster_om_clusterwide_operator_not_in_mesh_networking.py new file mode 100644 index 000000000..62f4d4bb5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_om/multicluster_om_clusterwide_operator_not_in_mesh_networking.py @@ -0,0 +1,294 @@ +from typing import Optional + +import kubernetes +from kubernetes.client.rest import ApiException +from kubetester import read_secret, read_service, try_load +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.common.constants import MEMBER_CLUSTER_2, MEMBER_CLUSTER_3 +from tests.common.ops_manager.multi_cluster import ( + ops_manager_multi_cluster_with_tls_s3_backups, +) +from tests.conftest import ( + create_appdb_certs, + create_issuer_ca_configmap, + get_aws_s3_client, + get_central_cluster_client, + get_central_cluster_name, + get_custom_appdb_version, + get_custom_om_version, + get_evergreen_task_id, + get_issuer_ca_filepath, + get_member_cluster_api_client, + get_member_cluster_clients, + get_multi_cluster_operator_installation_config, + get_namespace, + install_multi_cluster_operator_cluster_scoped, + update_coredns_hosts, +) +from tests.multicluster import prepare_multi_cluster_namespaces +from tests.multicluster.conftest import cluster_spec_list, create_namespace +from tests.multicluster_appdb.conftest import ( + create_s3_bucket_blockstore, + create_s3_bucket_oplog, +) + +# This test requires a cluster-wide operator. +# To run it locally you must specify the following in private-context: +# WATCH_NAMESPACE="mdb-om-mc,$NAMESPACE" +# When running in EVG or the operator is in a pod (LOCAL_OPERATOR=false) then it's handled automatically. + +# This test is for checking networking when: +# - OM deployed in different namespace than the operator +# - OM deployed in different clusters than the operator +# - The operator is not in the same Service Mesh as member clusters (headless service fqdn is not accessible) +# - TLS enabled with custom CAs for AppDB and OM +# - AppDB in MultiCluster mode, but limited to a single member cluster for simplicity +# - S3 backups enabled +# - OM's external connectivity enabled +# +# Test procedure: +# - deploy AppDB and OM +# - wait until the operator failed connecting to deployed OM API endpoint +# - configure external connectivity and point spec.opsManagerURL to external service's external IP using .interconnected hostname + + +class MultiClusterOMClusterWideTestHelper: + OM_NAMESPACE = "mdb-om-mc" + OM_NAME = "om-mc" + OM_CERT_PREFIX = "om-prefix" + APPDB_CERT_PREFIX = "appdb-prefix" + APPDB_CLUSTER_INDEX_WITH_MEMBERS = [(0, 3)] + + om_issuer_ca_configmap: str + appdb_issuer_ca_configmap: str + ops_manager_cert_secret_name: str + appdb_cert_secret_prefix: str + s3_bucket_blockstore: str + s3_bucket_oplog: str + + def prepare_namespaces(self): + print("MultiClusterOMClusterWideTestHelper: prepare_namespaces") + image_pull_secret_name = get_multi_cluster_operator_installation_config(get_namespace())[ + "registry.imagePullSecrets" + ] + image_pull_secret_data = read_secret( + get_namespace(), image_pull_secret_name, api_client=get_central_cluster_client() + ) + + create_namespace( + get_central_cluster_client(), + get_member_cluster_clients(), + get_evergreen_task_id(), + self.OM_NAMESPACE, + image_pull_secret_name, + image_pull_secret_data, + ) + + prepare_multi_cluster_namespaces( + self.OM_NAMESPACE, + get_multi_cluster_operator_installation_config(get_namespace()), + get_member_cluster_clients(), + get_central_cluster_name(), + True, + ) + + def create_tls_resources( + self, multi_cluster_cluster_issuer: str, additional_om_domains: Optional[list[str]] = None + ): + print("MultiClusterOMClusterWideTestHelper: create_tls_resources") + + self.om_issuer_ca_configmap = create_issuer_ca_configmap( + get_issuer_ca_filepath(), + namespace=self.OM_NAMESPACE, + name="om-issuer-ca", + api_client=get_central_cluster_client(), + ) + + self.appdb_issuer_ca_configmap = create_issuer_ca_configmap( + get_issuer_ca_filepath(), + namespace=self.OM_NAMESPACE, + name="appdb-issuer-ca", + api_client=get_central_cluster_client(), + ) + + self.ops_manager_cert_secret_name = create_ops_manager_tls_certs( + multi_cluster_cluster_issuer, + self.OM_NAMESPACE, + self.OM_NAME, + secret_name=f"{self.OM_CERT_PREFIX}-{self.OM_NAME}-cert", + clusterwide=True, + additional_domains=additional_om_domains, + ) + + self.appdb_cert_secret_prefix = create_appdb_certs( + self.OM_NAMESPACE, + multi_cluster_cluster_issuer, + self.OM_NAME + "-db", + cluster_index_with_members=self.APPDB_CLUSTER_INDEX_WITH_MEMBERS, + cert_prefix=self.APPDB_CERT_PREFIX, + clusterwide=True, + ) + + self.s3_bucket_blockstore = next( + create_s3_bucket_blockstore(self.OM_NAMESPACE, get_aws_s3_client(), api_client=get_central_cluster_client()) + ) + self.s3_bucket_oplog = next( + create_s3_bucket_oplog(self.OM_NAMESPACE, get_aws_s3_client(), api_client=get_central_cluster_client()) + ) + + def ops_manager(self) -> MongoDBOpsManager: + central_cluster_client = get_central_cluster_client() + + resource = ops_manager_multi_cluster_with_tls_s3_backups( + self.OM_NAMESPACE, + self.OM_NAME, + central_cluster_client, + get_custom_appdb_version(), + self.s3_bucket_blockstore, + self.s3_bucket_oplog, + ) + + if try_load(resource): + return resource + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = get_custom_om_version() + resource["spec"]["topology"] = "MultiCluster" + ## Force creating headless services for internal connectivity + resource["spec"]["internalConnectivity"] = { + "type": "ClusterIP", + "ClusterIP": "None", + } + resource["spec"]["clusterSpecList"] = cluster_spec_list( + [MEMBER_CLUSTER_2], [1], backup_configs=[{"members": 1}] + ) + resource["spec"]["security"] = { + "certsSecretPrefix": self.OM_CERT_PREFIX, + "tls": { + "ca": self.om_issuer_ca_configmap, + }, + } + + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list([MEMBER_CLUSTER_3], [3]), + "version": get_custom_appdb_version(), + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": self.APPDB_CERT_PREFIX, + "tls": { + "ca": self.appdb_issuer_ca_configmap, + }, + }, + } + + return resource + + +@fixture(scope="module") +def om_test_helper(multi_cluster_clusterissuer: str) -> MultiClusterOMClusterWideTestHelper: + test_helper = MultiClusterOMClusterWideTestHelper() + test_helper.prepare_namespaces() + test_helper.create_tls_resources( + multi_cluster_clusterissuer, + additional_om_domains=[get_om_external_host(test_helper.OM_NAMESPACE, test_helper.OM_NAME)], + ) + + return test_helper + + +@mark.e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking +def test_deploy_operator(om_test_helper: MultiClusterOMClusterWideTestHelper): + install_multi_cluster_operator_cluster_scoped(watch_namespaces=[get_namespace(), om_test_helper.OM_NAMESPACE]) + + +@mark.e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking +def test_deploy_ops_manager(om_test_helper: MultiClusterOMClusterWideTestHelper): + ops_manager = om_test_helper.ops_manager() + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Pending, msg_regexp="Enabling monitoring", timeout=900) + # the operator cannot connect to OM instance so it cannot finish configuring the deployment + ops_manager.om_status().assert_reaches_phase( + Phase.Failed, + msg_regexp="Failed to create an admin user in Ops Manager: Error sending POST request", + timeout=900, + ) + + +def get_external_service_ip(ops_manager: MongoDBOpsManager): + try: + svc = read_service( + ops_manager.namespace, + ops_manager.external_svc_name(MEMBER_CLUSTER_2), + get_member_cluster_api_client(MEMBER_CLUSTER_2), + ) + except ApiException as e: + if e.status == 404: + return None + else: + raise e + + external_ip = None + if svc.status.load_balancer.ingress: + external_ip = svc.status.load_balancer.ingress[0].ip + + return external_ip + + +@mark.e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking +def test_enable_external_connectivity(om_test_helper: MultiClusterOMClusterWideTestHelper): + ops_manager = om_test_helper.ops_manager() + # TODO make it only to work by overriding it in one member cluster + ops_manager["spec"]["externalConnectivity"] = { + "type": "LoadBalancer", + "port": 9000, + } + ops_manager.update() + + def external_ip_available(om: MongoDBOpsManager): + return get_external_service_ip(om) + + print("Waiting for external service's external IP address") + ops_manager.wait_for(external_ip_available, 60, should_raise=True) + + +def get_om_external_host(namespace: str, name: str): + return f"{name}-svc-ext.{namespace}.interconnected" + + +@mark.e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking +def test_configure_dns(om_test_helper: MultiClusterOMClusterWideTestHelper): + ops_manager = om_test_helper.ops_manager() + interconnected_field = get_om_external_host(ops_manager.namespace, ops_manager.name) + ip = get_external_service_ip(ops_manager) + + # let's make sure that every client can connect to OM. + for c in get_member_cluster_clients(): + update_coredns_hosts( + host_mappings=[(ip, interconnected_field)], + api_client=c.api_client, + cluster_name=c.cluster_name, + ) + + # let's make sure that the operator can connect to OM via that given address. + update_coredns_hosts( + host_mappings=[(ip, interconnected_field)], + api_client=get_central_cluster_client(), + cluster_name=get_central_cluster_name(), + ) + + +@mark.e2e_multi_cluster_om_clusterwide_operator_not_in_mesh_networking +def test_set_ops_manager_url(om_test_helper: MultiClusterOMClusterWideTestHelper): + ops_manager = om_test_helper.ops_manager() + host = get_om_external_host(ops_manager.namespace, ops_manager.name) + ops_manager["spec"]["opsManagerURL"] = f"https://{host}:9000" + ops_manager.update() + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/__init__.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/__init__.py new file mode 100644 index 000000000..0f36ca753 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/__init__.py @@ -0,0 +1,333 @@ +from typing import Dict, List + +import kubernetes +import pytest +from kubernetes.client import ApiClient +from kubetester import get_statefulset +from kubetester.kubetester import KubernetesTester, is_multi_cluster +from kubetester.mongodb import MongoDB +from kubetester.mongodb_multi import MultiClusterClient +from tests import test_logger +from tests.shardedcluster.conftest import ( + get_member_cluster_clients_using_cluster_mapping, +) + +logger = test_logger.get_test_logger(__name__) + + +# Expand shard overrides (they can contain multiple shard names) and build a mapping from shard name to +# its override configuration +def expand_shard_overrides(sc_spec) -> Dict: + resource_shard_overrides = sc_spec.get("shardOverrides", []) + resource_shard_override_map = {} + for resource_override in resource_shard_overrides: + for ac_shard_name in resource_override["shardNames"]: + resource_shard_override_map[ac_shard_name] = resource_override + return resource_shard_override_map + + +# Compare the applied resource to the automation config, and ensure shard, mongos, and config server counts are correct +def validate_member_count_in_ac(sharded_cluster: MongoDB, automation_config): + resource_spec = sharded_cluster["spec"] + + if is_multi_cluster(): + shard_count = resource_spec["shardCount"] + # Cfg serv and mongos count from cluster spec lists + mongos_cluster_specs = resource_spec.get("mongos", {}).get("clusterSpecList", []) + mongos_count = sum(spec.get("members", 1) for spec in mongos_cluster_specs) # Default to 1 if no member + config_cluster_specs = resource_spec.get("configSrv", {}).get("clusterSpecList", []) + config_server_count = sum(spec.get("members", 1) for spec in config_cluster_specs) + else: + shard_count = resource_spec["shardCount"] + mongos_count = resource_spec["mongosCount"] + config_server_count = resource_spec["configServerCount"] + + automation_processes = automation_config["processes"] + automation_replica_sets = automation_config["replicaSets"] + automation_sharding = automation_config["sharding"][0] + + # Verify shard count + automation_shards = automation_sharding["shards"] + assert shard_count == len( + automation_shards + ), f"Shard count mismatch: expected {shard_count}, got {len(automation_shards)}" + + # Verify mongos count + automation_mongos_processes = [p for p in automation_processes if p["processType"] == "mongos"] + assert mongos_count == len( + automation_mongos_processes + ), f"Mongos count mismatch: expected {mongos_count}, got {len(automation_mongos_processes)}" + + # Verify config server count + config_rs_list = [ + rs for rs in automation_replica_sets if rs["_id"] == f"{sharded_cluster['metadata']['name']}-config" + ] + assert len(config_rs_list) == 1, f"There must be exactly one config server replicaset, found {len(config_rs_list)}" + config_members = config_rs_list[0]["members"] + assert config_server_count == len( + config_members + ), f"Config server count mismatch: expected {config_server_count}, got {len(config_members)}" + + logger.info(f"Cluster {sharded_cluster.name} has the correct member counts") + logger.debug(f"{shard_count} shards, {mongos_count} mongos, {config_server_count} configs") + + +def assert_member_configs(expected_member_configs: List[Dict[str, str]], ac_members: List[Dict], ac_shard_name: str): + logger.debug(f"Ensuring member config correctness of shard {ac_shard_name}") + if expected_member_configs: + for idx, ac_member in enumerate(ac_members): + expected_config = expected_member_configs[idx] + expected_priority = int(expected_config.get("priority", "1")) + expected_votes = int(expected_config.get("votes", 1)) + actual_priority = ac_member["priority"] + actual_votes = ac_member["votes"] + assert ( + expected_priority == actual_priority + ), f"Shard {ac_shard_name} member {idx}: expected priority {expected_priority}, got {actual_priority}" + assert ( + expected_votes == actual_votes + ), f"Shard {ac_shard_name} member {idx}: expected votes {expected_votes}, got {actual_votes}" + else: + # If no member config, the default value for votes and priorities is 1 + for idx, ac_member in enumerate(ac_members): + assert ( + ac_member["priority"] == 1 + ), f"Shard {ac_shard_name} member {idx}: expected default priority 1, got {ac_member['priority']}" + assert ( + ac_member["votes"] == 1 + ), f"Shard {ac_shard_name} member {idx}: expected default votes 1, got {ac_member['votes']}" + logger.info(f"Shard {ac_shard_name} has the correct values for votes and priorities") + + +# Compare the applied resource to the automation config, and ensure Members, votes and priorities are correct +def validate_shard_configurations_in_ac(sharded_cluster: MongoDB, automation_config): + resource_spec = sharded_cluster["spec"] + resource_mongods_per_shard = resource_spec["mongodsPerShardCount"] + resource_shard_overrides = resource_spec.get("shardOverrides", []) + ac_replica_sets = automation_config["replicaSets"] + ac_sharding = automation_config["sharding"][0] + ac_automation_shards = ac_sharding["shards"] + # Build a mapping from shard name to its override configuration + resource_shard_override_map = {} + for resource_override in resource_shard_overrides: + for ac_shard_name in resource_override["shardNames"]: + resource_shard_override_map[ac_shard_name] = resource_override + for ac_shard in ac_automation_shards: + ac_shard_name = ac_shard["_id"] + ac_replica_set_name = ac_shard["rs"] + # Filter by name to get replicaset config + rs_list = list(filter(lambda rs: rs["_id"] == ac_replica_set_name, ac_replica_sets)) + if rs_list: + rs_config = rs_list[0] + else: + raise ValueError(f"Replica set {ac_replica_set_name} not found in automation config.") + ac_members = rs_config["members"] + resource_override = resource_shard_override_map.get(ac_shard_name) + if resource_override: + expected_member_configs = resource_override.get("memberConfig", []) + expected_members = resource_override.get("members", len(expected_member_configs)) + else: + expected_members = resource_mongods_per_shard + expected_member_configs = None # By default, there is no member config + assert expected_members == len( + ac_members + ), f"Shard {ac_shard_name}: expected {expected_members} members, got {len(ac_members)}" + # Verify member configurations + assert_member_configs(expected_member_configs, ac_members, ac_shard_name) + + +# Compare the applied resource to the automation config, and ensure Members, votes and priorities are correct +def validate_shard_configurations_in_ac_multi(sharded_cluster: MongoDB, automation_config): + resource_spec = sharded_cluster["spec"] + ac_replica_sets = automation_config["replicaSets"] + ac_sharding = automation_config["sharding"][0] + + ac_automation_shards = ac_sharding["shards"] + resource_shard_override_map = expand_shard_overrides(resource_spec) + + # Build default member configurations + default_shard_spec = resource_spec.get("shard", {}) + default_cluster_spec_list = default_shard_spec.get("clusterSpecList", []) + default_member_configs = [] + for cluster_spec in default_cluster_spec_list: + members = cluster_spec.get("members", 1) + member_configs = cluster_spec.get("memberConfig", [{"priority": "1", "votes": 1}] * members) + default_member_configs.extend(member_configs) + + for ac_shard in ac_automation_shards: + ac_shard_name = ac_shard["_id"] + ac_replica_set_name = ac_shard["rs"] + + # Get replicaset config from automation config + rs_list = list(filter(lambda rs: rs["_id"] == ac_replica_set_name, ac_replica_sets)) + if rs_list: + rs_config = rs_list[0] + else: + raise ValueError(f"Replica set {ac_replica_set_name} not found in automation config.") + + ac_members = rs_config["members"] + resource_override = resource_shard_override_map.get(ac_shard_name) + + # Build expected member configs + if resource_override: + cluster_spec_list = resource_override.get("clusterSpecList", []) + expected_member_configs = [] + for cluster_spec in cluster_spec_list: + members = cluster_spec.get("members", 1) + member_configs = cluster_spec.get("memberConfig", [{}] * members) + expected_member_configs.extend(member_configs) + else: + expected_member_configs = default_member_configs + + logger.debug(f"Testing {ac_shard_name} member count, expecting {len(expected_member_configs)} members") + assert len(ac_members) == len( + expected_member_configs + ), f"Shard {ac_shard_name}: expected {len(expected_member_configs)} members, got {len(ac_members)}" + logger.info(f"Shard {ac_shard_name} has the correct number of members: {len(expected_member_configs)}") + + # Verify member configurations + assert_member_configs(expected_member_configs, ac_members, ac_shard_name) + + +def build_expected_statefulsets(sc) -> Dict[str, int]: + sc_name = sc.name + sc_spec = sc["spec"] + shard_count = sc_spec["shardCount"] + mongods_per_shard = sc_spec["mongodsPerShardCount"] + shard_override_map = expand_shard_overrides(sc_spec) + + # Dict holding expected sts names and expected replica counts + expected_statefulsets = {} + + # Shards sts + for i in range(shard_count): + shard_sts_name = f"{sc_name}-{i}" + override = shard_override_map.get(shard_sts_name) + members = mongods_per_shard + if override and "members" in override: + # If 'members' is not specified in the override, we keep the default 'mongodsPerShardCount' + members = override.get("members") + expected_statefulsets[shard_sts_name] = members + + # Config server and mongos sts + expected_statefulsets[f"{sc_name}-config"] = sc_spec["configServerCount"] + expected_statefulsets[f"{sc_name}-mongos"] = sc_spec["mongosCount"] + + return expected_statefulsets + + +def build_expected_statefulsets_multi(sc: MongoDB, cluster_mapping: Dict[str, int]) -> Dict[str, Dict[str, int]]: + sc_name = sc.name + sc_spec = sc["spec"] + shard_count = sc_spec["shardCount"] + shard_override_map = expand_shard_overrides(sc_spec) + + # Dict holding expected sts per cluster: {cluster_name: {sts_name: replica_count}} + expected_clusters_sts = {} + + # Process each shard + for i in range(shard_count): + shard_name = f"{sc_name}-{i}" + override = shard_override_map.get(shard_name) + if override: + shard_cluster_spec_list = override.get("clusterSpecList", []) + else: + default_shard_spec = sc_spec.get("shard", {}) + shard_cluster_spec_list = default_shard_spec.get("clusterSpecList", []) + + update_expected_sts(shard_name, shard_cluster_spec_list, expected_clusters_sts, cluster_mapping) + + # Process config servers and mongos + config_spec = sc_spec.get("configSrv", {}) + config_cluster_spec_list = config_spec.get("clusterSpecList", []) + update_expected_sts(f"{sc_name}-config", config_cluster_spec_list, expected_clusters_sts, cluster_mapping) + + mongos_spec = sc_spec.get("mongos", {}) + mongos_cluster_spec_list = mongos_spec.get("clusterSpecList", []) + update_expected_sts(f"{sc_name}-mongos", mongos_cluster_spec_list, expected_clusters_sts, cluster_mapping) + + logger.debug(f"Expected statefulsets: {expected_clusters_sts}") + return expected_clusters_sts + + +def update_expected_sts(sts_prefix: str, clusterspeclist, expected_clusters_sts, cluster_mapping: Dict[str, int]): + for idx, cluster_spec in enumerate(clusterspeclist): + cluster_name = cluster_spec["clusterName"] + # The name of the sts is based on the unique cluster index, stored in the state configmap + cluster_idx_in_state = cluster_mapping.get(cluster_name) + if cluster_idx_in_state is None: + raise AssertionError(f"Cluster {cluster_name} ist not in the state, cluster mapping is {cluster_mapping}") + members = cluster_spec.get("members", 1) + sts_name = f"{sts_prefix}-{cluster_idx_in_state}" + if cluster_name not in expected_clusters_sts: + expected_clusters_sts[cluster_name] = {} + expected_clusters_sts[cluster_name][sts_name] = members + + +# Fetch each expected statefulset from the cluster and assert correct replica count +def validate_correct_sts_in_cluster( + expected_statefulsets: Dict[str, int], namespace: str, cluster_name: str, client: ApiClient +): + for sts_name, expected_replicas in expected_statefulsets.items(): + try: + sts = get_statefulset(namespace, sts_name, client) + except kubernetes.client.exceptions.ApiException as e: + if e.status == 404: + raise AssertionError( + f"StatefulSet {sts_name} not found in cluster {cluster_name} namespace {namespace}." + ) + else: + raise + + actual_replicas = sts.spec.replicas + assert ( + actual_replicas == expected_replicas + ), f"StatefulSet {sts_name} in cluster {cluster_name}: expected {expected_replicas} replicas, got {actual_replicas}" + logger.info( + f"StatefulSet {sts_name} in cluster {cluster_name} has the correct number of replicas: {actual_replicas}" + ) + + +def validate_correct_sts_in_cluster_multi( + expected_statefulsets_per_cluster: Dict[str, Dict[str, int]], + namespace: str, + member_cluster_clients: list[MultiClusterClient], +): + for cluster_name, sts_dict in expected_statefulsets_per_cluster.items(): + # Retrieve client from the list + client = None + for member_cluster_client in member_cluster_clients: + if member_cluster_client.cluster_name == cluster_name: + client = member_cluster_client + if not client: + raise AssertionError(f"ApiClient for cluster {cluster_name} not found.") + + validate_correct_sts_in_cluster(sts_dict, namespace, cluster_name, client.api_client) + + +def assert_correct_automation_config_after_scaling(sc: MongoDB): + config = KubernetesTester.get_automation_config() + validate_member_count_in_ac(sc, config) + validate_shard_configurations_in_ac_multi(sc, config) + + +def assert_shard_sts_members_count(sc: MongoDB, shard_in_cluster_distribution: List[List[int]]): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + for shard_idx, shard_distribution in enumerate(shard_in_cluster_distribution): + sts_name = sc.shard_statefulset_name(shard_idx, cluster_member_client.cluster_index) + expected_shard_members_in_cluster = shard_distribution[cluster_member_client.cluster_index] + cluster_member_client.assert_sts_members_count(sts_name, sc.namespace, expected_shard_members_in_cluster) + + +def assert_config_srv_sts_members_count(sc: MongoDB, config_srv_distribution: List[int]): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + sts_name = sc.config_srv_statefulset_name(cluster_member_client.cluster_index) + expected_config_srv_members_in_cluster = config_srv_distribution[cluster_member_client.cluster_index] + cluster_member_client.assert_sts_members_count(sts_name, sc.namespace, expected_config_srv_members_in_cluster) + + +def assert_mongos_sts_members_count(sc: MongoDB, mongos_distribution: List[int]): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + sts_name = sc.mongos_statefulset_name(cluster_member_client.cluster_index) + expected_mongos_members_in_cluster = mongos_distribution[cluster_member_client.cluster_index] + cluster_member_client.assert_sts_members_count(sts_name, sc.namespace, expected_mongos_members_in_cluster) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/e2e_multi_cluster_sharded_external_access_no_ext_domain.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/e2e_multi_cluster_sharded_external_access_no_ext_domain.py new file mode 100644 index 000000000..6c8b8d08b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/e2e_multi_cluster_sharded_external_access_no_ext_domain.py @@ -0,0 +1,113 @@ +from collections import defaultdict +from typing import Dict, List, Optional + +import kubernetes +from kubernetes import client +from kubetester import find_fixture, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import get_member_cluster_api_client, get_member_cluster_names +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + setup_external_access, +) + +MDB_RESOURCE_NAME = "sh" +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def sharded_cluster(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-multi-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + + enable_multi_cluster_deployment(resource=resource) + setup_external_access(resource=resource, enable_external_domain=False) + + resource.set_architecture_annotation() + + return resource + + +@mark.e2e_multi_cluster_sharded_external_access_no_ext_domain +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_external_access_no_ext_domain +def test_sharded_cluster(sharded_cluster: MongoDB): + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) + + +def service_exists(service_name: str, namespace: str, api_client: Optional[kubernetes.client.ApiClient] = None) -> bool: + try: + client.CoreV1Api(api_client=api_client).read_namespaced_service(service_name, namespace) + except client.rest.ApiException as e: + logger.error(f"Error reading {service_name}: {e}") + return False + return True + + +@mark.e2e_multi_cluster_sharded_external_access_no_ext_domain +def test_services_were_created(sharded_cluster: MongoDB, namespace: str): + resource_name = sharded_cluster.name + expected_services: Dict[str, List[str]] = defaultdict(list) + member_clusters = get_member_cluster_names() + + # Global services + for cluster in member_clusters: + expected_services[cluster].append(f"{resource_name}-svc") + expected_services[cluster].append(f"{resource_name}-{resource_name}") + + # All components get a headless service and per-pod services + # Config server also gets an additional headless service suffixed -cs + config_clusters = sharded_cluster["spec"]["configSrv"]["clusterSpecList"] + for idx, cluster_spec in enumerate(config_clusters): + members = cluster_spec["members"] + expected_services[cluster_spec["clusterName"]].append(f"{resource_name}-config-{idx}-svc") + expected_services[cluster_spec["clusterName"]].append(f"{resource_name}-{idx}-cs") + for pod in range(members): + expected_services[cluster_spec["clusterName"]].append(f"{resource_name}-config-{idx}-{pod}-svc") + + # Mongos also get an external service per pod + mongos_clusters = sharded_cluster["spec"]["mongos"]["clusterSpecList"] + for idx, cluster_spec in enumerate(mongos_clusters): + members = cluster_spec["members"] + cluster_name = cluster_spec["clusterName"] + expected_services[cluster_name].append(f"{resource_name}-mongos-{idx}-svc") + for pod in range(members): + expected_services[cluster_name].append(f"{resource_name}-mongos-{idx}-{pod}-svc") + expected_services[cluster_name].append(f"{resource_name}-mongos-{idx}-{pod}-svc-external") + + shard_count = sharded_cluster["spec"]["shardCount"] + shard_clusters = sharded_cluster["spec"]["shard"]["clusterSpecList"] + for shard in range(shard_count): + for idx, cluster_spec in enumerate(shard_clusters): + members = cluster_spec["members"] + cluster_name = cluster_spec["clusterName"] + expected_services[cluster_name].append(f"{resource_name}-{shard}-{idx}-svc") + for pod in range(members): + expected_services[cluster_name].append(f"{resource_name}-{shard}-{idx}-{pod}-svc") + + logger.debug("Asserting the following services exist:") + for cluster, services in expected_services.items(): + logger.debug(f"Cluster: {cluster}, service count: {len(services)}") + logger.debug(f"Services: {services}") + + # Assert that each expected service exists in its corresponding cluster. + for cluster, services in expected_services.items(): + api_client = get_member_cluster_api_client(cluster) # Retrieve the API client for the cluster + for svc in services: + assert service_exists( + svc, namespace, api_client + ), f"Service {svc} not found. Cluster: {cluster} Namespace: {namespace}" diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/fixtures/sharded-cluster-multi-cluster.yaml b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/fixtures/sharded-cluster-multi-cluster.yaml new file mode 100644 index 000000000..e9bff3428 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/fixtures/sharded-cluster-multi-cluster.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-base +spec: + shardCount: 1 + type: ShardedCluster + topology: MultiCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: WARN + persistent: true + mongos: {} + shard: {} + configSrv: {} + diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_disaster_recovery.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_disaster_recovery.py new file mode 100644 index 000000000..2a27b96bc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_disaster_recovery.py @@ -0,0 +1,285 @@ +import os +import time +from typing import Optional + +import kubernetes +import kubernetes.client +from kubetester import ( + delete_statefulset, + get_statefulset, + read_configmap, + try_load, + update_configmap, +) +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import ( + get_env_var_or_fail, + is_default_architecture_static, + is_multi_cluster, + run_periodically, + skip_if_local, +) +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import get_central_cluster_client, get_member_cluster_api_client +from tests.multicluster.conftest import cluster_spec_list +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_all_sharded_cluster_pod_names, +) + +MEMBER_CLUSTERS = ["kind-e2e-cluster-1", "kind-e2e-cluster-2", "kind-e2e-cluster-3"] +FAILED_MEMBER_CLUSTER_INDEX = 2 +FAILED_MEMBER_CLUSTER_NAME = MEMBER_CLUSTERS[FAILED_MEMBER_CLUSTER_INDEX] +RESOURCE_NAME = "sh-disaster-recovery" + +logger = test_logger.get_test_logger(__name__) + + +# We test a simple disaster recovery scenario: we lose one cluster without losing the majority. +# We ensure that the operator correctly ignores the unhealthy cluster in the subsequent reconciliation, +# and we can still scale. The DR procedure requires to first scale down all unhealthy members to be able +# to reconfigure the deployment further. + + +def is_cloud_qa() -> bool: + return os.getenv("ops_manager_version", "cloud_qa") == "cloud_qa" + + +@mark.e2e_multi_cluster_sharded_disaster_recovery +def test_install_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@fixture(scope="function") +def ops_manager( + namespace, + ops_manager_issuer_ca_configmap: str, + app_db_issuer_ca_configmap: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> Optional[MongoDBOpsManager]: + if is_cloud_qa(): + return None + + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_tls.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["security"] = {} + resource["spec"]["applicationDatabase"]["security"] = {} + resource["spec"]["backup"] = {"enabled": False} + + if is_multi_cluster(): + resource["spec"]["topology"] = "MultiCluster" + resource["spec"]["clusterSpecList"] = cluster_spec_list(["kind-e2e-cluster-1"], [1]) + resource["spec"]["applicationDatabase"]["topology"] = "MultiCluster" + resource["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list(["kind-e2e-cluster-1"], [3]) + resource.api = kubernetes.client.CustomObjectsApi(api_client=get_central_cluster_client()) + + return resource + + +@mark.skipif(is_cloud_qa(), reason="OM deployment is skipped if the test is executed against Cloud QA") +@mark.e2e_multi_cluster_sharded_disaster_recovery +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + def test_om_is_running( + self, + ops_manager: MongoDBOpsManager, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str, ops_manager: MongoDBOpsManager) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-scale-shards.yaml"), namespace=namespace, name=RESOURCE_NAME + ) + + if try_load(resource): + return resource + + # this allows us to reuse this test in both variants: with OMs and with Cloud QA + # if this is not executed, the resource uses default values for project and credentials (my-project/my-credentials) + # which are created up by the preparation scripts. + if not is_cloud_qa(): + resource.configure(ops_manager, RESOURCE_NAME, api_client=get_central_cluster_client()) + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[2, 1, 2], + mongos_members_array=[1, 0, 2], + configsrv_members_array=[2, 1, 2], + ) + + return resource.update() + + +@fixture(scope="module") +def config_version_store(): + class ConfigVersion: + version = 0 + + return ConfigVersion() + + +@mark.e2e_multi_cluster_sharded_disaster_recovery +def test_install_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_disaster_recovery +class TestDeployShardedClusterWithFailedCluster: + def test_create_sharded_cluster(self, sc: MongoDB, config_version_store): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + config_version_store.version = sc.get_automation_config_tester().automation_config["version"] + logger.debug(f"Automation Config Version after initial deployment: {config_version_store.version}") + + def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy( + self, namespace, central_cluster_client: kubernetes.client.ApiClient, multi_cluster_operator: Operator + ): + operator_cm_name = "mongodb-enterprise-operator-member-list" + logger.debug(f"Deleting cluster {FAILED_MEMBER_CLUSTER_NAME} from configmap {operator_cm_name}") + member_list_cm = read_configmap( + namespace, + operator_cm_name, + api_client=central_cluster_client, + ) + # this if is only for allowing re-running the test locally, without it the test function could be executed + # only once until the map is populated again by running prepare-local-e2e run again + if FAILED_MEMBER_CLUSTER_NAME in member_list_cm: + member_list_cm.pop(FAILED_MEMBER_CLUSTER_NAME) + + # this will trigger operators restart as it panics on changing the configmap + update_configmap( + namespace, + operator_cm_name, + member_list_cm, + api_client=central_cluster_client, + ) + + # sleeping to ensure the operator will suicide after config map is changed + # TODO: as part of https://jira.mongodb.org/browse/CLOUDP-288588, and when we re-activate this test, ensure + # this sleep is really nededed or if the subsquent call to multi_cluster_operator.assert_is_running() is enough + time.sleep(30) + + @skip_if_local + # Modifying the configmap triggers an (intentional) panic, the pod should restart. + # Operator process restart has to be done manually when running locally. + def test_operator_has_restarted(self, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + def test_delete_all_statefulsets_in_failed_cluster( + self, sc: MongoDB, central_cluster_client: kubernetes.client.ApiClient + ): + shards_sts_names = [ + sc.shard_statefulset_name(shard_idx, FAILED_MEMBER_CLUSTER_INDEX) + for shard_idx in range(sc["spec"]["shardCount"]) + ] + config_server_sts_name = sc.config_srv_statefulset_name(FAILED_MEMBER_CLUSTER_INDEX) + mongos_sts_name = sc.mongos_statefulset_name(FAILED_MEMBER_CLUSTER_INDEX) + + all_sts_names = shards_sts_names + [config_server_sts_name, mongos_sts_name] + logger.debug( + f"Deleting {len(all_sts_names)} statefulsets in failed cluster, statefulsets names: {all_sts_names}" + ) + + for sts_name in shards_sts_names + [config_server_sts_name, mongos_sts_name]: + try: + # delete all statefulsets in failed member cluster to simulate full cluster outage + delete_statefulset( + sc.namespace, + sts_name, + propagation_policy="Background", + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubernetes.client.ApiClient]): + try: + get_statefulset(namespace, name, api_client=api_client) + return False + except kubernetes.client.ApiException as e: + if e.status == 404: + return True + else: + raise e + + for sts_name in shards_sts_names + [config_server_sts_name, mongos_sts_name]: + run_periodically( + lambda: statefulset_is_deleted( + sc.namespace, + sts_name, + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + + def test_sharded_cluster_is_stable(self, sc: MongoDB, config_version_store): + sc.assert_reaches_phase(Phase.Running) + # Automation Config shouldn't change when we lose a cluster + expected_version = config_version_store.version + # in non-static, every restart of the operator increases version of ac due to agent upgrades + if not is_default_architecture_static(): + expected_version += 1 + + assert expected_version == sc.get_automation_config_tester().automation_config["version"] + + logger.debug(f"Automation Config Version after losing cluster: {config_version_store.version}") + + +@mark.e2e_multi_cluster_sharded_disaster_recovery +class TestScaleShardsAndMongosToZeroFirst: + def test_scale_shards_and_mongos_to_zero_first(self, sc: MongoDB): + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(MEMBER_CLUSTERS, [2, 1, 0]) # cluster3: 2->0 + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(MEMBER_CLUSTERS, [1, 0, 0]) # cluster3: 2->0 + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(MEMBER_CLUSTERS, [2, 1, 0]) # cluster3: 2->0 + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_expected_processes_in_ac(self, sc: MongoDB): + all_process_names = [p["name"] for p in sc.get_automation_config_tester().get_all_processes()] + assert set(get_all_sharded_cluster_pod_names(sc)) == set(all_process_names) + + +@mark.e2e_multi_cluster_sharded_disaster_recovery +class TestMoveFailedToHealthyClusters: + # simulate that we expand on the healthy clusters to have the same number of nodes as before "disaster" + def test_move_failed_to_healthy_clusters(self, sc: MongoDB): + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(MEMBER_CLUSTERS, [3, 2, 0]) # cluster1: 1->3 + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list( + MEMBER_CLUSTERS, [2, 1, 0] + ) # cluster1: 1->2, cluster2: 0->1 + # we don't get back to 6 members as adding each csrs node causes all mongos to perform rolling restart - it's taking too long + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list( + MEMBER_CLUSTERS, [3, 2, 0] + ) # cluster1: 2->3, cluster2: 1->2 + sc.update() + + # timeout is large due to scaling of config server, which is causing mongos rolling restart with each added member + sc.assert_reaches_phase(Phase.Running, timeout=2400) + + def test_expected_processes_in_ac(self, sc: MongoDB): + all_process_names = [p["name"] for p in sc.get_automation_config_tester().get_all_processes()] + assert set(get_all_sharded_cluster_pod_names(sc)) == set(all_process_names) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_geo_sharding.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_geo_sharding.py new file mode 100644 index 000000000..d8f060192 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_geo_sharding.py @@ -0,0 +1,157 @@ +from kubetester import find_fixture, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import get_member_cluster_names +from tests.multicluster.conftest import cluster_spec_list +from tests.multicluster_shardedcluster import ( + assert_config_srv_sts_members_count, + assert_correct_automation_config_after_scaling, + assert_mongos_sts_members_count, + assert_shard_sts_members_count, +) +from tests.shardedcluster.conftest import ( + get_member_cluster_clients_using_cluster_mapping, +) + +MDB_RESOURCE_NAME = "sh-geo-sharding" +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-multi-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME + ) + + if try_load(resource): + return resource + + resource.set_architecture_annotation() + + return resource + + +@mark.e2e_multi_cluster_sharded_geo_sharding +class TestShardedClusterGeoSharding: + """This test verifies scenario where user wants to have distributed local writes -> https://www.mongodb.com/docs/manual/tutorial/sharding-high-availability-writes/. + Assuming that writes to shard-x happen closer to cluster-y we can have primary for shard-x set in cluster-y. + We do this by setting high priority for particular replica members to elect primary (write) replicas in + desired clusters.""" + + def test_deploy_operator(self, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + def test_create_primary_for_each_shard_in_different_cluster( + self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str + ): + sc["spec"]["shardCount"] = 3 + # The distribution below is the default one but won't be used as all shards are overridden + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + + # All shards contain overrides + sc["spec"]["shardOverrides"] = [ + { + "shardNames": [f"{sc.name}-0"], + "clusterSpecList": cluster_spec_list( + get_member_cluster_names(), + [1, 1, 1], + [ + [ + { + "priority": "1", + } + ], + [ + { + "priority": "1", + } + ], + [ + { + "priority": "100", + } + ], + ], + ), + }, + { + "shardNames": [f"{sc.name}-1"], + "clusterSpecList": cluster_spec_list( + get_member_cluster_names(), + [1, 1, 1], + [ + [ + { + "priority": "1", + } + ], + [ + { + "priority": "100", + } + ], + [ + { + "priority": "1", + } + ], + ], + ), + }, + { + "shardNames": [f"{sc.name}-2"], + "clusterSpecList": cluster_spec_list( + get_member_cluster_names(), + [1, 1, 1], + [ + [ + { + "priority": "100", + } + ], + [ + { + "priority": "1", + } + ], + [ + { + "priority": "1", + } + ], + ], + ), + }, + ] + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_assert_correct_automation_config(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[1, 1, 1], [1, 1, 1], [1, 1, 1]]) + assert_mongos_sts_members_count(sc, [1, 1, 1]) + assert_config_srv_sts_members_count(sc, [1, 1, 1]) + + def test_assert_shard_primary_replicas(self, sc: MongoDB): + logger.info("Validating shard primaries in cluster(s)") + cluster_primary_member_mapping = { + 0: 2, # -> shard_idx: 0, cluster_idx: 2 + 1: 1, # -> shard_idx: 1, cluster_idx: 1 + 2: 0, # -> shard_idx: 2, cluster_idx: 0 + } + for shard_idx, cluster_idx in cluster_primary_member_mapping.items(): + shard_primary_hostname = sc.shard_hostname(shard_idx, 0, cluster_idx) + client = KubernetesTester.check_hosts_are_ready(hosts=[shard_primary_hostname]) + assert client.is_primary diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_scaling.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_scaling.py new file mode 100644 index 000000000..3be7c000f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_scaling.py @@ -0,0 +1,128 @@ +from kubetester import find_fixture, try_load +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import get_member_cluster_names +from tests.multicluster.conftest import cluster_spec_list +from tests.multicluster_shardedcluster import ( + assert_config_srv_sts_members_count, + assert_correct_automation_config_after_scaling, + assert_mongos_sts_members_count, + assert_shard_sts_members_count, +) + +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-multi-cluster.yaml"), + namespace=namespace, + name="sh-scaling", + ) + + if try_load(resource): + return resource + + resource.set_architecture_annotation() + + return resource + + +@mark.e2e_multi_cluster_sharded_scaling +class TestShardedClusterScalingInitial: + + def test_deploy_operator(self, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + def test_create(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[1, 1, 1]]) + assert_mongos_sts_members_count(sc, [1, 1, 1]) + assert_config_srv_sts_members_count(sc, [1, 1, 1]) + + +@mark.e2e_multi_cluster_sharded_scaling +class TestShardedClusterScalingUpscale: + + def test_upscale(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 3, 1]) + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [2, 2, 1]) + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=2300) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[1, 3, 1]]) + assert_config_srv_sts_members_count(sc, [2, 2, 1]) + assert_mongos_sts_members_count(sc, [1, 1, 1]) + + +@mark.e2e_multi_cluster_sharded_scaling +class TestShardedClusterScalingDownscale: + def test_downgrade_downscale(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 2, 1]) + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [2, 1, 1]) + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=3500) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[1, 2, 1]]) + assert_config_srv_sts_members_count(sc, [2, 1, 1]) + assert_mongos_sts_members_count(sc, [1, 1, 1]) + + +@mark.e2e_multi_cluster_sharded_scaling +class TestShardedClusterScalingDownscaleToZero: + def test_downscale_to_zero(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [0, 2, 1]) + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 0, 0]) + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=3500) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[0, 2, 1]]) + assert_config_srv_sts_members_count(sc, [1, 1, 1]) + assert_mongos_sts_members_count(sc, [1, 0, 0]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_scaling_all_shard_overrides.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_scaling_all_shard_overrides.py new file mode 100644 index 000000000..342a416ef --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_scaling_all_shard_overrides.py @@ -0,0 +1,126 @@ +from kubernetes.client import ApiClient +from kubetester import find_fixture, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import get_member_cluster_names +from tests.multicluster.conftest import cluster_spec_list +from tests.multicluster_shardedcluster import ( + assert_correct_automation_config_after_scaling, + assert_shard_sts_members_count, + validate_member_count_in_ac, + validate_shard_configurations_in_ac_multi, +) + +MDB_RESOURCE_NAME = "sh-scaling-shard-overrides" +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-multi-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME + ) + + if try_load(resource): + return resource + + resource.set_architecture_annotation() + + return resource + + +@mark.e2e_multi_cluster_sharded_scaling_all_shard_overrides +class TestShardedClusterScalingInitial: + + def test_deploy_operator(self, multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + def test_create(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shardCount"] = 3 + # The distribution below is the default one but won't be used as all shards are overridden + sc["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 2, 2]) + sc["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + sc["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1, 1]) + + # All shards contain overrides + sc["spec"]["shardOverrides"] = [ + { + "shardNames": [f"{sc.name}-0"], + "clusterSpecList": cluster_spec_list(get_member_cluster_names(), [1, 2, 0]), + }, + { + "shardNames": [f"{sc.name}-1", f"{sc.name}-2"], + "clusterSpecList": cluster_spec_list(get_member_cluster_names(), [0, 1, 2]), + }, + ] + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1600) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[1, 2, 0], [0, 1, 2], [0, 1, 2]]) + + +@mark.e2e_multi_cluster_sharded_scaling_all_shard_overrides +class TestShardedClusterScalingShardOverrides: + + def test_scale_overrides(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shardOverrides"] = [ + { + "shardNames": [f"{sc.name}-0"], + # cluster 3: 0->1 + "clusterSpecList": cluster_spec_list(get_member_cluster_names(), [1, 2, 1]), + }, + { + # cluster 1: 0->2 + "shardNames": [f"{sc.name}-1"], + "clusterSpecList": cluster_spec_list(get_member_cluster_names(), [2, 1, 2]), + }, + { + # cluster 1: 0->1 + "shardNames": [f"{sc.name}-2"], + "clusterSpecList": cluster_spec_list(get_member_cluster_names(), [1, 1, 2]), + }, + ] + + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1600) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + assert_shard_sts_members_count(sc, [[1, 2, 1], [2, 1, 2], [1, 1, 2]]) + + +@mark.e2e_multi_cluster_sharded_scaling_all_shard_overrides +class TestShardedClusterScalingAddShards: + def test_scale_shardcount(self, sc: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sc["spec"]["shardCount"] = sc["spec"]["shardCount"] + 2 + sc.update() + + def test_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=500) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + logger.info("Validating automation config correctness") + assert_correct_automation_config_after_scaling(sc) + + def test_assert_stateful_sets_after_scaling(self, sc: MongoDB): + logger.info("Validating statefulsets in cluster(s)") + # We added two shards, they are assigned the base distribution: [1, 2, 2] + assert_shard_sts_members_count(sc, [[1, 2, 1], [2, 1, 2], [1, 1, 2], [1, 2, 2], [1, 2, 2]]) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest.py new file mode 100644 index 000000000..d018f9c9f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest.py @@ -0,0 +1,42 @@ +from kubetester import find_fixture, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import get_member_cluster_names +from tests.multicluster.conftest import cluster_spec_list + +MDB_RESOURCE_NAME = "sh" + + +@fixture(scope="module") +def sharded_cluster(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-multi-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME + ) + + if try_load(resource): + return resource + + return resource + + +@mark.e2e_multi_cluster_sharded_simplest +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_simplest +def test_create(sharded_cluster: MongoDB, custom_mdb_version: str, issuer_ca_configmap: str): + sharded_cluster.set_version(ensure_ent_version(custom_mdb_version)) + + sharded_cluster["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [2, 2, 1]) + sharded_cluster["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [2, 2, 1]) + sharded_cluster["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 2, 1]) + sharded_cluster.set_architecture_annotation() + sharded_cluster.update() + + +@mark.e2e_multi_cluster_sharded_simplest +def test_sharded_cluster(sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=900) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest_no_mesh.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest_no_mesh.py new file mode 100644 index 000000000..503191bef --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_simplest_no_mesh.py @@ -0,0 +1,71 @@ +import logging + +import kubernetes +from kubetester import find_fixture, try_load +from kubetester.kubetester import ensure_ent_version, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import get_member_cluster_names, update_coredns_hosts +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_dns_hosts_for_external_access, + setup_external_access, +) + +MDB_RESOURCE_NAME = "sh" + + +@fixture(scope="module") +def sharded_cluster(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("sharded-cluster-multi-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE_NAME + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + + enable_multi_cluster_deployment(resource=resource) + setup_external_access(resource=resource, enable_external_domain=True) + + resource.set_architecture_annotation() + + return resource + + +@mark.e2e_multi_cluster_sharded_simplest_no_mesh +def test_disable_istio(disable_istio): + logging.info("Istio disabled") + + +@mark.e2e_multi_cluster_sharded_simplest_no_mesh +def test_update_coredns(cluster_clients: dict[str, kubernetes.client.ApiClient], sharded_cluster: MongoDB): + hosts = get_dns_hosts_for_external_access(resource=sharded_cluster, cluster_member_list=get_member_cluster_names()) + for cluster_name, cluster_api in cluster_clients.items(): + update_coredns_hosts(hosts, cluster_name, api_client=cluster_api) + + +@mark.e2e_multi_cluster_sharded_simplest_no_mesh +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_simplest_no_mesh +def test_sharded_cluster(sharded_cluster: MongoDB): + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) + + +# Testing connectivity with External Access requires using the same DNS as deployed in Kube within +# test_update_coredns. There's no easy way to set it up locally. +@skip_if_local() +@mark.e2e_multi_cluster_sharded_simplest_no_mesh +def test_shards_were_configured_and_accessible(sharded_cluster: MongoDB): + hosts = get_dns_hosts_for_external_access(resource=sharded_cluster, cluster_member_list=get_member_cluster_names()) + mongos_hostnames = [item[1] for item in hosts if "mongos" in item[1]] + # It's not obvious, but under the covers using Services and External Domain will ensure the tester respects + # the supplied hosts (and only them). + tester = sharded_cluster.tester(service_names=mongos_hostnames) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py new file mode 100644 index 000000000..85883e726 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_snippets.py @@ -0,0 +1,121 @@ +import os +from typing import List + +from kubetester import create_or_update_configmap, read_configmap, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import mark +from tests import test_logger + +SNIPPETS_DIR = "public/samples/sharded_multicluster/" +SNIPPETS_FILES = [ + "pod_template_shards_0.yaml", + "pod_template_shards_1.yaml", + "pod_template_config_servers.yaml", + "shardSpecificPodSpec_migration.yaml", + "example-sharded-cluster-deployment.yaml", +] + +logger = test_logger.get_test_logger(__name__) + + +def load_resource(namespace: str, file_path: str, resource_name: str = None) -> MongoDB: + resource = MongoDB.from_yaml(file_path, namespace=namespace, name=resource_name) + return resource + + +def get_project_directory() -> str: + project_dir = os.environ.get("PROJECT_DIR") + logger.debug(f"PROJECT_DIR: {project_dir}") + return project_dir + + +# To be able to access the snippets file from here, we added "COPY public /ops-manager-kubernetes/public" when building +# the mongodb-test container +# Then we set the env variable PROJECT_DIR to /ops-manager-kubernetes +# The test will also work locally if the variable is set correctly +def get_sharded_resources(namespace: str) -> List[MongoDB]: + resources = [] + project_directory = get_project_directory() + files_dir = os.path.join(project_directory, SNIPPETS_DIR) + for file_name in SNIPPETS_FILES: + file_path = os.path.join(files_dir, file_name) + logger.debug(f"Loading snippet file: {file_path}") + logger.debug(f"File found: {os.path.isfile(file_path)}") + # We set the resource name as the file name, but replace _ with - and lowercase, + # to respect kubernetes naming constraints + sc = load_resource(namespace, file_path) + sc["spec"]["opsManager"]["configMapRef"]["name"] = f"{file_to_resource_name(file_name)}-project-map" + resources.append(sc) + return resources + + +def file_to_resource_name(file_name: str) -> str: + return file_name.removesuffix(".yaml").replace("_", "-").lower() + + +@mark.e2e_multi_cluster_sharded_snippets +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_snippets +def test_create_projects_configmaps(namespace: str): + for file_name in SNIPPETS_FILES: + base_cm = read_configmap(namespace=namespace, name="my-project") + # Validate required keys + required_keys = ["baseUrl", "orgId", "projectName"] + for key in required_keys: + if key not in base_cm: + raise KeyError(f"The OM/CM project configmap is missing the key: {key}") + + create_or_update_configmap( + namespace=namespace, + name=f"{file_to_resource_name(file_name)}-project-map", + data={ + "baseUrl": base_cm["baseUrl"], + "orgId": base_cm["orgId"], + # In EVG, we generate a unique ID for the project name in the 'my-project' configmap when we set up a + # test. To avoid project name collisions in between two concurrently running tasks in CloudQA, + # we concatenate it to the name of the mdb resource + "projectName": f"{base_cm['projectName']}-{file_to_resource_name(file_name)}", + }, + ) + + +@mark.e2e_multi_cluster_sharded_snippets +def test_create(namespace: str, custom_mdb_version: str, issuer_ca_configmap: str): + for sc in get_sharded_resources(namespace): + sc.set_version(ensure_ent_version(custom_mdb_version)) + sc.update() + + +# All resources will be reconciled in parallel, we wait for all of them to reach Running to succeed +# Catching exceptions enables to display all failing resources instead of just the first, and makes debugging easier +@mark.e2e_multi_cluster_sharded_snippets +def test_running(namespace: str): + succeeded_resources = [] + failed_resources = [] + first_iter = True + + for sc in get_sharded_resources(namespace): + try: + logger.debug(f"Waiting for {sc.name} to reach Running phase") + # Once the first resource reached Running, it shouldn't take more than ~300s for the others to do so + sc.assert_reaches_phase(Phase.Running, timeout=900 if first_iter else 300) + succeeded_resources.append(sc.name) + first_iter = False + logger.info(f"{sc.name} reached Running phase") + except Exception as e: + logger.error(f"Error while waiting for {sc.name} to reach Running phase: {e}") + failed_resources.append(sc.name) + + if succeeded_resources: + logger.info(f"Resources that reached Running phase: {', '.join(succeeded_resources)}") + + # Ultimately fail the test if any resource failed to reconcile + if failed_resources: + raise AssertionError(f"Some resources failed to reach Running phase: {', '.join(failed_resources)}") + else: + logger.info(f"All resources reached Running phase") diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls.py new file mode 100644 index 000000000..0780621e4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls.py @@ -0,0 +1,140 @@ +import kubernetes +from kubetester import try_load +from kubetester.certs import ( + SetPropertiesMultiCluster, + create_multi_cluster_tls_certs, + generate_cert, + get_agent_x509_subject, + get_mongodb_x509_subject, +) +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + get_central_cluster_client, + get_member_cluster_clients, + get_member_cluster_names, +) +from tests.multicluster.conftest import cluster_spec_list + +MDB_RESOURCE = "sharded-cluster-custom-certs" +SUBJECT = {"organizations": ["MDB Tests"], "organizationalUnits": ["Servers"]} +SERVER_SETS = frozenset( + [ + SetPropertiesMultiCluster(MDB_RESOURCE + "-0", MDB_RESOURCE + "-sh", 3, 2), + SetPropertiesMultiCluster(MDB_RESOURCE + "-config", MDB_RESOURCE + "-cs", 3, 2), + SetPropertiesMultiCluster(MDB_RESOURCE + "-mongos", MDB_RESOURCE + "-svc", 2, 2), + ] +) + + +@fixture(scope="module") +def all_certs(issuer, namespace) -> None: + """Generates all required TLS certificates: Servers and Client/Member.""" + + for server_set in SERVER_SETS: + service_fqdns = [] + for cluster_idx in range(server_set.number_of_clusters): + for pod_idx in range(server_set.replicas): + service_fqdns.append(f"{server_set.name}-{cluster_idx}-{pod_idx}-svc.{namespace}.svc.cluster.local") + + create_multi_cluster_tls_certs( + multi_cluster_issuer=issuer, + central_cluster_client=get_central_cluster_client(), + member_clients=get_member_cluster_clients(), + secret_name="prefix-" + server_set.name + "-cert", + mongodb_multi=None, + namespace=namespace, + additional_domains=None, + service_fqdns=service_fqdns, + clusterwide=False, + spec=get_mongodb_x509_subject(namespace), + ) + + create_multi_cluster_tls_certs( + multi_cluster_issuer=issuer, + central_cluster_client=get_central_cluster_client(), + member_clients=get_member_cluster_clients(), + secret_name="prefix-" + server_set.name + "-clusterfile", + mongodb_multi=None, + namespace=namespace, + additional_domains=None, + service_fqdns=service_fqdns, + clusterwide=False, + spec=get_mongodb_x509_subject(namespace), + ) + + +@fixture(scope="module") +def agent_certs( + namespace: str, + issuer: str, +): + spec = get_agent_x509_subject(namespace) + return generate_cert( + namespace=namespace, + pod="tmp", + dns="", + issuer=issuer, + spec=spec, + multi_cluster_mode=True, + api_client=get_central_cluster_client(), + secret_name=f"prefix-{MDB_RESOURCE}-agent-certs", + ) + + +@fixture(scope="function") +def sharded_cluster( + namespace: str, + all_certs, + agent_certs, + issuer_ca_configmap: str, +) -> MongoDB: + mdb: MongoDB = MongoDB.from_yaml( + _fixture("test-tls-base-sc-require-ssl.yaml"), + name=MDB_RESOURCE, + namespace=namespace, + ) + mdb.api = kubernetes.client.CustomObjectsApi(get_central_cluster_client()) + if try_load(mdb): + return mdb + + mdb["spec"]["security"] = { + "authentication": { + "enabled": True, + "modes": ["X509"], + "agents": {"mode": "X509"}, + "internalCluster": "X509", + }, + "tls": { + "enabled": True, + "ca": issuer_ca_configmap, + }, + "certsSecretPrefix": "prefix", + } + mdb["spec"]["mongodsPerShardCount"] = 0 + mdb["spec"]["mongosCount"] = 0 + mdb["spec"]["configServerCount"] = 0 + mdb["spec"]["topology"] = "MultiCluster" + mdb["spec"]["shard"] = {} + mdb["spec"]["configSrv"] = {} + mdb["spec"]["mongos"] = {} + mdb["spec"]["shard"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [3, 1]) + mdb["spec"]["configSrv"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [3, 1]) + mdb["spec"]["mongos"]["clusterSpecList"] = cluster_spec_list(get_member_cluster_names(), [1, 1]) + + mdb.set_architecture_annotation() + + return mdb + + +@mark.e2e_multi_cluster_sharded_tls +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_tls +def test_sharded_cluster_with_prefix_gets_to_running_state(sharded_cluster: MongoDB): + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) diff --git a/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls_no_mesh.py b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls_no_mesh.py new file mode 100644 index 000000000..d6265e112 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/multicluster_shardedcluster/multi_cluster_sharded_tls_no_mesh.py @@ -0,0 +1,170 @@ +import logging + +import kubernetes +from kubetester import try_load +from kubetester.certs import ( + SetPropertiesMultiCluster, + create_multi_cluster_tls_certs, + generate_cert, + get_agent_x509_subject, + get_mongodb_x509_subject, +) +from kubetester.kubetester import fixture as _fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import with_tls +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + get_central_cluster_client, + get_member_cluster_clients, + get_member_cluster_names, + update_coredns_hosts, +) +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_dns_hosts_for_external_access, + setup_external_access, +) + +MDB_RESOURCE = "sharded-cluster-custom-certs" +SUBJECT = {"organizations": ["MDB Tests"], "organizationalUnits": ["Servers"]} +SERVER_SETS = frozenset( + [ + SetPropertiesMultiCluster(MDB_RESOURCE + "-0", MDB_RESOURCE + "-sh", 3, 3), + SetPropertiesMultiCluster(MDB_RESOURCE + "-config", MDB_RESOURCE + "-cs", 3, 3), + SetPropertiesMultiCluster(MDB_RESOURCE + "-mongos", MDB_RESOURCE + "-svc", 3, 3), + ] +) + + +@fixture(scope="module") +def all_certs(issuer, namespace, sharded_cluster: MongoDB) -> None: + """Generates all required TLS certificates: Servers and Client/Member.""" + + for server_set in SERVER_SETS: + # TODO: Think about optimizing this. For simplicity, we enable each cert to be valid for each Service + # This way, all components can talk to each other. We may want to be more strict here and enable + # communication between the same components only. + service_fqdns = [ + external_address[1] + for external_address in get_dns_hosts_for_external_access( + resource=sharded_cluster, cluster_member_list=get_member_cluster_names() + ) + ] + + create_multi_cluster_tls_certs( + multi_cluster_issuer=issuer, + central_cluster_client=get_central_cluster_client(), + member_clients=get_member_cluster_clients(), + secret_name="prefix-" + server_set.name + "-cert", + mongodb_multi=None, + namespace=namespace, + additional_domains=None, + service_fqdns=service_fqdns, + clusterwide=False, + spec=get_mongodb_x509_subject(namespace), + ) + + create_multi_cluster_tls_certs( + multi_cluster_issuer=issuer, + central_cluster_client=get_central_cluster_client(), + member_clients=get_member_cluster_clients(), + secret_name="prefix-" + server_set.name + "-clusterfile", + mongodb_multi=None, + namespace=namespace, + additional_domains=None, + service_fqdns=service_fqdns, + clusterwide=False, + spec=get_mongodb_x509_subject(namespace), + ) + + +@fixture(scope="module") +def agent_certs( + namespace: str, + issuer: str, +): + spec = get_agent_x509_subject(namespace) + return generate_cert( + namespace=namespace, + pod="tmp", + dns="", + issuer=issuer, + spec=spec, + multi_cluster_mode=True, + api_client=get_central_cluster_client(), + secret_name=f"prefix-{MDB_RESOURCE}-agent-certs", + ) + + +@fixture(scope="module") +def sharded_cluster( + namespace: str, + issuer_ca_configmap: str, +) -> MongoDB: + mdb: MongoDB = MongoDB.from_yaml( + _fixture("test-tls-base-sc-require-ssl.yaml"), + name=MDB_RESOURCE, + namespace=namespace, + ) + if try_load(mdb): + return mdb + + mdb["spec"]["security"] = { + "authentication": { + "enabled": True, + "modes": ["X509"], + "agents": {"mode": "X509"}, + "internalCluster": "X509", + }, + "tls": { + "enabled": True, + "ca": issuer_ca_configmap, + }, + "certsSecretPrefix": "prefix", + } + + enable_multi_cluster_deployment(resource=mdb) + setup_external_access(resource=mdb) + mdb.set_architecture_annotation() + + return mdb + + +@mark.e2e_multi_cluster_sharded_tls_no_mesh +def test_update_coredns(cluster_clients: dict[str, kubernetes.client.ApiClient], sharded_cluster: MongoDB): + hosts = get_dns_hosts_for_external_access(resource=sharded_cluster, cluster_member_list=get_member_cluster_names()) + for cluster_name, cluster_api in cluster_clients.items(): + update_coredns_hosts(hosts, cluster_name, api_client=cluster_api) + + +@mark.e2e_multi_cluster_sharded_tls_no_mesh +def test_deploy_operator(multi_cluster_operator: Operator): + multi_cluster_operator.assert_is_running() + + +@mark.e2e_multi_cluster_sharded_tls_no_mesh +def test_deploy_certs(all_certs, agent_certs): + logging.info(f"Certificates deployed successfully") + + +@mark.e2e_multi_cluster_sharded_tls_no_mesh +def test_sharded_cluster_with_prefix_gets_to_running_state(sharded_cluster: MongoDB): + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=800) + + +# TODO: (slaskawi) clearly the client tries to connect to mongos without TLS (we can see this in the logs). +# The Server rejects the connection (also we can see this in the logs) and everything ends with a timeout. Why? +# # Testing connectivity with External Access requires using the same DNS as deployed in Kube within +# # test_update_coredns. There's no easy way to set it up locally. +# @skip_if_local() +# @mark.e2e_multi_cluster_sharded_tls_no_mesh +# def test_shards_were_configured_and_accessible(sharded_cluster: MongoDB, ca_path: str): +# hosts = get_dns_hosts_for_external_access(resource=sharded_cluster, cluster_member_list=get_member_cluster_names()) +# mongos_hostnames = [item[1] for item in hosts if "mongos" in item[1]] +# # It's not obvious, but under the covers using Services and External Domain will ensure the tester respects +# # the supplied hosts (and only them). +# tester = sharded_cluster.tester(service_names=mongos_hostnames) +# tester.assert_connectivity(opts=[with_tls(use_tls=True, ca_path=ca_path)]) diff --git a/docker/mongodb-enterprise-tests/tests/olm/__init__.py b/docker/mongodb-enterprise-tests/tests/olm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_ops_manager_backup.yaml b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_ops_manager_backup.yaml new file mode 100644 index 000000000..716c31347 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_ops_manager_backup.yaml @@ -0,0 +1,47 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup +spec: + replicas: 2 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + headDB: + storage: 500M + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + blockStores: + - name: blockStore1 + mongodbResourceRef: + name: my-mongodb-blockstore + s3Stores: + - name: s3Store1 + mongodbResourceRef: + name: my-mongodb-s3 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + + applicationDatabase: + members: 3 + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + diff --git a/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_replica_set_for_om.yaml b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_replica_set_for_om.yaml new file mode 100644 index 000000000..3f429bc36 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_replica_set_for_om.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: the-replica-set +spec: + members: 3 + type: ReplicaSet + opsManager: + configMapRef: + name: om-rs-configmap + credentials: my-credentials + persistent: true + logLevel: DEBUG + diff --git a/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_scram_sha_user_backing_db.yaml b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_scram_sha_user_backing_db.yaml new file mode 100644 index 000000000..c4103400a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_scram_sha_user_backing_db.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mms-user-1 +spec: + passwordSecretKeyRef: + name: mms-user-1-password + key: password + username: "mms-user-1@/" + db: "admin" + mongodbResourceRef: + name: "my-replica-set" + roles: + - db: "admin" + name: "clusterMonitor" + - db: "admin" + name: "readWriteAnyDatabase" + - db: "admin" + name: "dbAdminAnyDatabase" diff --git a/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_sharded_cluster_for_om.yaml b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_sharded_cluster_for_om.yaml new file mode 100644 index 000000000..af1cb7502 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/fixtures/olm_sharded_cluster_for_om.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: the-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 2 + type: ShardedCluster + opsManager: + configMapRef: + name: om-sc-configmap + credentials: my-credentials + persistent: true + logLevel: DEBUG + diff --git a/docker/mongodb-enterprise-tests/tests/olm/olm_operator_upgrade.py b/docker/mongodb-enterprise-tests/tests/olm/olm_operator_upgrade.py new file mode 100644 index 000000000..22d7837a4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/olm_operator_upgrade.py @@ -0,0 +1,101 @@ +import pytest +from kubernetes.client.rest import ApiException +from kubetester import MongoDB, read_service, wait_for_webhook +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import ( + get_default_architecture, + is_default_architecture_static, +) +from kubetester.opsmanager import MongoDBOpsManager +from tests.olm.olm_test_commons import ( + get_catalog_image, + get_catalog_source_resource, + get_current_operator_version, + get_latest_released_operator_version, + get_operator_group_resource, + get_subscription_custom_object, + increment_patch_version, + wait_for_operator_ready, +) + +# See docs how to run this locally: https://wiki.corp.mongodb.com/display/MMS/E2E+Tests+Notes#E2ETestsNotes-OLMtests + +# This tests only OLM upgrade of the operator without deploying any resources. + + +@pytest.mark.e2e_olm_operator_upgrade +def test_upgrade_operator_only(namespace: str, version_id: str): + latest_released_operator_version = get_latest_released_operator_version() + current_operator_version = get_current_operator_version() + incremented_operator_version = increment_patch_version(current_operator_version) + + get_operator_group_resource(namespace, namespace).update() + catalog_source_resource = get_catalog_source_resource( + namespace, get_catalog_image(f"{incremented_operator_version}-{version_id}") + ) + catalog_source_resource.update() + + static_value = get_default_architecture() + subscription = get_subscription_custom_object( + "mongodb-enterprise-operator", + namespace, + { + "channel": "stable", # stable channel contains latest released operator in RedHat's certified repository + "name": "mongodb-enterprise", + "source": catalog_source_resource.name, + "sourceNamespace": namespace, + "installPlanApproval": "Automatic", + # In certified OpenShift bundles we have this enabled, so the operator is not defining security context (it's managed globally by OpenShift). + # In Kind this will result in empty security contexts and problems deployments with filesystem permissions. + "config": { + "env": [ + {"name": "MANAGED_SECURITY_CONTEXT", "value": "false"}, + {"name": "OPERATOR_ENV", "value": "dev"}, + {"name": "MDB_DEFAULT_ARCHITECTURE", "value": static_value}, + {"name": "MDB_OPERATOR_TELEMETRY_SEND_ENABLED", "value": "false"}, + ] + }, + }, + ) + + subscription.update() + + wait_for_operator_ready(namespace, f"mongodb-enterprise.v{latest_released_operator_version}") + + subscription.load() + subscription["spec"]["channel"] = "fast" # fast channel contains operator build from the current branch + subscription.update() + + wait_for_operator_ready(namespace, f"mongodb-enterprise.v{incremented_operator_version}") + + +@pytest.mark.e2e_olm_operator_upgrade +def test_operator_webhook_is_deleted_and_not_installed_anymore(namespace: str): + # in the first release of OLM webhooks, the previous version will have this webhook installed + # in subsequent releases we will only test here that it's no longer installed + with pytest.raises(ApiException) as e: + read_service(namespace, "operator-webhook") + assert e.value.status == 404 + + +@pytest.mark.e2e_olm_operator_upgrade +def test_wait_for_webhook(namespace: str): + wait_for_webhook(namespace=namespace, service_name="mongodb-enterprise-operator-service") + + +@pytest.mark.e2e_olm_operator_upgrade +def test_opsmanager_webhook(namespace: str): + resource = MongoDBOpsManager.from_yaml(yaml_fixture("om_validation.yaml"), namespace=namespace) + resource["spec"]["version"] = "4.4.4.4" + + with pytest.raises(ApiException, match=r"is an invalid value for spec.version"): + resource.create() + + +@pytest.mark.e2e_olm_operator_upgrade +def test_mongodb_webhook(namespace: str): + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), namespace=namespace) + resource["spec"]["members"] = 0 + + with pytest.raises(ApiException, match=r"'spec.members' must be specified if"): + resource.create() diff --git a/docker/mongodb-enterprise-tests/tests/olm/olm_operator_upgrade_with_resources.py b/docker/mongodb-enterprise-tests/tests/olm/olm_operator_upgrade_with_resources.py new file mode 100644 index 000000000..310e9e5e0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/olm_operator_upgrade_with_resources.py @@ -0,0 +1,413 @@ +import kubernetes +import pytest +from kubeobject import CustomObject +from kubetester import ( + MongoDB, + create_or_update_secret, + get_default_storage_class, + try_load, +) +from kubetester.awss3client import AwsS3Client +from kubetester.certs import create_sharded_cluster_certs +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import ( + get_default_architecture, + is_default_architecture_static, + run_periodically, +) +from kubetester.mongodb import Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.olm.olm_test_commons import ( + get_catalog_image, + get_catalog_source_resource, + get_current_operator_version, + get_latest_released_operator_version, + get_operator_group_resource, + get_subscription_custom_object, + increment_patch_version, + wait_for_operator_ready, +) +from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket + +# See docs how to run this locally: https://wiki.corp.mongodb.com/display/MMS/E2E+Tests+Notes#E2ETestsNotes-OLMtests + +# This test performs operator upgrade of the operator while having OM and MongoDB resources deployed. +# It performs the following actions: +# - deploy latest released operator using OLM +# - deploy OM +# - deploy backup-required MongoDB: oplog, s3, blockstore +# - deploy TLS-enabled sharded MongoDB +# - check everything is running +# - upgrade the operator to the version built from the current branch +# - wait for resources to be rolling-updated due to updated stateful sets by the new operator +# - check everything is running and connectable + + +@fixture +def catalog_source(namespace: str, version_id: str): + current_operator_version = get_current_operator_version() + incremented_operator_version = increment_patch_version(current_operator_version) + + get_operator_group_resource(namespace, namespace).update() + catalog_source_resource = get_catalog_source_resource( + namespace, get_catalog_image(f"{incremented_operator_version}-{version_id}") + ) + catalog_source_resource.update() + + return catalog_source_resource + + +@fixture +def subscription(namespace: str, catalog_source: CustomObject): + static_value = get_default_architecture() + return get_subscription_custom_object( + "mongodb-enterprise-operator", + namespace, + { + "channel": "stable", # stable channel contains latest released operator in RedHat's certified repository + "name": "mongodb-enterprise", + "source": catalog_source.name, + "sourceNamespace": namespace, + "installPlanApproval": "Automatic", + # In certified OpenShift bundles we have this enabled, so the operator is not defining security context (it's managed globally by OpenShift). + # In Kind this will result in empty security contexts and problems deployments with filesystem permissions. + "config": { + "env": [ + {"name": "MANAGED_SECURITY_CONTEXT", "value": "false"}, + {"name": "OPERATOR_ENV", "value": "dev"}, + {"name": "MDB_DEFAULT_ARCHITECTURE", "value": static_value}, + {"name": "MDB_OPERATOR_TELEMETRY_SEND_ENABLED", "value": "false"}, + ] + }, + }, + ) + + +@fixture +def current_operator_version(): + return get_latest_released_operator_version() + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_install_stable_operator_version( + namespace: str, + version_id: str, + current_operator_version: str, + catalog_source: CustomObject, + subscription: CustomObject, +): + subscription.update() + wait_for_operator_ready(namespace, f"mongodb-enterprise.v{current_operator_version}") + + +# install resources on the latest released version of the operator + + +@fixture(scope="module") +def ops_manager(namespace: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("olm_ops_manager_backup.yaml"), namespace=namespace + ) + + try_load(resource) + return resource + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, "my-s3-secret", namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-s3") + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_create_om( + ops_manager: MongoDBOpsManager, + s3_bucket: str, + custom_version: str, + custom_appdb_version: str, +): + try_load(ops_manager) + ops_manager["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + ops_manager["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + ops_manager["spec"]["backup"]["members"] = 2 + + ops_manager.set_version(custom_version) + ops_manager.set_appdb_version(custom_appdb_version) + ops_manager.allow_mdb_rc_versions() + + ops_manager.update() + + +def wait_for_om_healthy_response(ops_manager: MongoDBOpsManager): + def wait_for_om_healthy_response_fn(): + status_code, status_response = ops_manager.get_om_tester().check_healthiness() + if status_code == 200: + return True, f"Got healthy status_code=200: {status_response}" + else: + return False, f"Got unhealthy status_code={status_code}: {status_response}" + + run_periodically( + wait_for_om_healthy_response_fn, + timeout=300, + msg=f"OM returning healthy response", + ) + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_om_connectivity(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Pending) + + wait_for_om_healthy_response(ops_manager) + + +# sharded mongodb fixtures + + +@fixture(scope="module") +def mdb_sharded_certs(issuer: str, namespace: str): + create_sharded_cluster_certs( + namespace, + "mdb-sharded", + shards=1, + mongod_per_shard=2, + config_servers=1, + mongos=1, + secret_prefix="prefix-", + ) + + +@fixture +def mdb_sharded( + ops_manager: MongoDBOpsManager, + namespace, + custom_mdb_version: str, + issuer_ca_configmap: str, + mdb_sharded_certs, +): + resource = MongoDB.from_yaml( + yaml_fixture("olm_sharded_cluster_for_om.yaml"), + namespace=namespace, + name="mdb-sharded", + ).configure(ops_manager, "mdb-sharded") + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["security"] = { + "tls": { + "ca": issuer_ca_configmap, + }, + } + resource.configure_backup(mode="disabled") + resource.update() + return resource + + +# OpsManager backup-backing databases + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("olm_replica_set_for_om.yaml"), + namespace=namespace, + name="my-mongodb-oplog", + ).configure(ops_manager, "oplog") + resource.set_version(custom_mdb_version) + return resource.update() + + +@fixture(scope="module") +def s3_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("olm_replica_set_for_om.yaml"), + namespace=namespace, + name="my-mongodb-s3", + ).configure(ops_manager, "s3metadata") + resource.set_version(custom_mdb_version) + return resource.update() + + +@fixture(scope="module") +def blockstore_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("olm_replica_set_for_om.yaml"), + namespace=namespace, + name="my-mongodb-blockstore", + ).configure(ops_manager, "blockstore") + resource.set_version(custom_mdb_version) + return resource.update() + + +@fixture(scope="module") +def blockstore_user(namespace, blockstore_replica_set: MongoDB) -> MongoDBUser: + return create_secret_and_user( + namespace, + "blockstore-user", + blockstore_replica_set.name, + "blockstore-user-password-secret", + "Passw0rd.", + ) + + +@fixture(scope="module") +def oplog_user(namespace, oplog_replica_set: MongoDB) -> MongoDBUser: + return create_secret_and_user( + namespace, + "oplog-user", + oplog_replica_set.name, + "oplog-user-password-secret", + "Passw0rd.", + ) + + +def create_secret_and_user( + namespace: str, name: str, replica_set_name: str, secret_name: str, password: str +) -> MongoDBUser: + resource = MongoDBUser.from_yaml( + yaml_fixture("olm_scram_sha_user_backing_db.yaml"), + namespace=namespace, + name=name, + ) + resource["spec"]["mongodbResourceRef"]["name"] = replica_set_name + resource["spec"]["passwordSecretKeyRef"]["name"] = secret_name + create_or_update_secret(namespace, secret_name, {"password": password}) + return resource.update() + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_resources_created( + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + mdb_sharded: MongoDB, + blockstore_user: MongoDBUser, + oplog_user: MongoDBUser, +): + """Creates mongodb databases all at once""" + oplog_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + mdb_sharded.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_set_backup_users( + ops_manager: MongoDBOpsManager, + oplog_user: MongoDBUser, + blockstore_user: MongoDBUser, +): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager["spec"]["backup"]["blockStores"][0]["mongodbUserRef"] = {"name": blockstore_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase(Phase.Running, ignore_errors=True) + + assert ops_manager.backup_status().get_message() is None + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_om_connectivity_with_backup( + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, +): + wait_for_om_healthy_response(ops_manager) + + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_resources_in_running_state_before_upgrade( + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + s3_replica_set: MongoDB, + mdb_sharded: MongoDB, +): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + mdb_sharded.assert_reaches_phase(Phase.Running) + + +# upgrade the operator + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_operator_upgrade_to_fast( + namespace: str, + version_id: str, + catalog_source: CustomObject, + subscription: CustomObject, +): + current_operator_version = get_current_operator_version() + incremented_operator_version = increment_patch_version(current_operator_version) + + # It is very likely that OLM will be doing a series of status updates during this time. + # It's better to employ a retry mechanism and spin here for a while before failing. + def update_subscription() -> bool: + try: + subscription.load() + subscription["spec"]["channel"] = "fast" # fast channel contains operator build from the current branch + subscription.update() + return True + except kubernetes.client.ApiException as e: + if e.status == 409: + return False + else: + raise e + + run_periodically(update_subscription, timeout=100, msg="Subscription to be updated") + + wait_for_operator_ready(namespace, f"mongodb-enterprise.v{incremented_operator_version}") + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_one_resources_not_in_running_state(ops_manager: MongoDBOpsManager, mdb_sharded: MongoDB): + # Wait for the first resource to become reconciling after operator upgrade. + # Only then wait for all to not get a false positive when all resources are ready, + # because the upgraded operator haven't started reconciling + ops_manager.om_status().assert_reaches_phase(Phase.Pending, timeout=600) + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_resources_in_running_state_after_upgrade( + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + s3_replica_set: MongoDB, + mdb_sharded: MongoDB, +): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + oplog_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + blockstore_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + s3_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + mdb_sharded.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_olm_operator_upgrade_with_resources +def test_resources_connectivity_after_upgrade( + ca_path: str, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + s3_replica_set: MongoDB, + mdb_sharded: MongoDB, +): + wait_for_om_healthy_response(ops_manager) + + oplog_replica_set.assert_connectivity() + blockstore_replica_set.assert_connectivity() + s3_replica_set.assert_connectivity() + mdb_sharded.assert_connectivity(ca_path=ca_path) diff --git a/docker/mongodb-enterprise-tests/tests/olm/olm_test_commons.py b/docker/mongodb-enterprise-tests/tests/olm/olm_test_commons.py new file mode 100644 index 000000000..897dd0115 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/olm/olm_test_commons.py @@ -0,0 +1,194 @@ +import json +import os +import re +import tempfile +import time +from typing import Callable + +import kubetester +import requests +import yaml +from kubeobject import CustomObject +from kubernetes import client +from kubetester import run_periodically + + +def custom_object_from_yaml(yaml_string: str) -> CustomObject: + tmpfile_name = tempfile.mkstemp()[1] + try: + with open(tmpfile_name, "w") as tmpfile: + tmpfile.write(yaml_string) + return CustomObject.from_yaml(tmpfile_name) + finally: + os.remove(tmpfile_name) + + +def get_operator_group_resource(namespace: str, target_namespace: str) -> CustomObject: + resource = CustomObject( + "mongodb-group", + namespace, + "OperatorGroup", + "operatorgroups", + "operators.coreos.com", + "v1", + ) + resource["spec"] = {"targetNamespaces": [target_namespace]} + + return resource + + +def get_catalog_source_custom_object(namespace: str, name: str): + return CustomObject( + name, + namespace, + "CatalogSource", + "catalogsources", + "operators.coreos.com", + "v1alpha1", + ) + + +def get_catalog_source_resource(namespace: str, image: str) -> CustomObject: + resource = get_catalog_source_custom_object(namespace, "mongodb-operator-catalog") + resource["spec"] = { + "image": image, + "sourceType": "grpc", + "displayName": "MongoDB Enterprise Operator upgrade test", + "publisher": "MongoDB", + "updateStrategy": {"registryPoll": {"interval": "5m"}}, + } + + return resource + + +def get_package_manifest_resource(namespace: str, manifest_name: str = "mongodb-enterprise") -> CustomObject: + return CustomObject( + manifest_name, + namespace, + "PackageManifest", + "packagemanifests", + "packages.operators.coreos.com", + "v1", + ) + + +def get_subscription_custom_object(name: str, namespace: str, spec: dict[str, str]) -> CustomObject: + resource = CustomObject( + name, + namespace, + "Subscription", + "subscriptions", + "operators.coreos.com", + "v1alpha1", + ) + resource["spec"] = spec + return resource + + +def get_registry(): + registry = os.getenv("REGISTRY") + if registry is None: + raise Exception("Cannot get base registry url, specify it in REGISTRY env variable.") + + return registry + + +def get_catalog_image(version: str): + return f"{get_registry()}/mongodb-enterprise-operator-certified-catalog:{version}" + + +def list_operator_pods(namespace: str, name: str) -> list[client.V1Pod]: + return client.CoreV1Api().list_namespaced_pod(namespace, label_selector=f"app.kubernetes.io/name={name}") + + +def check_operator_pod_ready_and_with_condition_version( + namespace: str, name: str, expected_condition_version +) -> tuple[bool, str]: + pod = kubetester.is_pod_ready(namespace=namespace, label_selector=f"app.kubernetes.io/name={name}") + if pod is None: + return False, f"pod {namespace}/{name} is not ready yet" + + condition_env_var = get_pod_condition_env_var(pod) + if condition_env_var != expected_condition_version: + return ( + False, + f"incorrect condition env var: expected {expected_condition_version}, got {condition_env_var}", + ) + + return True, "" + + +def get_pod_condition_env_var(pod): + operator_container = pod.spec.containers[0] + operator_condition_env = [e for e in operator_container.env if e.name == "OPERATOR_CONDITION_NAME"] + if len(operator_condition_env) == 0: + return None + return operator_condition_env[0].value + + +def get_release_json_path() -> str: + # when running in pod, release.json will be available in /release.json (it's copied there in Dockerfile) + if os.path.exists("release.json"): + return "release.json" + else: + # when running locally, we try to read it from the project's dir + release_json_path = os.path.join(os.environ["PROJECT_DIR"], "release.json") + print(f"release.json not found in current path, checking {release_json_path}") + if os.path.exists(release_json_path): + return release_json_path + else: + raise Exception( + "release.json file not found, ensure it's copied into test pod or $PROJECT_DIR ({os.environ['PROJECT_DIR']}) is set to ops-manager-kubernetes dir" + ) + + +def get_release_json() -> dict[str, any]: + with open(get_release_json_path()) as f: + return json.load(f) + + +def get_current_operator_version() -> str: + return get_release_json()["mongodbOperator"] + + +def get_latest_released_operator_version() -> str: + released_operators_url = f"https://api.github.com/repos/redhat-openshift-ecosystem/certified-operators/contents/operators/mongodb-enterprise" + response = requests.get(released_operators_url, headers={"Accept": "application/vnd.github.v3+json"}) + + if response.status_code != 200: + raise Exception( + f"Error getting contents of released operators dir {released_operators_url} in certified operators repo: {response.status_code}" + ) + + data = response.json() + version_pattern = re.compile(r"(\d+\.\d+\.\d+)") + versioned_directories = [ + item["name"] for item in data if item["type"] == "dir" and version_pattern.match(item["name"]) + ] + if not versioned_directories: + raise Exception( + f"Error getting contents of released operators dir {released_operators_url} in certified operators repo: there are no versions" + ) + + print(f"Received list of versions from {released_operators_url}: {versioned_directories}") + + # GitHub is returning sorted directories, so the last one is the latest released operator + return versioned_directories[-1] + + +def increment_patch_version(version: str): + major, minor, patch = version.split(".") + return ".".join([major, minor, str(int(patch) + 1)]) + + +def wait_for_operator_ready(namespace: str, expected_operator_version: str): + def wait_for_operator_ready_fn(): + return check_operator_pod_ready_and_with_condition_version( + namespace, "mongodb-enterprise-operator", expected_operator_version + ) + + run_periodically( + wait_for_operator_ready_fn, + timeout=120, + msg=f"operator ready and with {expected_operator_version} version", + ) diff --git a/docker/mongodb-enterprise-tests/tests/operator/__init__.py b/docker/mongodb-enterprise-tests/tests/operator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/operator/fixtures/squid-proxy.yaml b/docker/mongodb-enterprise-tests/tests/operator/fixtures/squid-proxy.yaml new file mode 100644 index 000000000..a48abdd3a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/operator/fixtures/squid-proxy.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: squid-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: squid + template: + metadata: + labels: + app: squid + spec: + containers: + - name: squid + image: ubuntu/squid:edge + ports: + - containerPort: 3128 + name: squid + protocol: TCP + volumeMounts: + - name: squid-config-volume + mountPath: /etc/squid/squid.conf + subPath: squid.conf + volumes: + - name: squid-config-volume + configMap: + name: squid-config + items: + - key: squid + path: squid.conf +--- +apiVersion: v1 +kind: Service +metadata: + name: squid-service + labels: + app: squid +spec: + ports: + - port: 3128 + selector: + app: squid diff --git a/docker/mongodb-enterprise-tests/tests/operator/fixtures/squid.conf b/docker/mongodb-enterprise-tests/tests/operator/fixtures/squid.conf new file mode 100644 index 000000000..04fac707b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/operator/fixtures/squid.conf @@ -0,0 +1,39 @@ +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localhost manager +http_access deny manager +http_access allow localhost +http_access allow localnet +http_access deny all +http_port 3128 +coredump_dir /var/spool/squid +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern . 0 20% 4320 +logfile_rotate 0 +cache deny all diff --git a/docker/mongodb-enterprise-tests/tests/operator/operator_clusterwide.py b/docker/mongodb-enterprise-tests/tests/operator/operator_clusterwide.py new file mode 100644 index 000000000..53dfecc9e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/operator/operator_clusterwide.py @@ -0,0 +1,218 @@ +import time +from typing import Dict + +import pytest +from kubernetes import client +from kubetester import create_secret, read_secret, try_load +from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml +from kubetester.helm import helm_template +from kubetester.kubetester import create_testing_namespace +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import running_locally +from kubetester.mongodb import MongoDB, Phase, generic_replicaset +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture + +""" +This is the test that verifies the procedure of configuring Operator in cluster-wide scope. +See https://docs.mongodb.com/kubernetes-operator/stable/tutorial/plan-k8s-operator-install/#cluster-wide-scope + +""" + + +@fixture(scope="module") +def ops_manager_namespace(evergreen_task_id: str) -> str: + # Note, that it's safe to create the namespace with constant name as the test must be run in isolated environment + # and no collisions may happen + return create_testing_namespace(evergreen_task_id, "om-namespace") + + +@fixture(scope="module") +def mdb_namespace(evergreen_task_id: str) -> str: + return create_testing_namespace(evergreen_task_id, "mdb-namespace") + + +@fixture(scope="module") +def unmanaged_namespace(evergreen_task_id: str) -> str: + return create_testing_namespace(evergreen_task_id, "unmanaged-namespace") + + +@fixture(scope="module") +def ops_manager(ops_manager_namespace: str, custom_version: str, custom_appdb_version: str) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=ops_manager_namespace) + resource["spec"]["backup"]["enabled"] = True + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + try_load(resource) + + return resource + + +@fixture(scope="module") +def mdb(ops_manager: MongoDBOpsManager, mdb_namespace: str, namespace: str, custom_mdb_prev_version: str) -> MongoDB: + # we need to copy credentials secret - as the global api key secret exists in Operator namespace only + data = read_secret(namespace, ops_manager.api_key_secret(namespace)) + # we are now copying the secret from operator to mdb_namespace and the api_key_secret should therefore check for mdb_namespace, later + # mongodb.configure will reference this new secret + create_secret(mdb_namespace, ops_manager.api_key_secret(mdb_namespace), data) + + resource = ( + MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=mdb_namespace, + name="my-replica-set", + ) + .configure(ops_manager, "development") + .set_version(custom_mdb_prev_version) + ) + + try_load(resource) + + return resource + + +@fixture(scope="module") +def unmanaged_mdb(ops_manager: MongoDBOpsManager, unmanaged_namespace: str) -> MongoDB: + rs = generic_replicaset(unmanaged_namespace, "5.0.0", "unmanaged-mdb", ops_manager).create() + + yield rs + + rs.delete() + + +@pytest.mark.e2e_operator_clusterwide +def test_install_clusterwide_operator(operator_clusterwide: Operator): + operator_clusterwide.assert_is_running() + + +@pytest.mark.e2e_operator_multi_namespaces +def test_install_multi_namespace_operator( + operator_installation_config: Dict[str, str], + ops_manager_namespace: str, + mdb_namespace: str, + namespace: str, +): + """ + Installs the operator in default namespace and watches over both OM and MDB + namespaces. + """ + + helm_args = operator_installation_config.copy() + helm_args["operator.watchNamespace"] = ops_manager_namespace + "," + mdb_namespace + + Operator(namespace=namespace, helm_args=helm_args).install().assert_is_running() + + +@pytest.mark.e2e_operator_clusterwide +def test_configure_ops_manager_namespace(ops_manager_namespace: str, operator_installation_config: Dict[str, str]): + """create a new namespace and configures all necessary service accounts there""" + yaml_file = helm_template( + helm_args={ + "registry.imagePullSecrets": operator_installation_config["registry.imagePullSecrets"], + }, + templates="templates/database-roles.yaml", + helm_options=[f"--namespace {ops_manager_namespace}"], + ) + create_or_replace_from_yaml(client.api_client.ApiClient(), yaml_file) + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_create_image_pull_secret_ops_manager_namespace( + namespace: str, + ops_manager_namespace: str, + operator_installation_config: Dict[str, str], +): + """We need to copy image pull secrets to om namespace""" + secret_name = operator_installation_config["registry.imagePullSecrets"] + data = read_secret(namespace, secret_name) + create_secret(ops_manager_namespace, secret_name, data, type="kubernetes.io/dockerconfigjson") + + +@pytest.mark.e2e_operator_clusterwide +def test_configure_mdb_namespace(mdb_namespace: str, operator_installation_config: Dict[str, str]): + yaml_file = helm_template( + helm_args={ + "registry.imagePullSecrets": operator_installation_config["registry.imagePullSecrets"], + }, + templates="templates/database-roles.yaml", + helm_options=[f"--namespace {mdb_namespace}"], + ) + create_or_replace_from_yaml(client.api_client.ApiClient(), yaml_file) + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_create_image_pull_secret_mdb_namespace( + namespace: str, mdb_namespace: str, operator_installation_config: Dict[str, str] +): + secret_name = operator_installation_config["registry.imagePullSecrets"] + data = read_secret(namespace, secret_name) + create_secret(mdb_namespace, secret_name, data, type="kubernetes.io/dockerconfigjson") + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_create_om_in_separate_namespace(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.create_admin_secret() + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, msg_regexp=".*configuration is required for backup", timeout=900 + ) + ops_manager.get_om_tester().assert_healthiness() + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_check_k8s_resources(ops_manager: MongoDBOpsManager, ops_manager_namespace: str, namespace: str): + """Verifying that all the K8s resources were created in an ops manager namespace""" + assert ops_manager.read_statefulset().metadata.namespace == ops_manager_namespace + assert ops_manager.read_backup_statefulset().metadata.namespace == ops_manager_namespace + # api key secret is created in the Operator namespace for better access control + ops_manager.read_api_key_secret(namespace) + assert ops_manager.read_gen_key_secret().metadata.namespace == ops_manager_namespace + assert ops_manager.read_appdb_generated_password_secret().metadata.namespace == ops_manager_namespace + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_create_mdb_in_separate_namespace(mdb: MongoDB, mdb_namespace: str): + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=600) + mdb.assert_connectivity() + assert mdb.read_statefulset().metadata.namespace == mdb_namespace + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_upgrade_mdb(mdb: MongoDB, custom_mdb_version): + mdb.set_version(custom_mdb_version) + + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + mdb.assert_connectivity() + mdb.tester().assert_version(custom_mdb_version) + + +@pytest.mark.e2e_operator_clusterwide +@pytest.mark.e2e_operator_multi_namespaces +def test_delete_mdb(mdb: MongoDB): + mdb.delete() + + time.sleep(10) + with pytest.raises(client.rest.ApiException): + mdb.read_statefulset() + + +@pytest.mark.e2e_operator_multi_namespaces +def test_resources_on_unmanaged_namespaces_stay_cold(unmanaged_mdb: MongoDB): + """ + For an unmanaged resource, the status should not be updated! + """ + for i in range(10): + time.sleep(5) + + unmanaged_mdb.reload() + assert "status" not in unmanaged_mdb diff --git a/docker/mongodb-enterprise-tests/tests/operator/operator_partial_crd.py b/docker/mongodb-enterprise-tests/tests/operator/operator_partial_crd.py new file mode 100644 index 000000000..63b65a234 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/operator/operator_partial_crd.py @@ -0,0 +1,88 @@ +# Dev note: remove all the CRDs before running the test locally! +from typing import Dict + +import pytest +from kubernetes import client +from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml +from kubetester.operator import Operator, delete_operator_crds, list_operator_crds +from pytest import fixture + + +@fixture(scope="module") +def ops_manager_and_mongodb_crds(): + """Installs OM and MDB CRDs only (we need to do this manually as Helm 3 doesn't support templating for CRDs""" + create_or_replace_from_yaml(client.api_client.ApiClient(), "helm_chart/crds/mongodb.com_mongodb.yaml") + create_or_replace_from_yaml(client.api_client.ApiClient(), "helm_chart/crds/mongodb.com_opsmanagers.yaml") + + +@fixture(scope="module") +def operator_only_ops_manager_and_mongodb( + ops_manager_and_mongodb_crds, + namespace: str, + operator_installation_config: Dict[str, str], +) -> Operator: + helm_args = operator_installation_config.copy() + helm_args["operator.watchedResources"] = "{opsmanagers,mongodb}" + + return Operator( + namespace=namespace, + helm_args=helm_args, + helm_options=["--skip-crds"], + ).install() + + +@fixture(scope="module") +def mongodb_crds(): + """Installs OM and MDB CRDs only (we need to do this manually as Helm 3 doesn't support templating for CRDs""" + create_or_replace_from_yaml(client.api_client.ApiClient(), "helm_chart/crds/mongodb.com_mongodb.yaml") + + +@fixture(scope="module") +def operator_only_mongodb( + mongodb_crds, + namespace: str, + operator_installation_config: Dict[str, str], +) -> Operator: + helm_args = operator_installation_config.copy() + helm_args["operator.watchedResources"] = "{mongodb}" + + return Operator( + namespace=namespace, + helm_args=helm_args, + helm_options=["--skip-crds"], + ).install() + + +@pytest.mark.e2e_operator_partial_crd +def test_install_operator_ops_manager_and_mongodb_only( + operator_only_ops_manager_and_mongodb: Operator, +): + """Note, that currently it's not possible to install OpsManager only as it requires MongoDB resources + (it watches them internally)""" + operator_only_ops_manager_and_mongodb.assert_is_running() + + +@pytest.mark.e2e_operator_partial_crd +def test_only_ops_manager_and_mongodb_crds_exist(): + operator_crds = list_operator_crds() + assert len(operator_crds) == 2 + assert operator_crds[0].metadata.name == "mongodb.mongodb.com" + assert operator_crds[1].metadata.name == "opsmanagers.mongodb.com" + + +@pytest.mark.e2e_operator_partial_crd +def test_remove_operator_and_crds(operator_only_ops_manager_and_mongodb: Operator): + delete_operator_crds() + operator_only_ops_manager_and_mongodb.uninstall() + + +@pytest.mark.e2e_operator_partial_crd +def test_install_operator_mongodb_only(operator_only_mongodb: Operator): + operator_only_mongodb.assert_is_running() + + +@pytest.mark.e2e_operator_partial_crd +def test_only_mongodb_and_users_crds_exists(): + operator_crds = list_operator_crds() + assert len(operator_crds) == 1 + assert operator_crds[0].metadata.name == "mongodb.mongodb.com" diff --git a/docker/mongodb-enterprise-tests/tests/operator/operator_proxy.py b/docker/mongodb-enterprise-tests/tests/operator/operator_proxy.py new file mode 100644 index 000000000..755cfb476 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/operator/operator_proxy.py @@ -0,0 +1,100 @@ +import os + +import yaml +from kubernetes import client +from kubetester import create_or_update_configmap +from kubetester.create_or_replace_from_yaml import ( + create_or_replace_from_yaml as apply_yaml, +) +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as _fixture +from kubetester.kubetester import get_pods +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from oauthlib.oauth1.rfc5849.endpoints import resource +from pytest import fixture, mark + +MDB_RESOURCE = "replica-set" +PROXY_SVC_NAME = "squid-service" +PROXY_SVC_PORT = 3128 + + +@fixture(scope="module") +def squid_proxy(namespace: str) -> str: + with open(_fixture("squid.conf"), "r") as conf_file: + squid_conf = conf_file.read() + create_or_update_configmap(namespace=namespace, name="squid-config", data={"squid": squid_conf}) + + apply_yaml(client.api_client.ApiClient(), _fixture("squid-proxy.yaml"), namespace=namespace) + + def check_svc_endpoints(): + try: + endpoint = client.CoreV1Api().read_namespaced_endpoints("squid-service", namespace) + assert len(endpoint.subsets[0].addresses) == 1 + return True + except: + return False + + KubernetesTester.wait_until(check_svc_endpoints, timeout=30) + return f"http://{PROXY_SVC_NAME}:{PROXY_SVC_PORT}" + + +@fixture(scope="module") +def operator_with_proxy(namespace: str, operator_installation_config: dict[str, str], squid_proxy: str) -> Operator: + os.environ["HTTP_PROXY"] = os.environ["HTTPS_PROXY"] = squid_proxy + helm_args = operator_installation_config.copy() + helm_args["customEnvVars"] += ( + f"\&MDB_PROPAGATE_PROXY_ENV=true" + f"\&HTTP_PROXY={squid_proxy}" + f"\&HTTPS_PROXY={squid_proxy}" + ) + return Operator(namespace=namespace, helm_args=helm_args).install() + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(_fixture("replica-set-basic.yaml"), namespace=namespace, name=MDB_RESOURCE) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + resource.update() + + return resource + + +@mark.e2e_operator_proxy +def test_install_operator_with_proxy( + operator_with_proxy: Operator, +): + operator_with_proxy.assert_is_running() + + +@mark.e2e_operator_proxy +def test_replica_set_reconciles(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) + + +@mark.e2e_operator_proxy +def test_proxy_logs_requests(namespace: str): + proxy_pods = client.CoreV1Api().list_namespaced_pod(namespace, label_selector="app=squid").items + pod_name = proxy_pods[0].metadata.name + container_name = "squid" + pod_logs = KubernetesTester.read_pod_logs(namespace, pod_name, container_name) + assert "cloud-qa.mongodb.com" in pod_logs + assert "api-agents-qa.mongodb.com" in pod_logs + assert "api-backup-qa.mongodb.com" in pod_logs + + +@mark.e2e_operator_proxy +def test_proxy_env_vars_set_in_pod(namespace: str): + for pod_name in get_pods(MDB_RESOURCE + "-{}", 3): + pod = client.CoreV1Api().read_namespaced_pod(pod_name, namespace) + env_vars = {var.name: var.value for var in pod.spec.containers[0].env} + assert "http_proxy" in env_vars + assert "HTTP_PROXY" in env_vars + assert "https_proxy" in env_vars + assert "HTTPS_PROXY" in env_vars + assert ( + env_vars["HTTP_PROXY"] + == env_vars["http_proxy"] + == env_vars["HTTPS_PROXY"] + == env_vars["https_proxy"] + == f"http://{PROXY_SVC_NAME}:{PROXY_SVC_PORT}" + ) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/__init__.py b/docker/mongodb-enterprise-tests/tests/opsmanager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/backup_snapshot_schedule_tests.py b/docker/mongodb-enterprise-tests/tests/opsmanager/backup_snapshot_schedule_tests.py new file mode 100644 index 000000000..ec2162b73 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/backup_snapshot_schedule_tests.py @@ -0,0 +1,200 @@ +from typing import Dict + +import pytest +from kubernetes.client import ApiException +from kubetester import try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.omtester import OMTester +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture + + +class BackupSnapshotScheduleTests: + """Test executes snapshot schedule tests on top of existing ops_manager. + + This test class is intended to be reused by inheriting from it and overriding fixtures: + mdb - for providing base MongoDB resource + mdb_version - for customizing MongoDB versions. + om_project_name - for customizing project name - for multiple tests running on top of single OM. + """ + + @fixture + def mdb(self, ops_manager: MongoDBOpsManager): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name="mdb-backup-snapshot-schedule", + ) + + try_load(resource) + return resource + + @fixture + def mdb_version(self, custom_mdb_version): + return custom_mdb_version + + @fixture + def om_project_name(self): + return "backupSnapshotSchedule" + + def test_create_mdb_with_backup_enabled_and_configured_snapshot_schedule( + self, + mdb: MongoDB, + ops_manager: MongoDBOpsManager, + mdb_version: str, + om_project_name: str, + ): + mdb.configure(ops_manager, om_project_name) + + mdb["spec"]["version"] = ensure_ent_version(mdb_version) + mdb.configure_backup(mode="enabled") + mdb["spec"]["backup"]["snapshotSchedule"] = { + "fullIncrementalDayOfWeek": "MONDAY", + } + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + mdb.assert_backup_reaches_status("STARTED") + + self.assert_snapshot_schedule_in_ops_manager( + mdb.get_om_tester(), + { + "fullIncrementalDayOfWeek": "MONDAY", + }, + ) + + @pytest.mark.skip( + reason="Backup termination is not handled properly by the operator: https://jira.mongodb.org/browse/CLOUDP-149270" + ) + def test_when_backup_terminated_snapshot_schedule_is_ignored(self, mdb: MongoDB): + mdb.configure_backup(mode="disabled") + mdb.update() + mdb.assert_reaches_phase(Phase.Running) + mdb.assert_backup_reaches_status("STOPPED") + + mdb.load() + mdb.configure_backup(mode="terminated") + mdb.update() + mdb.assert_reaches_phase(Phase.Running) + mdb.assert_backup_reaches_status("TERMINATING") + + try: + mdb.get_om_tester().api_read_backup_snapshot_schedule() + assert False, "exception about missing backup configuration should be raised" + except Exception: + pass + + mdb.configure_backup(mode="enabled") + mdb.update() + mdb.assert_reaches_phase(Phase.Running) + mdb.assert_backup_reaches_status("STARTED") + + def test_stop_backup_and_change_snapshot_schedule(self, mdb: MongoDB): + mdb.configure_backup(mode="disabled") + self.update_snapshot_schedule( + mdb, + { + "fullIncrementalDayOfWeek": "WEDNESDAY", + }, + ) + mdb.assert_backup_reaches_status("STOPPED") + + self.assert_snapshot_schedule_in_ops_manager( + mdb.get_om_tester(), + { + "fullIncrementalDayOfWeek": "WEDNESDAY", + }, + ) + + def test_enable_backup_and_change_snapshot_schedule(self, mdb: MongoDB): + mdb.configure_backup(mode="enabled") + self.update_snapshot_schedule( + mdb, + { + "fullIncrementalDayOfWeek": "TUESDAY", + }, + ) + mdb.assert_backup_reaches_status("STARTED") + + self.assert_snapshot_schedule_in_ops_manager( + mdb.get_om_tester(), + { + "fullIncrementalDayOfWeek": "TUESDAY", + }, + ) + + def test_only_one_field_is_set(self, mdb: MongoDB): + prev_snapshot_schedule = mdb.get_om_tester().api_read_backup_snapshot_schedule() + + self.update_snapshot_schedule( + mdb, + { + "fullIncrementalDayOfWeek": "THURSDAY", + }, + ) + + expected_snapshot_schedule = dict(prev_snapshot_schedule) + expected_snapshot_schedule["fullIncrementalDayOfWeek"] = "THURSDAY" + + self.assert_snapshot_schedule_in_ops_manager(mdb.get_om_tester(), expected_snapshot_schedule) + + def test_check_all_fields_are_set(self, mdb: MongoDB): + self.update_and_assert_snapshot_schedule( + mdb, + { + "snapshotIntervalHours": 12, + "snapshotRetentionDays": 3, + "dailySnapshotRetentionDays": 5, + "weeklySnapshotRetentionWeeks": 4, + "monthlySnapshotRetentionMonths": 5, + "pointInTimeWindowHours": 6, + "referenceHourOfDay": 1, + "referenceMinuteOfHour": 3, + "fullIncrementalDayOfWeek": "THURSDAY", + }, + ) + + def test_validations(self, mdb: MongoDB): + # we're smoke-testing if any of the CRD validations works + try: + self.update_snapshot_schedule( + mdb, + { + "fullIncrementalDayOfWeek": "January", + }, + ) + except ApiException as e: + assert e.status == 422 # "Unprocessable Entity" + pass + + # revert state back to running + self.update_snapshot_schedule( + mdb, + { + "fullIncrementalDayOfWeek": "WEDNESDAY", + }, + ) + + @staticmethod + def update_snapshot_schedule(mdb: MongoDB, snapshot_schedule: Dict): + last_transition = mdb.get_status_last_transition_time() + + mdb["spec"]["backup"]["snapshotSchedule"] = snapshot_schedule + mdb.update() + + mdb.assert_state_transition_happens(last_transition) + mdb.assert_reaches_phase(Phase.Running, ignore_errors=True) + + @staticmethod + def assert_snapshot_schedule_in_ops_manager(om_tester: OMTester, expected_snapshot_schedule: Dict): + snapshot_schedule = om_tester.api_read_backup_snapshot_schedule() + + for k, v in expected_snapshot_schedule.items(): + assert k in snapshot_schedule + assert snapshot_schedule[k] == v + + @staticmethod + def update_and_assert_snapshot_schedule(mdb: MongoDB, snapshot_schedule: Dict): + BackupSnapshotScheduleTests.update_snapshot_schedule(mdb, snapshot_schedule) + BackupSnapshotScheduleTests.assert_snapshot_schedule_in_ops_manager(mdb.get_om_tester(), snapshot_schedule) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/conftest.py b/docker/mongodb-enterprise-tests/tests/opsmanager/conftest.py new file mode 100644 index 000000000..fd506f484 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/conftest.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +import os +from pathlib import Path +from typing import Dict, Optional + +from kubernetes import client +from kubetester import get_pod_when_ready +from kubetester.helm import helm_install_from_chart +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import is_multi_cluster + +MINIO_OPERATOR = "minio-operator" +MINIO_TENANT = "minio-tenant" + + +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if is_multi_cluster(): + if item.fixturenames not in ( + "multi_cluster_operator_with_monitored_appdb", + "multi_cluster_operator", + ): + print("\nAdding operator installation fixture: multi_cluster_operator") + item.fixturenames.insert(0, "multi_cluster_operator_with_monitored_appdb") + elif item.fixturenames not in [ + "default_operator", + "operator_with_monitored_appdb", + "multi_cluster_operator_with_monitored_appdb", + "multi_cluster_operator", + ]: + item.fixturenames.insert(0, "default_operator") + + +@fixture(scope="module") +def custom_om_prev_version() -> str: + """Returns a CUSTOM_OM_PREV_VERSION for OpsManager to be created/upgraded.""" + return os.getenv("CUSTOM_OM_PREV_VERSION", "6.0.0") + + +@fixture(scope="module") +def custom_mdb_prev_version() -> str: + """Returns a CUSTOM_MDB_PREV_VERSION for Mongodb to be created/upgraded to for testing. + Used for backup mainly (to test backup for different mdb versions). + Defaults to 4.4.24 (simplifies testing locally)""" + return os.getenv("CUSTOM_MDB_PREV_VERSION", "5.0.15") + + +@fixture(scope="module") +def gen_key_resource_version(ops_manager: MongoDBOpsManager) -> str: + secret = ops_manager.read_gen_key_secret() + return secret.metadata.resource_version + + +@fixture(scope="module") +def admin_key_resource_version(ops_manager: MongoDBOpsManager) -> str: + secret = ops_manager.read_api_key_secret() + return secret.metadata.resource_version + + +def mino_operator_install( + namespace: str, + operator_name: str = MINIO_OPERATOR, + cluster_client: Optional[client.ApiClient] = None, + cluster_name: Optional[str] = None, + helm_args: Dict[str, str] = None, + version="5.0.6", +): + if cluster_name is not None: + os.environ["HELM_KUBECONTEXT"] = cluster_name + + if helm_args is None: + helm_args = {} + helm_args.update( + { + "namespace": namespace, + "fullnameOverride": operator_name, + "nameOverride": operator_name, + } + ) + + # check if the pod exists, if not do a helm upgrade + operator_pod = client.CoreV1Api(api_client=cluster_client).list_namespaced_pod( + namespace, label_selector=f"app.kubernetes.io/instance={operator_name}" + ) + # check if the console exists, if not do a helm upgrade + console_pod = client.CoreV1Api(api_client=cluster_client).list_namespaced_pod( + namespace, label_selector=f"app.kubernetes.io/instance=minio-operator-console" + ) + if not operator_pod.items or not console_pod: + print(f"Performing helm upgrade of minio-operator") + + helm_install_from_chart( + release=operator_name, + namespace=namespace, + helm_args=helm_args, + version=version, + custom_repo=("minio", "https://operator.min.io/"), + chart=f"minio/operator", + ) + else: + print(f"Minio operator already installed, skipping helm installation!") + + get_pod_when_ready( + namespace, + f"app.kubernetes.io/instance={operator_name}", + api_client=cluster_client, + ) + get_pod_when_ready( + namespace, + f"app.kubernetes.io/instance=minio-operator-console", + api_client=cluster_client, + ) + + +def mino_tenant_install( + namespace: str, + tenant_name: str = MINIO_TENANT, + cluster_client: Optional[client.ApiClient] = None, + cluster_name: Optional[str] = None, + helm_args: Dict[str, str] = None, + version="5.0.6", +): + if cluster_name is not None: + os.environ["HELM_KUBECONTEXT"] = cluster_name + + # check if the minio pod exists, if not do a helm upgrade + pods = client.CoreV1Api(api_client=cluster_client).list_namespaced_pod(namespace, label_selector=f"app=minio") + if not pods.items: + print(f"Performing helm upgrade of minio-tenant") + + path = f"{Path(__file__).parent}/fixtures/minio/values-tenant.yaml" + helm_install_from_chart( + release=tenant_name, + namespace=namespace, + helm_args=helm_args, + version=version, + custom_repo=("minio", "https://operator.min.io/"), + chart=f"minio/tenant", + override_path=path, + ) + else: + print(f"Minio tenant already installed, skipping helm installation!") + + get_pod_when_ready(namespace, f"app=minio", api_client=cluster_client) + + +def get_appdb_member_cluster_names(): + return ["kind-e2e-cluster-2", "kind-e2e-cluster-3"] diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/amazon-ca-1.pem b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/amazon-ca-1.pem new file mode 100644 index 000000000..a6f3e92af --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/amazon-ca-1.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/amazon-ca-2.pem b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/amazon-ca-2.pem new file mode 100644 index 000000000..efe3c9fed --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/amazon-ca-2.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK +gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ +W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg +1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K +8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r +2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me +z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR +8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj +mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz +7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 ++XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI +0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm +UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 +LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY ++gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS +k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl +7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm +btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl +urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ +fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 +n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE +76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H +9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT +4PsJYGw= +-----END CERTIFICATE----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls-full-chain.crt b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls-full-chain.crt new file mode 100644 index 000000000..5dd75939e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls-full-chain.crt @@ -0,0 +1,131 @@ +-----BEGIN CERTIFICATE----- +MIIDDjCCAfagAwIBAgIJAPifkd/1ZoEXMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV +BAYTAklSMB4XDTIzMDExMDEzNDMwMVoXDTMzMDEwNzEzNDMwMVowDTELMAkGA1UE +BhMCSVIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCzsVQpiIqdBO87 +SVJsgXjJ7paWsnXDCSmFjOSELrGw1l2pnoYxc9IDysP4rsdIw3THWp3w3Wv/kqQf +PD8cY6r4RMbPWn6lN3/vGfgQz3XaosfobLuJFZRORZQY870FH21nBE8iNqapsxPy +NVu70TyqbThmM1bUZ0sB+RIaBUEY0e+M7bVxxFdt1t+dmNdmJg0R4RuJWz/UbUYo +wyNhqTe98NE7SKGFti7q3mhOEg1s6zvtqO6EILTg0i0Ndf8rKgJVds6h+HRUFoxt +rssoxwycDxPhCKlRFMCzYmPXE+PGJ5BXXctlj8Wh5gGwQZ8pMboS/GqFoRQfW1tI +HwvVnednAgMBAAGjcTBvMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI10Lxi6 +GKcBbc0+zqDXFRepKpEbMD0GA1UdIwQ2MDSAFI10Lxi6GKcBbc0+zqDXFRepKpEb +oRGkDzANMQswCQYDVQQGEwJJUoIJAPifkd/1ZoEXMA0GCSqGSIb3DQEBCwUAA4IB +AQCdimbB5spMgpIv+NPGoeJTXWlojWZ9SBvYTPG5wGjcoPMvM81f3+XRrXPfyl+e +iozVDYL8YE3IGPtqlSYdlS4gg43+sNvXCv0O/dT/T/587LuRLP3iYCyVcn/1UcXJ +IDfYUJR6JVkKtI6Rk0WDTuDmfUAmn5S35Ll8fvZnC2WcWSRqnYhKAqcmac/t89AX +M9mhOT3qOKMfkgT94wgtz5DTR9lAEAVfEgrmQ+zbHDfAfzPD7fDZ5WXIhhU15Wkp +0My5C/ob54EonfnxQCqc/Xge6oSRmNnPIMBhBffPvDpJK23MCIXm5yyWaYhqD3M+ +K3ASw6isSK98ICi/ujWSHWC0 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIF9zCCBN+gAwIBAgIQDTiokZJgKoaLCOzt0K3ulzANBgkqhkiG9w0BAQsFADBG +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRUwEwYDVQQLEwxTZXJ2ZXIg +Q0EgMUIxDzANBgNVBAMTBkFtYXpvbjAeFw0yMjA2MjIwMDAwMDBaFw0yMzA3MjEy +MzU5NTlaMCAxHjAcBgNVBAMTFWRvd25sb2Fkcy5tb25nb2RiLmNvbTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBANZyoobPRnuB0Y8g1MNre1ptStvETcBw +bXh3g7QtgEoFUryEQPnrkPQJUh7vWmVAhwyfYb3NjoGPoUfCC7vc2kVqXUfydV45 +/V8mUhs/8dbpZXn//sfoFum5PrbX+GydWrvtC17USkWto+Miw046dqEntmill7J4 +znR3EHmvNZiXMkf67QCyBjpTT71LYecJFjKVxjUdC3xe8TOsCsx+2IPMVNvIeqxt +G3aW0rPxuFlQ7oIyXgdRXtlD5vdoPioxk+P2vj1gmKtTGbKUTDqac2r2cwP0uuGo +xV0gN/JGmufEu6bcngIj2DXpMEtBgHcYXk8IgQpe4QLiRdrgaK+DMz8CAwEAAaOC +AwUwggMBMB8GA1UdIwQYMBaAFFmkZgZSoHuVkjyjlAcnlnRb+T3QMB0GA1UdDgQW +BBQQ/1E/6lSuAqx97QEK8lVn0L81bjA1BgNVHREELjAsghVkb3dubG9hZHMubW9u +Z29kYi5jb22CE2Rvd25sb2Fkcy4xMGdlbi5jb20wDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKgMKAuhixo +dHRwOi8vY3JsLnNjYTFiLmFtYXpvbnRydXN0LmNvbS9zY2ExYi0xLmNybDATBgNV +HSAEDDAKMAgGBmeBDAECATB1BggrBgEFBQcBAQRpMGcwLQYIKwYBBQUHMAGGIWh0 +dHA6Ly9vY3NwLnNjYTFiLmFtYXpvbnRydXN0LmNvbTA2BggrBgEFBQcwAoYqaHR0 +cDovL2NydC5zY2ExYi5hbWF6b250cnVzdC5jb20vc2NhMWIuY3J0MAwGA1UdEwEB +/wQCMAAwggF+BgorBgEEAdZ5AgQCBIIBbgSCAWoBaAB2AOg+0No+9QY1MudXKLyJ +a8kD08vREWvs62nhd31tBr1uAAABgYyc2+4AAAQDAEcwRQIgJcz+XaaaFAcPiteo +AaunoWepexB4vim6CmH5RjUxp9gCIQCD4rMXUFwiUbwrCvzYDNxoLmApMCWOwxgF +Sg0KpEFcyQB2ADXPGRu/sWxXvw+tTG1Cy7u2JyAmUeo/4SrvqAPDO9ZMAAABgYyc +3CEAAAQDAEcwRQIhAMKh1TL/yOlbSCIGf1NBMy3y+TLSjg672TAbxks+i/w/AiBX +RMleFwVvl0rfn7mtjx4sNg/KV7jxGWlSsE9HVE59tQB2ALc++yTfnE26dfI5xbpY +9Gxd/ELPep81xJ4dCYEl7bSZAAABgYyc3CQAAAQDAEcwRQIhAMcmDntNQU28qZd0 +8SaSdXKsb6DxJC/aZF+aYCi2dv14AiBHPCw6fOGgPSYM4yhs5cdqwK9wUH23G+22 +yYj5PrHTtTANBgkqhkiG9w0BAQsFAAOCAQEAooTXKMDE9j5bXGpGB8pIyYcDJVZK +Xe14ClMwJdcIr2dJM1HY++2+x9taTT1+C/bxwOfU+S5O68cGTcSY5/HZKbhIMHo4 +oGBzpfREIxMLl9n74USGBbf7h8bI+m7wJK+68HhH3La6Hd1tic6DkPmpVxrRB5ux +as1tbGN26JON/UqKxUZwcx+szhdEApWkmEJBm9S3A6/jN/yIPW5IiS084jRSk9YG +p6TInSGnHnf3M5CnlSUEG1tgMNl1tbvAf1NIsAdMSDwahTtZqO39N8XD7X/IriT2 +z20gl9DxUae6etCoGbo44+giCkqDfIeZFcabLqJiD8bNz6wj6ac17pysXQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIESTCCAzGgAwIBAgITBn+UV4WH6Kx33rJTMlu8mYtWDTANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MTAyMjAwMDAwMFoXDTI1MTAxOTAwMDAwMFowRjEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEVMBMGA1UECxMMU2VydmVyIENB +IDFCMQ8wDQYDVQQDEwZBbWF6b24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDCThZn3c68asg3Wuw6MLAd5tES6BIoSMzoKcG5blPVo+sDORrMd4f2AbnZ +cMzPa43j4wNxhplty6aUKk4T1qe9BOwKFjwK6zmxxLVYo7bHViXsPlJ6qOMpFge5 +blDP+18x+B26A0piiQOuPkfyDyeR4xQghfj66Yo19V+emU3nazfvpFA+ROz6WoVm +B5x+F2pV8xeKNR7u6azDdU5YVX1TawprmxRC1+WsAYmz6qP+z8ArDITC2FMVy2fw +0IjKOtEXc/VfmtTFch5+AfGYMGMqqvJ6LcXiAhqG5TI+Dr0RtM88k+8XUBCeQ8IG +KuANaL7TiItKZYxK1MMuTJtV9IblAgMBAAGjggE7MIIBNzASBgNVHRMBAf8ECDAG +AQH/AgEAMA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUWaRmBlKge5WSPKOUByeW +dFv5PdAwHwYDVR0jBBgwFoAUhBjMhTTsvAyUlC4IWZzHshBOCggwewYIKwYBBQUH +AQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8vb2NzcC5yb290Y2ExLmFtYXpvbnRy +dXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDovL2NydC5yb290Y2ExLmFtYXpvbnRy +dXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8EODA2MDSgMqAwhi5odHRwOi8vY3Js +LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY3JsMBMGA1UdIAQMMAow +CAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IBAQCFkr41u3nPo4FCHOTjY3NTOVI1 +59Gt/a6ZiqyJEi+752+a1U5y6iAwYfmXss2lJwJFqMp2PphKg5625kXg8kP2CN5t +6G7bMQcT8C8xDZNtYTd7WPD8UZiRKAJPBXa30/AbwuZe0GaFEQ8ugcYQgSn+IGBI +8/LwhBNTZTUVEWuCUUBVV18YtbAiPq3yXqMB48Oz+ctBWuZSkbvkNodPLamkB2g1 +upRyzQ7qDn1X8nn8N8V7YJ6y68AtkHcNSRAnpTitxBKjtKPISLMVCx7i4hncxHZS +yLyKQXhw2W2Xs0qLeC1etA+jTGDK4UfLeC0SF7FSi8o5LL21L8IzApar2pR/ +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEkjCCA3qgAwIBAgITBn+USionzfP6wq4rAfkI7rnExjANBgkqhkiG9w0BAQsF +ADCBmDELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNj +b3R0c2RhbGUxJTAjBgNVBAoTHFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4x +OzA5BgNVBAMTMlN0YXJmaWVsZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1 +dGhvcml0eSAtIEcyMB4XDTE1MDUyNTEyMDAwMFoXDTM3MTIzMTAxMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaOCATEwggEtMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgGGMB0GA1UdDgQWBBSEGMyFNOy8DJSULghZnMeyEE4KCDAfBgNVHSMEGDAW +gBScXwDfqgHXMCs4iKK4bUqc8hGRgzB4BggrBgEFBQcBAQRsMGowLgYIKwYBBQUH +MAGGImh0dHA6Ly9vY3NwLnJvb3RnMi5hbWF6b250cnVzdC5jb20wOAYIKwYBBQUH +MAKGLGh0dHA6Ly9jcnQucm9vdGcyLmFtYXpvbnRydXN0LmNvbS9yb290ZzIuY2Vy +MD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly9jcmwucm9vdGcyLmFtYXpvbnRydXN0 +LmNvbS9yb290ZzIuY3JsMBEGA1UdIAQKMAgwBgYEVR0gADANBgkqhkiG9w0BAQsF +AAOCAQEAYjdCXLwQtT6LLOkMm2xF4gcAevnFWAu5CIw+7bMlPLVvUOTNNWqnkzSW +MiGpSESrnO09tKpzbeR/FoCJbM8oAxiDR3mjEH4wW6w7sGDgd9QIpuEdfF7Au/ma +eyKdpwAJfqxGF4PcnCZXmTA5YpaP7dreqsXMGz7KQ2hsVxa81Q4gLv7/wmpdLqBK +bRRYh5TmOTFffHPLkIhqhBGWJ6bt2YFGpn6jcgAKUj6DiAdjd4lpFw85hdKrCEVN +0FE6/V1dN2RMfjCyVSRCnTawXZwXgWHxyvkQAiSr6w10kY17RSlQOYiypok1JR4U +akcjMS9cmvqtmg5iUaQqqcT5NJ0hGA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEdTCCA12gAwIBAgIJAKcOSkw0grd/MA0GCSqGSIb3DQEBCwUAMGgxCzAJBgNV +BAYTAlVTMSUwIwYDVQQKExxTdGFyZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTIw +MAYDVQQLEylTdGFyZmllbGQgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eTAeFw0wOTA5MDIwMDAwMDBaFw0zNDA2MjgxNzM5MTZaMIGYMQswCQYDVQQGEwJV +UzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTElMCMGA1UE +ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjE7MDkGA1UEAxMyU3RhcmZp +ZWxkIFNlcnZpY2VzIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVDDrEKvlO4vW+GZdfjohTsR8/ +y8+fIBNtKTrID30892t2OGPZNmCom15cAICyL1l/9of5JUOG52kbUpqQ4XHj2C0N +Tm/2yEnZtvMaVq4rtnQU68/7JuMauh2WLmo7WJSJR1b/JaCTcFOD2oR0FMNnngRo +Ot+OQFodSk7PQ5E751bWAHDLUu57fa4657wx+UX2wmDPE1kCK4DMNEffud6QZW0C +zyyRpqbn3oUYSXxmTqM6bam17jQuug0DuDPfR+uxa40l2ZvOgdFFRjKWcIfeAg5J +Q4W2bHO7ZOphQazJ1FTfhy/HIrImzJ9ZVGif/L4qL8RVHHVAYBeFAlU5i38FAgMB +AAGjgfAwge0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0O +BBYEFJxfAN+qAdcwKziIorhtSpzyEZGDMB8GA1UdIwQYMBaAFL9ft9HO3R+G9FtV +rNzXEMIOqYjnME8GCCsGAQUFBwEBBEMwQTAcBggrBgEFBQcwAYYQaHR0cDovL28u +c3MyLnVzLzAhBggrBgEFBQcwAoYVaHR0cDovL3guc3MyLnVzL3guY2VyMCYGA1Ud +HwQfMB0wG6AZoBeGFWh0dHA6Ly9zLnNzMi51cy9yLmNybDARBgNVHSAECjAIMAYG +BFUdIAAwDQYJKoZIhvcNAQELBQADggEBACMd44pXyn3pF3lM8R5V/cxTbj5HD9/G +VfKyBDbtgB9TxF00KGu+x1X8Z+rLP3+QsjPNG1gQggL4+C/1E2DUBc7xgQjB3ad1 +l08YuW3e95ORCLp+QCztweq7dp4zBncdDQh/U90bZKuCJ/Fp1U1ervShw3WnWEQt +8jxwmKy6abaVd38PMV4s/KCHOkdp8Hlf9BRUpJVeEXgSYCfOn8J3/yNTd126/+pZ +59vPr5KW7ySaNRB6nJHGDn2Z9j8Z3/VyVOEVqQdZe4O/Ui5GjLIAZHYcSNPYeehu +VsyuLAOQ1xk4meTKCRlb/weWsKh/NEnfVqn3sF/tM+2MR7cwA130A4w= +-----END CERTIFICATE----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.crt b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.crt new file mode 100644 index 000000000..2dfe19859 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDDjCCAfagAwIBAgIJAPifkd/1ZoEXMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV +BAYTAklSMB4XDTIzMDExMDEzNDMwMVoXDTMzMDEwNzEzNDMwMVowDTELMAkGA1UE +BhMCSVIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCzsVQpiIqdBO87 +SVJsgXjJ7paWsnXDCSmFjOSELrGw1l2pnoYxc9IDysP4rsdIw3THWp3w3Wv/kqQf +PD8cY6r4RMbPWn6lN3/vGfgQz3XaosfobLuJFZRORZQY870FH21nBE8iNqapsxPy +NVu70TyqbThmM1bUZ0sB+RIaBUEY0e+M7bVxxFdt1t+dmNdmJg0R4RuJWz/UbUYo +wyNhqTe98NE7SKGFti7q3mhOEg1s6zvtqO6EILTg0i0Ndf8rKgJVds6h+HRUFoxt +rssoxwycDxPhCKlRFMCzYmPXE+PGJ5BXXctlj8Wh5gGwQZ8pMboS/GqFoRQfW1tI +HwvVnednAgMBAAGjcTBvMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI10Lxi6 +GKcBbc0+zqDXFRepKpEbMD0GA1UdIwQ2MDSAFI10Lxi6GKcBbc0+zqDXFRepKpEb +oRGkDzANMQswCQYDVQQGEwJJUoIJAPifkd/1ZoEXMA0GCSqGSIb3DQEBCwUAA4IB +AQCdimbB5spMgpIv+NPGoeJTXWlojWZ9SBvYTPG5wGjcoPMvM81f3+XRrXPfyl+e +iozVDYL8YE3IGPtqlSYdlS4gg43+sNvXCv0O/dT/T/587LuRLP3iYCyVcn/1UcXJ +IDfYUJR6JVkKtI6Rk0WDTuDmfUAmn5S35Ll8fvZnC2WcWSRqnYhKAqcmac/t89AX +M9mhOT3qOKMfkgT94wgtz5DTR9lAEAVfEgrmQ+zbHDfAfzPD7fDZ5WXIhhU15Wkp +0My5C/ob54EonfnxQCqc/Xge6oSRmNnPIMBhBffPvDpJK23MCIXm5yyWaYhqD3M+ +K3ASw6isSK98ICi/ujWSHWC0 +-----END CERTIFICATE----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.key b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.key new file mode 100644 index 000000000..7cd89c24d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCzsVQpiIqdBO87 +SVJsgXjJ7paWsnXDCSmFjOSELrGw1l2pnoYxc9IDysP4rsdIw3THWp3w3Wv/kqQf +PD8cY6r4RMbPWn6lN3/vGfgQz3XaosfobLuJFZRORZQY870FH21nBE8iNqapsxPy +NVu70TyqbThmM1bUZ0sB+RIaBUEY0e+M7bVxxFdt1t+dmNdmJg0R4RuJWz/UbUYo +wyNhqTe98NE7SKGFti7q3mhOEg1s6zvtqO6EILTg0i0Ndf8rKgJVds6h+HRUFoxt +rssoxwycDxPhCKlRFMCzYmPXE+PGJ5BXXctlj8Wh5gGwQZ8pMboS/GqFoRQfW1tI +HwvVnednAgMBAAECggEAeZkth/GjQ4B8V5VVlqHC2HuBIjdf43zGwV5HoX9rtWxK +86aXzs0+uFw1Y4r6xq2lz+XtbXqZQ9i7AXwmhRKZNupr0xO9EhbNl0LukImjijGP +sCQsgCa/Nnx1LLF8HwRWZ1kOJ+vtuna5r7UV/7InKHlCqj5hqti/dHVVH5Cgrab4 +4HBTAnwX2YVKaU163GKTTJyVGzGwN9AOONnLuGpGs8j700McKe8Ydn461W5TKdoG +RbPcS6lbQztNE6Nn8oXNuFRsm8Wf9ZwvPqypQqgv0CIYxxUt4EEtMngNWQgJhPAc +uZr5htR3+rrFi+exlfm/gn6UZ2GWxZtN+w1/xZSa8QKBgQDcGkFg5IQUuiwTub9L +Ki3ShApfIqkdSsF26T3PoeF9mASzQslXxQSj6dMdFPlwbq6baIRxygmkA2LzBgDp +uUvPxoRKQGcpddzmsYLQIht43S1R12kN9JR3jcaIH4dxYecTTyjgdvBWYm1aGQwW +Nga8GMIl/CnrEeOkwqkgmvrdaQKBgQDQ/+BZOQUFtQdViQEjxCZFbCJs29v+c552 +rVFEtH7DbIymUDKWeqlMNt4ibnSL+Z4pmUgaWE7GlRdwNBM/9ACWiRfh10J0Ybyi +rS5yHVGK1759N9H4pYKUngUCYUihKlzYKB+/43QG+WbQ/INm2TUho05s7rOo81F3 +q172iuB0TwKBgQCTJMZSZVLbnH69DS+Sq3cIxpc8dKqEV6awvUtCVOGvmgKCaQK7 +t43bmwU06wG7JXN7l8r7W2tIh68N8xSHLAY/uGJWVWniMNZmL4PZawPcsFiM3ypv +VvQuXMy90f41UZMuuHwGW91ektyyIA6RhrrH4vFgfYz0hvgd/LkegB14CQKBgQCf +vRIR35zREd27KG2wknjV0qI1JY1tW50gA7P7mSDR6KNPcjhX/wRqdf0tv9JgMbcL +AFa1nA0JhmZVodecp7fTVpDkUgw+u3zbsRWwrmvmfKLhPcrECmxVfrlBam2CkMhJ +hdFObl/9/Jzy2izsbNNJFHIanA7A8MexeU+pi9elzQKBgAE5mh67F00AQjC8zbq6 +332EUeAZNcDNdiBbX7P6PVB7BCyjNV4+33/4U8nvjR0aXvtrVWLaxXP5l8yo9Z8E +Uk7xhRR8z1IX2IyxMmLOYSfQnDu1PWDGYiQcvSEYpTsDpwBIRgFt0LgvScjgm1mM +4zpzDXaoMZHsUFCgX6BMtUn8 +-----END PRIVATE KEY----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/custom_logback.xml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/custom_logback.xml new file mode 100644 index 000000000..5c37e770a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/custom_logback.xml @@ -0,0 +1,69 @@ + + + + + + + + true + + + + + + + + + ${log_path}.log + + + %d{"yyyy-MM-dd'T'HH:mm:ss.SSSZ"} [%thread] %XF{groupId, "gid:%s "}%XF{jobId, "jobId:%s "}%XF{planId, "planId:%s "}%-5level ${marker}%logger [%caller] - %msg%n + + + + + ${log_path}.%d{yyyyMMdd}.log.gz + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/downloads.mongodb.com.chained+root.crt b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/downloads.mongodb.com.chained+root.crt new file mode 100644 index 000000000..89562a069 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/downloads.mongodb.com.chained+root.crt @@ -0,0 +1,78 @@ +-----BEGIN CERTIFICATE----- +MIIFnzCCBIegAwIBAgIQDuqCvyfH3Q13NSQK0X77FDANBgkqhkiG9w0BAQsFADBG +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRUwEwYDVQQLEwxTZXJ2ZXIg +Q0EgMUIxDzANBgNVBAMTBkFtYXpvbjAeFw0yMDA3MjYwMDAwMDBaFw0yMTA4MjYx +MjAwMDBaMCAxHjAcBgNVBAMTFWRvd25sb2Fkcy5tb25nb2RiLmNvbTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAL3hiqdT/B4M++HrnEVhmKvA3Jg1/1uI +YMKuIzL53AjdYfeHojw0Kx7EMkPxFTz1hztivJTDdXhd3FiiZDKm10OgUFHHaqLZ +MVdWpSRL8ftmMvKnN+JeN0IFst3+wh5r2hQEnPLPpFZy+O1DHaep8YlaPRSN5QA+ +UTraj6JcRsUcjEyXVtrX5FOxRVNw8eYwzAzktdGAy09OekO5VRmZmKoNOXPfMQQ2 +YiJNboKrEc9u5fN28cWJhO9eC+Of21qIVG2/Uln2823ZGWTmZ4bXi7Vg+UruiNNo +pUHQx6O/QIVLtom+dh7x3XszCUlZHwpnOlV2zdmp5srIjyu3XJVtTY8CAwEAAaOC +Aq0wggKpMB8GA1UdIwQYMBaAFFmkZgZSoHuVkjyjlAcnlnRb+T3QMB0GA1UdDgQW +BBQpx6kAjV7Otgz6vi0cdL4LnMQALTBLBgNVHREERDBCghVkb3dubG9hZHMubW9u +Z29kYi5jb22CEmZhc3RkbC5tb25nb2RiLm9yZ4IVZG93bmxvYWRzLm1vbmdvZGIu +b3JnMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH +AwIwOwYDVR0fBDQwMjAwoC6gLIYqaHR0cDovL2NybC5zY2ExYi5hbWF6b250cnVz +dC5jb20vc2NhMWIuY3JsMCAGA1UdIAQZMBcwCwYJYIZIAYb9bAECMAgGBmeBDAEC +ATB1BggrBgEFBQcBAQRpMGcwLQYIKwYBBQUHMAGGIWh0dHA6Ly9vY3NwLnNjYTFi +LmFtYXpvbnRydXN0LmNvbTA2BggrBgEFBQcwAoYqaHR0cDovL2NydC5zY2ExYi5h +bWF6b250cnVzdC5jb20vc2NhMWIuY3J0MAwGA1UdEwEB/wQCMAAwggEFBgorBgEE +AdZ5AgQCBIH2BIHzAPEAdwD2XJQv0XcwIhRUGAgwlFaO400TGTO/3wwvIAvMTvFk +4wAAAXOIwjoPAAAEAwBIMEYCIQCO4BXmrdIGgh9DzIIxTh3vbUEX1c5ACGVhOlKy +kOwGMgIhAI89gFYczRzYZBWMCu4GYsZNyCqdnPQYKGaSAl8V7j0hAHYAXNxDkv7m +q0VEsV6a1FbmEDf71fpH3KFzlLJe5vbHDsoAAAFziMI6PAAABAMARzBFAiEAiGfq +xNmVD0l5igz6lwZsbZIVOuRzI9MRrqKgN52Ok8oCIAtg+SFfP3grDkYjPxqErjat +mX6ZowWVnyeLPL9SWrlhMA0GCSqGSIb3DQEBCwUAA4IBAQAwlVcR5ONm1lTKIo6v +k9eq0AJ/CpPlGB8ZtDmeJ8+kbHe89o2eSC9jcDdZjBYGQ1zVSzwj+VJ46FZ1rvd8 +6jTR3+jhN3JkxjmCbW6JkXJo+JQuOWf1c4o7+dFqzpBaV1bHDB8b+PWkkLlf9hOx +sPadHt9KTURP65/KK/G5bMvoFQnview7RJxILhr55D0YnpDYWVtVVSS6Nt2siORf +C8YFa3xsMBTWXzzPjhUiEC/kz2Hhru4rNss/eGh5VA13xonT+ohgMgncj3SXMWOu +qYT9XPG+/+i2m5dtzQqDe46AgM+zsejyyrdeJ7QYNyNIxqIH56wyvOSrJlQLuCqI +4L9U +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIESTCCAzGgAwIBAgITBn+UV4WH6Kx33rJTMlu8mYtWDTANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MTAyMjAwMDAwMFoXDTI1MTAxOTAwMDAwMFowRjEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEVMBMGA1UECxMMU2VydmVyIENB +IDFCMQ8wDQYDVQQDEwZBbWF6b24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDCThZn3c68asg3Wuw6MLAd5tES6BIoSMzoKcG5blPVo+sDORrMd4f2AbnZ +cMzPa43j4wNxhplty6aUKk4T1qe9BOwKFjwK6zmxxLVYo7bHViXsPlJ6qOMpFge5 +blDP+18x+B26A0piiQOuPkfyDyeR4xQghfj66Yo19V+emU3nazfvpFA+ROz6WoVm +B5x+F2pV8xeKNR7u6azDdU5YVX1TawprmxRC1+WsAYmz6qP+z8ArDITC2FMVy2fw +0IjKOtEXc/VfmtTFch5+AfGYMGMqqvJ6LcXiAhqG5TI+Dr0RtM88k+8XUBCeQ8IG +KuANaL7TiItKZYxK1MMuTJtV9IblAgMBAAGjggE7MIIBNzASBgNVHRMBAf8ECDAG +AQH/AgEAMA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUWaRmBlKge5WSPKOUByeW +dFv5PdAwHwYDVR0jBBgwFoAUhBjMhTTsvAyUlC4IWZzHshBOCggwewYIKwYBBQUH +AQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8vb2NzcC5yb290Y2ExLmFtYXpvbnRy +dXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDovL2NydC5yb290Y2ExLmFtYXpvbnRy +dXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8EODA2MDSgMqAwhi5odHRwOi8vY3Js +LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY3JsMBMGA1UdIAQMMAow +CAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IBAQCFkr41u3nPo4FCHOTjY3NTOVI1 +59Gt/a6ZiqyJEi+752+a1U5y6iAwYfmXss2lJwJFqMp2PphKg5625kXg8kP2CN5t +6G7bMQcT8C8xDZNtYTd7WPD8UZiRKAJPBXa30/AbwuZe0GaFEQ8ugcYQgSn+IGBI +8/LwhBNTZTUVEWuCUUBVV18YtbAiPq3yXqMB48Oz+ctBWuZSkbvkNodPLamkB2g1 +upRyzQ7qDn1X8nn8N8V7YJ6y68AtkHcNSRAnpTitxBKjtKPISLMVCx7i4hncxHZS +yLyKQXhw2W2Xs0qLeC1etA+jTGDK4UfLeC0SF7FSi8o5LL21L8IzApar2pR/ +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/minio/values-tenant.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/minio/values-tenant.yaml new file mode 100644 index 000000000..832b487bf --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/minio/values-tenant.yaml @@ -0,0 +1,264 @@ +## Secret with default environment variable configurations to be used by MinIO Tenant. +## Not recommended for production deployments! Create the secret manually instead. +secrets: + name: myminio-env-configuration + # MinIO root user and password + accessKey: minio + secretKey: minio123 + ## Set the value for existingSecret to use a pre created secret and dont create default one + # existingSecret: random-env-configuration +## MinIO Tenant Definition +tenant: + # Tenant name + name: tenant-0 + ## Registry location and Tag to download MinIO Server image + image: + repository: quay.io/minio/minio + tag: RELEASE.2023-06-23T20-26-00Z + pullPolicy: IfNotPresent + ## Customize any private registry image pull secret. + ## currently only one secret registry is supported + imagePullSecret: { } + ## If a scheduler is specified here, Tenant pods will be dispatched by specified scheduler. + ## If not specified, the Tenant pods will be dispatched by default scheduler. + scheduler: { } + ## Secret name that contains additional environment variable configurations. + ## The secret is expected to have a key named config.env containing environment variables exports. + configuration: + name: myminio-env-configuration + ## Specification for MinIO Pool(s) in this Tenant. + pools: + ## Servers specifies the number of MinIO Tenant Pods / Servers in this pool. + ## For standalone mode, supply 1. For distributed mode, supply 4 or more. + ## Note that the operator does not support upgrading from standalone to distributed mode. + - servers: 1 + ## custom name for the pool + name: pool-0 + ## volumesPerServer specifies the number of volumes attached per MinIO Tenant Pod / Server. + volumesPerServer: 1 + ## size specifies the capacity per volume + size: 2Gi + ## storageClass specifies the storage class name to be used for this pool + storageClassName: standard + ## Used to specify annotations for pods + annotations: { } + ## Used to specify labels for pods + labels: { } + ## Used to specify a toleration for a pod + tolerations: [ ] + ## nodeSelector parameters for MinIO Pods. It specifies a map of key-value pairs. For the pod to be + ## eligible to run on a node, the node must have each of the + ## indicated key-value pairs as labels. + ## Read more here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: { } + ## Affinity settings for MinIO pods. Read more about affinity + ## here: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity. + affinity: { } + ## Configure resource requests and limits for MinIO containers + resources: { } + ## Configure security context + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + runAsNonRoot: true + ## Configure container security context + containerSecurityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + ## Configure topology constraints + topologySpreadConstraints: [ ] + ## Configure Runtime Class + # runtimeClassName: "" + ## Mount path where PV will be mounted inside container(s). + mountPath: /export + ## Sub path inside Mount path where MinIO stores data. + ## WARNING: + ## We recommend you to keep the same mountPath and the same subPath once the + ## Tenant has been deployed over your different PVs. + ## This is because if you change these values once Tenant is deployed, then + ## you will end up with multiple paths for different buckets. So please, be + ## very careful to keep same value for the life of the Tenant. + subPath: /data + # pool metrics to be read by Prometheus + metrics: + enabled: false + port: 9000 + protocol: http + certificate: + ## Use this field to provide one or more external CA certificates. This is used by MinIO + ## to verify TLS connections with other applications: + ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + externalCaCertSecret: [ ] + ## Use this field to provide a list of Secrets with external certificates. This can be used to configure + ## TLS for MinIO Tenant pods. Create secrets as explained here: + ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + externalCertSecret: + - name: tls-ssl-minio + type: kubernetes.io/tls + ## Enable automatic Kubernetes based certificate generation and signing as explained in + ## https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster + requestAutoCert: true + ## This field is used only when "requestAutoCert" is set to true. Use this field to set CommonName + ## for the auto-generated certificate. Internal DNS name for the pod will be used if CommonName is + ## not provided. DNS name format is *.minio.default.svc.cluster.local + certConfig: { } + ## MinIO features to enable or disable in the MinIO Tenant + ## https://github.com/minio/operator/blob/master/docs/tenant_crd.adoc#features + features: + bucketDNS: false + domains: { } + ## List of bucket definitions to create during tenant provisioning. + ## Example: + # - name: my-minio-bucket + # objectLock: false # optional + # region: us-east-1 # optional + buckets: + - name: oplog-s3-bucket + - name: s3-store-bucket + + ## List of secret names to use for generating MinIO users during tenant provisioning + users: [ ] + ## PodManagement policy for MinIO Tenant Pods. Can be "OrderedReady" or "Parallel" + ## Refer https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy + ## for details. + podManagementPolicy: Parallel + # Liveness Probe for container liveness. Container will be restarted if the probe fails. + # Refer https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes. + liveness: { } + # Readiness Probe for container readiness. Container will be removed from service endpoints if the probe fails. + # Refer https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + readiness: { } + # Startup Probe for container startup. Container will be restarted if the probe fails. + # Refer https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + startup: { } + ## exposeServices defines the exposure of the MinIO object storage and Console services. + ## service is exposed as a loadbalancer in k8s service. + exposeServices: { } + # kubernetes service account associated with a specific tenant + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccountName: "" + # Tenant scrape configuration will be added to prometheus managed by the prometheus-operator. + prometheusOperator: false + # Enable JSON, Anonymous logging for MinIO tenants. + # Refer https://github.com/minio/operator/blob/master/pkg/apis/minio.min.io/v2/types.go#L303 + # How logs will look: + # $ k logs myminio-pool-0-0 -n default + # {"level":"INFO","errKind":"","time":"2022-04-07T21:49:33.740058549Z","message":"All MinIO sub-systems initialized successfully"} + # Notice they are in JSON format to be consumed + logging: + anonymous: true + json: true + quiet: true + ## serviceMetadata allows passing additional labels and annotations to MinIO and Console specific + ## services created by the operator. + serviceMetadata: { } + ## Add environment variables to be set in MinIO container (https://github.com/minio/minio/tree/master/docs/config) + env: [ ] + ## PriorityClassName indicates the Pod priority and hence importance of a Pod relative to other Pods. + ## This is applied to MinIO pods only. + ## Refer Kubernetes documentation for details https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass/ + priorityClassName: "" + ## Define configuration for KES (stateless and distributed key-management system) + ## Refer https://github.com/minio/kes + #kes: + # image: "" # minio/kes:2023-05-02T22-48-10Z + # env: [ ] + # replicas: 2 + # configuration: |- + # address: :7373 + # root: _ # Effectively disabled since no root identity necessary. + # tls: + # key: /tmp/kes/server.key # Path to the TLS private key + # cert: /tmp/kes/server.crt # Path to the TLS certificate + # proxy: + # identities: [] + # header: + # cert: X-Tls-Client-Cert + # policy: + # my-policy: + # paths: + # - /v1/key/create/* + # - /v1/key/generate/* + # - /v1/key/decrypt/* + # identities: + # - ${MINIO_KES_IDENTITY} + # cache: + # expiry: + # any: 5m0s + # unused: 20s + # log: + # error: on + # audit: off + # keys: + # ## KES configured with fs (File System mode) doesnt work in Kubernetes environments and it's not recommended + # ## use a real KMS + # # fs: + # # path: "./keys" # Path to directory. Keys will be stored as files. Not Recommended for Production. + # vault: + # endpoint: "http://vault.default.svc.cluster.local:8200" # The Vault endpoint + # namespace: "" # An optional Vault namespace. See: https://www.vaultproject.io/docs/enterprise/namespaces/index.html + # prefix: "my-minio" # An optional K/V prefix. The server will store keys under this prefix. + # approle: # AppRole credentials. See: https://www.vaultproject.io/docs/auth/approle.html + # id: "" # Your AppRole Role ID + # secret: "" # Your AppRole Secret ID + # retry: 15s # Duration until the server tries to re-authenticate after connection loss. + # tls: # The Vault client TLS configuration for mTLS authentication and certificate verification + # key: "" # Path to the TLS client private key for mTLS authentication to Vault + # cert: "" # Path to the TLS client certificate for mTLS authentication to Vault + # ca: "" # Path to one or multiple PEM root CA certificates + # status: # Vault status configuration. The server will periodically reach out to Vault to check its status. + # ping: 10s # Duration until the server checks Vault's status again. + # # aws: + # # # The AWS SecretsManager key store. The server will store + # # # secret keys at the AWS SecretsManager encrypted with + # # # AWS-KMS. See: https://aws.amazon.com/secrets-manager + # # secretsmanager: + # # endpoint: "" # The AWS SecretsManager endpoint - e.g.: secretsmanager.us-east-2.amazonaws.com + # # region: "" # The AWS region of the SecretsManager - e.g.: us-east-2 + # # kmskey: "" # The AWS-KMS key ID used to en/decrypt secrets at the SecretsManager. By default (if not set) the default AWS-KMS key will be used. + # # credentials: # The AWS credentials for accessing secrets at the AWS SecretsManager. + # # accesskey: "" # Your AWS Access Key + # # secretkey: "" # Your AWS Secret Key + # # token: "" # Your AWS session token (usually optional) + # imagePullPolicy: "IfNotPresent" + # externalCertSecret: null + # clientCertSecret: null + # ## Key name to be created on the KMS, default is "my-minio-key" + # keyName: "" + # resources: { } + # nodeSelector: { } + # affinity: + # nodeAffinity: { } + # podAffinity: { } + # podAntiAffinity: { } + # tolerations: [ ] + # annotations: { } + # labels: { } + # serviceAccountName: "" + # securityContext: + # runAsUser: 1000 + # runAsGroup: 1000 + # runAsNonRoot: true + # fsGroup: 1000 + +ingress: + api: + enabled: false + ingressClassName: "" + labels: { } + annotations: { } + tls: [ ] + host: minio.local + path: / + pathType: Prefix + console: + enabled: false + ingressClassName: "" + labels: { } + annotations: { } + tls: [ ] + host: minio-console.local + path: / + pathType: Prefix diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb-download.crt b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb-download.crt new file mode 100644 index 000000000..1fe9cdd17 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb-download.crt @@ -0,0 +1,112 @@ +-----BEGIN CERTIFICATE----- +MIIF9zCCBN+gAwIBAgIQDTiokZJgKoaLCOzt0K3ulzANBgkqhkiG9w0BAQsFADBG +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRUwEwYDVQQLEwxTZXJ2ZXIg +Q0EgMUIxDzANBgNVBAMTBkFtYXpvbjAeFw0yMjA2MjIwMDAwMDBaFw0yMzA3MjEy +MzU5NTlaMCAxHjAcBgNVBAMTFWRvd25sb2Fkcy5tb25nb2RiLmNvbTCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBANZyoobPRnuB0Y8g1MNre1ptStvETcBw +bXh3g7QtgEoFUryEQPnrkPQJUh7vWmVAhwyfYb3NjoGPoUfCC7vc2kVqXUfydV45 +/V8mUhs/8dbpZXn//sfoFum5PrbX+GydWrvtC17USkWto+Miw046dqEntmill7J4 +znR3EHmvNZiXMkf67QCyBjpTT71LYecJFjKVxjUdC3xe8TOsCsx+2IPMVNvIeqxt +G3aW0rPxuFlQ7oIyXgdRXtlD5vdoPioxk+P2vj1gmKtTGbKUTDqac2r2cwP0uuGo +xV0gN/JGmufEu6bcngIj2DXpMEtBgHcYXk8IgQpe4QLiRdrgaK+DMz8CAwEAAaOC +AwUwggMBMB8GA1UdIwQYMBaAFFmkZgZSoHuVkjyjlAcnlnRb+T3QMB0GA1UdDgQW +BBQQ/1E/6lSuAqx97QEK8lVn0L81bjA1BgNVHREELjAsghVkb3dubG9hZHMubW9u +Z29kYi5jb22CE2Rvd25sb2Fkcy4xMGdlbi5jb20wDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA9BgNVHR8ENjA0MDKgMKAuhixo +dHRwOi8vY3JsLnNjYTFiLmFtYXpvbnRydXN0LmNvbS9zY2ExYi0xLmNybDATBgNV +HSAEDDAKMAgGBmeBDAECATB1BggrBgEFBQcBAQRpMGcwLQYIKwYBBQUHMAGGIWh0 +dHA6Ly9vY3NwLnNjYTFiLmFtYXpvbnRydXN0LmNvbTA2BggrBgEFBQcwAoYqaHR0 +cDovL2NydC5zY2ExYi5hbWF6b250cnVzdC5jb20vc2NhMWIuY3J0MAwGA1UdEwEB +/wQCMAAwggF+BgorBgEEAdZ5AgQCBIIBbgSCAWoBaAB2AOg+0No+9QY1MudXKLyJ +a8kD08vREWvs62nhd31tBr1uAAABgYyc2+4AAAQDAEcwRQIgJcz+XaaaFAcPiteo +AaunoWepexB4vim6CmH5RjUxp9gCIQCD4rMXUFwiUbwrCvzYDNxoLmApMCWOwxgF +Sg0KpEFcyQB2ADXPGRu/sWxXvw+tTG1Cy7u2JyAmUeo/4SrvqAPDO9ZMAAABgYyc +3CEAAAQDAEcwRQIhAMKh1TL/yOlbSCIGf1NBMy3y+TLSjg672TAbxks+i/w/AiBX +RMleFwVvl0rfn7mtjx4sNg/KV7jxGWlSsE9HVE59tQB2ALc++yTfnE26dfI5xbpY +9Gxd/ELPep81xJ4dCYEl7bSZAAABgYyc3CQAAAQDAEcwRQIhAMcmDntNQU28qZd0 +8SaSdXKsb6DxJC/aZF+aYCi2dv14AiBHPCw6fOGgPSYM4yhs5cdqwK9wUH23G+22 +yYj5PrHTtTANBgkqhkiG9w0BAQsFAAOCAQEAooTXKMDE9j5bXGpGB8pIyYcDJVZK +Xe14ClMwJdcIr2dJM1HY++2+x9taTT1+C/bxwOfU+S5O68cGTcSY5/HZKbhIMHo4 +oGBzpfREIxMLl9n74USGBbf7h8bI+m7wJK+68HhH3La6Hd1tic6DkPmpVxrRB5ux +as1tbGN26JON/UqKxUZwcx+szhdEApWkmEJBm9S3A6/jN/yIPW5IiS084jRSk9YG +p6TInSGnHnf3M5CnlSUEG1tgMNl1tbvAf1NIsAdMSDwahTtZqO39N8XD7X/IriT2 +z20gl9DxUae6etCoGbo44+giCkqDfIeZFcabLqJiD8bNz6wj6ac17pysXQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIESTCCAzGgAwIBAgITBn+UV4WH6Kx33rJTMlu8mYtWDTANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MTAyMjAwMDAwMFoXDTI1MTAxOTAwMDAwMFowRjEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEVMBMGA1UECxMMU2VydmVyIENB +IDFCMQ8wDQYDVQQDEwZBbWF6b24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDCThZn3c68asg3Wuw6MLAd5tES6BIoSMzoKcG5blPVo+sDORrMd4f2AbnZ +cMzPa43j4wNxhplty6aUKk4T1qe9BOwKFjwK6zmxxLVYo7bHViXsPlJ6qOMpFge5 +blDP+18x+B26A0piiQOuPkfyDyeR4xQghfj66Yo19V+emU3nazfvpFA+ROz6WoVm +B5x+F2pV8xeKNR7u6azDdU5YVX1TawprmxRC1+WsAYmz6qP+z8ArDITC2FMVy2fw +0IjKOtEXc/VfmtTFch5+AfGYMGMqqvJ6LcXiAhqG5TI+Dr0RtM88k+8XUBCeQ8IG +KuANaL7TiItKZYxK1MMuTJtV9IblAgMBAAGjggE7MIIBNzASBgNVHRMBAf8ECDAG +AQH/AgEAMA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUWaRmBlKge5WSPKOUByeW +dFv5PdAwHwYDVR0jBBgwFoAUhBjMhTTsvAyUlC4IWZzHshBOCggwewYIKwYBBQUH +AQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8vb2NzcC5yb290Y2ExLmFtYXpvbnRy +dXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDovL2NydC5yb290Y2ExLmFtYXpvbnRy +dXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8EODA2MDSgMqAwhi5odHRwOi8vY3Js +LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY3JsMBMGA1UdIAQMMAow +CAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IBAQCFkr41u3nPo4FCHOTjY3NTOVI1 +59Gt/a6ZiqyJEi+752+a1U5y6iAwYfmXss2lJwJFqMp2PphKg5625kXg8kP2CN5t +6G7bMQcT8C8xDZNtYTd7WPD8UZiRKAJPBXa30/AbwuZe0GaFEQ8ugcYQgSn+IGBI +8/LwhBNTZTUVEWuCUUBVV18YtbAiPq3yXqMB48Oz+ctBWuZSkbvkNodPLamkB2g1 +upRyzQ7qDn1X8nn8N8V7YJ6y68AtkHcNSRAnpTitxBKjtKPISLMVCx7i4hncxHZS +yLyKQXhw2W2Xs0qLeC1etA+jTGDK4UfLeC0SF7FSi8o5LL21L8IzApar2pR/ +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEkjCCA3qgAwIBAgITBn+USionzfP6wq4rAfkI7rnExjANBgkqhkiG9w0BAQsF +ADCBmDELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNj +b3R0c2RhbGUxJTAjBgNVBAoTHFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4x +OzA5BgNVBAMTMlN0YXJmaWVsZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1 +dGhvcml0eSAtIEcyMB4XDTE1MDUyNTEyMDAwMFoXDTM3MTIzMTAxMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaOCATEwggEtMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ +BAQDAgGGMB0GA1UdDgQWBBSEGMyFNOy8DJSULghZnMeyEE4KCDAfBgNVHSMEGDAW +gBScXwDfqgHXMCs4iKK4bUqc8hGRgzB4BggrBgEFBQcBAQRsMGowLgYIKwYBBQUH +MAGGImh0dHA6Ly9vY3NwLnJvb3RnMi5hbWF6b250cnVzdC5jb20wOAYIKwYBBQUH +MAKGLGh0dHA6Ly9jcnQucm9vdGcyLmFtYXpvbnRydXN0LmNvbS9yb290ZzIuY2Vy +MD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly9jcmwucm9vdGcyLmFtYXpvbnRydXN0 +LmNvbS9yb290ZzIuY3JsMBEGA1UdIAQKMAgwBgYEVR0gADANBgkqhkiG9w0BAQsF +AAOCAQEAYjdCXLwQtT6LLOkMm2xF4gcAevnFWAu5CIw+7bMlPLVvUOTNNWqnkzSW +MiGpSESrnO09tKpzbeR/FoCJbM8oAxiDR3mjEH4wW6w7sGDgd9QIpuEdfF7Au/ma +eyKdpwAJfqxGF4PcnCZXmTA5YpaP7dreqsXMGz7KQ2hsVxa81Q4gLv7/wmpdLqBK +bRRYh5TmOTFffHPLkIhqhBGWJ6bt2YFGpn6jcgAKUj6DiAdjd4lpFw85hdKrCEVN +0FE6/V1dN2RMfjCyVSRCnTawXZwXgWHxyvkQAiSr6w10kY17RSlQOYiypok1JR4U +akcjMS9cmvqtmg5iUaQqqcT5NJ0hGA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEdTCCA12gAwIBAgIJAKcOSkw0grd/MA0GCSqGSIb3DQEBCwUAMGgxCzAJBgNV +BAYTAlVTMSUwIwYDVQQKExxTdGFyZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTIw +MAYDVQQLEylTdGFyZmllbGQgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eTAeFw0wOTA5MDIwMDAwMDBaFw0zNDA2MjgxNzM5MTZaMIGYMQswCQYDVQQGEwJV +UzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTElMCMGA1UE +ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjE7MDkGA1UEAxMyU3RhcmZp +ZWxkIFNlcnZpY2VzIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVDDrEKvlO4vW+GZdfjohTsR8/ +y8+fIBNtKTrID30892t2OGPZNmCom15cAICyL1l/9of5JUOG52kbUpqQ4XHj2C0N +Tm/2yEnZtvMaVq4rtnQU68/7JuMauh2WLmo7WJSJR1b/JaCTcFOD2oR0FMNnngRo +Ot+OQFodSk7PQ5E751bWAHDLUu57fa4657wx+UX2wmDPE1kCK4DMNEffud6QZW0C +zyyRpqbn3oUYSXxmTqM6bam17jQuug0DuDPfR+uxa40l2ZvOgdFFRjKWcIfeAg5J +Q4W2bHO7ZOphQazJ1FTfhy/HIrImzJ9ZVGif/L4qL8RVHHVAYBeFAlU5i38FAgMB +AAGjgfAwge0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0O +BBYEFJxfAN+qAdcwKziIorhtSpzyEZGDMB8GA1UdIwQYMBaAFL9ft9HO3R+G9FtV +rNzXEMIOqYjnME8GCCsGAQUFBwEBBEMwQTAcBggrBgEFBQcwAYYQaHR0cDovL28u +c3MyLnVzLzAhBggrBgEFBQcwAoYVaHR0cDovL3guc3MyLnVzL3guY2VyMCYGA1Ud +HwQfMB0wG6AZoBeGFWh0dHA6Ly9zLnNzMi51cy9yLmNybDARBgNVHSAECjAIMAYG +BFUdIAAwDQYJKoZIhvcNAQELBQADggEBACMd44pXyn3pF3lM8R5V/cxTbj5HD9/G +VfKyBDbtgB9TxF00KGu+x1X8Z+rLP3+QsjPNG1gQggL4+C/1E2DUBc7xgQjB3ad1 +l08YuW3e95ORCLp+QCztweq7dp4zBncdDQh/U90bZKuCJ/Fp1U1ervShw3WnWEQt +8jxwmKy6abaVd38PMV4s/KCHOkdp8Hlf9BRUpJVeEXgSYCfOn8J3/yNTd126/+pZ +59vPr5KW7ySaNRB6nJHGDn2Z9j8Z3/VyVOEVqQdZe4O/Ui5GjLIAZHYcSNPYeehu +VsyuLAOQ1xk4meTKCRlb/weWsKh/NEnfVqn3sF/tM+2MR7cwA130A4w= +-----END CERTIFICATE----- diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb_versions_claim.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb_versions_claim.yaml new file mode 100644 index 000000000..f9d790ab1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb_versions_claim.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongodb-versions-claim +spec: + storageClassName: gp2 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_configure_all_images.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_configure_all_images.yaml new file mode 100644 index 000000000..f45592c32 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_configure_all_images.yaml @@ -0,0 +1,18 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-upgrade +spec: + replicas: 1 + # version is configured in the test + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + + backup: + enabled: false + + applicationDatabase: + # version is configured in the test + members: 3 + version: "4.4.20-ent" diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_scale_up_down.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_scale_up_down.yaml new file mode 100644 index 000000000..27153601e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_scale_up_down.yaml @@ -0,0 +1,17 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-scale +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + + applicationDatabase: + version: "4.4.20-ent" + members: 3 + agent: + logLevel: DEBUG + + backup: + enabled: false diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_scram.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_scram.yaml new file mode 100644 index 000000000..4c3292629 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_scram.yaml @@ -0,0 +1,15 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-scram +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + + applicationDatabase: + members: 3 + version: "4.4.20-ent" + + backup: + enabled: false diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_upgrade.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_upgrade.yaml new file mode 100644 index 000000000..216b87a6a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_appdb_upgrade.yaml @@ -0,0 +1,33 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-upgrade +spec: + replicas: 1 + version: 4.4.0 + adminCredentials: ops-manager-admin-secret + + applicationDatabase: + members: 3 + version: 4.4.20-ent + agent: + logLevel: DEBUG + additionalMongodConfig: + operationProfiling: + mode: slowOp + + backup: + enabled: false + + # adding this just to avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_backup_delete_sts.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_backup_delete_sts.yaml new file mode 100644 index 000000000..3b0ed92c7 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_backup_delete_sts.yaml @@ -0,0 +1,37 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup +spec: + replicas: 1 + version: 4.2.13 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + blockStores: + - name: blockStore1 + mongodbResourceRef: + name: my-mongodb-blockstore + + applicationDatabase: + members: 3 + version: 4.4.20-ent + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_https_enabled.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_https_enabled.yaml new file mode 100644 index 000000000..2eb7c1573 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_https_enabled.yaml @@ -0,0 +1,161 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-with-https + +spec: + replicas: 1 + version: 4.2.15 + adminCredentials: ops-manager-admin-secret + + configuration: + mms.testUtil.enabled: "true" + automation.versions.source: local + + applicationDatabase: + members: 3 + version: "4.4.20-ent" + + statefulSet: + spec: + template: + spec: + volumes: + - name: mongodb-versions + emptyDir: {} + containers: + - name: mongodb-ops-manager + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + + # The initContainers will download the require 4.2+ and 4.4+ versions of MongoDB + # allowing Ops Manager to act as download endpoint for the automation agent + # for this particular version. + # This is required because of public Internet downloads not being + # possible after using a custom-ca for the OM HTTPS server. + initContainers: + - name: setting-up-rhel-mongodb + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.2.8.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-4.2.8.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-4-4 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.4.11.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-4.4.11.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-5-0 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-5.0.5.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-5.0.5.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-6-0 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-6.0.5.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-6.0.5.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-6-0-sig + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-6.0.5.tgz.sig + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-6.0.5.tgz.sig + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-6-0-16 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-6.0.16.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-6.0.16.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-6-0-16-sig + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-6.0.16.tgz.sig + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-6.0.16.tgz.sig + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + + - name: setting-up-rhel-mongodb-7-0 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-7.0.2.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-7.0.2.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-7-0-sig + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-7.0.2.tgz.sig + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-7.0.2.tgz.sig + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-8-0 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel8-8.0.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel8-8.0.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-8-0-sig + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel8-8.0.0.tgz.sig + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel8-8.0.0.tgz.sig + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + backup: + enabled: false diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_localmode-multiple-pv.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_localmode-multiple-pv.yaml new file mode 100644 index 000000000..b7291fb52 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_localmode-multiple-pv.yaml @@ -0,0 +1,38 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-localmode-multiple-pv +spec: + replicas: 2 + version: 4.2.12 + adminCredentials: ops-manager-admin-secret + configuration: + automation.versions.source: local + + statefulSet: + spec: + volumeClaimTemplates: + - metadata: + name: mongodb-versions + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20G + template: + spec: + containers: + - name: mongodb-ops-manager + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + + + backup: + enabled: false + + applicationDatabase: + version: "4.4.20-ent" + members: 3 + + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_localmode-single-pv.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_localmode-single-pv.yaml new file mode 100644 index 000000000..e2e0d2998 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_localmode-single-pv.yaml @@ -0,0 +1,85 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-localmode-single-pv +spec: + replicas: 1 + version: 4.4.1 + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + automation.versions.source: local + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + + statefulSet: + spec: + template: + spec: + volumes: + - name: mongodb-versions + persistentVolumeClaim: + claimName: mongodb-versions-claim + containers: + - name: mongodb-ops-manager + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + initContainers: + - name: setting-up-rhel-mongodb-4-2-8 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.2.8.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-4.2.8.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-6-0-16 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-6.0.16.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-6.0.16.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-7-0-2 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-7.0.2.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel80-7.0.2.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + - name: setting-up-rhel-mongodb-8-0-0 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel8-8.0.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel8-8.0.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + backup: + enabled: false + + applicationDatabase: + version: "4.4.20-ent" + members: 3 diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_more_orgs.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_more_orgs.yaml new file mode 100644 index 000000000..9368f7c08 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_more_orgs.yaml @@ -0,0 +1,30 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-validate +spec: + replicas: 1 + version: 5.0.0 + adminCredentials: ops-manager-admin-secret + + backup: + enabled: false + + applicationDatabase: + members: 3 + version: 4.4.20-ent + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + limits: + memory: 15G + + configuration: + mms.limits.maxGroupsPerOrg: "5000" + mms.limits.maxGroupsPerUser: "5000" + mms.limits.maxOrgsPerUser: "5000" diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_appdb_monitoring_tls.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_appdb_monitoring_tls.yaml new file mode 100644 index 000000000..01a4e4107 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_appdb_monitoring_tls.yaml @@ -0,0 +1,39 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-tls-monitored-appdb +spec: + replicas: 1 + version: 4.4.1 + adminCredentials: ops-manager-admin-secret + security: + tls: + secretRef: + name: certs-for-ops-manager + ca: issuer-ca + backup: + enabled: false + applicationDatabase: + members: 3 + version: 5.0.14-ent + + passwordSecretKeyRef: + name: appdb-secret + + security: + certsSecretPrefix: appdb + tls: + ca: issuer-ca + + # adding this just to avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_appdb_upgrade_tls.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_appdb_upgrade_tls.yaml new file mode 100644 index 000000000..d6320d900 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_appdb_upgrade_tls.yaml @@ -0,0 +1,33 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-appdb-upgrade-tls +spec: + replicas: 1 + version: 5.0.2 + adminCredentials: ops-manager-admin-secret + backup: + enabled: false + security: + tls: + ca: issuer-ca + applicationDatabase: + members: 3 + version: 5.0.14-ent + security: + tls: + ca: issuer-ca + + # adding this just to avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup.yaml new file mode 100644 index 000000000..c1f6651c6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup.yaml @@ -0,0 +1,72 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup +spec: + replicas: 1 + version: 6.0.13 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + headDB: + storage: 500M + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + blockStores: + - name: blockStore1 + mongodbResourceRef: + name: my-mongodb-blockstore + s3Stores: + - name: s3Store1 + mongodbResourceRef: + name: my-mongodb-s3 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-backup-daemon + resources: + requests: + memory: 10G + limits: + memory: 10G + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + requests: + memory: 15G + limits: + memory: 15G + + applicationDatabase: + members: 3 + version: 6.0.5-ent + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_irsa.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_irsa.yaml new file mode 100644 index 000000000..74afb6479 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_irsa.yaml @@ -0,0 +1,69 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup +spec: + replicas: 1 + version: 5.0.4 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + headDB: + storage: 500M + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + s3Stores: + - name: s3Store1 + mongodbResourceRef: + name: my-mongodb-s3 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-backup-daemon + resources: + requests: + memory: 10G + limits: + memory: 10G + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + requests: + memory: 15G + limits: + memory: 15G + + applicationDatabase: + members: 3 + version: 4.4.20-ent + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + brs.store.s3.iam.flavor: web-identity-token + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_kmip.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_kmip.yaml new file mode 100644 index 000000000..04835a5cb --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_kmip.yaml @@ -0,0 +1,59 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup-kmip +spec: + replicas: 1 + version: 4.2.13 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + encryption: + kmip: + server: + url: "kmip-svc:5696" + ca: "" + s3OpLogStores: + - name: s3Store2 + s3SecretRef: + name: my-s3-secret-oplog + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: "" + pathStyleAccessEnabled: true + s3Stores: + - name: s3Store1 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + applicationDatabase: + members: 3 + version: 4.4.20-ent + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + requests: + memory: 15G + limits: + memory: 15G + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + mms.preflight.run: "false" diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_light.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_light.yaml new file mode 100644 index 000000000..66e59f74c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_light.yaml @@ -0,0 +1,65 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup + labels: + label1: val1 + label2: val2 +spec: + replicas: 1 + version: 4.2.13 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + s3Stores: + - name: s3Store1 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-backup-daemon + resources: + requests: + memory: 10G + limits: + memory: 10G + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + requests: + memory: 15G + limits: + memory: 15G + applicationDatabase: + members: 3 + + version: 4.4.20-ent + + # Dev: adding this just to avoid wizard when opening OM UI + # (note, that to debug issues in OM you need to add 'spec.externalConnectivity.type=NodePort' + # and specify some port: 'port: 32400'. Don't forget to open it in AWS) + configuration: + # this property is critical to make backup S3 work in OM 4.2.10 and 4.2.12 + brs.s3.validation.testing: disabled + + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_tls.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_tls.yaml new file mode 100644 index 000000000..530e65301 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_tls.yaml @@ -0,0 +1,65 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup-tls +spec: + replicas: 1 + version: 4.2.10 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + blockStores: + - name: blockStore1 + mongodbResourceRef: + name: my-mongodb-blockstore + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-backup-daemon + resources: + requests: + memory: 10G + limits: + memory: 10G + security: + tls: + ca: issuer-ca + applicationDatabase: + version: 4.4.20-ent + members: 3 + security: + certsSecretPrefix: appdb + tls: + ca: issuer-ca + secretRef: + prefix: appdb + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + requests: + memory: 15G + limits: + memory: 15G + + # adding this just to avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_tls_s3.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_tls_s3.yaml new file mode 100644 index 000000000..22e912b51 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_backup_tls_s3.yaml @@ -0,0 +1,49 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-backup-tls-s3 +spec: + replicas: 1 + version: 5.0.20 + adminCredentials: ops-manager-admin-secret + backup: + enabled: true + s3Stores: + - name: my-s3-block-store + s3SecretRef: + name: "" + pathStyleAccessEnabled: true + s3BucketEndpoint: "" + s3BucketName: "" + s3OpLogStores: + - name: my-s3-oplog-store + s3SecretRef: + name: "" + s3BucketEndpoint: "" + s3BucketName: "" + pathStyleAccessEnabled: true + + security: + tls: + ca: issuer-ca + applicationDatabase: + version: 4.4.20-ent + members: 3 + security: + tls: + ca: issuer-ca + secretRef: + prefix: appdb + + # adding this just to avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_basic.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_basic.yaml new file mode 100644 index 000000000..daa8b85f5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_basic.yaml @@ -0,0 +1,17 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-basic +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + + applicationDatabase: + members: 3 + version: 4.4.20-ent + agent: + logLevel: DEBUG + + backup: + enabled: false diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_full.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_full.yaml new file mode 100644 index 000000000..f0823192d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_full.yaml @@ -0,0 +1,33 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-full +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + + backup: + enabled: true + headDB: + storage: 500M + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + s3Stores: + - name: s3Store1 + mongodbResourceRef: + name: my-mongodb-s3 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + + applicationDatabase: + members: 3 + version: 5.0.14-ent + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_jvm_params.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_jvm_params.yaml new file mode 100644 index 000000000..edfe47108 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_jvm_params.yaml @@ -0,0 +1,28 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-jvm-params +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-ops-manager + resources: + requests: + memory: "400M" + limits: + memory: "5G" + + applicationDatabase: + members: 3 + version: 4.4.20-ent + + backup: + enabled: true + jvmParameters: ["-Xmx4352m","-Xms4352m"] diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_pod_spec.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_pod_spec.yaml new file mode 100644 index 000000000..347c67fca --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_pod_spec.yaml @@ -0,0 +1,81 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-pod-spec +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + backup: + enabled: true + statefulSet: + spec: + template: + spec: + hostAliases: + - ip: "1.2.3.4" + hostnames: ["hostname"] + containers: + - name: "mongodb-backup-daemon" + resources: + requests: + cpu: "0.50" + memory: "4500M" + statefulSet: + spec: + template: + metadata: + annotations: + key1: value1 + spec: + volumes: + - name: test-volume + emptyDir: {} + tolerations: + - key: "key" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: mongodb-ops-manager + readinessProbe: + failureThreshold: 20 + startupProbe: + periodSeconds: 25 + volumeMounts: + - mountPath: /somewhere + name: test-volume + resources: + limits: + cpu: "0.70" + memory: "6G" + applicationDatabase: + members: 3 + version: 4.4.20-ent + podSpec: + persistence: + single: + storage: 1G + podTemplate: + spec: + # TO kill sidecar right away + terminationGracePeriodSeconds: 3 + # This container will be added to each pod as a sidecar + containers: + - name: appdb-sidecar + image: busybox + command: ["sleep"] + args: ["infinity"] + resources: + limits: + cpu: "1" + requests: + cpu: 500m + - name: mongodb-agent + resources: + requests: + memory: 500M + limits: + cpu: "0.75" + memory: 850M diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_scale.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_scale.yaml new file mode 100644 index 000000000..485ca4795 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_scale.yaml @@ -0,0 +1,39 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-scale +spec: + replicas: 2 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + mms.monitoring.rrd.maintenanceEnabled: "false" + mms.monitoring.accessLogs.maintenanceEnabled: "false" + mms.monitoring.slowlogs.maintenanceEnabled: "false" + applicationDatabase: + members: 3 + version: 4.4.20-ent + podSpec: + persistence: + multiple: + data: + storage: 1Gi + storageClass: csi-hostpath-sc + journal: + storage: 1Gi + storageClass: csi-hostpath-sc + logs: + storage: 1Gi + storageClass: csi-hostpath-sc + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_secure_config.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_secure_config.yaml new file mode 100644 index 000000000..296912e0b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_secure_config.yaml @@ -0,0 +1,35 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-secure-config +spec: + replicas: 1 + version: 4.2.10 + adminCredentials: ops-manager-admin-secret + backup: + enabled: false # enabled during test + opLogStores: + - name: oplog1 + mongodbResourceRef: + name: my-mongodb-oplog + blockStores: + - name: blockStore1 + mongodbResourceRef: + name: my-mongodb-blockstore + applicationDatabase: + members: 3 + version: 4.4.20-ent + + # adding this just to avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_upgrade.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_upgrade.yaml new file mode 100644 index 000000000..adbeb7e9e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_ops_manager_upgrade.yaml @@ -0,0 +1,37 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-upgrade +spec: + replicas: 1 + # version is configured in the test + adminCredentials: ops-manager-admin-secret + + backup: + enabled: true + s3Stores: + - name: s3Store1 + s3SecretRef: + name: my-s3-secret + pathStyleAccessEnabled: true + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: test-bucket + + applicationDatabase: + # version is configured in the test + members: 3 + version: "4.4.20-ent" + + # avoid wizard when opening OM UI + configuration: + automation.versions.source: mongodb + mms.testUtil.enabled: "true" + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_s3store_validation.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_s3store_validation.yaml new file mode 100644 index 000000000..1b4678618 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_s3store_validation.yaml @@ -0,0 +1,20 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-s3-validate +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + + applicationDatabase: + members: 3 + version: 4.2.0 + backup: + enabled: true + opLogStores: + - name: "oplog-store-1" + mongodbResourceRef: + name: "my-oplog-mdb" + s3Stores: [] + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_validation.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_validation.yaml new file mode 100644 index 000000000..507f001e6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/om_validation.yaml @@ -0,0 +1,15 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-validate +spec: + replicas: 1 + version: 5.0.0 + adminCredentials: ops-manager-admin-secret + + backup: + enabled: false + + applicationDatabase: + members: 3 + version: 4.4.20-ent diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx-config.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx-config.yaml new file mode 100644 index 000000000..b3b37bd75 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx-config.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf +data: + nginx.conf: | + events {} + http { + server { + server_name localhost; + listen 80; + location /linux/ { + alias /mongodb-ops-manager/mongodb-releases/linux/; + } + location /compass/ { + alias /mongodb-ops-manager/mongodb-releases/compass/; + } + } + } diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx-svc.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx-svc.yaml new file mode 100644 index 000000000..b4b042afe --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx-svc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-svc + labels: + app: nginx +spec: + ports: + - port: 80 + protocol: TCP + selector: + app: nginx diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx.yaml new file mode 100644 index 000000000..a7946e79c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/nginx.yaml @@ -0,0 +1,131 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - image: nginx:1.14.2 + imagePullPolicy: IfNotPresent + name: nginx + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /mongodb-ops-manager/mongodb-releases/linux + name: mongodb-versions + - mountPath: /mongodb-ops-manager/mongodb-releases/compass + name: mongosh-versions + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + initContainers: + - name: setting-up-mongosh-1-4-1 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-1.4.1-linux-x64.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-1-9-1 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-1.9.1-linux-x64.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-1-10-4 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-1.10.4-linux-x64.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-0-0 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.0.0-linux-x64.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-0-2 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.0.2-linux-x64-openssl11.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-0-2-om7 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.0.2-linux-x64.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-1-5-om7 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.1.5-linux-x64-openssl11.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-2-3-om7 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.2.3-linux-x64-openssl11.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-2-4-om7 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.2.4-linux-x64-openssl11.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + - name: setting-up-mongosh-2-4-0 + image: curlimages/curl:latest + command: + - sh + - -c + - curl -LO https://downloads.mongodb.com/compass/mongosh-2.4.0-linux-x64-openssl11.tgz --output-dir /mongodb-ops-manager/mongodb-releases/compass && true + volumeMounts: + - name: mongosh-versions + mountPath: /mongodb-ops-manager/mongodb-releases/compass + + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - name: mongodb-versions + emptyDir: {} + - name: mongosh-versions + emptyDir: {} + - configMap: + name: nginx-conf + name: nginx-conf diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/om_remotemode.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/om_remotemode.yaml new file mode 100644 index 000000000..ddf83a136 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/remote_fixtures/om_remotemode.yaml @@ -0,0 +1,27 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-remotemode +spec: + replicas: 1 + version: 4.4.0-rc2 # OM version with "remote" mode support + adminCredentials: ops-manager-admin-secret + configuration: + # CLOUDP-83792: as of OM 4.4.9 this property is critical to make remote mode work with enterprise build + automation.versions.download.baseUrl.allowOnlyAvailableBuilds: "false" + automation.versions.source: mongodb + mms.adminEmailAddr: cloud-manager-support@mongodb.com + mms.fromEmailAddr: cloud-manager-support@mongodb.com + mms.ignoreInitialUiSetup: "true" + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + mms.replyToEmailAddr: cloud-manager-support@mongodb.com + backup: + enabled: false + + applicationDatabase: + members: 3 + version: "4.4.20-ent" diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-for-om.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-for-om.yaml new file mode 100644 index 000000000..fe27611e5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-for-om.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: the-replica-set +spec: + members: 3 + version: 4.4.11 + type: ReplicaSet + opsManager: + configMapRef: + name: om-rs-configmap + credentials: my-credentials + persistent: true + logLevel: DEBUG + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-kmip-static.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-kmip-static.yaml new file mode 100644 index 000000000..c65e6f8e5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-kmip-static.yaml @@ -0,0 +1,54 @@ +--- +# This fixture contains KMIP OpsManager settings delivered via https://jira.mongodb.org/browse/CLOUDP-135429 +# and cluster encryption settings described in https://kb.corp.mongodb.com/article/000020599/ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-latest +spec: + members: 3 + version: 4.4.11 + type: ReplicaSet + opsManager: + configMapRef: + name: om-rs-configmap + credentials: my-credentials + persistent: false + logLevel: DEBUG + backup: + encryption: + kmip: + client: + clientCertificatePrefix: "test-prefix" + additionalMongodConfig: + systemLog: + component: + network: + verbosity: 5 + security: + enableEncryption: true + kmip: + clientCertificateFile: "/kmip/cert/tls.crt" + serverCAFile: "/kmip/ca/ca.pem" + serverName: kmip-svc + port: 5696 + podSpec: + podTemplate: + spec: + containers: + - name: mongodb-agent + volumeMounts: + - name: mongodb-kmip-client-pem + mountPath: /kmip/cert + - name: mongodb-kmip-certificate-authority-pem + mountPath: /kmip/ca + volumes: + - name: mongodb-kmip-client-pem + secret: + secretName: test-prefix-mdb-latest-kmip-client + - name: mongodb-kmip-certificate-authority-pem + configMap: + name: issuer-ca + items: + - key: ca-pem + path: ca.pem diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-kmip.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-kmip.yaml new file mode 100644 index 000000000..26c1df31b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/replica-set-kmip.yaml @@ -0,0 +1,54 @@ +--- +# This fixture contains KMIP OpsManager settings delivered via https://jira.mongodb.org/browse/CLOUDP-135429 +# and cluster encryption settings described in https://kb.corp.mongodb.com/article/000020599/ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-latest +spec: + members: 3 + version: 4.4.11 + type: ReplicaSet + opsManager: + configMapRef: + name: om-rs-configmap + credentials: my-credentials + persistent: false + logLevel: DEBUG + backup: + encryption: + kmip: + client: + clientCertificatePrefix: "test-prefix" + additionalMongodConfig: + systemLog: + component: + network: + verbosity: 5 + security: + enableEncryption: true + kmip: + clientCertificateFile: "/kmip/cert/tls.crt" + serverCAFile: "/kmip/ca/ca.pem" + serverName: kmip-svc + port: 5696 + podSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + volumeMounts: + - name: mongodb-kmip-client-pem + mountPath: /kmip/cert + - name: mongodb-kmip-certificate-authority-pem + mountPath: /kmip/ca + volumes: + - name: mongodb-kmip-client-pem + secret: + secretName: test-prefix-mdb-latest-kmip-client + - name: mongodb-kmip-certificate-authority-pem + configMap: + name: issuer-ca + items: + - key: ca-pem + path: ca.pem diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/scram-sha-user-backing-db.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/scram-sha-user-backing-db.yaml new file mode 100644 index 000000000..c4103400a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/scram-sha-user-backing-db.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: mms-user-1 +spec: + passwordSecretKeyRef: + name: mms-user-1-password + key: password + username: "mms-user-1@/" + db: "admin" + mongodbResourceRef: + name: "my-replica-set" + roles: + - db: "admin" + name: "clusterMonitor" + - db: "admin" + name: "readWriteAnyDatabase" + - db: "admin" + name: "dbAdminAnyDatabase" diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/sharded-cluster-for-om.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/sharded-cluster-for-om.yaml new file mode 100644 index 000000000..ef527a5da --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/sharded-cluster-for-om.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: the-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 1 + configServerCount: 1 + version: 4.4.21 + type: ShardedCluster + opsManager: + configMapRef: + name: om-sc-configmap + credentials: my-credentials + persistent: true + logLevel: DEBUG + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/upgrade_appdb.yaml b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/upgrade_appdb.yaml new file mode 100644 index 000000000..3646f3398 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/upgrade_appdb.yaml @@ -0,0 +1,16 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: om-full +spec: + replicas: 1 + version: 5.0.1 + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + backup: + enabled: false + applicationDatabase: + members: 3 + version: 4.4.20-ent + diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/kubernetes_tester/__init__.py b/docker/mongodb-enterprise-tests/tests/opsmanager/kubernetes_tester/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/kubernetes_tester/om_appdb_validation.py b/docker/mongodb-enterprise-tests/tests/opsmanager/kubernetes_tester/om_appdb_validation.py new file mode 100644 index 000000000..90ac814bd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/kubernetes_tester/om_appdb_validation.py @@ -0,0 +1,186 @@ +from typing import Optional + +import pytest +from kubernetes.client import ApiException +from kubetester import wait_for_webhook +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.opsmanager import MongoDBOpsManager + +# This test still uses KubernetesTester therefore it wasn't used for Multi-Cluster AppDB tests. +# It was also moved to separate directory to clearly indicate this test is unmanageable until converting to normal imperative test. + + +def om_resource(namespace: str) -> MongoDBOpsManager: + return MongoDBOpsManager.from_yaml(yaml_fixture("om_validation.yaml"), namespace=namespace) + + +@pytest.mark.e2e_om_appdb_validation +def test_wait_for_webhook(namespace: str): + wait_for_webhook(namespace) + + +@pytest.mark.e2e_om_appdb_validation +def test_wrong_appdb_version(namespace: str, custom_version: Optional[str]): + om = om_resource(namespace) + om["spec"]["applicationDatabase"]["version"] = "3.6.12" + om.set_version(custom_version) + with pytest.raises( + ApiException, + match=r"the version of Application Database must be .* 4.0", + ): + om.create() + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerAppDbWrongSize(KubernetesTester): + """ + name: Wrong size of AppDb + description: | + AppDB with members < 3 is not allowed + create: + file: om_validation.yaml + patch: '[{"op":"replace","path":"/spec/applicationDatabase/members","value":2}]' + exception: 'spec.applicationDatabase.members in body should be greater than or equal to 3' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerBackupEnabledNotSpecified(KubernetesTester): + """ + name: Backup 'enabled' check + description: | + Backup specified but 'enabled' field is missing - it is required + create: + file: om_validation.yaml + patch: '[{"op":"add","path":"/spec/backup","value":{}}]' + exception: 'spec.backup.enabled in body is required' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerBackupOplogStoreNameRequired(KubernetesTester): + """ + name: Backup 'enabled' check + description: | + Backup oplog store specified but missing 'name' field + create: + file: om_validation.yaml + patch: '[{"op":"add","path":"/spec/backup","value":{"enabled": true, "opLogStores": [{"mongodbResourceRef": {}}]}}]' + exception: 'spec.backup.opLogStores[0].name: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerBackupOplogStoreMongodbRefRequired(KubernetesTester): + """ + name: Backup 'enabled' check + description: | + Backup oplog store specified but missing 'mongodbResourceRef' field + create: + file: om_validation.yaml + patch: '[{"op":"add","path":"/spec/backup","value":{"enabled": true, "opLogStores": [{"name": "foo"}]}}]' + exception: 'spec.backup.opLogStores[0].mongodbResourceRef: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerS3StoreNameRequired(KubernetesTester): + """ + description: | + S3 store specified but missing 'name' field + create: + file: om_s3store_validation.yaml + patch: '[ { "op":"add","path":"/spec/backup/s3Stores/-","value": {} }]' + exception: 'spec.backup.s3Stores[0].name: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerS3StorePathStyleAccessEnabledRequired(KubernetesTester): + """ + description: | + S3 store specified but missing 'pathStyleAccessEnabled' field + create: + file: om_s3store_validation.yaml + patch: '[{"op":"add","path":"/spec/backup/s3Stores/-","value":{ "name": "foo", "mongodbResourceRef": {"name":"my-rs" }}}]' + exception: 'spec.backup.s3Stores[0].pathStyleAccessEnabled: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerS3StoreS3BucketEndpointRequired(KubernetesTester): + """ + description: | + S3 store specified but missing 's3BucketEndpoint' field + create: + file: om_s3store_validation.yaml + patch: '[{"op":"add","path":"/spec/backup/s3Stores/-","value":{ "name": "foo", "mongodbResourceRef": {"name":"my-rs" }, "pathStyleAccessEnabled": true }}]' + exception: 'spec.backup.s3Stores[0].s3BucketEndpoint: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerS3StoreS3BucketNameRequired(KubernetesTester): + """ + description: | + S3 store specified but missing 's3BucketName' field + create: + file: om_s3store_validation.yaml + patch: '[{"op":"add","path":"/spec/backup/s3Stores/-","value":{ "name": "foo", "mongodbResourceRef": {"name":"my-rs" }, "pathStyleAccessEnabled": true , "s3BucketEndpoint": "my-endpoint"}}]' + exception: 'spec.backup.s3Stores[0].s3BucketName: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerExternalConnectivityTypeRequired(KubernetesTester): + """ + description: | + 'spec.externalConnectivity.type' is a required field + create: + file: om_validation.yaml + patch: '[{"op":"add","path":"/spec/externalConnectivity","value":{}}]' + exception: 'spec.externalConnectivity.type: Required value' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_om_appdb_validation +class TestOpsManagerExternalConnectivityWrongType(KubernetesTester): + """ + description: | + 'spec.externalConnectivity.type' must be either "LoadBalancer" or "NodePort" + create: + file: om_validation.yaml + patch: '[{"op":"add","path":"/spec/externalConnectivity","value":{"type": "nginx"}}]' + exception: 'spec.externalConnectivity.type in body should be one of [LoadBalancer NodePort]' + """ + + def test_validation_ok(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_appdb_multi_change.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_appdb_multi_change.py new file mode 100644 index 000000000..2457bd8fd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_appdb_multi_change.py @@ -0,0 +1,48 @@ +from kubetester import find_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: str, custom_appdb_version: str) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(find_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@mark.e2e_om_appdb_multi_change +def test_appdb(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_appdb_multi_change +def test_change_appdb(ops_manager: MongoDBOpsManager): + """This change affects both the StatefulSet spec (agent flags) and the AutomationConfig (mongod config). + Appdb controller is expected to perform wait after the automation config push so that all the pods got to not ready + status and the next StatefulSet spec change didn't result in the immediate rolling upgrade. + See CLOUDP-73296 for more details.""" + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["agent"] = {"startupOptions": {"maxLogFiles": "30"}} + ops_manager["spec"]["applicationDatabase"]["additionalMongodConfig"] = { + "replication": {"enableMajorityReadConcern": "true"} + } + ops_manager.update() + + ops_manager.appdb_status().assert_abandons_phase(Phase.Running, timeout=100) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_om_appdb_multi_change +def test_om_ok(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=800) + ops_manager.get_om_tester().assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_appdb_scram.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_appdb_scram.py new file mode 100644 index 000000000..736f59cda --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_appdb_scram.py @@ -0,0 +1,249 @@ +from typing import Optional + +import pytest +from kubetester import create_or_update_secret +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import get_central_cluster_client, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +OM_RESOURCE_NAME = "om-scram" +OM_USER_NAME = "mongodb-ops-manager" +USER_DEFINED_PASSWORD = "@my-scram-password#:" +UPDATED_USER_DEFINED_PASSWORD = f"updated-{USER_DEFINED_PASSWORD}" +EXPECTED_OM_USER_ROLES = { + ("admin", "readWriteAnyDatabase"), + ("admin", "dbAdminAnyDatabase"), + ("admin", "clusterMonitor"), + ("admin", "hostManager"), + ("admin", "backup"), + ("admin", "restore"), +} + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml(yaml_fixture("om_appdb_scram.yaml"), namespace=namespace) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def auto_generated_password(ops_manager: MongoDBOpsManager) -> str: + return ops_manager.read_appdb_generated_password() + + +@pytest.mark.e2e_om_appdb_scram +class TestOpsManagerCreation: + """ + Creates an Ops Manager instance with AppDB of size 3. This test waits until Ops Manager + is ready to avoid changing password before Ops Manager has reached ready state + """ + + def test_appdb(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + assert ops_manager.appdb_status().get_members() == 3 + assert ops_manager.appdb_status().get_version() == custom_appdb_version + + def test_admin_config_map(self, ops_manager: MongoDBOpsManager): + ops_manager.get_automation_config_tester().reached_version(1) + + def test_ops_manager_spec(self, ops_manager: MongoDBOpsManager): + """security (and authentication inside it) are not show in spec""" + assert "security" not in ops_manager["spec"]["applicationDatabase"] + + def test_mongod(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + mdb_tester = ops_manager.get_appdb_tester() + mdb_tester.assert_connectivity() + mdb_tester.assert_version(custom_appdb_version) + + def test_appdb_automation_config(self, ops_manager: MongoDBOpsManager): + # only user should be the Ops Manager user + tester = ops_manager.get_automation_config_tester() + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + tester.assert_has_user(OM_USER_NAME) + tester.assert_user_has_roles(OM_USER_NAME, EXPECTED_OM_USER_ROLES) + tester.assert_expected_users(1) + tester.assert_authoritative_set(False) + + def test_scram_secrets_exists_with_correct_owner_reference(self, ops_manager: MongoDBOpsManager): + password_secret = ops_manager.read_appdb_agent_password_secret() + keyfile_secret = ops_manager.read_appdb_agent_keyfile_secret() + omUID = ops_manager.backing_obj["metadata"]["uid"] + + assert len(password_secret.metadata.owner_references) == 1 + assert password_secret.metadata.owner_references[0].uid == omUID + + assert len(keyfile_secret.metadata.owner_references) == 1 + assert keyfile_secret.metadata.owner_references[0].uid == omUID + + def test_appdb_scram_sha(self, ops_manager: MongoDBOpsManager, auto_generated_password: str): + app_db_tester = ops_manager.get_appdb_tester() + + # should be possible to auth as the operator will have auto generated a password + app_db_tester.assert_scram_sha_authentication( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-256" + ) + + def test_om_is_created(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(phase=Phase.Running) + # Let the monitoring get registered + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_appdb_scram +class TestChangeOpsManagerUserPassword: + """ + Creates a secret with a new password that the Ops Manager user should use and ensures that + SCRAM is configured correctly with the new password + """ + + def test_upgrade_om(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + create_or_update_secret( + ops_manager.namespace, + "my-password", + {"new-key": USER_DEFINED_PASSWORD}, + api_client=get_central_cluster_client(), + ) + + ops_manager["spec"]["applicationDatabase"]["passwordSecretKeyRef"] = { + "name": "my-password", + "key": "new-key", + } + ops_manager.update() + + # Swapping the password can lead to a race where we check for the status before om reconciler was able to swap + # the password. + ops_manager.om_status().assert_reaches_phase(Phase.Running, ignore_errors=True) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + @pytest.mark.xfail(reason="the auto generated password should have been deleted once the user creates their own") + def test_auto_generated_password_exists(self, ops_manager: MongoDBOpsManager): + ops_manager.read_appdb_generated_password_secret() + + def test_config_map_reached_v2(self, ops_manager: MongoDBOpsManager): + # should reach version 2 as a password has changed, resulting in new ScramShaCreds + ops_manager.get_automation_config_tester().reached_version(2) + + def test_appdb_automation_config(self, ops_manager: MongoDBOpsManager): + tester = ops_manager.get_automation_config_tester() + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + tester.assert_has_user(OM_USER_NAME) + tester.assert_user_has_roles(OM_USER_NAME, EXPECTED_OM_USER_ROLES) + tester.assert_expected_users(1) + tester.assert_authoritative_set(False) + + def test_authenticate_with_user_password(self, ops_manager: MongoDBOpsManager): + app_db_tester = ops_manager.get_appdb_tester() + password = KubernetesTester.read_secret(ops_manager.namespace, "my-password")["new-key"] + assert password == USER_DEFINED_PASSWORD + app_db_tester.assert_scram_sha_authentication(OM_USER_NAME, password, auth_mechanism="SCRAM-SHA-256") + + def test_cannot_authenticate_with_old_autogenerated_password( + self, ops_manager: MongoDBOpsManager, auto_generated_password: str + ): + app_db_tester = ops_manager.get_appdb_tester() + app_db_tester.assert_scram_sha_authentication_fails( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-256" + ) + + +@pytest.mark.e2e_om_appdb_scram +class TestChangeOpsManagerExistingUserPassword: + """ + Updating the secret should trigger another reconciliation because the + Operator should be watching the user created secret. + """ + + def test_user_update_password(self, namespace: str): + KubernetesTester.update_secret( + namespace, + "my-password", + {"new-key": UPDATED_USER_DEFINED_PASSWORD}, + ) + + def test_om_reconciled(self, ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_abandons_phase(Phase.Running) + ops_manager.om_status().assert_abandons_phase(Phase.Running) + # Swapping the password can lead to a race where we check for the status before om reconciler was able to swap + # the password. + ops_manager.om_status().assert_reaches_phase(Phase.Running, ignore_errors=True) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + @pytest.mark.xfail(reason="the auto generated password should have been deleted once the user creates their own") + def test_auto_generated_password_exists(self, ops_manager: MongoDBOpsManager): + ops_manager.read_appdb_generated_password_secret() + + def test_config_map_reached_v3(self, ops_manager: MongoDBOpsManager): + KubernetesTester.wait_until( + lambda: ops_manager.get_automation_config_tester().reached_version(3), + timeout=180, + ) + + def test_appdb_automation_config(self, ops_manager: MongoDBOpsManager): + tester = ops_manager.get_automation_config_tester() + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + tester.assert_has_user(OM_USER_NAME) + tester.assert_user_has_roles(OM_USER_NAME, EXPECTED_OM_USER_ROLES) + tester.assert_authoritative_set(False) + tester.assert_expected_users(1) + + def test_authenticate_with_user_password(self, ops_manager: MongoDBOpsManager): + app_db_tester = ops_manager.get_appdb_tester() + password = KubernetesTester.read_secret(ops_manager.namespace, "my-password")["new-key"] + assert password == UPDATED_USER_DEFINED_PASSWORD + app_db_tester.assert_scram_sha_authentication(OM_USER_NAME, password, auth_mechanism="SCRAM-SHA-256") + + def test_cannot_authenticate_with_old_autogenerated_password( + self, ops_manager: MongoDBOpsManager, auto_generated_password: str + ): + app_db_tester = ops_manager.get_appdb_tester() + app_db_tester.assert_scram_sha_authentication_fails( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-256" + ) + + +@pytest.mark.e2e_om_appdb_scram +class TestOpsManagerGeneratesNewPasswordIfNoneSpecified: + def test_upgrade_om(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["passwordSecretKeyRef"] = { + "name": "", + "key": "", + } + ops_manager.update() + # Swapping the password can lead to a race when the Agent deploys new password to AppDB and the + # Operator updates the connection string (which leads to restarting OM). Since there are no time + # or order guarantees, the entire system may go through an error state. In that case, + # the recovery will happen soon, but it needs a bit more time. + ops_manager.om_status().assert_reaches_phase(Phase.Running, ignore_errors=True, timeout=1800) + + def test_new_password_was_created(self, ops_manager: MongoDBOpsManager): + assert ops_manager.read_appdb_generated_password() != "" + + def test_wait_for_config_map_reached_v4(self, ops_manager: MongoDBOpsManager): + # should reach version 4 as password should change back + assert ops_manager.get_automation_config_tester().reached_version(4) + + def test_cannot_authenticate_with_old_password(self, ops_manager: MongoDBOpsManager): + app_db_tester = ops_manager.get_appdb_tester() + app_db_tester.assert_scram_sha_authentication_fails( + OM_USER_NAME, USER_DEFINED_PASSWORD, auth_mechanism="SCRAM-SHA-256" + ) + + def test_authenticate_with_user_password(self, ops_manager: MongoDBOpsManager, auto_generated_password: str): + app_db_tester = ops_manager.get_appdb_tester() + password = ops_manager.read_appdb_generated_password() + assert password != auto_generated_password, "new password should have been generated" + app_db_tester.assert_scram_sha_authentication(OM_USER_NAME, password, auth_mechanism="SCRAM-SHA-256") diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_external_connectivity.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_external_connectivity.py new file mode 100644 index 000000000..aaa218a79 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_external_connectivity.py @@ -0,0 +1,195 @@ +import random +from typing import Optional + +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def opsmanager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + ## Force creating headless services for internal connectivity + resource["spec"]["internalConnectivity"] = { + "type": "ClusterIP", + "ClusterIP": "None", + } + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@mark.e2e_om_external_connectivity +def test_reaches_goal_state(opsmanager: MongoDBOpsManager): + opsmanager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + # some time for monitoring to be finished + opsmanager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + opsmanager.om_status().assert_reaches_phase(Phase.Running, timeout=50) + + for ( + cluster_idx, + cluster_spec_item, + ) in opsmanager.get_om_indexed_cluster_spec_items(): + internal, external = opsmanager.services(cluster_spec_item["clusterName"]) + assert internal is not None + assert external is None + assert internal.spec.type == "ClusterIP" + assert internal.spec.cluster_ip == "None" + + +@mark.e2e_om_external_connectivity +def test_set_external_connectivity_load_balancer_with_default_port( + opsmanager: MongoDBOpsManager, +): + ext_connectivity = { + "type": "LoadBalancer", + "loadBalancerIP": "172.18.255.211", + "externalTrafficPolicy": "Local", + "annotations": { + "first-annotation": "first-value", + "second-annotation": "second-value", + }, + } + opsmanager.load() + opsmanager["spec"]["externalConnectivity"] = ext_connectivity + opsmanager.update() + + opsmanager.om_status().assert_reaches_phase(Phase.Running) + + for _, cluster_spec_item in opsmanager.get_om_indexed_cluster_spec_items(): + internal, external = opsmanager.services(cluster_spec_item["clusterName"]) + + assert internal is not None + assert internal.spec.type == "ClusterIP" + assert internal.spec.cluster_ip == "None" + + assert external is not None + assert external.spec.type == "LoadBalancer" + assert len(external.spec.ports) == 1 + assert external.spec.ports[0].port == 8080 # if not specified it will be the default port + assert external.spec.load_balancer_ip == "172.18.255.211" + assert external.spec.external_traffic_policy == "Local" + + +@mark.e2e_om_external_connectivity +def test_set_external_connectivity(opsmanager: MongoDBOpsManager): + ext_connectivity = { + "type": "LoadBalancer", + "loadBalancerIP": "172.18.255.211", + "externalTrafficPolicy": "Local", + "port": 443, + "annotations": { + "first-annotation": "first-value", + "second-annotation": "second-value", + }, + } + opsmanager.load() + opsmanager["spec"]["externalConnectivity"] = ext_connectivity + opsmanager.update() + + opsmanager.om_status().assert_reaches_phase(Phase.Running) + + for _, cluster_spec_item in opsmanager.get_om_indexed_cluster_spec_items(): + internal, external = opsmanager.services(cluster_spec_item["clusterName"]) + + assert internal is not None + assert internal.spec.type == "ClusterIP" + assert internal.spec.cluster_ip == "None" + + assert external is not None + assert external.spec.type == "LoadBalancer" + assert len(external.spec.ports) == 1 + assert external.spec.ports[0].port == 443 + assert external.spec.load_balancer_ip == "172.18.255.211" + assert external.spec.external_traffic_policy == "Local" + + +@mark.e2e_om_external_connectivity +def test_add_annotations(opsmanager: MongoDBOpsManager): + """Makes sure annotations are updated properly.""" + annotations = {"second-annotation": "edited-value", "added-annotation": "new-value"} + opsmanager.load() + opsmanager["spec"]["externalConnectivity"]["annotations"] = annotations + opsmanager.update() + + opsmanager.om_status().assert_reaches_phase(Phase.Running) + + for _, cluster_spec_item in opsmanager.get_om_indexed_cluster_spec_items(): + internal, external = opsmanager.services(cluster_spec_item["clusterName"]) + + ant = external.metadata.annotations + assert len(ant) == 3 + assert "first-annotation" in ant + assert "second-annotation" in ant + assert "added-annotation" in ant + + assert ant["second-annotation"] == "edited-value" + assert ant["added-annotation"] == "new-value" + + +@mark.e2e_om_external_connectivity +def test_service_set_node_port(opsmanager: MongoDBOpsManager): + """Changes externalConnectivity to type NodePort.""" + node_port = random.randint(30000, 32700) + opsmanager["spec"]["externalConnectivity"] = { + "type": "NodePort", + "port": node_port, + } + opsmanager.update() + + opsmanager.assert_reaches(lambda om: service_is_changed_to_nodeport(om)) + + for _, cluster_spec_item in opsmanager.get_om_indexed_cluster_spec_items(): + internal, external = opsmanager.services(cluster_spec_item["clusterName"]) + assert internal.spec.type == "ClusterIP" + assert external.spec.type == "NodePort" + assert external.spec.ports[0].node_port == node_port + assert external.spec.ports[0].port == node_port + assert external.spec.ports[0].target_port == 8080 + + opsmanager.load() + opsmanager["spec"]["externalConnectivity"] = { + "type": "LoadBalancer", + "port": 443, + } + opsmanager.update() + + opsmanager.assert_reaches(lambda om: service_is_changed_to_loadbalancer(om)) + + for _, cluster_spec_item in opsmanager.get_om_indexed_cluster_spec_items(): + _, external = opsmanager.services(cluster_spec_item["clusterName"]) + assert external.spec.type == "LoadBalancer" + assert external.spec.ports[0].port == 443 + assert external.spec.ports[0].target_port == 8080 + + +def service_is_changed_to_nodeport(om: MongoDBOpsManager) -> bool: + for _, cluster_spec_item in om.get_om_indexed_cluster_spec_items(): + svc = om.services(cluster_spec_item["clusterName"])[1] + if svc.spec.type != "NodePort": + return False + + return True + + +def service_is_changed_to_loadbalancer(om: MongoDBOpsManager) -> bool: + for _, cluster_spec_item in om.get_om_indexed_cluster_spec_items(): + svc = om.services(cluster_spec_item["clusterName"])[1] + if svc.spec.type != "LoadBalancer": + return False + + return True diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_jvm_params.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_jvm_params.py new file mode 100644 index 000000000..91d4a5ec8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_jvm_params.py @@ -0,0 +1,180 @@ +import re +from typing import Optional + +from dateutil.parser import parse +from kubetester import try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import assert_log_rotation_process, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +OM_CONF_PATH_DIR = "mongodb-ops-manager/conf/mms.conf" +APPDB_LOG_DIR = "/data" +JAVA_MMS_UI_OPTS = "JAVA_MMS_UI_OPTS" +JAVA_DAEMON_OPTS = "JAVA_DAEMON_OPTS" + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + """The fixture for Ops Manager to be created.""" + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_jvm_params.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + om["spec"]["applicationDatabase"]["agent"] = { + "logRotate": { + "sizeThresholdMB": "0.0001", + "percentOfDiskspace": "10", + "numTotal": 10, + "timeThresholdHrs": 1, + "numUncompressed": 2, + }, + "systemLog": { + "destination": "file", + "path": APPDB_LOG_DIR + "/mongodb.log", + "logAppend": False, + }, + } + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + try_load(om) + return om + + +def is_date(file_name) -> bool: + try: + parse(file_name, fuzzy=True) + return True + + except ValueError: + return False + + +@mark.e2e_om_jvm_params +class TestOpsManagerCreationWithJvmParams: + def test_om_created(self, ops_manager: MongoDBOpsManager): + ops_manager.update() + # Backup is not fully configured so we wait until Pending phase + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + timeout=900, + msg_regexp="Oplog Store configuration is required for backup.*", + ) + + def test_om_jvm_params_configured(self, ops_manager: MongoDBOpsManager): + for api_client, pod in ops_manager.read_om_pods(): + cmd = ["/bin/sh", "-c", "cat " + OM_CONF_PATH_DIR] + + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + ops_manager.namespace, + cmd, + container="mongodb-ops-manager", + api_client=api_client, + ) + java_params = self.parse_java_params(result, JAVA_MMS_UI_OPTS) + assert "-Xmx4291m" in java_params + assert "-Xms343m" in java_params + + def test_om_process_mem_scales(self, ops_manager: MongoDBOpsManager): + for api_client, pod in ops_manager.read_om_pods(): + + cmd = ["/bin/sh", "-c", "ps aux"] + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + ops_manager.namespace, + cmd, + container="mongodb-ops-manager", + api_client=api_client, + ) + rss = self.parse_rss(result) + + # rss is in kb, we want to ensure that it is > 400mb + # this is to ensure that OM can grow properly with it's container + assert int(rss) / 1024 > 400 + + def test_om_jvm_backup_params_configured(self, ops_manager: MongoDBOpsManager): + for api_client, pod in ops_manager.read_backup_pods(): + + cmd = ["/bin/sh", "-c", "cat " + OM_CONF_PATH_DIR] + + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + ops_manager.namespace, + cmd, + container="mongodb-backup-daemon", + api_client=api_client, + ) + + java_params = self.parse_java_params(result, JAVA_DAEMON_OPTS) + assert "-Xmx4352m" in java_params + assert "-Xms4352m" in java_params + + def test_om_log_rotate_configured(self, ops_manager: MongoDBOpsManager): + processes = ops_manager.get_automation_config_tester().automation_config["processes"] + expected = { + "timeThresholdHrs": 1, + "numUncompressed": 2, + "numTotal": 10, + "sizeThresholdMB": 0.0001, + "percentOfDiskspace": 10, + } + for p in processes: + assert p["logRotate"] == expected + + def test_update_appdb_log_rotation_keep_deprecated_fields(self, ops_manager): + # configuration over mongod takes precedence over deprecated logRotation directly under agent + ops_manager["spec"]["applicationDatabase"]["agent"]["mongod"] = { + "logRotate": { + "sizeThresholdMB": "1", + "percentOfDiskspace": "1", + "numTotal": 1, + "timeThresholdHrs": 1, + "numUncompressed": 1, + } + } + + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + timeout=900, + msg_regexp="Oplog Store configuration is required for backup.*", + ) + + def test_om_log_rotate_has_changed(self, ops_manager: MongoDBOpsManager): + processes = ops_manager.get_automation_config_tester().automation_config["processes"] + expected = { + "timeThresholdHrs": 1, + "numUncompressed": 1, + "numTotal": 1, + "sizeThresholdMB": 1, + "percentOfDiskspace": 1, + } + for p in processes: + assert p["logRotate"] == expected + + def parse_java_params(self, conf: str, opts_key: str) -> str: + java_params = "" + for line in conf.split("\n"): + if not line.startswith("#") and opts_key in line: + param_line = line.split("=", 1) + assert len(param_line) != 0, "Expected key=value format" + java_params = param_line[1] + + return java_params + + def parse_rss(self, ps_output: str) -> int: + rss = 0 + sep = re.compile("[\s]+") + + for row in ps_output.split("\n"): + columns = sep.split(row) + if "ops-manager" in columns[10]: + # RSS + rss = int(columns[5]) + break + + return rss diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_localmode_multiple_pv.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_localmode_multiple_pv.py new file mode 100644 index 000000000..686a54aa5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_localmode_multiple_pv.py @@ -0,0 +1,104 @@ +from typing import Optional + +from kubetester import get_default_storage_class +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_localmode-multiple-pv.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def replica_set(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + ).configure(ops_manager, "my-replica-set") + resource.set_version(custom_mdb_version) + resource["spec"]["members"] = 2 + + resource.update() + return resource + + +@mark.e2e_om_localmode_multiple_pv +class TestOpsManagerCreation: + def test_ops_manager_ready(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1000) + + def test_volume_mounts(self, ops_manager: MongoDBOpsManager): + statefulset = ops_manager.read_statefulset() + + # pod template has volume mount request + assert ("/mongodb-ops-manager/mongodb-releases", "mongodb-versions") in ( + (mount.mount_path, mount.name) for mount in statefulset.spec.template.spec.containers[0].volume_mounts + ) + + def test_pvcs(self, ops_manager: MongoDBOpsManager): + for api_client, pod in ops_manager.read_om_pods(): + claims = [volume for volume in pod.spec.volumes if getattr(volume, "persistent_volume_claim")] + assert len(claims) == 1 + + KubernetesTester.check_single_pvc( + namespace=ops_manager.namespace, + volume=claims[0], + expected_name="mongodb-versions", + expected_claim_name="mongodb-versions-{}".format(pod.metadata.name), + expected_size="20G", + storage_class=get_default_storage_class(), + api_client=api_client, + ) + + def test_replica_set_reaches_failed_phase(self, replica_set: MongoDB): + # CLOUDP-61573 - we don't get the validation error on automation config submission if the OM has no + # distros for local mode - so just wait until the agents don't reach goal state + replica_set.assert_reaches_phase(Phase.Failed, timeout=300) + + def test_add_mongodb_distros(self, ops_manager: MongoDBOpsManager, custom_mdb_version: str): + ops_manager.download_mongodb_binaries(custom_mdb_version) + + def test_replica_set_reaches_running_phase(self, replica_set: MongoDB): + # note that the Replica Set may sometimes still get to Failed error + # ("Status: 400 (Bad Request), Detail: Invalid config: MongoDB version 4.2.0 is not available.") + # so we are ignoring errors during this wait + replica_set.assert_reaches_phase(Phase.Running, timeout=300, ignore_errors=True) + + def test_client_can_connect_to_mongodb(self, replica_set: MongoDB, custom_mdb_version: str): + replica_set.assert_connectivity() + replica_set.tester().assert_version(custom_mdb_version) + + +@mark.e2e_om_localmode_multiple_pv +class TestOpsManagerRestarted: + def test_restart_ops_manager_pod(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["configuration"]["mms.testUtil.enabled"] = "false" + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + def test_can_scale_replica_set(self, replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["members"] = 4 + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=200) + + def test_client_can_still_connect(self, replica_set: MongoDB): + replica_set.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_localmode_single_pv.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_localmode_single_pv.py new file mode 100644 index 000000000..6d426425f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_localmode_single_pv.py @@ -0,0 +1,116 @@ +from typing import Optional + +import yaml +from kubetester import get_default_storage_class, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import get_member_cluster_api_client, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import ( + enable_multi_cluster_deployment, + get_om_member_cluster_names, +) + +# This version is not supported by the Ops Manager and is not present in the local mode. +VERSION_NOT_IN_OPS_MANAGER = "6.0.4" + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + with open(yaml_fixture("mongodb_versions_claim.yaml"), "r") as f: + pvc_body = yaml.safe_load(f.read()) + + """ The fixture for Ops Manager to be created.""" + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_localmode-single-pv.yaml"), namespace=namespace + ) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + for member_cluster_name in get_om_member_cluster_names(): + member_client = get_member_cluster_api_client(member_cluster_name=member_cluster_name) + KubernetesTester.create_or_update_pvc( + namespace, + body=pvc_body, + storage_class_name=get_default_storage_class(), + api_client=member_client, + ) + else: + KubernetesTester.create_or_update_pvc(namespace, body=pvc_body, storage_class_name=get_default_storage_class()) + + try_load(om) + return om + + +@fixture(scope="module") +def replica_set(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + ).configure(ops_manager, "my-replica-set") + resource["spec"]["version"] = custom_mdb_version + try_load(resource) + return resource + + +@mark.e2e_om_localmode +def test_ops_manager_reaches_running_phase(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=800) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_localmode +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_localmode +def test_replica_set_version_upgraded_reaches_failed_phase(replica_set: MongoDB): + replica_set["spec"]["version"] = VERSION_NOT_IN_OPS_MANAGER + replica_set.update() + replica_set.assert_reaches_phase( + Phase.Failed, + msg_regexp=f".*Invalid config: MongoDB version {VERSION_NOT_IN_OPS_MANAGER} is not available.*", + ) + + +@mark.e2e_om_localmode +def test_replica_set_recovers(replica_set: MongoDB, custom_mdb_version: str): + replica_set["spec"]["version"] = custom_mdb_version + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@skip_if_local +@mark.e2e_om_localmode +def test_client_can_connect_to_mongodb(replica_set: MongoDB): + replica_set.assert_connectivity() + + +@mark.e2e_om_localmode +def test_restart_ops_manager_pod(ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["configuration"]["mms.testUtil.enabled"] = "false" + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_localmode +def test_can_scale_replica_set(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["members"] = 5 + replica_set.update() + # We should retry if we are running into errors while adding new members. + replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + + +@skip_if_local +@mark.e2e_om_localmode +def test_client_can_still_connect(replica_set: MongoDB): + replica_set.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_migration.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_migration.py new file mode 100644 index 000000000..c22ee1047 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_migration.py @@ -0,0 +1,40 @@ +import re +from typing import Optional + +from dateutil.parser import parse +from kubetester import try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + """The fixture for Ops Manager to be created.""" + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + try_load(om) + return om + + +@mark.e2e_om_migration +class TestOpsManagerOmMigration: + def test_om_created(self, ops_manager: MongoDBOpsManager): + ops_manager.update() + # Backup is not fully configured so we wait until Pending phase + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + def test_migrate_architecture(self, ops_manager: MongoDBOpsManager): + ops_manager.trigger_architecture_migration() + ops_manager.appdb_status().assert_abandons_phase(Phase.Running, timeout=1000) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=1000) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup.py new file mode 100644 index 000000000..ed5764076 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup.py @@ -0,0 +1,829 @@ +from operator import attrgetter +from typing import Dict, Optional + +from kubernetes import client +from kubernetes.client import ApiException +from kubetester import ( + assert_pod_container_security_context, + assert_pod_security_context, + create_or_update_configmap, + create_or_update_secret, + get_default_storage_class, + run_periodically, + try_load, +) +from kubetester.awss3client import AwsS3Client, s3_endpoint +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import running_locally, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.omtester import OMTester +from kubetester.opsmanager import MongoDBOpsManager +from kubetester.test_identifiers import set_test_identifier +from pytest import fixture, mark +from tests.conftest import AWS_REGION, is_multi_cluster +from tests.opsmanager.backup_snapshot_schedule_tests import BackupSnapshotScheduleTests +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +HEAD_PATH = "/head/" +S3_SECRET_NAME = "my-s3-secret" +OPLOG_RS_NAME = "my-mongodb-oplog" +S3_RS_NAME = "my-mongodb-s3" +BLOCKSTORE_RS_NAME = "my-mongodb-blockstore" +USER_PASSWORD = "/qwerty@!#:" + +""" +Current test focuses on backup capabilities. It creates an explicit MDBs for S3 snapshot metadata, Blockstore and Oplog +databases. Tests backup enabled for both MDB 4.0 and 4.2, snapshots created +""" + + +def new_om_s3_store( + mdb: MongoDB, + s3_id: str, + s3_bucket_name: str, + aws_s3_client: AwsS3Client, + assignment_enabled: bool = True, + path_style_access_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "id": s3_id, + "pathStyleAccessEnabled": path_style_access_enabled, + "s3BucketEndpoint": s3_endpoint(AWS_REGION), + "s3BucketName": s3_bucket_name, + "awsAccessKey": aws_s3_client.aws_access_key, + "awsSecretKey": aws_s3_client.aws_secret_access_key, + "assignmentEnabled": assignment_enabled, + } + + +def new_om_data_store( + mdb: MongoDB, + id: str, + assignment_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "id": id, + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "ssl": mdb.is_tls_enabled(), + "assignmentEnabled": assignment_enabled, + } + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-s3") + + +def create_aws_secret( + aws_s3_client, + secret_name: str, + namespace: str, + api_client: Optional[client.ApiClient] = None, +): + create_or_update_secret( + namespace, + secret_name, + { + "accessKey": aws_s3_client.aws_access_key, + "secretKey": aws_s3_client.aws_secret_access_key, + }, + api_client=api_client, + ) + print("\nCreated a secret for S3 credentials", secret_name) + + +def create_s3_bucket(aws_s3_client, bucket_prefix: str = "test-bucket-"): + """creates a s3 bucket and a s3 config""" + bucket_name = set_test_identifier( + f"create_s3_bucket_{bucket_prefix}", + KubernetesTester.random_k8s_name(bucket_prefix), + ) + + try: + aws_s3_client.create_s3_bucket(bucket_name) + print("Created S3 bucket", bucket_name) + + yield bucket_name + if not running_locally(): + print("\nRemoving S3 bucket", bucket_name) + try: + aws_s3_client.delete_s3_bucket(bucket_name) + except Exception as e: + print( + f"Caught exception while removing S3 bucket {bucket_name}, but ignoring to not obscure the test result: {e}" + ) + except Exception as e: + if running_locally(): + print(f"Local run: skipping creating bucket {bucket_name} because it already exists.") + yield bucket_name + else: + raise e + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup.yaml"), namespace=namespace + ) + + try_load(resource) + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + resource.set_version(custom_mdb_version) + + # TODO: Remove when CLOUDP-60443 is fixed + # This test will update oplog to have SCRAM enabled + # Currently this results in OM failure when enabling backup for a project, backup seems to do some caching resulting in the + # mongoURI not being updated unless pod is killed. This is documented in CLOUDP-60443, once resolved this skip & comment can be deleted + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + yield resource.update() + + +@fixture(scope="module") +def s3_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=S3_RS_NAME, + ).configure(ops_manager, "s3metadata") + + resource.set_version(custom_mdb_version) + yield resource.update() + + +@fixture(scope="module") +def blockstore_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + resource.set_version(custom_mdb_version) + yield resource.update() + + +@fixture(scope="module") +def blockstore_user(namespace, blockstore_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user-backing-db.yaml"), namespace=namespace) + resource["spec"]["mongodbResourceRef"]["name"] = blockstore_replica_set.name + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + yield resource.update() + + +@fixture(scope="module") +def oplog_user(namespace, oplog_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user-backing-db.yaml"), + namespace=namespace, + name="mms-user-2", + ) + resource["spec"]["mongodbResourceRef"]["name"] = oplog_replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "mms-user-2-password" + resource["spec"]["username"] = "mms-user-2" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + yield resource.update() + + +@mark.e2e_om_ops_manager_backup +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. The OM is expected to get to 'Pending' state + eventually as it will wait for oplog db to be created + """ + + def test_create_access_logback_xml_configmap(self, namespace: str, custom_logback_file_path: str): + logback = open(custom_logback_file_path).read() + data = {"logback-access.xml": logback} + create_or_update_configmap(namespace, "logback-access-config", data) + + def test_create_om( + self, + ops_manager: MongoDBOpsManager, + s3_bucket: str, + custom_version: str, + custom_appdb_version: str, + ): + """creates a s3 bucket, s3 config and an OM resource (waits until Backup gets to Pending state)""" + + ops_manager["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + ops_manager["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + ops_manager["spec"]["backup"]["members"] = 2 + + ops_manager.set_version(custom_version) + ops_manager.set_appdb_version(custom_appdb_version) + ops_manager.allow_mdb_rc_versions() + + if is_multi_cluster(): + enable_multi_cluster_deployment(ops_manager) + + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + def test_daemon_statefulset(self, ops_manager: MongoDBOpsManager): + def stateful_sets_become_ready(): + total_ready_replicas = 0 + total_current_replicas = 0 + for member_cluster_name, _ in ops_manager.get_backup_sts_names_in_member_clusters(): + stateful_set = ops_manager.read_backup_statefulset(member_cluster_name=member_cluster_name) + ready_replicas = ( + stateful_set.status.ready_replicas if stateful_set.status.ready_replicas is not None else 0 + ) + total_ready_replicas += ready_replicas + current_replicas = ( + stateful_set.status.current_replicas if stateful_set.status.current_replicas is not None else 0 + ) + total_current_replicas += current_replicas + return total_ready_replicas == 2 and total_current_replicas == 2 + + KubernetesTester.wait_until(stateful_sets_become_ready, timeout=300) + + for member_cluster_name, _ in ops_manager.get_backup_sts_names_in_member_clusters(): + stateful_set = ops_manager.read_backup_statefulset(member_cluster_name=member_cluster_name) + # pod template has volume mount request + assert (HEAD_PATH, "head") in ( + (mount.mount_path, mount.name) for mount in stateful_set.spec.template.spec.containers[0].volume_mounts + ) + + def test_daemon_pvc(self, ops_manager: MongoDBOpsManager, namespace: str): + """Verifies the PVCs mounted to the pod""" + for api_client, pod in ops_manager.read_backup_pods(): + claims = [volume for volume in pod.spec.volumes if getattr(volume, "persistent_volume_claim")] + assert len(claims) == 1 + claims.sort(key=attrgetter("name")) + pod_name = pod.metadata.name + default_sc = get_default_storage_class() + KubernetesTester.check_single_pvc( + namespace, + claims[0], + "head", + f"head-{pod_name}", + "500M", + default_sc, + api_client=api_client, + ) + + def test_backup_daemon_services_created(self, ops_manager: MongoDBOpsManager, namespace: str): + """Backup creates two additional services for queryable backup""" + service_count = 0 + for api_client, _ in ops_manager.read_backup_pods(): + services = client.CoreV1Api(api_client=api_client).list_namespaced_service(namespace).items + + # If running locally in 'default' namespace, there might be more + # services on it. Let's make sure we only count those that we care of. + # For now we allow this test to fail, because it is too broad to be significant + # and it is easy to break it. + backup_services = [s for s in services if s.metadata.name.startswith("om-backup")] + service_count += len(backup_services) + + assert service_count >= 3 + + @skip_if_local + def test_om(self, ops_manager: MongoDBOpsManager): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + # No oplog stores were created in Ops Manager by this time + om_tester.assert_oplog_stores([]) + om_tester.assert_s3_stores([]) + + def test_generations(self, ops_manager: MongoDBOpsManager): + assert ops_manager.appdb_status().get_observed_generation() == 1 + assert ops_manager.om_status().get_observed_generation() == 1 + assert ops_manager.backup_status().get_observed_generation() == 1 + + def test_security_contexts_appdb( + self, + ops_manager: MongoDBOpsManager, + operator_installation_config: Dict[str, str], + ): + managed = operator_installation_config["managedSecurityContext"] == "true" + for _, pod in ops_manager.read_appdb_pods(): + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + def test_security_contexts_om( + self, + ops_manager: MongoDBOpsManager, + operator_installation_config: Dict[str, str], + ): + managed = operator_installation_config["managedSecurityContext"] == "true" + for _, pod in ops_manager.read_om_pods(): + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + +@mark.e2e_om_ops_manager_backup +class TestBackupDatabasesAdded: + """name: Creates three mongodb resources for oplog, s3 and blockstore and waits until OM resource gets to + running state""" + + def test_backup_mdbs_created( + self, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + ): + """Creates mongodb databases all at once""" + oplog_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_oplog_user_created(self, oplog_user: MongoDBUser): + oplog_user.assert_reaches_phase(Phase.Updated) + + def test_oplog_updated_scram_sha_enabled(self, oplog_replica_set: MongoDB): + oplog_replica_set.load() + oplog_replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + oplog_replica_set.update() + oplog_replica_set.assert_reaches_phase(Phase.Running) + + def test_om_failed_oplog_no_user_ref(self, ops_manager: MongoDBOpsManager): + """Waits until Backup is in failed state as blockstore doesn't have reference to the user""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + msg_regexp=".*is configured to use SCRAM-SHA authentication mode, the user " + "must be specified using 'mongodbUserRef'", + ) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, oplog_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + + assert ops_manager.backup_status().get_message() is None + + @skip_if_local + def test_om( + self, + s3_bucket: str, + aws_s3_client: AwsS3Client, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + oplog_user: MongoDBUser, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + # Nothing has changed for daemon + + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + + om_tester.assert_block_stores([new_om_data_store(blockstore_replica_set, "blockStore1")]) + # oplog store has authentication enabled + om_tester.assert_oplog_stores( + [ + new_om_data_store( + oplog_replica_set, + "oplog1", + user_name=oplog_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + om_tester.assert_s3_stores([new_om_s3_store(s3_replica_set, "s3Store1", s3_bucket, aws_s3_client)]) + + def test_generations(self, ops_manager: MongoDBOpsManager): + """There have been an update to the OM spec - all observed generations are expected to be updated""" + assert ops_manager.appdb_status().get_observed_generation() == 2 + assert ops_manager.om_status().get_observed_generation() == 2 + assert ops_manager.backup_status().get_observed_generation() == 2 + + def test_security_contexts_backup( + self, + ops_manager: MongoDBOpsManager, + operator_installation_config: Dict[str, str], + ): + managed = operator_installation_config["managedSecurityContext"] == "true" + pods = ops_manager.read_backup_pods() + for _, pod in pods: + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + +@mark.e2e_om_ops_manager_backup +class TestOpsManagerWatchesBlockStoreUpdates: + def test_om_running(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=40) + + def test_scramsha_enabled_for_blockstore(self, blockstore_replica_set: MongoDB): + """Enables SCRAM for the blockstore replica set. Note that until CLOUDP-67736 is fixed + the order of operations (scram first, MongoDBUser - next) is important""" + blockstore_replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + blockstore_replica_set.update() + + # timeout of 600 is required when enabling SCRAM in mdb5.0.0 + blockstore_replica_set.assert_reaches_phase(Phase.Running, timeout=900) + + def test_blockstore_user_was_added_to_om(self, blockstore_user: MongoDBUser): + blockstore_user.assert_reaches_phase(Phase.Updated) + + def test_om_failed_oplog_no_user_ref(self, ops_manager: MongoDBOpsManager): + """Waits until Ops manager is in failed state as blockstore doesn't have reference to the user""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + msg_regexp=".*is configured to use SCRAM-SHA authentication mode, the user " + "must be specified using 'mongodbUserRef'", + ) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, blockstore_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["blockStores"][0]["mongodbUserRef"] = {"name": blockstore_user.name} + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + assert ops_manager.backup_status().get_message() is None + + @skip_if_local + def test_om( + self, + s3_bucket: str, + aws_s3_client: AwsS3Client, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + oplog_user: MongoDBUser, + blockstore_user: MongoDBUser, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + # Nothing has changed for daemon + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + + # block store has authentication enabled + om_tester.assert_block_stores( + [ + new_om_data_store( + blockstore_replica_set, + "blockStore1", + user_name=blockstore_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + + # oplog has authentication enabled + om_tester.assert_oplog_stores( + [ + new_om_data_store( + oplog_replica_set, + "oplog1", + user_name=oplog_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + om_tester.assert_s3_stores([new_om_s3_store(s3_replica_set, "s3Store1", s3_bucket, aws_s3_client)]) + + +@mark.e2e_om_ops_manager_backup +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created. + Both latest and the one before the latest are tested (as the backup process for them may differ significantly)""" + + @fixture(scope="class") + def mdb_latest(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-four-two", + ).configure(ops_manager, "firstProject") + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="disabled") + + try_load(resource) + return resource + + @fixture(scope="class") + def mdb_prev(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_prev_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-four-zero", + ).configure(ops_manager, "secondProject") + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.configure_backup(mode="disabled") + + try_load(resource) + return resource + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + mdb_latest.update() + mdb_prev.update() + + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_mdbs_enable_backup(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + mdb_latest.load() + mdb_latest.configure_backup(mode="enabled") + mdb_latest.update() + + mdb_prev.load() + mdb_prev.configure_backup(mode="enabled") + mdb_prev.update() + + mdb_prev.assert_reaches_phase(Phase.Running) + mdb_latest.assert_reaches_phase(Phase.Running) + + def test_mdbs_backuped(self, ops_manager: MongoDBOpsManager): + om_tester_first = ops_manager.get_om_tester(project_name="firstProject") + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + + # wait until a first snapshot is ready for both + om_tester_first.wait_until_backup_snapshots_are_ready(expected_count=1) + om_tester_second.wait_until_backup_snapshots_are_ready(expected_count=1) + + def test_can_transition_from_started_to_stopped(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # a direction transition from enabled to disabled is a single + # step for the operator + mdb_prev.assert_backup_reaches_status("STARTED", timeout=100) + mdb_prev.configure_backup(mode="disabled") + mdb_prev.update() + mdb_prev.assert_backup_reaches_status("STOPPED", timeout=600) + + def test_can_transition_from_started_to_terminated_0(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # a direct transition from enabled to terminated is not possible + # the operator should handle the transition from STARTED -> STOPPED -> TERMINATING + mdb_latest.assert_backup_reaches_status("STARTED", timeout=100) + mdb_latest.configure_backup(mode="terminated") + mdb_latest.update() + mdb_latest.assert_backup_reaches_status("TERMINATING", timeout=600) + + def test_backup_terminated_for_deleted_resource(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB): + # re-enable backup + mdb_prev.configure_backup(mode="enabled") + mdb_prev["spec"]["backup"]["autoTerminateOnDeletion"] = True + mdb_prev.update() + mdb_prev.assert_backup_reaches_status("STARTED", timeout=600) + mdb_prev.delete() + + def resource_is_deleted() -> bool: + try: + mdb_prev.load() + return False + except ApiException: + return True + + # wait until the resource is deleted + run_periodically(resource_is_deleted, timeout=300) + + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + om_tester_second.wait_until_backup_snapshots_are_ready(expected_count=0) + + # assert backup is deactivated + om_tester_second.wait_until_backup_deactivated() + + def test_hosts_were_removed(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB): + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + om_tester_second.wait_until_hosts_are_empty() + + def test_deploy_same_mdb_again_with_orphaned_backup(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB): + mdb_prev.configure_backup(mode="enabled") + mdb_prev["spec"]["backup"]["autoTerminateOnDeletion"] = False + # we need to make sure to use a clean resource since we might have loaded it + del mdb_prev["metadata"]["resourceVersion"] + mdb_prev.create() + mdb_prev.assert_reaches_phase(Phase.Running) + + mdb_prev.assert_backup_reaches_status("STARTED", timeout=600) + mdb_prev.delete() + + def resource_is_deleted() -> bool: + try: + mdb_prev.load() + return False + except ApiException: + return True + + # wait until the resource is deleted + run_periodically(resource_is_deleted, timeout=300) + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + + # assert backup is orphaned + om_tester_second.wait_until_backup_running() + + def test_hosts_were_not_removed(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB): + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + om_tester_second.wait_until_hosts_are_not_empty() + + +@mark.e2e_om_ops_manager_backup +class TestBackupSnapshotSchedule(BackupSnapshotScheduleTests): + pass + + +@mark.e2e_om_ops_manager_backup +class TestAssignmentLabels: + @fixture(scope="class") + def project_name(self): + return "mdb-assignment-labels" + + @fixture(scope="class") + def mdb_assignment_labels( + self, + ops_manager: MongoDBOpsManager, + namespace, + project_name, + custom_mdb_version: str, + ): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=project_name, + ).configure(ops_manager, project_name) + resource["spec"]["members"] = 1 + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["backup"] = {} + resource["spec"]["backup"]["assignmentLabels"] = ["test"] + resource.configure_backup(mode="enabled") + return resource.update() + + @fixture(scope="class") + def mdb_assignment_labels_om_tester(self, ops_manager: MongoDBOpsManager, project_name: str) -> OMTester: + ops_manager.load() + return ops_manager.get_om_tester(project_name=project_name) + + def test_add_assignment_labels_to_the_om(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["backup"]["assignmentLabels"] = ["test"] + ops_manager["spec"]["backup"]["blockStores"][0]["assignmentLabels"] = ["test"] + ops_manager["spec"]["backup"]["opLogStores"][0]["assignmentLabels"] = ["test"] + ops_manager["spec"]["backup"]["s3Stores"][0]["assignmentLabels"] = ["test"] + + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, ignore_errors=True) + + def test_mdb_assignment_labels_created(self, mdb_assignment_labels: MongoDB): + # In order to configure backup on the project level, the labels + # on both Backup Daemons and project need to match. Otherwise, this will fail. + mdb_assignment_labels.assert_reaches_phase(Phase.Running, ignore_errors=True) + + def test_assignment_labels_in_om(self, mdb_assignment_labels_om_tester: OMTester): + # Those labels are set on the Ops Manager CR level + mdb_assignment_labels_om_tester.api_read_backup_configs() + mdb_assignment_labels_om_tester.assert_s3_stores([{"id": "s3Store1", "labels": ["test"]}]) + mdb_assignment_labels_om_tester.assert_oplog_stores([{"id": "oplog1", "labels": ["test"]}]) + mdb_assignment_labels_om_tester.assert_block_stores([{"id": "blockStore1", "labels": ["test"]}]) + # This one is set on the MongoDB CR level + assert mdb_assignment_labels_om_tester.api_backup_group()["labelFilter"] == ["test"] + + +@mark.e2e_om_ops_manager_backup +class TestBackupConfigurationAdditionDeletion: + def test_oplog_store_is_added( + self, + ops_manager: MongoDBOpsManager, + s3_bucket: str, + aws_s3_client: AwsS3Client, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + oplog_user: MongoDBUser, + ): + ops_manager.reload() + ops_manager["spec"]["backup"]["opLogStores"].append( + {"name": "oplog2", "mongodbResourceRef": {"name": S3_RS_NAME}} + ) + + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=600) + + om_tester = ops_manager.get_om_tester() + om_tester.assert_oplog_stores( + [ + new_om_data_store( + oplog_replica_set, + "oplog1", + user_name=oplog_user.get_user_name(), + password=USER_PASSWORD, + ), + new_om_data_store(s3_replica_set, "oplog2"), + ] + ) + om_tester.assert_s3_stores([new_om_s3_store(s3_replica_set, "s3Store1", s3_bucket, aws_s3_client)]) + + def test_oplog_store_is_deleted_correctly( + self, + ops_manager: MongoDBOpsManager, + s3_bucket: str, + aws_s3_client: AwsS3Client, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + blockstore_user: MongoDBUser, + oplog_user: MongoDBUser, + ): + ops_manager.reload() + ops_manager["spec"]["backup"]["opLogStores"].pop() + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=600) + + om_tester = ops_manager.get_om_tester() + + om_tester.assert_oplog_stores( + [ + new_om_data_store( + oplog_replica_set, + "oplog1", + user_name=oplog_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + om_tester.assert_s3_stores([new_om_s3_store(s3_replica_set, "s3Store1", s3_bucket, aws_s3_client)]) + om_tester.assert_block_stores( + [ + new_om_data_store( + blockstore_replica_set, + "blockStore1", + user_name=blockstore_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + + def test_error_on_s3store_removal( + self, + ops_manager: MongoDBOpsManager, + ): + """Removing the s3 store when there are backups running is an error""" + ops_manager.reload() + ops_manager["spec"]["backup"]["s3Stores"] = [] + ops_manager.update() + + try: + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + msg_regexp=".*BACKUP_CANNOT_REMOVE_S3_STORE_CONFIG.*", + timeout=200, + ) + except Exception: + # Some backup internal logic: if more than 1 snapshot stores are configured in OM (CM?) then + # the random one will be picked for backup of mongodb resource. + # There's no way to specify the config to be used when enabling backup for a mongodb deployment. + # So this means that either the removal of s3 will fail as it's used already or it will succeed as + # the blockstore snapshot was used instead and the OM would get to Running phase + assert ops_manager.backup_status().get_phase() == Phase.Running diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_delete_sts.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_delete_sts.py new file mode 100644 index 000000000..02c682ba4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_delete_sts.py @@ -0,0 +1,125 @@ +from typing import Optional + +import semver +from kubetester import MongoDB, wait_until +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + assert_log_rotation_backup_monitoring, + assert_log_rotation_process, + get_custom_om_version, + is_multi_cluster, + setup_log_rotate_for_agents, +) +from tests.opsmanager.om_ops_manager_backup import BLOCKSTORE_RS_NAME, OPLOG_RS_NAME +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +DEFAULT_APPDB_USER_NAME = "mongodb-ops-manager" +supports_process_log_rotation = semver.VersionInfo.parse(get_custom_om_version()).match( + ">=7.0.4" +) or semver.VersionInfo.parse(get_custom_om_version()).match(">=6.0.24") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_backup_delete_sts.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + resource.set_version(ensure_ent_version(custom_mdb_version)) + + setup_log_rotate_for_agents(resource, supports_process_log_rotation) + + return resource.update() + + +@fixture(scope="module") +def blockstore_replica_set( + ops_manager, + custom_mdb_version, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + resource.set_version(ensure_ent_version(custom_mdb_version)) + + return resource.update() + + +@mark.e2e_om_ops_manager_backup_delete_sts_and_log_rotation +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_om_ops_manager_backup_delete_sts_and_log_rotation +def test_create_backing_replica_sets(oplog_replica_set: MongoDB, blockstore_replica_set: MongoDB): + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + +@mark.e2e_om_ops_manager_backup_delete_sts_and_log_rotation +def test_automation_log_rotation(ops_manager: MongoDBOpsManager): + config = ops_manager.get_om_tester(project_name="development").get_automation_config_tester() + processes = config.automation_config["processes"] + for p in processes: + assert_log_rotation_process(p, supports_process_log_rotation) + + +@mark.e2e_om_ops_manager_backup_delete_sts_and_log_rotation +def test_backup_log_rotation(ops_manager: MongoDBOpsManager): + bvk = ops_manager.get_om_tester(project_name="development").get_backup_config() + assert_log_rotation_backup_monitoring(bvk) + + +@mark.e2e_om_ops_manager_backup_delete_sts_and_log_rotation +def test_monitoring_log_rotation(ops_manager: MongoDBOpsManager): + m = ops_manager.get_om_tester(project_name="development").get_monitoring_config() + assert_log_rotation_backup_monitoring(m) + + +@mark.e2e_om_ops_manager_backup_delete_sts_and_log_rotation +def test_backup_statefulset_gets_recreated( + ops_manager: MongoDBOpsManager, +): + # Wait for the the backup to be fully running + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + ops_manager.load() + ops_manager["spec"]["backup"]["statefulSet"] = {"spec": {"revisionHistoryLimit": 15}} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + # the backup statefulset should have been recreated + def check_backup_sts(): + try: + ops_manager.read_backup_statefulset() + return True + except: + return False + + wait_until(check_backup_sts, timeout=90, sleep_time=5) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_kmip.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_kmip.py new file mode 100644 index 000000000..496da8dcf --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_kmip.py @@ -0,0 +1,178 @@ +from typing import Optional + +import pymongo +from kubetester import MongoDB, create_or_update_secret, read_secret +from kubetester.awss3client import AwsS3Client +from kubetester.certs import create_tls_certs +from kubetester.kmip import KMIPDeployment +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static +from kubetester.mongodb import Phase +from kubetester.omtester import OMTester +from kubetester.opsmanager import MongoDBOpsManager +from pymongo import ReadPreference +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import ( + S3_SECRET_NAME, + create_aws_secret, + create_s3_bucket, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} +OPLOG_SECRET_NAME = S3_SECRET_NAME + "-oplog" + +MONGODB_CR_NAME = "mdb-latest" +MONGODB_CR_KMIP_TEST_PREFIX = "test-prefix" + + +@fixture(scope="module") +def kmip(issuer, issuer_ca_configmap, namespace: str) -> KMIPDeployment: + return KMIPDeployment(namespace, issuer, "ca-key-pair", issuer_ca_configmap).deploy() + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket") + + +@fixture(scope="module") +def oplog_s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, OPLOG_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-oplog") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, + oplog_s3_bucket: str, + issuer_ca_configmap: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_kmip.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["backup"]["encryption"]["kmip"]["server"]["ca"] = issuer_ca_configmap + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + + resource["spec"]["backup"]["s3OpLogStores"] = [ + { + "name": "s3Store2", + "s3SecretRef": { + "name": OPLOG_SECRET_NAME, + }, + "pathStyleAccessEnabled": True, + "s3BucketEndpoint": "s3.us-east-1.amazonaws.com", + "s3BucketName": oplog_s3_bucket, + } + ] + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def mdb_latest( + ops_manager: MongoDBOpsManager, + mdb_latest_kmip_secrets, + namespace, + custom_mdb_version: str, +): + fixture_file_name = "replica-set-kmip.yaml" + if is_default_architecture_static(): + fixture_file_name = "replica-set-kmip-static.yaml" + resource = MongoDB.from_yaml( + yaml_fixture(fixture_file_name), + namespace=namespace, + name=MONGODB_CR_NAME, + ).configure(ops_manager, "mdbLatestProject") + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="enabled") + + return resource.update() + + +@fixture(scope="module") +def mdb_latest_kmip_secrets(aws_s3_client: AwsS3Client, namespace, issuer, issuer_ca_configmap: str) -> str: + mdb_latest_generated_kmip_certs_secret_name = create_tls_certs( + issuer, + namespace, + MONGODB_CR_NAME, + replicas=3, + common_name=MONGODB_CR_NAME, + ) + mdb_latest_generated_kmip_certs_secret = read_secret(namespace, mdb_latest_generated_kmip_certs_secret_name) + mdb_secret_name = MONGODB_CR_KMIP_TEST_PREFIX + "-" + MONGODB_CR_NAME + "-kmip-client" + create_or_update_secret( + namespace, + mdb_secret_name, + { + "tls.crt": mdb_latest_generated_kmip_certs_secret["tls.key"] + + mdb_latest_generated_kmip_certs_secret["tls.crt"], + }, + "tls", + ) + return mdb_secret_name + + +@fixture(scope="module") +def mdb_latest_test_collection(mdb_latest): + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient(mdb_latest.tester().cnx_string, **mdb_latest.tester().default_opts)["testdb"] + return collection["testcollection"].with_options(read_preference=ReadPreference.PRIMARY_PREFERRED) + + +@fixture(scope="module") +def mdb_latest_project(ops_manager: MongoDBOpsManager) -> OMTester: + return ops_manager.get_om_tester(project_name="mdbLatestProject") + + +@mark.e2e_om_ops_manager_backup_kmip +class TestOpsManagerCreation: + def test_create_kmip(self, kmip: KMIPDeployment): + kmip.status().assert_is_running() + + def test_create_om(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.backup_status().assert_reaches_phase(Phase.Pending) + + def test_s3_oplog_created(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=900) + + def test_mdbs_created(self, mdb_latest: MongoDB, ops_manager: MongoDBOpsManager): + # Once MDB is created, the OpsManager will be redeployed which may cause HTTP errors. + # This is required to mount new secrets for KMIP. Having said that, we also need longer timeout. + mdb_latest.assert_reaches_phase(Phase.Running, timeout=1800, ignore_errors=True) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + +@mark.e2e_om_ops_manager_backup_kmip +class TestBackupForMongodb: + def test_mdbs_created(self, mdb_latest: MongoDB): + mdb_latest.assert_reaches_phase(Phase.Running) + + def test_add_test_data(self, mdb_latest_test_collection): + mdb_latest_test_collection.insert_one(TEST_DATA) + + def test_mdbs_backed_up(self, mdb_latest_project: OMTester): + # If OM is misconfigured, this will never become ready + mdb_latest_project.wait_until_backup_snapshots_are_ready(expected_count=1, timeout=3500) + + def test_mdbs_backup_encrypted(self, mdb_latest_project: OMTester): + # This type of testing has been agreed with Ops Manager / Backup Team + cluster_id = mdb_latest_project.get_backup_cluster_id(expected_config_count=1) + snapshots = mdb_latest_project.api_get_snapshots(cluster_id) + assert snapshots[0]["parts"][0]["encryptionEnabled"] diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_manual.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_manual.py new file mode 100644 index 000000000..f01810a9d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_manual.py @@ -0,0 +1,371 @@ +""" + +This test is meant to be run manually. + +This is a companion test for docs/investigation/pod-is-killed-while-agent-restores.md + +""" + +from typing import Dict, Optional + +from kubetester import create_or_update_secret, get_default_storage_class +from kubetester.awss3client import AwsS3Client, s3_endpoint +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +HEAD_PATH = "/head/" +S3_SECRET_NAME = "my-s3-secret" +OPLOG_RS_NAME = "my-mongodb-oplog" +S3_RS_NAME = "my-mongodb-s3" +BLOCKSTORE_RS_NAME = "my-mongodb-blockstore" +USER_PASSWORD = "/qwerty@!#:" + + +def new_om_s3_store( + mdb: MongoDB, + s3_id: str, + s3_bucket_name: str, + aws_s3_client: AwsS3Client, + assignment_enabled: bool = True, + path_style_access_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "id": s3_id, + "pathStyleAccessEnabled": path_style_access_enabled, + "s3BucketEndpoint": s3_endpoint(AWS_REGION), + "s3BucketName": s3_bucket_name, + "awsAccessKey": aws_s3_client.aws_access_key, + "awsSecretKey": aws_s3_client.aws_secret_access_key, + "assignmentEnabled": assignment_enabled, + } + + +def new_om_data_store( + mdb: MongoDB, + id: str, + assignment_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "id": id, + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "ssl": mdb.is_tls_enabled(), + "assignmentEnabled": assignment_enabled, + } + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + return create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-") + + +def create_aws_secret(aws_s3_client, secret_name: str, namespace: str): + create_or_update_secret( + namespace, + secret_name, + { + "accessKey": aws_s3_client.aws_access_key, + "secretKey": aws_s3_client.aws_secret_access_key, + }, + ) + print("\nCreated a secret for S3 credentials", secret_name) + + +def create_s3_bucket(aws_s3_client, bucket_prefix: str = "test-bucket-"): + """creates a s3 bucket and a s3 config""" + bucket_prefix = KubernetesTester.random_k8s_name(bucket_prefix) + aws_s3_client.create_s3_bucket(bucket_prefix) + print("Created S3 bucket", bucket_prefix) + + return bucket_prefix + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup.yaml"), namespace=namespace + ) + + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + resource["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + resource["spec"]["externalConnectivity"] = {"type": "LoadBalancer"} + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + resource.set_version(custom_mdb_version) + + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + return resource.create() + + +@fixture(scope="module") +def s3_replica_set(ops_manager, namespace) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=S3_RS_NAME, + ).configure(ops_manager, "s3metadata") + + return resource.create() + + +@fixture(scope="module") +def blockstore_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + resource.set_version(custom_mdb_version) + return resource.create() + + +@fixture(scope="module") +def blockstore_user(namespace, blockstore_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user-backing-db.yaml"), namespace=namespace) + resource["spec"]["mongodbResourceRef"]["name"] = blockstore_replica_set.name + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + yield resource.create() + + +@fixture(scope="module") +def oplog_user(namespace, oplog_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user-backing-db.yaml"), + namespace=namespace, + name="mms-user-2", + ) + resource["spec"]["mongodbResourceRef"]["name"] = oplog_replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "mms-user-2-password" + resource["spec"]["username"] = "mms-user-2" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + yield resource.create() + + +@mark.e2e_om_ops_manager_backup_manual +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. The OM is expected to get to 'Pending' state + eventually as it will wait for oplog db to be created + """ + + def test_create_om(self, ops_manager: MongoDBOpsManager): + """creates a s3 bucket, s3 config and an OM resource (waits until Backup gets to Pending state)""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + def test_daemon_statefulset(self, ops_manager: MongoDBOpsManager): + def stateful_set_becomes_ready(): + stateful_set = ops_manager.read_backup_statefulset() + return stateful_set.status.ready_replicas == 1 and stateful_set.status.current_replicas == 1 + + KubernetesTester.wait_until(stateful_set_becomes_ready, timeout=300) + + stateful_set = ops_manager.read_backup_statefulset() + # pod template has volume mount request + assert (HEAD_PATH, "head") in ( + (mount.mount_path, mount.name) for mount in stateful_set.spec.template.spec.containers[0].volume_mounts + ) + + def test_om(self, ops_manager: MongoDBOpsManager): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + + # No oplog stores were created in Ops Manager by this time + om_tester.assert_oplog_stores([]) + om_tester.assert_s3_stores([]) + + +@mark.e2e_om_ops_manager_backup_manual +class TestBackupDatabasesAdded: + def test_backup_mdbs_created( + self, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + ): + """Creates mongodb databases all at once""" + oplog_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_oplog_user_created(self, oplog_user: MongoDBUser): + oplog_user.assert_reaches_phase(Phase.Updated) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, oplog_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + + +@mark.e2e_om_ops_manager_backup_manual +class TestOpsManagerWatchesBlockStoreUpdates: + def test_om_running(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=40) + + def test_scramsha_enabled_for_blockstore(self, blockstore_replica_set: MongoDB): + """Enables SCRAM for the blockstore replica set. Note that until CLOUDP-67736 is fixed + the order of operations (scram first, MongoDBUser - next) is important""" + blockstore_replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + blockstore_replica_set.update() + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_blockstore_user_was_added_to_om(self, blockstore_user: MongoDBUser): + blockstore_user.assert_reaches_phase(Phase.Updated) + + def test_configure_blockstore_user(self, ops_manager: MongoDBOpsManager, blockstore_user: MongoDBUser): + ops_manager.reload() + ops_manager["spec"]["backup"]["blockStores"][0]["mongodbUserRef"] = {"name": blockstore_user.name} + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + assert ops_manager.backup_status().get_message() is None + + +@mark.e2e_om_ops_manager_backup_manual +class TestBackupForMongodb: + @fixture(scope="class") + def mdb_latest(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="rs-fixed", + ).configure(ops_manager, "firstProject") + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource["spec"]["podSpec"] = {"podTemplate": {"spec": {}}} + resource["spec"]["podSpec"]["podTemplate"]["spec"]["containers"] = [ + { + "name": "mongodb-enterprise-database", + "resources": { + "requests": { + "memory": "6Gi", + "cpu": "1", + } + }, + } + ] + resource["spec"]["podSpec"]["podTemplate"]["spec"]["initContainers"] = [ + { + "name": "mongodb-enterprise-init-database", + "image": "268558157000.dkr.ecr.us-east-1.amazonaws.com/rodrigo/ubuntu/mongodb-enterprise-init-database:latest-fixed-probe", + } + ] + + resource.configure_backup(mode="disabled") + return resource.create() + + @fixture(scope="class") + def mdb_non_fixed(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="rs-not-fixed", + ).configure(ops_manager, "secondProject") + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource["spec"]["podSpec"] = {"podTemplate": {"spec": {}}} + resource["spec"]["podSpec"]["podTemplate"]["spec"]["containers"] = [ + { + "name": "mongodb-enterprise-database", + "resources": { + "requests": { + "memory": "6Gi", + "cpu": "1", + } + }, + } + ] + + resource["spec"]["podSpec"]["podTemplate"]["spec"]["initContainers"] = [ + { + "name": "mongodb-enterprise-init-database", + "image": "268558157000.dkr.ecr.us-east-1.amazonaws.com/rodrigo/ubuntu/mongodb-enterprise-init-database:non-fixed-probe", + } + ] + + resource.configure_backup(mode="disabled") + return resource.create() + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_non_fixed: MongoDB): + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_non_fixed.assert_reaches_phase(Phase.Running) + + def test_mdbs_enable_backup(self, mdb_latest: MongoDB, mdb_non_fixed: MongoDB): + mdb_latest.load() + mdb_latest.configure_backup(mode="enabled") + mdb_latest.update() + mdb_latest.assert_reaches_phase(Phase.Running) + + mdb_non_fixed.load() + mdb_non_fixed.configure_backup(mode="enabled") + mdb_non_fixed.update() + mdb_non_fixed.assert_reaches_phase(Phase.Running) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_restore.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_restore.py new file mode 100644 index 000000000..97fd6e49c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_restore.py @@ -0,0 +1,256 @@ +import datetime +import time +from typing import Optional + +import pymongo +from kubetester import MongoDB, try_load +from kubetester.awss3client import AwsS3Client +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.omtester import OMTester +from kubetester.opsmanager import MongoDBOpsManager +from pymongo import ReadPreference +from pymongo.errors import ServerSelectionTimeoutError +from pytest import fixture, mark +from tests.conftest import assert_data_got_restored, is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import ( + S3_SECRET_NAME, + create_aws_secret, + create_s3_bucket, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +""" +The test checks the backup for MongoDB 4.0 and 4.2, checks that snapshots are built and PIT restore and +restore from snapshot are working. +""" + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +OPLOG_SECRET_NAME = S3_SECRET_NAME + "-oplog" + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-s3") + + +@fixture(scope="module") +def oplog_s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, OPLOG_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-oplog") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_light.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def mdb_latest(ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-latest", + ).configure(ops_manager, "mdbLatestProject") + # MongoD versions greater than 4.2.0 must be enterprise build to enable backup + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="enabled") + + try_load(resource) + + return resource + + +@fixture(scope="module") +def mdb_prev(ops_manager: MongoDBOpsManager, namespace, custom_mdb_prev_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-previous", + ).configure(ops_manager, "mdbPreviousProject") + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.configure_backup(mode="enabled") + + try_load(resource) + return resource + + +@fixture(scope="function") +def mdb_prev_test_collection(mdb_prev): + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient(mdb_prev.tester().cnx_string, **mdb_prev.tester().default_opts)["testdb"] + return collection["testcollection"].with_options(read_preference=ReadPreference.PRIMARY_PREFERRED) + + +@fixture(scope="function") +def mdb_latest_test_collection(mdb_latest): + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient(mdb_latest.tester().cnx_string, **mdb_latest.tester().default_opts)["testdb"] + return collection["testcollection"].with_options(read_preference=ReadPreference.PRIMARY_PREFERRED) + + +@fixture(scope="module") +def mdb_prev_project(ops_manager: MongoDBOpsManager) -> OMTester: + return ops_manager.get_om_tester(project_name="mdbPreviousProject") + + +@fixture(scope="module") +def mdb_latest_project(ops_manager: MongoDBOpsManager) -> OMTester: + return ops_manager.get_om_tester(project_name="mdbLatestProject") + + +@mark.e2e_om_ops_manager_backup_restore +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + """creates a s3 bucket and an OM resource, the S3 configs get created using AppDB. Oplog store is still required.""" + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="Oplog Store configuration is required for backup", + timeout=300, + ) + + def test_s3_oplog_created(self, ops_manager: MongoDBOpsManager, oplog_s3_bucket: str): + ops_manager.load() + + ops_manager["spec"]["backup"]["s3OpLogStores"] = [ + { + "name": "s3Store2", + "s3SecretRef": { + "name": OPLOG_SECRET_NAME, + }, + "pathStyleAccessEnabled": True, + "s3BucketEndpoint": "s3.us-east-1.amazonaws.com", + "s3BucketName": oplog_s3_bucket, + } + ] + + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=500, + ignore_errors=True, + ) + + +@mark.e2e_om_ops_manager_backup_restore +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created. + Both Mdb 4.0 and 4.2 are tested (as the backup process for them differs significantly)""" + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + mdb_latest.update() + mdb_prev.update() + + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_add_test_data(self, mdb_prev_test_collection, mdb_latest_test_collection): + mdb_prev_test_collection.insert_one(TEST_DATA) + mdb_latest_test_collection.insert_one(TEST_DATA) + + def test_mdbs_backed_up(self, mdb_prev_project: OMTester, mdb_latest_project: OMTester): + # wait until a first snapshot is ready for both + mdb_prev_project.wait_until_backup_snapshots_are_ready(expected_count=1) + mdb_latest_project.wait_until_backup_snapshots_are_ready(expected_count=1) + + +@mark.e2e_om_ops_manager_backup_restore +class TestBackupRestorePIT: + """This part checks the work of PIT restore.""" + + def test_mdbs_change_data(self, mdb_prev_test_collection, mdb_latest_test_collection): + """Changes the MDB documents to check that restore rollbacks this change later. + Note, that we need to wait for some time to ensure the PIT timestamp gets to the range + [snapshot_created <= PIT <= changes_applied]""" + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + time.sleep(30) + + mdb_prev_test_collection.insert_one({"foo": "bar"}) + mdb_latest_test_collection.insert_one({"foo": "bar"}) + + def test_mdbs_pit_restore(self, mdb_prev_project: OMTester, mdb_latest_project: OMTester): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + + pit_datetme = datetime.datetime.now() - datetime.timedelta(seconds=15) + pit_millis = time_to_millis(pit_datetme) + print("Restoring back to the moment 15 seconds ago (millis): {}".format(pit_millis)) + + mdb_prev_project.create_restore_job_pit(pit_millis) + mdb_latest_project.create_restore_job_pit(pit_millis) + + def test_mdbs_ready(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # Note: that we are not waiting for the restore jobs to get finished as PIT restore jobs get FINISHED status + # right away. + # But the agent might still do work on the cluster, so we need to wait for that to happen. + time.sleep(5) + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_data_got_restored(self, mdb_prev_test_collection, mdb_latest_test_collection): + assert_data_got_restored(TEST_DATA, mdb_prev_test_collection, mdb_latest_test_collection) + + +@mark.e2e_om_ops_manager_backup_restore +class TestBackupRestoreFromSnapshot: + """This part tests the restore to the snapshot built once the backup has been enabled.""" + + def test_mdbs_change_data(self, mdb_prev_test_collection, mdb_latest_test_collection): + """Changes the MDB documents to check that restore rollbacks this change later""" + mdb_prev_test_collection.delete_many({}) + mdb_prev_test_collection.insert_one({"foo": "bar"}) + + mdb_latest_test_collection.delete_many({}) + mdb_latest_test_collection.insert_one({"foo": "bar"}) + + def test_mdbs_automated_restore(self, mdb_prev_project: OMTester, mdb_latest_project: OMTester): + restore_prev_id = mdb_prev_project.create_restore_job_snapshot() + mdb_prev_project.wait_until_restore_job_is_ready(restore_prev_id) + + restore_latest_id = mdb_latest_project.create_restore_job_snapshot() + mdb_latest_project.wait_until_restore_job_is_ready(restore_latest_id) + + def test_mdbs_ready(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # Note: that we are not waiting for the restore jobs to get finished as PIT restore jobs get FINISHED status + # right away. + # But the agent might still do work on the cluster, so we need to wait for that to happen. + time.sleep(5) + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_data_got_restored(self, mdb_prev_test_collection, mdb_latest_test_collection): + """The data in the db has been restored to the initial""" + records = list(mdb_prev_test_collection.find()) + assert records == [TEST_DATA] + + records = list(mdb_latest_test_collection.find()) + assert records == [TEST_DATA] + + +def time_to_millis(date_time) -> int: + """https://stackoverflow.com/a/11111177/614239""" + epoch = datetime.datetime.utcfromtimestamp(0) + pit_millis = (date_time - epoch).total_seconds() * 1000 + return pit_millis diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_restore_minio.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_restore_minio.py new file mode 100644 index 000000000..22d9a598c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_restore_minio.py @@ -0,0 +1,444 @@ +import datetime +import os +import time +from typing import List, Optional + +import pymongo +from kubetester import ( + MongoDB, + create_or_update_namespace, + create_or_update_secret, + read_secret, + try_load, +) +from kubetester.certs import create_mongodb_tls_certs, create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.omtester import OMTester +from kubetester.opsmanager import MongoDBOpsManager +from pymongo import ReadPreference +from pymongo.errors import ServerSelectionTimeoutError +from pytest import fixture, mark +from tests.conftest import ( + assert_data_got_restored, + create_appdb_certs, + is_multi_cluster, +) +from tests.opsmanager.conftest import mino_operator_install, mino_tenant_install +from tests.opsmanager.om_ops_manager_backup import S3_SECRET_NAME +from tests.opsmanager.om_ops_manager_backup_tls_custom_ca import ( + FIRST_PROJECT_RS_NAME, + SECOND_PROJECT_RS_NAME, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +OPLOG_SECRET_NAME = S3_SECRET_NAME + "-oplog" + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str) -> str: + return create_appdb_certs(namespace, issuer, "om-backup-db") + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str, tenant_domains: List[str]): + return create_ops_manager_tls_certs(issuer, namespace, "om-backup", additional_domains=tenant_domains) + + +# In the helm-chart we hard-code the server secret which contains the secrets for the clients. +# To make it work, we generate these from our existing tls secret and copy them over. +@fixture(scope="module") +def copy_manager_certs_for_minio(namespace: str, ops_manager_certs: str, tenant_name: str) -> str: + create_or_update_namespace(tenant_name) + + data = read_secret(namespace, ops_manager_certs) + crt = data["tls.crt"] + key = data["tls.key"] + new_data = dict() + new_data["tls.crt"] = crt + new_data["tls.key"] = key + return create_or_update_secret( + namespace=tenant_name, + type="kubernetes.io/tls", + name="tls-ssl-minio", + data=new_data, + ) + + +@fixture(scope="module") +def first_project_certs(namespace: str, issuer: str): + create_mongodb_tls_certs( + issuer, + namespace, + FIRST_PROJECT_RS_NAME, + f"first-project-{FIRST_PROJECT_RS_NAME}-cert", + ) + return "first-project" + + +@fixture(scope="module") +def second_project_certs(namespace: str, issuer: str): + create_mongodb_tls_certs( + issuer, + namespace, + SECOND_PROJECT_RS_NAME, + f"second-project-{SECOND_PROJECT_RS_NAME}-cert", + ) + return "second-project" + + +@fixture(scope="module") +def tenant_name() -> str: + return "tenant-0" + + +@fixture(scope="module") +def tenant_domains() -> List[str]: + return [ + "tenant-0-pool-0-{0...3}.tenant-0-hl.tenant-0.svc.cluster.local", + "minio.tenant-0.svc.cluster.local", + "minio.tenant-0", + "minio.tenant-0.svc", + ".tenant-0-hl.tenant-0.svc.cluster.local", + ".tenant-0.svc.cluster.local", + ] + + +@fixture(scope="module") +def s3_bucket_endpoint(tenant_name: str) -> str: + return f"minio.{tenant_name}.svc.cluster.local" + + +@fixture(scope="module") +def oplog_s3_bucket_name() -> str: + return "oplog-s3-bucket" + + +@fixture(scope="module") +def s3_store_bucket_name() -> str: + return "s3-store-bucket" + + +@fixture(scope="module") +def minio_s3_access_key() -> str: + return "minio" # this is defined in the values.yaml in the helm-chart + + +@fixture(scope="module") +def minio_s3_secret_key() -> str: + return "minio123" # this is defined in the values.yaml in the helm-chart + + +@fixture(scope="module") +def ops_manager( + namespace: str, + issuer_ca_configmap: str, + custom_version: Optional[str], + custom_appdb_version: str, + ops_manager_certs: str, + appdb_certs: str, + s3_bucket_endpoint: str, + s3_store_bucket_name: str, + minio_s3_access_key: str, + minio_s3_secret_key: str, + issuer_ca_filepath: str, +) -> MongoDBOpsManager: + # ensure the requests library will use this CA when communicating with Ops Manager + os.environ["REQUESTS_CA_BUNDLE"] = issuer_ca_filepath + + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_light.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + # these values come from the tenant creation in minio. + create_or_update_secret( + namespace, + S3_SECRET_NAME, + { + "accessKey": minio_s3_access_key, + "secretKey": minio_s3_secret_key, + }, + ) + + create_or_update_secret( + namespace, + OPLOG_SECRET_NAME, + { + "accessKey": minio_s3_access_key, + "secretKey": minio_s3_secret_key, + }, + ) + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["configuration"]["mms.preflight.run"] = "false" + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_store_bucket_name + # This ensures that we are using our own customCertificate and not rely on the jvm trusted store + resource["spec"]["backup"]["s3Stores"][0]["customCertificateSecretRefs"] = [ + {"name": ops_manager_certs, "key": "tls.crt"} + ] + resource["spec"]["backup"]["s3Stores"][0]["s3BucketEndpoint"] = s3_bucket_endpoint + + resource["spec"]["security"] = {"tls": {"ca": issuer_ca_configmap, "secretRef": {"name": ops_manager_certs}}} + resource["spec"]["applicationDatabase"]["security"] = { + "tls": {"ca": issuer_ca_configmap, "secretRef": {"prefix": appdb_certs}} + } + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource + + +@fixture(scope="module") +def mdb_latest( + ops_manager: MongoDBOpsManager, + namespace, + custom_mdb_version: str, + issuer_ca_configmap: str, + first_project_certs: str, +): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=FIRST_PROJECT_RS_NAME, + ).configure(ops_manager, "mdbLatestProject") + # MongoD versions greater than 4.2.0 must be enterprise build to enable backup + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="enabled") + resource.configure_custom_tls(issuer_ca_configmap, first_project_certs) + + try_load(resource) + return resource + + +@fixture(scope="module") +def mdb_prev( + ops_manager: MongoDBOpsManager, + namespace, + custom_mdb_prev_version: str, + issuer_ca_configmap: str, + second_project_certs: str, +): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=SECOND_PROJECT_RS_NAME, + ).configure(ops_manager, "mdbPreviousProject") + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.configure_backup(mode="enabled") + resource.configure_custom_tls(issuer_ca_configmap, second_project_certs) + + try_load(resource) + return resource + + +@fixture(scope="function") +def mdb_prev_test_collection(mdb_prev, ca_path: str): + tester = mdb_prev.tester(ca_path=ca_path, use_ssl=True) + + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient(tester.cnx_string, **tester.default_opts)["testdb"] + return collection["testcollection"].with_options(read_preference=ReadPreference.PRIMARY_PREFERRED) + + +@fixture(scope="function") +def mdb_latest_test_collection(mdb_latest, ca_path: str): + tester = mdb_latest.tester(ca_path=ca_path, use_ssl=True) + + # we instantiate the pymongo client per test to avoid flakiness as the primary and secondary might swap + collection = pymongo.MongoClient(tester.cnx_string, **tester.default_opts)["testdb"] + return collection["testcollection"].with_options(read_preference=ReadPreference.PRIMARY_PREFERRED) + + +@fixture(scope="module") +def mdb_prev_project(ops_manager: MongoDBOpsManager) -> OMTester: + return ops_manager.get_om_tester(project_name="mdbPreviousProject") + + +@fixture(scope="module") +def mdb_latest_project(ops_manager: MongoDBOpsManager) -> OMTester: + return ops_manager.get_om_tester(project_name="mdbLatestProject") + + +@mark.e2e_om_ops_manager_backup_restore_minio +class TestMinioCreation: + def test_install_minio( + self, + tenant_name: str, + issuer_ca_configmap: str, + copy_manager_certs_for_minio: str, + ): + mino_operator_install(namespace=tenant_name) + mino_tenant_install(namespace=tenant_name) + + +@mark.e2e_om_ops_manager_backup_restore_minio +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + """creates a s3 bucket and an OM resource, the S3 configs get created using AppDB. Oplog store is still required.""" + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="Oplog Store configuration is required for backup", + timeout=600, + ) + + def test_s3_oplog_created( + self, + ops_manager: MongoDBOpsManager, + ops_manager_certs: str, + oplog_s3_bucket_name: str, + s3_bucket_endpoint: str, + ): + ops_manager.load() + + # Backups rely on the JVM default keystore if no customCertificate has been uploaded + ops_manager["spec"]["backup"]["s3OpLogStores"] = [ + { + "name": "s3Store2", + "s3SecretRef": { + "name": OPLOG_SECRET_NAME, + }, + "pathStyleAccessEnabled": True, + "s3BucketEndpoint": s3_bucket_endpoint, + "s3BucketName": oplog_s3_bucket_name, + "customCertificateSecretRefs": [{"name": ops_manager_certs, "key": "tls.crt"}], + } + ] + + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=500, + ignore_errors=True, + ) + + def test_add_custom_ca_to_project_configmap( + self, ops_manager: MongoDBOpsManager, issuer_ca_plus: str, namespace: str + ): + projects = [ + ops_manager.get_or_create_mongodb_connection_config_map(FIRST_PROJECT_RS_NAME, "firstProject"), + ops_manager.get_or_create_mongodb_connection_config_map(SECOND_PROJECT_RS_NAME, "secondProject"), + ] + + data = { + "sslMMSCAConfigMap": issuer_ca_plus, + } + + for project in projects: + KubernetesTester.update_configmap(namespace, project, data) + + # Give a few seconds for the operator to catch the changes on + # the project ConfigMaps + time.sleep(10) + + +@mark.e2e_om_ops_manager_backup_restore_minio +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created. + Both Mdb 4.0 and 4.2 are tested (as the backup process for them differs significantly)""" + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + mdb_latest.update() + mdb_prev.update() + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_add_test_data(self, mdb_prev_test_collection, mdb_latest_test_collection): + mdb_prev_test_collection.insert_one(TEST_DATA) + mdb_latest_test_collection.insert_one(TEST_DATA) + + def test_mdbs_backed_up(self, mdb_prev_project: OMTester, mdb_latest_project: OMTester): + # wait until a first snapshot is ready for both + mdb_prev_project.wait_until_backup_snapshots_are_ready(expected_count=1) + mdb_latest_project.wait_until_backup_snapshots_are_ready(expected_count=1) + + +@mark.e2e_om_ops_manager_backup_restore_minio +class TestBackupRestorePIT: + """This part checks the work of PIT restore.""" + + def test_mdbs_change_data(self, mdb_prev_test_collection, mdb_latest_test_collection): + """Changes the MDB documents to check that restore rollbacks this change later. + Note, that we need to wait for some time to ensure the PIT timestamp gets to the range + [snapshot_created <= PIT <= changes_applied]""" + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + time.sleep(30) + + mdb_prev_test_collection.insert_one({"foo": "bar"}) + mdb_latest_test_collection.insert_one({"foo": "bar"}) + + def test_mdbs_pit_restore(self, mdb_prev_project: OMTester, mdb_latest_project: OMTester): + now_millis = time_to_millis(datetime.datetime.now()) + print("\nCurrent time (millis): {}".format(now_millis)) + + pit_datetme = datetime.datetime.now() - datetime.timedelta(seconds=15) + pit_millis = time_to_millis(pit_datetme) + print("Restoring back to the moment 15 seconds ago (millis): {}".format(pit_millis)) + + mdb_prev_project.create_restore_job_pit(pit_millis) + mdb_latest_project.create_restore_job_pit(pit_millis) + + def test_mdbs_ready(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # Note: that we are not waiting for the restore jobs to get finished as PIT restore jobs get FINISHED status + # right away. + # But the agent might still do work on the cluster, so we need to wait for that to happen. + time.sleep(5) + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_data_got_restored(self, mdb_prev_test_collection, mdb_latest_test_collection): + assert_data_got_restored(TEST_DATA, mdb_prev_test_collection, mdb_latest_test_collection) + + +@mark.e2e_om_ops_manager_backup_restore_minio +class TestBackupRestoreFromSnapshot: + """This part tests the restore to the snapshot built once the backup has been enabled.""" + + def test_mdbs_change_data(self, mdb_prev_test_collection, mdb_latest_test_collection): + """Changes the MDB documents to check that restore rollbacks this change later""" + mdb_prev_test_collection.delete_many({}) + mdb_prev_test_collection.insert_one({"foo": "bar"}) + + mdb_latest_test_collection.delete_many({}) + mdb_latest_test_collection.insert_one({"foo": "bar"}) + + def test_mdbs_automated_restore(self, mdb_prev_project: OMTester, mdb_latest_project: OMTester): + restore_prev_id = mdb_prev_project.create_restore_job_snapshot() + mdb_prev_project.wait_until_restore_job_is_ready(restore_prev_id) + + restore_latest_id = mdb_latest_project.create_restore_job_snapshot() + mdb_latest_project.wait_until_restore_job_is_ready(restore_latest_id) + + def test_mdbs_ready(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # Note: that we are not waiting for the restore jobs to get finished as PIT restore jobs get FINISHED status + # right away. + # But the agent might still do work on the cluster, so we need to wait for that to happen. + time.sleep(5) + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_data_got_restored(self, mdb_prev_test_collection, mdb_latest_test_collection): + """The data in the db has been restored to the initial""" + records = list(mdb_prev_test_collection.find()) + assert records == [TEST_DATA] + + records = list(mdb_latest_test_collection.find()) + assert records == [TEST_DATA] + + +def time_to_millis(date_time) -> int: + """https://stackoverflow.com/a/11111177/614239""" + epoch = datetime.datetime.utcfromtimestamp(0) + pit_millis = (date_time - epoch).total_seconds() * 1000 + return pit_millis diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_s3_tls.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_s3_tls.py new file mode 100644 index 000000000..1fd54cb02 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_s3_tls.py @@ -0,0 +1,150 @@ +from typing import Optional + +from kubetester import create_or_update_secret, try_load +from kubetester.awss3client import AwsS3Client, s3_endpoint +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.common.constants import S3_BLOCKSTORE_NAME, S3_OPLOG_NAME +from tests.conftest import AWS_REGION, create_appdb_certs, is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +""" +This test checks the work with TLS-enabled backing databases (oplog & blockstore) +""" + +S3_TEST_CA1 = "s3-test-ca-1" +S3_TEST_CA2 = "s3-test-ca-2" +S3_NOT_WORKING_CA = "not-working-ca" + + +@fixture(scope="module") +def appdb_certs_secret(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, "om-backup-tls-s3-db") + + +@fixture(scope="module") +def s3_bucket_oplog(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_OPLOG_NAME + "-secret", namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-oplog-") + + +@fixture(scope="module") +def s3_bucket_blockstore(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_BLOCKSTORE_NAME + "-secret", namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-blockstorage-") + + +@fixture(scope="module") +def duplicate_configmap_ca(namespace, amazon_ca_1_filepath, amazon_ca_2_filepath, ca_path): + ca = open(amazon_ca_1_filepath).read() + data = {"ca-pem": ca} + create_or_update_secret(namespace, S3_TEST_CA1, data) + + ca = open(amazon_ca_2_filepath).read() + data = {"ca-pem": ca} + create_or_update_secret(namespace, S3_TEST_CA2, data) + + ca = open(ca_path).read() + data = {"ca-pem": ca} + create_or_update_secret(namespace, S3_NOT_WORKING_CA, data) + + +@fixture(scope="module") +def ops_manager( + namespace, + duplicate_configmap_ca, + issuer_ca_configmap: str, + appdb_certs_secret: str, + custom_version: Optional[str], + custom_appdb_version: str, + s3_bucket_oplog: str, + s3_bucket_blockstore: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_tls_s3.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + + custom_certificate = {"name": S3_NOT_WORKING_CA, "key": "ca-pem"} + + resource["spec"]["backup"]["s3Stores"][0]["name"] = S3_BLOCKSTORE_NAME + resource["spec"]["backup"]["s3Stores"][0]["s3SecretRef"]["name"] = S3_BLOCKSTORE_NAME + "-secret" + resource["spec"]["backup"]["s3Stores"][0]["s3BucketEndpoint"] = s3_endpoint(AWS_REGION) + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket_blockstore + resource["spec"]["backup"]["s3Stores"][0]["s3RegionOverride"] = AWS_REGION + resource["spec"]["backup"]["s3Stores"][0]["customCertificateSecretRefs"] = [custom_certificate] + resource["spec"]["backup"]["s3OpLogStores"][0]["name"] = S3_OPLOG_NAME + resource["spec"]["backup"]["s3OpLogStores"][0]["s3SecretRef"]["name"] = S3_OPLOG_NAME + "-secret" + resource["spec"]["backup"]["s3OpLogStores"][0]["s3BucketEndpoint"] = s3_endpoint(AWS_REGION) + resource["spec"]["backup"]["s3OpLogStores"][0]["s3BucketName"] = s3_bucket_oplog + resource["spec"]["backup"]["s3OpLogStores"][0]["s3RegionOverride"] = AWS_REGION + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource + + +@mark.e2e_om_ops_manager_backup_s3_tls +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + def test_om_backup_is_failed( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + timeout=600, + msg_regexp=".* valid certification path to requested target.*", + ) + + def test_om_with_correct_custom_cert(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + custom_certificate = [ + {"name": S3_TEST_CA1, "key": "ca-pem"}, + {"name": S3_TEST_CA2, "key": "ca-pem"}, + ] + + ops_manager["spec"]["backup"]["s3OpLogStores"][0]["customCertificateSecretRefs"] = custom_certificate + ops_manager["spec"]["backup"]["s3Stores"][0]["customCertificateSecretRefs"] = custom_certificate + + ops_manager.update() + + def test_om_is_running( + self, + ops_manager: MongoDBOpsManager, + ): + # this takes more time, since the change of the custom certs requires a restart of om + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=1200, ignore_errors=True) + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + + def test_om_s3_stores( + self, + ops_manager: MongoDBOpsManager, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_s3_stores([{"id": S3_BLOCKSTORE_NAME, "s3RegionOverride": AWS_REGION}]) + om_tester.assert_oplog_s3_stores([{"id": S3_OPLOG_NAME, "s3RegionOverride": AWS_REGION}]) + + # verify that we were able to setup (and no error) certificates + a = om_tester.get_s3_stores() + assert a["results"][0]["customCertificates"][0]["filename"] == f"{S3_TEST_CA1}/ca-pem" + assert a["results"][0]["customCertificates"][1]["filename"] == f"{S3_TEST_CA2}/ca-pem" + b = om_tester.get_oplog_s3_stores() + assert b["results"][0]["customCertificates"][0]["filename"] == f"{S3_TEST_CA1}/ca-pem" + assert b["results"][0]["customCertificates"][1]["filename"] == f"{S3_TEST_CA2}/ca-pem" diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_sharded_cluster.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_sharded_cluster.py new file mode 100644 index 000000000..ac7c1dd69 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_sharded_cluster.py @@ -0,0 +1,398 @@ +import time +from typing import Optional + +from kubernetes.client.rest import ApiException +from kubetester import ( + create_or_update_secret, + get_default_storage_class, + try_load, + wait_until, +) +from kubetester.awss3client import AwsS3Client +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.mongotester import MongoTester +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment as enable_multi_cluster_deployment_mdb, +) +from tests.shardedcluster.conftest import get_mongos_service_names + +HEAD_PATH = "/head/" +S3_SECRET_NAME = "my-s3-secret" +OPLOG_RS_NAME = "my-mongodb-oplog" +S3_RS_NAME = "my-mongodb-s3" +BLOCKSTORE_RS_NAME = "my-mongodb-blockstore" +USER_PASSWORD = "/qwerty@!#:" + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-sharded-") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup.yaml"), namespace=namespace + ) + + try_load(resource) + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + resource.set_version(custom_mdb_version) + + # TODO: Remove when CLOUDP-60443 is fixed + # This test will update oplog to have SCRAM enabled + # Currently this results in OM failure when enabling backup for a project, backup seems to do some caching resulting in the + # mongoURI not being updated unless pod is killed. This is documented in CLOUDP-60443, once resolved this skip & comment can be deleted + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + return resource.update() + + +@fixture(scope="module") +def s3_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=S3_RS_NAME, + ).configure(ops_manager, "s3metadata") + resource.set_version(custom_mdb_version) + + return resource.update() + + +@fixture(scope="module") +def blockstore_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + resource.set_version(custom_mdb_version) + + return resource.update() + + +@fixture(scope="module") +def blockstore_user(namespace, blockstore_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user-backing-db.yaml"), namespace=namespace) + resource["spec"]["mongodbResourceRef"]["name"] = blockstore_replica_set.name + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_user(namespace, oplog_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user-backing-db.yaml"), + namespace=namespace, + name="mms-user-2", + ) + resource["spec"]["mongodbResourceRef"]["name"] = oplog_replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "mms-user-2-password" + resource["spec"]["username"] = "mms-user-2" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + yield resource.update() + + +@mark.e2e_om_ops_manager_backup_sharded_cluster +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. The OM is expected to get to 'Pending' state + eventually as it will wait for oplog db to be created + """ + + def test_create_om( + self, + ops_manager: MongoDBOpsManager, + s3_bucket: str, + custom_version: str, + custom_appdb_version: str, + ): + """creates a s3 bucket, s3 config and an OM resource (waits until Backup gets to Pending state)""" + + ops_manager["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + ops_manager["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + ops_manager["spec"]["backup"]["members"] = 2 + + ops_manager.set_version(custom_version) + ops_manager.set_appdb_version(custom_appdb_version) + ops_manager.allow_mdb_rc_versions() + + if is_multi_cluster(): + enable_multi_cluster_deployment(ops_manager) + + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + def test_daemon_statefulset(self, ops_manager: MongoDBOpsManager): + def stateful_sets_become_ready(): + total_ready_replicas = 0 + total_current_replicas = 0 + for member_cluster_name, _ in ops_manager.get_backup_sts_names_in_member_clusters(): + stateful_set = ops_manager.read_backup_statefulset(member_cluster_name=member_cluster_name) + ready_replicas = ( + stateful_set.status.ready_replicas if stateful_set.status.ready_replicas is not None else 0 + ) + total_ready_replicas += ready_replicas + current_replicas = ( + stateful_set.status.current_replicas if stateful_set.status.current_replicas is not None else 0 + ) + total_current_replicas += current_replicas + return total_ready_replicas == 2 and total_current_replicas == 2 + + KubernetesTester.wait_until(stateful_sets_become_ready, timeout=300) + + for member_cluster_name, _ in ops_manager.get_backup_sts_names_in_member_clusters(): + stateful_set = ops_manager.read_backup_statefulset(member_cluster_name=member_cluster_name) + # pod template has volume mount request + assert (HEAD_PATH, "head") in ( + (mount.mount_path, mount.name) for mount in stateful_set.spec.template.spec.containers[0].volume_mounts + ) + + +@mark.e2e_om_ops_manager_backup_sharded_cluster +class TestBackupDatabasesAdded: + """name: Creates three mongodb resources for oplog, s3 and blockstore and waits until OM resource gets to + running state""" + + def test_backup_mdbs_created( + self, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + ): + """Creates mongodb databases all at once. Concurrent AC modifications may happen from time to time""" + oplog_replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + s3_replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + blockstore_replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + + def test_oplog_user_created(self, oplog_user: MongoDBUser): + oplog_user.assert_reaches_phase(Phase.Updated) + + def test_oplog_updated_scram_sha_enabled(self, oplog_replica_set: MongoDB): + oplog_replica_set.load() + oplog_replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + oplog_replica_set.update() + oplog_replica_set.assert_reaches_phase(Phase.Running) + + def test_om_failed_oplog_no_user_ref(self, ops_manager: MongoDBOpsManager): + """Waits until Backup is in failed state as blockstore doesn't have reference to the user""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Failed, + msg_regexp=".*is configured to use SCRAM-SHA authentication mode, the user " + "must be specified using 'mongodbUserRef'", + ) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, oplog_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + + assert ops_manager.backup_status().get_message() is None + + +@mark.e2e_om_ops_manager_backup_sharded_cluster +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created. + Both latest and the one before the latest are tested (as the backup process for them may differ significantly)""" + + @fixture(scope="class") + def mdb_latest(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-for-om.yaml"), + namespace=namespace, + name="mdb-four-two", + ).configure(ops_manager, "firstProject") + + if try_load(resource): + return resource + + if is_multi_cluster(): + enable_multi_cluster_deployment_mdb( + resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, None, None], + configsrv_members_array=[None, 1, None], + ) + + resource.configure_backup(mode="disabled") + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + return resource.update() + + @fixture(scope="class") + def mdb_prev(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_prev_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-for-om.yaml"), + namespace=namespace, + name="mdb-four-zero", + ).configure(ops_manager, "secondProject") + + if try_load(resource): + return resource + + if is_multi_cluster(): + enable_multi_cluster_deployment_mdb( + resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, None, None], + configsrv_members_array=[None, 1, None], + ) + + resource.configure_backup(mode="disabled") + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.set_architecture_annotation() + + return resource.update() + + @fixture(scope="class") + def mdb_latest_tester(self, mdb_latest: MongoDB) -> MongoTester: + service_names = get_mongos_service_names(mdb_latest) + + return mdb_latest.tester(service_names=service_names) + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + mdb_latest.assert_reaches_phase(Phase.Running, timeout=1200) + mdb_prev.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_mdbs_enable_backup(self, mdb_latest: MongoDB, mdb_prev: MongoDB, mdb_latest_tester: MongoTester): + def until_shards_are_here(): + shards = mdb_latest_tester.client.admin.command("listShards") + if len(shards["shards"]) == 2: + return True + else: + print(f"shards are not configured yet: {shards}") + + wait_until(until_shards_are_here, 500) + + # we need to sleep here to give OM some time to recognize the shards. + # otherwise, if you start a backup during a topology change will lead the backup to be aborted. + time.sleep(30) + mdb_latest.load() + mdb_latest.configure_backup(mode="enabled") + mdb_latest.update() + + mdb_prev.load() + mdb_prev.configure_backup(mode="enabled") + mdb_prev.update() + + mdb_prev.assert_reaches_phase(Phase.Running, timeout=1200) + mdb_latest.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_mdbs_backuped(self, ops_manager: MongoDBOpsManager): + om_tester_first = ops_manager.get_om_tester(project_name="firstProject") + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + + # wait until a first snapshot is ready for both + om_tester_first.wait_until_backup_snapshots_are_ready( + expected_count=1, expected_config_count=4, is_sharded_cluster=True, timeout=1600 + ) + om_tester_second.wait_until_backup_snapshots_are_ready( + expected_count=1, expected_config_count=4, is_sharded_cluster=True, timeout=1600 + ) + + def test_can_transition_from_started_to_stopped(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # a direction transition from enabled to disabled is a single + # step for the operator + mdb_prev.assert_backup_reaches_status("STARTED", timeout=100) + mdb_prev.configure_backup(mode="disabled") + mdb_prev.update() + mdb_prev.assert_backup_reaches_status("STOPPED", timeout=1600) + + def test_can_transition_from_started_to_terminated_0(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + # a direct transition from enabled to terminated is not possible + # the operator should handle the transition from STARTED -> STOPPED -> TERMINATING + mdb_latest.assert_backup_reaches_status("STARTED", timeout=100) + mdb_latest.configure_backup(mode="terminated") + mdb_latest.update() + mdb_latest.assert_backup_reaches_status("TERMINATING", timeout=1600) + + def test_backup_terminated_for_deleted_resource(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB): + # re-enable backup + mdb_prev.configure_backup(mode="enabled") + mdb_prev["spec"]["backup"]["autoTerminateOnDeletion"] = True + mdb_prev.update() + mdb_prev.assert_backup_reaches_status("STARTED", timeout=1600) + mdb_prev.delete() + + def resource_is_deleted() -> bool: + try: + mdb_prev.load() + return False + except ApiException: + return True + + # wait until the resource is deleted + run_periodically(resource_is_deleted, timeout=300) + + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + om_tester_second.wait_until_backup_snapshots_are_ready( + expected_count=0, expected_config_count=4, is_sharded_cluster=True + ) + om_tester_second.wait_until_backup_deactivated(is_sharded_cluster=True, expected_config_count=4) + + def test_hosts_were_removed(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB): + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + om_tester_second.wait_until_hosts_are_empty() + + # Note: as of right now, we cannot deploy the same mdb again, because we will run into the error: Backup failed + # to start: Config server : 27017 has no startup parameters. diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_tls.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_tls.py new file mode 100644 index 000000000..f9d5f020b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_tls.py @@ -0,0 +1,239 @@ +from typing import Optional + +from kubetester import MongoDB, create_or_update_configmap +from kubetester.certs import create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + create_appdb_certs, + get_member_cluster_api_client, + is_multi_cluster, +) +from tests.opsmanager.om_ops_manager_backup import ( + BLOCKSTORE_RS_NAME, + OPLOG_RS_NAME, + new_om_data_store, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +""" +This test checks the work with TLS-enabled backing databases (oplog & blockstore) +""" + + +@fixture(scope="module") +def appdb_certs_secret(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, "om-backup-tls-db") + + +@fixture(scope="module") +def oplog_certs_secret(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, OPLOG_RS_NAME, f"oplog-{OPLOG_RS_NAME}-cert") + return "oplog" + + +@fixture(scope="module") +def blockstore_certs_secret(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, BLOCKSTORE_RS_NAME, f"blockstore-{BLOCKSTORE_RS_NAME}-cert") + return "blockstore" + + +@fixture(scope="module") +def ops_manager( + namespace, + ops_manager_issuer_ca_configmap: str, + app_db_issuer_ca_configmap: str, + appdb_certs_secret: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_tls.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["security"]["tls"]["ca"] = ops_manager_issuer_ca_configmap + resource["spec"]["applicationDatabase"]["security"]["tls"]["ca"] = app_db_issuer_ca_configmap + resource["spec"]["backup"]["logging"] = { + "LogBackAccessRef": {"name": "logback-access-config"}, + "LogBackRef": {"name": "logback-config"}, + } + resource["spec"]["logging"] = { + "LogBackAccessRef": {"name": "logback-access-config"}, + "LogBackRef": {"name": "logback-config"}, + } + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set( + ops_manager, app_db_issuer_ca_configmap: str, oplog_certs_secret: str, custom_mdb_version: str +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + resource.configure_custom_tls(app_db_issuer_ca_configmap, oplog_certs_secret) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource.update() + return resource + + +@fixture(scope="module") +def blockstore_replica_set( + ops_manager, app_db_issuer_ca_configmap: str, blockstore_certs_secret: str, custom_mdb_version: str +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + resource.configure_custom_tls(app_db_issuer_ca_configmap, blockstore_certs_secret) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.update() + return resource + + +@mark.e2e_om_ops_manager_backup_tls +class TestOpsManagerCreation: + def test_create_logback_xml_configmap(self, namespace: str, custom_logback_file_path: str, central_cluster_client): + logback = open(custom_logback_file_path).read() + data = {"logback.xml": logback} + create_or_update_configmap(namespace, "logback-config", data, api_client=central_cluster_client) + + def test_create_logback_access_xml_configmap( + self, namespace: str, custom_logback_file_path: str, central_cluster_client + ): + logback = open(custom_logback_file_path).read() + data = {"logback-access.xml": logback} + create_or_update_configmap(namespace, "logback-access-config", data, api_client=central_cluster_client) + + def test_create_om(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + def test_backing_dbs_created(self, oplog_replica_set: MongoDB, blockstore_replica_set: MongoDB): + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_oplog_running(self, oplog_replica_set: MongoDB, ca_path: str): + oplog_replica_set.assert_connectivity(ca_path=ca_path) + + def test_blockstore_running(self, blockstore_replica_set: MongoDB, ca_path: str): + blockstore_replica_set.assert_connectivity(ca_path=ca_path) + + def test_om_is_running( + self, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + ): + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_oplog_stores([new_om_data_store(oplog_replica_set, "oplog1")]) + om_tester.assert_block_stores([new_om_data_store(blockstore_replica_set, "blockStore1")]) + + def test_logback_xml_mounted_correctly_om(self, ops_manager: MongoDBOpsManager, namespace): + cmd_normal = [ + "/bin/sh", + "-c", + "cat /mongodb-ops-manager/conf/logback.xml", + ] + cmd_access = [ + "/bin/sh", + "-c", + "cat /mongodb-ops-manager/conf/logback-access.xml", + ] + + for member_cluster_name, pod_name in ops_manager.get_om_pod_names_in_member_clusters(): + member_api_client = get_member_cluster_api_client(member_cluster_name) + result = KubernetesTester.run_command_in_pod_container( + pod_name, namespace, cmd_normal, container="mongodb-ops-manager", api_client=member_api_client + ) + assert "" in result + result = KubernetesTester.run_command_in_pod_container( + pod_name, namespace, cmd_access, container="mongodb-ops-manager", api_client=member_api_client + ) + assert "" in result + + def test_logback_xml_mounted_correctly_backup(self, ops_manager: MongoDBOpsManager, namespace): + cmd_normal = [ + "/bin/sh", + "-c", + "cat /mongodb-ops-manager/conf/logback.xml", + ] + cmd_access = [ + "/bin/sh", + "-c", + "cat /mongodb-ops-manager/conf/logback-access.xml", + ] + + for member_cluster_name, pod_name in ops_manager.backup_daemon_pod_names(): + member_api_client = get_member_cluster_api_client(member_cluster_name) + result = KubernetesTester.run_command_in_pod_container( + pod_name, namespace, cmd_normal, container="mongodb-backup-daemon", api_client=member_api_client + ) + assert "" in result + result = KubernetesTester.run_command_in_pod_container( + pod_name, namespace, cmd_access, container="mongodb-backup-daemon", api_client=member_api_client + ) + assert "" in result + + +@mark.e2e_om_ops_manager_backup_tls +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created.""" + + @fixture(scope="class") + def mdb_latest(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-four-two", + ).configure(ops_manager, "firstProject") + # MongoD versions greater than 4.2.0 must be enterprise build to enable backup + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="enabled") + resource.update() + return resource + + @fixture(scope="class") + def mdb_prev(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_prev_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-four-zero", + ).configure(ops_manager, "secondProject") + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.configure_backup(mode="enabled") + resource.update() + return resource + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB): + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + + def test_mdbs_backed_up(self, ops_manager: MongoDBOpsManager): + om_tester_first = ops_manager.get_om_tester(project_name="firstProject") + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + + # wait until a first snapshot is ready for both + om_tester_first.wait_until_backup_snapshots_are_ready(expected_count=1) + om_tester_second.wait_until_backup_snapshots_are_ready(expected_count=1) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_tls_custom_ca.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_tls_custom_ca.py new file mode 100644 index 000000000..a99966a9e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_tls_custom_ca.py @@ -0,0 +1,316 @@ +import os +import time +from typing import Optional + +from kubetester import MongoDB +from kubetester.certs import create_mongodb_tls_certs, create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + create_appdb_certs, + default_external_domain, + external_domain_fqdns, + is_multi_cluster, + update_coredns_hosts, +) +from tests.opsmanager.om_ops_manager_backup import ( + BLOCKSTORE_RS_NAME, + OPLOG_RS_NAME, + S3_SECRET_NAME, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +OPLOG_SECRET_NAME = S3_SECRET_NAME + "-oplog" +FIRST_PROJECT_RS_NAME = "mdb-four-two" +SECOND_PROJECT_RS_NAME = "mdb-four-zero" +EXTERNAL_DOMAIN_RS_NAME = "my-replica-set" + +""" +This test checks the work with TLS-enabled backing databases (oplog & blockstore) + +Tests checking externalDomain (consider updating all of them when changing test logic): + tls/tls_replica_set_process_hostnames.py + replicaset/replica_set_process_hostnames.py + opsmanager/om_ops_manager_backup_tls_custom_ca.py +""" + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + prefix = "prefix" + create_ops_manager_tls_certs(issuer, namespace, "om-backup-tls", secret_name=f"{prefix}-om-backup-tls-cert") + return prefix + + +@fixture(scope="module") +def appdb_certs_secret(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, "om-backup-tls-db") + + +@fixture(scope="module") +def oplog_certs_secret(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, OPLOG_RS_NAME, f"oplog-{OPLOG_RS_NAME}-cert") + return "oplog" + + +@fixture(scope="module") +def blockstore_certs_secret(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, BLOCKSTORE_RS_NAME, f"blockstore-{BLOCKSTORE_RS_NAME}-cert") + return "blockstore" + + +@fixture(scope="module") +def first_project_certs(namespace: str, issuer: str): + create_mongodb_tls_certs( + issuer, + namespace, + FIRST_PROJECT_RS_NAME, + f"first-project-{FIRST_PROJECT_RS_NAME}-cert", + ) + return "first-project" + + +@fixture(scope="module") +def second_project_certs(namespace: str, issuer: str): + create_mongodb_tls_certs( + issuer, + namespace, + SECOND_PROJECT_RS_NAME, + f"second-project-{SECOND_PROJECT_RS_NAME}-cert", + ) + return "second-project" + + +@fixture(scope="module") +def mdb_external_domain_certs(namespace: str, issuer: str): + create_mongodb_tls_certs( + issuer, + namespace, + EXTERNAL_DOMAIN_RS_NAME, + f"external-domain-{EXTERNAL_DOMAIN_RS_NAME}-cert", + process_hostnames=external_domain_fqdns(EXTERNAL_DOMAIN_RS_NAME, 3), + ) + # prefix for certificates + return "external-domain" + + +@fixture(scope="module") +def ops_manager( + namespace, + issuer_ca_configmap: str, + appdb_certs_secret: str, + custom_version: Optional[str], + custom_appdb_version: str, + ops_manager_certs: str, + issuer_ca_filepath: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_tls.yaml"), namespace=namespace + ) + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["security"]["certsSecretPrefix"] = ops_manager_certs + + # ensure the requests library will use this CA when communicating with Ops Manager + os.environ["REQUESTS_CA_BUNDLE"] = issuer_ca_filepath + + # configure OM to be using hybrid mode + resource["spec"]["configuration"]["automation.versions.source"] = "hybrid" + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set( + ops_manager, issuer_ca_configmap: str, oplog_certs_secret: str, custom_mdb_version: str +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "oplog") + resource.configure_custom_tls(issuer_ca_configmap, oplog_certs_secret) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.update() + return resource + + +@fixture(scope="module") +def blockstore_replica_set( + ops_manager, issuer_ca_configmap: str, blockstore_certs_secret: str, custom_mdb_version: str +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + resource.configure_custom_tls(issuer_ca_configmap, blockstore_certs_secret) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + resource.update() + return resource + + +@mark.e2e_om_ops_manager_backup_tls_custom_ca +def test_update_coredns(): + if is_multi_cluster(): + hosts = [ + ("172.18.255.211", "my-replica-set-0.mongodb.interconnected"), + ("172.18.255.212", "my-replica-set-1.mongodb.interconnected"), + ("172.18.255.213", "my-replica-set-2.mongodb.interconnected"), + ("172.18.255.214", "my-replica-set-3.mongodb.interconnected"), + ] + else: + hosts = [ + ("172.18.255.200", "my-replica-set-0.mongodb.interconnected"), + ("172.18.255.201", "my-replica-set-1.mongodb.interconnected"), + ("172.18.255.202", "my-replica-set-2.mongodb.interconnected"), + ("172.18.255.203", "my-replica-set-3.mongodb.interconnected"), + ] + + update_coredns_hosts(hosts) + + +@mark.e2e_om_ops_manager_backup_tls_custom_ca +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + def test_add_custom_ca_to_project_configmap( + self, ops_manager: MongoDBOpsManager, issuer_ca_plus: str, namespace: str + ): + projects = [ + ops_manager.get_or_create_mongodb_connection_config_map(OPLOG_RS_NAME, "oplog"), + ops_manager.get_or_create_mongodb_connection_config_map(BLOCKSTORE_RS_NAME, "blockstore"), + ops_manager.get_or_create_mongodb_connection_config_map(FIRST_PROJECT_RS_NAME, "firstProject"), + ops_manager.get_or_create_mongodb_connection_config_map(SECOND_PROJECT_RS_NAME, "secondProject"), + ops_manager.get_or_create_mongodb_connection_config_map(EXTERNAL_DOMAIN_RS_NAME, "externalDomain"), + ] + + data = { + "sslMMSCAConfigMap": issuer_ca_plus, + } + + for project in projects: + KubernetesTester.update_configmap(namespace, project, data) + + # Give a few seconds for the operator to catch the changes on + # the project ConfigMaps + time.sleep(10) + + def test_backing_dbs_created(self, blockstore_replica_set: MongoDB, oplog_replica_set: MongoDB): + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + def test_oplog_running(self, oplog_replica_set: MongoDB, ca_path: str): + oplog_replica_set.assert_connectivity(ca_path=ca_path) + + def test_blockstore_running(self, blockstore_replica_set: MongoDB, ca_path: str): + blockstore_replica_set.assert_connectivity(ca_path=ca_path) + + def test_om_is_running( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_om_ops_manager_backup_tls_custom_ca +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created.""" + + @fixture(scope="class") + def mdb_latest( + self, + ops_manager: MongoDBOpsManager, + namespace, + custom_mdb_version: str, + issuer_ca_configmap: str, + first_project_certs: str, + ): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=FIRST_PROJECT_RS_NAME, + ).configure(ops_manager, "firstProject") + # MongoD versions greater than 4.2.0 must be enterprise build to enable backup + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="enabled") + resource.configure_custom_tls(issuer_ca_configmap, first_project_certs) + resource.update() + return resource + + @fixture(scope="class") + def mdb_prev( + self, + ops_manager: MongoDBOpsManager, + namespace, + custom_mdb_prev_version: str, + issuer_ca_configmap: str, + second_project_certs: str, + ): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=SECOND_PROJECT_RS_NAME, + ).configure(ops_manager, "secondProject") + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.configure_backup(mode="enabled") + resource.configure_custom_tls(issuer_ca_configmap, second_project_certs) + resource.update() + return resource + + @fixture(scope="class") + def mdb_external_domain( + self, + ops_manager: MongoDBOpsManager, + namespace, + custom_mdb_prev_version: str, + issuer_ca_configmap: str, + mdb_external_domain_certs: str, + ): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=EXTERNAL_DOMAIN_RS_NAME, + ).configure(ops_manager, "externalDomain") + + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.configure_backup(mode="enabled") + resource.configure_custom_tls(issuer_ca_configmap, mdb_external_domain_certs) + resource["spec"]["members"] = 3 + resource["spec"]["externalAccess"] = {} + resource["spec"]["externalAccess"]["externalDomain"] = default_external_domain() + resource.update() + return resource + + def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB, mdb_external_domain: MongoDB): + mdb_latest.assert_reaches_phase(Phase.Running) + mdb_prev.assert_reaches_phase(Phase.Running) + mdb_external_domain.assert_reaches_phase(Phase.Running) + + def test_mdbs_backed_up(self, ops_manager: MongoDBOpsManager): + om_tester_first = ops_manager.get_om_tester(project_name="firstProject") + om_tester_second = ops_manager.get_om_tester(project_name="secondProject") + om_tester_external_domain = ops_manager.get_om_tester(project_name="externalDomain") + + # wait until a first snapshot is ready for both + om_tester_first.wait_until_backup_snapshots_are_ready(expected_count=1) + om_tester_second.wait_until_backup_snapshots_are_ready(expected_count=1) + om_tester_external_domain.wait_until_backup_snapshots_are_ready(expected_count=1) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_feature_controls.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_feature_controls.py new file mode 100644 index 000000000..d8e34c5e6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_feature_controls.py @@ -0,0 +1,180 @@ +from typing import Optional + +import kubernetes +from kubetester import try_load, wait_until +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + try_load(resource) + return resource + + +@fixture(scope="module") +def replica_set(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb", + ).configure(ops_manager, "mdb") + resource.set_version(custom_mdb_version) + + try_load(resource) + return resource + + +@mark.e2e_om_feature_controls +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_feature_controls +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + + +@mark.e2e_om_feature_controls +def test_authentication_is_owned_by_opsmanager(replica_set: MongoDB): + """ + There is no authentication, so feature controls API should allow + authentication changes from Ops Manager UI. + """ + fc = replica_set.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + assert len(fc["policies"]) == 2 + policies = [p["policy"] for p in fc["policies"]] + assert "EXTERNALLY_MANAGED_LOCK" in policies + assert "DISABLE_SET_MONGOD_VERSION" in policies + + for p in fc["policies"]: + if p["policy"] == "EXTERNALLY_MANAGED_LOCK": + assert p["disabledParams"] == [] + + +@mark.e2e_om_feature_controls +def test_authentication_disabled_is_owned_by_operator(replica_set: MongoDB): + """ + Authentication has been added to the Spec, on "disabled" mode, + this makes the Operator to *own* authentication and thus + making Feature controls API to restrict any + """ + replica_set["spec"]["security"] = {"authentication": {"enabled": False}} + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + fc = replica_set.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert len(fc["policies"]) == 3 + + assert policies[0]["disabledParams"] == [] + assert policies[2]["disabledParams"] == [] + + policies = [p["policy"] for p in fc["policies"]] + assert "EXTERNALLY_MANAGED_LOCK" in policies + assert "DISABLE_AUTHENTICATION_MECHANISMS" in policies + assert "DISABLE_SET_MONGOD_VERSION" in policies + + +@mark.e2e_om_feature_controls +def test_authentication_enabled_is_owned_by_operator(replica_set: MongoDB): + """ + Authentication has been enabled on the Operator. Authentication is still + owned by the operator so feature controls should be kept the same. + """ + replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + fc = replica_set.get_om_tester().get_feature_controls() + + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + assert len(fc["policies"]) == 3 + # sort the policies to have pre-determined order + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert policies[0]["disabledParams"] == [] + assert policies[2]["disabledParams"] == [] + + policies = [p["policy"] for p in fc["policies"]] + assert "EXTERNALLY_MANAGED_LOCK" in policies + assert "DISABLE_AUTHENTICATION_MECHANISMS" in policies + assert "DISABLE_SET_MONGOD_VERSION" in policies + + +@mark.e2e_om_feature_controls +def test_authentication_disabled_owned_by_opsmanager(replica_set: MongoDB): + """ + Authentication has been disabled (removed) on the Operator. Authentication + is now "owned" by Ops Manager. + """ + last_transition = replica_set.get_status_last_transition_time() + replica_set["spec"]["security"] = None + replica_set.update() + + replica_set.assert_state_transition_happens(last_transition) + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + fc = replica_set.get_om_tester().get_feature_controls() + + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + # sort the policies to have pre-determined order + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert len(fc["policies"]) == 2 + assert policies[0]["policy"] == "DISABLE_SET_MONGOD_VERSION" + assert policies[1]["policy"] == "EXTERNALLY_MANAGED_LOCK" + assert policies[1]["disabledParams"] == [] + + +@mark.e2e_om_feature_controls +def test_feature_controls_cleared_on_replica_set_deletion(replica_set: MongoDB): + """ + Replica set was deleted from the cluster. Policies are removed from the OpsManager group. + """ + replica_set.delete() + + def replica_set_deleted() -> bool: + k8s_resource_deleted = None + try: + replica_set.load() + k8s_resource_deleted = False + except kubernetes.client.ApiException: + k8s_resource_deleted = True + automation_config_deleted = None + tester = replica_set.get_automation_config_tester() + try: + tester.assert_empty() + automation_config_deleted = True + except AssertionError: + automation_config_deleted = False + return k8s_resource_deleted and automation_config_deleted + + wait_until(replica_set_deleted, timeout=60) + + fc = replica_set.get_om_tester().get_feature_controls() + + # after deleting the replicaset the policies in the feature control are removed + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + assert len(fc["policies"]) == 0 diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https.py new file mode 100644 index 000000000..6143f868a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https.py @@ -0,0 +1,222 @@ +import time +from typing import Optional + +from kubetester import try_load +from kubetester.certs import create_ops_manager_tls_certs, rotate_cert +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import create_appdb_certs, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str) -> str: + return create_appdb_certs(namespace, issuer, "om-with-https-db") + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + return create_ops_manager_tls_certs(issuer, namespace, "om-with-https", "prefix-om-with-https-cert") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + issuer_ca_configmap: str, + appdb_certs: str, + ops_manager_certs: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml(_fixture("om_https_enabled.yaml"), namespace=namespace) + + if try_load(om): + return om + + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + om.allow_mdb_rc_versions() + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + return om + + +@fixture(scope="module") +def replicaset0(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str): + """The First replicaset to be created before Ops Manager is configured with HTTPS.""" + resource = MongoDB.from_yaml(_fixture("replica-set.yaml"), name="replicaset0", namespace=namespace).configure( + ops_manager, "replicaset0" + ) + + resource.set_version(custom_mdb_version) + + try_load(resource) + return resource + + +@fixture(scope="module") +def replicaset1(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str): + """Second replicaset to be created when Ops Manager was restarted with HTTPS.""" + resource = MongoDB.from_yaml(_fixture("replica-set.yaml"), name="replicaset1", namespace=namespace).configure( + ops_manager, "replicaset1" + ) + + # NOTE: If running a test using a version different from 6.0.5 for OM6 means we will need to + # also download the respective signature (tgz.sig) as seen in om_https_enabled.yaml + resource.set_version(custom_mdb_version) + + try_load(resource) + return resource + + +@mark.e2e_om_ops_manager_https_enabled +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.update() + + +@mark.e2e_om_ops_manager_https_enabled +def test_om_created_no_tls(ops_manager: MongoDBOpsManager): + """Ops Manager is started over plain HTTP. AppDB also doesn't have TLS enabled""" + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + assert ops_manager.om_status().get_url().startswith("http://") + assert ops_manager.om_status().get_url().endswith(":8080") + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_ops_manager_https_enabled +def test_appdb_running_no_tls(ops_manager: MongoDBOpsManager): + ops_manager.get_appdb_tester().assert_connectivity() + + +@mark.e2e_om_ops_manager_https_enabled +def test_appdb_enable_tls(ops_manager: MongoDBOpsManager, issuer_ca_configmap: str, appdb_certs: str): + """Enable TLS for the AppDB (not for OM though).""" + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["security"] = { + "certsSecretPrefix": appdb_certs, + "tls": {"ca": issuer_ca_configmap}, + } + ops_manager.update() + ops_manager.appdb_status().assert_abandons_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_ops_manager_https_enabled +def test_appdb_running_over_tls(ops_manager: MongoDBOpsManager, ca_path: str): + ops_manager.get_appdb_tester(ssl=True, ca_path=ca_path).assert_connectivity() + + +@mark.e2e_om_ops_manager_https_enabled +def test_appdb_no_connection_without_tls(ops_manager: MongoDBOpsManager): + ops_manager.get_appdb_tester().assert_no_connection() + + +@mark.e2e_om_ops_manager_https_enabled +def test_replica_set_over_non_https_ops_manager(replicaset0: MongoDB): + """First replicaset is started over non-HTTPS Ops Manager.""" + replicaset0.update() + replicaset0.assert_reaches_phase(Phase.Running) + replicaset0.assert_connectivity() + + +@mark.e2e_om_ops_manager_https_enabled +def test_enable_https_on_opsmanager( + ops_manager: MongoDBOpsManager, + issuer_ca_configmap: str, + ops_manager_certs: str, + custom_version: Optional[str], +): + """Ops Manager is restarted with HTTPS enabled.""" + ops_manager["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": {"ca": issuer_ca_configmap}, + } + + # this enables download verification for om with https + # probably need to be done above and if only test replicaset1 since that one already has tls setup or test below + # custom ca setup + # only run this test if om > 6.0.18 + if custom_version >= "6.0.18": + print("verifying download signature for OM!") + ops_manager["spec"]["configuration"]["mms.featureFlag.automation.verifyDownloads"] = "enabled" + + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + assert ops_manager.om_status().get_url().startswith("https://") + assert ops_manager.om_status().get_url().endswith(":8443") + + +@mark.e2e_om_ops_manager_https_enabled +def test_project_is_configured_with_custom_ca( + ops_manager: MongoDBOpsManager, + namespace: str, + issuer_ca_configmap: str, +): + """Both projects are configured with the new HTTPS enabled Ops Manager.""" + project1 = ops_manager.get_or_create_mongodb_connection_config_map("replicaset0", "replicaset0") + project2 = ops_manager.get_or_create_mongodb_connection_config_map("replicaset1", "replicaset1") + + data = { + "sslMMSCAConfigMap": issuer_ca_configmap, + } + KubernetesTester.update_configmap(namespace, project1, data) + KubernetesTester.update_configmap(namespace, project2, data) + + # Give a few seconds for the operator to catch the changes on + # the project ConfigMaps + time.sleep(10) + + +@mark.e2e_om_ops_manager_https_enabled +def test_mongodb_replicaset_over_https_ops_manager(replicaset0: MongoDB, replicaset1: MongoDB): + """Both replicasets get to running state and are reachable. + Note that 'replicaset1' is created just now.""" + + replicaset1.update() + + # This would fail if there are no, sig files provided for the respective mongodb which the agent downloads. + replicaset1.assert_reaches_phase(Phase.Running, timeout=360) + replicaset1.assert_connectivity() + + +@mark.e2e_om_ops_manager_https_enabled +def test_change_om_certificate_and_wait_for_running(ops_manager: MongoDBOpsManager, namespace: str): + rotate_cert(namespace, certificate_name="prefix-om-with-https-cert") + ops_manager.om_status().assert_abandons_phase(Phase.Running, timeout=600) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + assert ops_manager.om_status().get_url().startswith("https://") + assert ops_manager.om_status().get_url().endswith(":8443") + + +@mark.e2e_om_ops_manager_https_enabled +def test_change_appdb_certificate_and_wait_for_running(ops_manager: MongoDBOpsManager, namespace: str): + rotate_cert(namespace, certificate_name="appdb-om-with-https-db-cert") + ops_manager.appdb_status().assert_abandons_phase(Phase.Running, timeout=600) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_ops_manager_https_enabled +def test_change_om_certificate_with_sts_restarting(ops_manager: MongoDBOpsManager, namespace: str): + ops_manager.trigger_om_sts_restart() + rotate_cert(namespace, certificate_name="prefix-om-with-https-cert") + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + assert ops_manager.om_status().get_url().startswith("https://") + assert ops_manager.om_status().get_url().endswith(":8443") + + +@mark.e2e_om_ops_manager_https_enabled +def test_change_appdb_certificate_with_sts_restarting(ops_manager: MongoDBOpsManager, namespace: str): + ops_manager.trigger_appdb_sts_restart() + rotate_cert(namespace, certificate_name="appdb-om-with-https-db-cert") + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_hybrid_mode.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_hybrid_mode.py new file mode 100644 index 000000000..6541157ba --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_hybrid_mode.py @@ -0,0 +1,109 @@ +import time +from typing import Optional + +from kubetester.certs import create_mongodb_tls_certs, create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import create_appdb_certs, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def certs_secret_prefix(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, "replicaset0", "certs-replicaset0-cert") + return "certs" + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str) -> str: + return create_appdb_certs(namespace, issuer, "om-with-https-db") + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + return create_ops_manager_tls_certs(issuer, namespace, "om-with-https", secret_name="prefix-om-with-https-cert") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + issuer_ca_configmap: str, + appdb_certs: str, + ops_manager_certs: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml(_fixture("om_https_enabled.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + om.allow_mdb_rc_versions() + del om["spec"]["statefulSet"] + om["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": issuer_ca_configmap, + }, + } + om["spec"]["configuration"]["automation.versions.source"] = "hybrid" + om["spec"]["applicationDatabase"]["security"] = { + "tls": { + "ca": issuer_ca_configmap, + }, + "certsSecretPrefix": appdb_certs, + } + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + om.update() + return om + + +@fixture(scope="module") +def replicaset0( + ops_manager: MongoDBOpsManager, + namespace: str, + custom_mdb_version: str, + issuer_ca_configmap: str, + certs_secret_prefix: str, +): + """First replicaset to be created before Ops Manager is configured with HTTPS.""" + resource = MongoDB.from_yaml(_fixture("replica-set.yaml"), name="replicaset0", namespace=namespace).configure( + ops_manager, "replicaset0" + ) + resource.set_version(custom_mdb_version) + resource.configure_custom_tls(issuer_ca_configmap, certs_secret_prefix) + return resource.create() + + +@mark.e2e_om_ops_manager_https_enabled_hybrid +def test_appdb_running_over_tls(ops_manager: MongoDBOpsManager, ca_path: str): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.get_appdb_tester(ssl=True, ca_path=ca_path).assert_connectivity() + + +@mark.e2e_om_ops_manager_https_enabled_hybrid +def test_om_reaches_running_state(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_om_ops_manager_https_enabled_hybrid +def test_config_map_has_ca_set_correctly(ops_manager: MongoDBOpsManager, issuer_ca_plus: str, namespace: str): + project1 = ops_manager.get_or_create_mongodb_connection_config_map("replicaset0", "replicaset0") + data = { + "sslMMSCAConfigMap": issuer_ca_plus, + } + KubernetesTester.update_configmap(namespace, project1, data) + + # Give a few seconds for the operator to catch the changes on + # the project ConfigMaps + time.sleep(10) + + +@mark.e2e_om_ops_manager_https_enabled_hybrid +def test_mongodb_replicaset_over_https_ops_manager(replicaset0: MongoDB, ca_path: str): + replicaset0.assert_reaches_phase(Phase.Running, timeout=400, ignore_errors=True) + replicaset0.assert_connectivity(ca_path=ca_path) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_internet_mode.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_internet_mode.py new file mode 100644 index 000000000..610c7ed84 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_internet_mode.py @@ -0,0 +1,120 @@ +import time +from typing import Optional + +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import create_appdb_certs, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, "om-with-https-db") + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + return create_ops_manager_tls_certs(issuer, namespace, "om-with-https", secret_name="prefix-om-with-https-cert") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + issuer_ca_plus: str, + appdb_certs: str, + ops_manager_certs: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml(_fixture("om_https_enabled.yaml"), namespace=namespace) + om.set_version(custom_version) + + # do not use local mode + del om["spec"]["configuration"]["automation.versions.source"] + del om["spec"]["statefulSet"] + + om.set_appdb_version(custom_appdb_version) + # configure CA + tls secrets for AppDB members to community with each other + om["spec"]["applicationDatabase"]["security"] = { + "tls": {"ca": issuer_ca_plus, "secretRef": {"prefix": appdb_certs}} + } + + # configure the CA that will be used to communicate with Ops Manager + om["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": issuer_ca_plus, + }, + } + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + om.update() + return om + + +@fixture(scope="module") +def replicaset0(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_prev_version: str): + resource = MongoDB.from_yaml(_fixture("replica-set.yaml"), name="replicaset0", namespace=namespace).configure( + ops_manager, "replicaset0" + ) + resource["spec"]["version"] = custom_mdb_prev_version + + return resource.create() + + +@fixture(scope="module") +def replicaset1(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str): + resource = MongoDB.from_yaml(_fixture("replica-set.yaml"), name="replicaset1", namespace=namespace).configure( + ops_manager, "replicaset1" + ) + resource["spec"]["version"] = custom_mdb_version + + return resource.create() + + +@mark.e2e_om_ops_manager_https_enabled_internet_mode +def test_enable_https_on_opsmanager(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + assert ops_manager.om_status().get_url().startswith("https://") + assert ops_manager.om_status().get_url().endswith(":8443") + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_ops_manager_https_enabled_internet_mode +def test_project_is_configured_with_custom_ca( + ops_manager: MongoDBOpsManager, + namespace: str, + issuer_ca_plus: str, +): + """Both projects are configured with the new HTTPS enabled Ops Manager.""" + project1 = ops_manager.get_or_create_mongodb_connection_config_map("replicaset0", "replicaset0") + project2 = ops_manager.get_or_create_mongodb_connection_config_map("replicaset1", "replicaset1") + + data = { + "sslMMSCAConfigMap": issuer_ca_plus, + } + KubernetesTester.update_configmap(namespace, project1, data) + KubernetesTester.update_configmap(namespace, project2, data) + + # Give a few seconds for the operator to catch the changes on + # the project ConfigMaps + time.sleep(10) + + +@mark.e2e_om_ops_manager_https_enabled_internet_mode +def test_mongodb_replicaset_over_https_ops_manager(replicaset0: MongoDB, replicaset1: MongoDB): + """Both replicasets get to running state and are reachable.""" + + replicaset0.assert_reaches_phase(Phase.Running, timeout=360) + replicaset0.assert_connectivity() + + replicaset1.assert_reaches_phase(Phase.Running, timeout=360) + replicaset1.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_prefix.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_prefix.py new file mode 100644 index 000000000..4bed89004 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_https_prefix.py @@ -0,0 +1,51 @@ +from typing import Optional + +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + return create_ops_manager_tls_certs(issuer, namespace, "om-with-https", secret_name="prefix-om-with-https-cert") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + issuer_ca_configmap: str, + ops_manager_certs: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml(_fixture("om_https_enabled.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + om["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": { + "ca": issuer_ca_configmap, + }, + } + + # No need to mount the additional mongods because they are not used + # in this test, and make the test slower. + om["spec"]["statefulSet"]["spec"]["template"]["spec"] = {} + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + om.update() + return om + + +@mark.e2e_om_ops_manager_https_enabled_prefix +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + assert ops_manager.om_status().get_url().startswith("https://") + assert ops_manager.om_status().get_url().endswith(":8443") diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_local_mode_enable_and_disable_manually_deleting_sts.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_local_mode_enable_and_disable_manually_deleting_sts.py new file mode 100644 index 000000000..eecedde0e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_local_mode_enable_and_disable_manually_deleting_sts.py @@ -0,0 +1,111 @@ +from typing import Optional + +from kubetester import MongoDB, delete_pod, delete_statefulset, get_pod_when_ready +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_static_containers +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import get_member_cluster_api_client, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace + ) + + resource["spec"]["replicas"] = 2 + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource.update() + + +@fixture(scope="module") +def replica_set(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + ).configure(ops_manager, "my-replica-set") + resource.set_version(custom_mdb_version) + + resource.update() + return resource + + +@skip_if_static_containers +@mark.e2e_om_ops_manager_enable_local_mode_running_om +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1000) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=1000) + + +@skip_if_static_containers +@mark.e2e_om_ops_manager_enable_local_mode_running_om +def test_enable_local_mode(ops_manager: MongoDBOpsManager, namespace: str): + + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_localmode-multiple-pv.yaml"), namespace=namespace) + + for member_cluster_name, sts_name in ops_manager.get_om_sts_names_in_member_clusters(): + # We manually delete the ops manager sts, it won't delete the pods as + # the function by default does cascade=false + delete_statefulset(namespace, sts_name, api_client=get_member_cluster_api_client(member_cluster_name)) + + ops_manager.load() + ops_manager["spec"]["configuration"] = {"automation.versions.source": "local"} + ops_manager["spec"]["statefulSet"] = om["spec"]["statefulSet"] + ops_manager.update() + + # At this point the operator has created a new sts but the existing pods can't be bound to + # it because podspecs are immutable so the volumes field can't be changed + # and thus we can't rollout + + for member_cluster_name, pod_name in ops_manager.get_om_pod_names_in_member_clusters(): + # So we manually delete one, wait for it to be ready + # and do the same for the second one + delete_pod(namespace, pod_name, api_client=get_member_cluster_api_client(member_cluster_name)) + get_pod_when_ready(namespace, f"statefulset.kubernetes.io/pod-name={pod_name}") + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@skip_if_static_containers +@mark.e2e_om_ops_manager_enable_local_mode_running_om +def test_add_mongodb_distros(ops_manager: MongoDBOpsManager, custom_mdb_version: str): + ops_manager.download_mongodb_binaries(custom_mdb_version) + + +@skip_if_static_containers +@mark.e2e_om_ops_manager_enable_local_mode_running_om +def test_new_binaries_are_present(ops_manager: MongoDBOpsManager, namespace: str): + cmd = [ + "/bin/sh", + "-c", + "ls /mongodb-ops-manager/mongodb-releases/*.tgz", + ] + for member_cluster_name, pod_name in ops_manager.get_om_pod_names_in_member_clusters(): + result = KubernetesTester.run_command_in_pod_container( + pod_name, + namespace, + cmd, + container="mongodb-ops-manager", + api_client=get_member_cluster_api_client(member_cluster_name), + ) + assert result != "0" + + +@skip_if_static_containers +@mark.e2e_om_ops_manager_enable_local_mode_running_om +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=600) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_prometheus.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_prometheus.py new file mode 100644 index 000000000..41ce204f2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_prometheus.py @@ -0,0 +1,241 @@ +import time + +from kubernetes import client +from kubetester import MongoDB, create_or_update_secret, random_k8s_name +from kubetester.certs import create_mongodb_tls_certs +from kubetester.http import https_endpoint_is_reachable +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase, generic_replicaset +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +def certs_for_prometheus(issuer: str, namespace: str, resource_name: str) -> str: + secret_name = random_k8s_name(resource_name + "-") + "-cert" + + return create_mongodb_tls_certs( + issuer, + namespace, + resource_name, + secret_name, + ) + + +CONFIGURED_PROMETHEUS_PORT = 9999 + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_appdb_version: str, + custom_version: str, + issuer: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace + ) + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + resource["spec"]["replicas"] = 1 + + create_or_update_secret(namespace, "appdb-prom-secret", {"password": "prom-password"}) + + prom_cert_secret = certs_for_prometheus(issuer, namespace, resource.name + "-db") + resource["spec"]["applicationDatabase"]["prometheus"] = { + "username": "prom-user", + "passwordSecretRef": {"name": "appdb-prom-secret"}, + "tlsSecretKeyRef": { + "name": prom_cert_secret, + }, + } + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def sharded_cluster(ops_manager: MongoDBOpsManager, namespace: str, issuer: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster.yaml"), + namespace=namespace, + ) + prom_cert_secret = certs_for_prometheus(issuer, namespace, resource.name) + + create_or_update_secret(namespace, "cluster-secret", {"password": "cluster-prom-password"}) + + resource["spec"]["prometheus"] = { + "username": "prom-user", + "passwordSecretRef": { + "name": "cluster-secret", + }, + "tlsSecretKeyRef": { + "name": prom_cert_secret, + }, + "port": CONFIGURED_PROMETHEUS_PORT, + } + del resource["spec"]["cloudManager"] + resource.configure(ops_manager, namespace) + resource.set_version(ensure_ent_version(custom_mdb_version)) + + yield resource.create() + + +@fixture(scope="module") +def replica_set( + ops_manager: MongoDBOpsManager, + namespace: str, + custom_mdb_version: str, + issuer: str, +) -> MongoDB: + + create_or_update_secret(namespace, "rs-secret", {"password": "prom-password"}) + + resource = generic_replicaset(namespace, custom_mdb_version, "replica-set-with-prom", ops_manager) + + prom_cert_secret = certs_for_prometheus(issuer, namespace, resource.name) + resource["spec"]["prometheus"] = { + "username": "prom-user", + "passwordSecretRef": { + "name": "rs-secret", + }, + "tlsSecretKeyRef": { + "name": prom_cert_secret, + }, + } + yield resource.create() + + +@mark.e2e_om_ops_manager_prometheus +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_ops_manager_prometheus +def test_create_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_ops_manager_prometheus +def test_prometheus_endpoint_works_on_every_pod(replica_set: MongoDB, namespace: str): + members = replica_set["spec"]["members"] + name = replica_set.name + + auth = ("prom-user", "prom-password") + + for idx in range(members): + member_url = f"https://{name}-{idx}.{name}-svc.{namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(member_url, auth, tls_verify=False) + + +@mark.e2e_om_ops_manager_prometheus +def test_prometheus_can_change_credentials(replica_set: MongoDB): + replica_set["spec"]["prometheus"] = {"username": "prom-user-but-changed-this-time"} + replica_set.update() + + # TODO: is the resource even being moved away from Running phase? + time.sleep(20) + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_ops_manager_prometheus +def test_prometheus_endpoint_works_on_every_pod_with_changed_username(replica_set: MongoDB, namespace: str): + members = replica_set["spec"]["members"] + name = replica_set.name + + auth = ("prom-user-but-changed-this-time", "prom-password") + + for idx in range(members): + member_url = f"https://{name}-{idx}.{name}-svc.{namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(member_url, auth, tls_verify=False) + + +@mark.e2e_om_ops_manager_prometheus +def test_create_sharded_cluster(sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_ops_manager_prometheus +def test_prometheus_endpoint_works_on_every_pod_on_the_cluster(sharded_cluster: MongoDB, namespace: str): + """ + Checks that all of the Prometheus endpoints that we expect are up and listening. + """ + + auth = ("prom-user", "cluster-prom-password") + name = sharded_cluster.name + + port = sharded_cluster["spec"]["prometheus"]["port"] + mongos_count = sharded_cluster["spec"]["mongosCount"] + for idx in range(mongos_count): + url = f"https://{name}-mongos-{idx}.{name}-svc.{namespace}.svc.cluster.local:{port}/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + shard_count = sharded_cluster["spec"]["shardCount"] + mongodbs_per_shard_count = sharded_cluster["spec"]["mongodsPerShardCount"] + for shard in range(shard_count): + for mongodb in range(mongodbs_per_shard_count): + url = f"https://{name}-{shard}-{mongodb}.{name}-sh.{namespace}.svc.cluster.local:{port}/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + config_server_count = sharded_cluster["spec"]["configServerCount"] + for idx in range(config_server_count): + url = f"https://{name}-config-{idx}.{name}-cs.{namespace}.svc.cluster.local:{port}/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + +@mark.e2e_om_ops_manager_prometheus +def test_sharded_cluster_service_has_been_updated_with_prometheus_port(replica_set: MongoDB, sharded_cluster: MongoDB): + # Check that the service that belong to the Replica Set has the + # the default Prometheus port. + assert_mongodb_prometheus_port_exist( + replica_set.name + "-svc", + replica_set.namespace, + port=9216, + ) + + # Checks that the Services that belong to the Sharded cluster have + # the configured Prometheus port. + assert_mongodb_prometheus_port_exist( + sharded_cluster.name + "-svc", + sharded_cluster.namespace, + port=CONFIGURED_PROMETHEUS_PORT, + ) + assert_mongodb_prometheus_port_exist( + sharded_cluster.name + "-cs", + sharded_cluster.namespace, + port=CONFIGURED_PROMETHEUS_PORT, + ) + assert_mongodb_prometheus_port_exist( + sharded_cluster.name + "-sh", + sharded_cluster.namespace, + port=CONFIGURED_PROMETHEUS_PORT, + ) + + +@mark.e2e_om_ops_manager_prometheus +def test_prometheus_endpoint_works_on_every_pod_on_appdb(ops_manager: MongoDB): + auth = ("prom-user", "prom-password") + name = ops_manager.name + "-db" + + for idx in range(ops_manager["spec"]["applicationDatabase"]["members"]): + url = f"https://{name}-{idx}.{name}-svc.{ops_manager.namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + assert_mongodb_prometheus_port_exist(name + "-svc", ops_manager.namespace, 9216) + + +def assert_mongodb_prometheus_port_exist(service_name: str, namespace: str, port: int): + services = client.CoreV1Api().read_namespaced_service(name=service_name, namespace=namespace) + assert len(services.spec.ports) == 2 + ports = ((p.name, p.port) for p in services.spec.ports) + + assert ("mongodb", 27017) in ports + assert ("prometheus", port) in ports diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_queryable_backup.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_queryable_backup.py new file mode 100644 index 000000000..5f9ee694a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_queryable_backup.py @@ -0,0 +1,517 @@ +import logging +import os +from operator import attrgetter +from typing import Dict, Optional + +from kubernetes import client +from kubetester import ( + assert_pod_container_security_context, + assert_pod_security_context, + create_or_update_secret, + get_default_storage_class, +) +from kubetester.awss3client import AwsS3Client, s3_endpoint +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import running_locally, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.om_queryable_backups import generate_queryable_pem +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import AWS_REGION, is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +TEST_DB = "testdb" +TEST_COLLECTION = "testcollection" +TEST_DATA = {"_id": "unique_id", "name": "John", "address": "Highway 37", "age": 30} + +PROJECT_NAME = "firstProject" + +LOGLEVEL = os.environ.get("LOGLEVEL", "DEBUG").upper() +logging.basicConfig(level=LOGLEVEL) + +HEAD_PATH = "/head/" +S3_SECRET_NAME = "my-s3-secret" +OPLOG_RS_NAME = "my-mongodb-oplog" +S3_RS_NAME = "my-mongodb-s3" +BLOCKSTORE_RS_NAME = "my-mongodb-blockstore" +USER_PASSWORD = "/qwerty@!#:" + + +""" +Current test focuses on backup capabilities. It creates an explicit MDBs for S3 snapshot metadata, Blockstore and Oplog +databases. Tests backup enabled for both MDB 4.0 and 4.2, snapshots created +""" + + +@fixture(scope="module") +def queryable_pem_secret(namespace: str) -> str: + return create_or_update_secret( + namespace, + "queryable-bkp-pem", + {"queryable.pem": generate_queryable_pem(namespace)}, + ) + + +def new_om_s3_store( + mdb: MongoDB, + s3_id: str, + s3_bucket_name: str, + aws_s3_client: AwsS3Client, + assignment_enabled: bool = True, + path_style_access_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "id": s3_id, + "pathStyleAccessEnabled": path_style_access_enabled, + "s3BucketEndpoint": s3_endpoint(AWS_REGION), + "s3BucketName": s3_bucket_name, + "awsAccessKey": aws_s3_client.aws_access_key, + "awsSecretKey": aws_s3_client.aws_secret_access_key, + "assignmentEnabled": assignment_enabled, + } + + +def new_om_data_store( + mdb: MongoDB, + id: str, + assignment_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "id": id, + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "ssl": mdb.is_tls_enabled(), + "assignmentEnabled": assignment_enabled, + } + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, "test-bucket-s3") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, + queryable_pem_secret: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup.yaml"), namespace=namespace + ) + + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + resource["spec"]["backup"]["headDB"]["storageClass"] = get_default_storage_class() + resource["spec"]["backup"]["members"] = 1 + resource["spec"]["backup"]["queryableBackupSecretRef"] = {"name": queryable_pem_secret} + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource.allow_mdb_rc_versions() + + resource["spec"]["configuration"]["mongodb.release.autoDownload.enterprise"] = "true" + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + resource.set_version(custom_mdb_version) + + # TODO: Remove when CLOUDP-60443 is fixed + # This test will update oplog to have SCRAM enabled + # Currently this results in OM failure when enabling backup for a project, backup seems to do some caching resulting in the + # mongoURI not being updated unless pod is killed. This is documented in CLOUDP-60443, once resolved this skip & comment can be deleted + resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + + resource.update() + return resource + + +@fixture(scope="module") +def s3_replica_set(ops_manager, namespace) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=S3_RS_NAME, + ).configure(ops_manager, "s3metadata") + + resource.update() + return resource + + +@fixture(scope="module") +def blockstore_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + + resource.set_version(custom_mdb_version) + + resource.update() + return resource + + +@fixture(scope="module") +def blockstore_user(namespace, blockstore_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml(yaml_fixture("scram-sha-user-backing-db.yaml"), namespace=namespace) + resource["spec"]["mongodbResourceRef"]["name"] = blockstore_replica_set.name + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_user(namespace, oplog_replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user-backing-db.yaml"), + namespace=namespace, + name="mms-user-2", + ) + resource["spec"]["mongodbResourceRef"]["name"] = oplog_replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "mms-user-2-password" + resource["spec"]["username"] = "mms-user-2" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + resource.update() + return resource + + +@fixture(scope="module") +def mdb42(ops_manager: MongoDBOpsManager, namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="mdb-four-two", + ).configure(ops_manager, PROJECT_NAME) + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.configure_backup(mode="enabled") + resource.update() + return resource + + +@fixture(scope="module") +def mdb_test_collection(mdb42: MongoDB): + dbClient = mdb42.tester().client[TEST_DB] + return dbClient[TEST_COLLECTION] + + +@mark.e2e_om_ops_manager_queryable_backup +class TestOpsManagerCreation: + """ + name: Ops Manager successful creation with backup and oplog stores enabled + description: | + Creates an Ops Manager instance with backup enabled. The OM is expected to get to 'Pending' state + eventually as it will wait for oplog db to be created + """ + + def test_create_om(self, ops_manager: MongoDBOpsManager): + """creates a s3 bucket, s3 config and an OM resource (waits until Backup gets to Pending state)""" + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=1200, + ) + + def test_daemon_statefulset(self, ops_manager: MongoDBOpsManager): + def stateful_set_becomes_ready(): + stateful_set = ops_manager.read_backup_statefulset() + return stateful_set.status.ready_replicas == 1 and stateful_set.status.current_replicas == 1 + + KubernetesTester.wait_until(stateful_set_becomes_ready, timeout=300) + + stateful_set = ops_manager.read_backup_statefulset() + # pod template has volume mount request + assert (HEAD_PATH, "head") in ( + (mount.mount_path, mount.name) for mount in stateful_set.spec.template.spec.containers[0].volume_mounts + ) + + def test_daemon_pvc(self, ops_manager: MongoDBOpsManager, namespace: str): + """Verifies the PVCs mounted to the pod""" + pods = ops_manager.read_backup_pods() + idx = 0 + for api_client, pod in pods: + claims = [volume for volume in pod.spec.volumes if getattr(volume, "persistent_volume_claim")] + assert len(claims) == 1 + claims.sort(key=attrgetter("name")) + + default_sc = get_default_storage_class() + KubernetesTester.check_single_pvc( + namespace, + claims[0], + "head", + "head-{}-{}".format(ops_manager.backup_daemon_sts_name(), idx), + "500M", + default_sc, + api_client=api_client, + ) + idx += 1 + + def test_backup_daemon_services_created(self, namespace): + """Backup creates two additional services for queryable backup""" + services = client.CoreV1Api().list_namespaced_service(namespace).items + + # If running locally in 'default' namespace, there might be more + # services on it. Let's make sure we only count those that we care of. + # For now we allow this test to fail, because it is too broad to be significant + # and it is easy to break it. + backup_services = [s for s in services if s.metadata.name.startswith("om-backup")] + + assert len(backup_services) >= 3 + + @skip_if_local + def test_om(self, ops_manager: MongoDBOpsManager): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + # No oplog stores were created in Ops Manager by this time + om_tester.assert_oplog_stores([]) + om_tester.assert_s3_stores([]) + + def test_generations(self, ops_manager: MongoDBOpsManager): + assert ops_manager.appdb_status().get_observed_generation() == 1 + assert ops_manager.om_status().get_observed_generation() == 1 + assert ops_manager.backup_status().get_observed_generation() == 1 + + def test_security_contexts_appdb( + self, + ops_manager: MongoDBOpsManager, + operator_installation_config: Dict[str, str], + ): + managed = operator_installation_config["managedSecurityContext"] == "true" + for _, pod in ops_manager.read_appdb_pods(): + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + def test_security_contexts_om( + self, + ops_manager: MongoDBOpsManager, + operator_installation_config: Dict[str, str], + ): + managed = operator_installation_config["managedSecurityContext"] == "true" + for api_client, pod in ops_manager.read_om_pods(): + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + +@mark.e2e_om_ops_manager_queryable_backup +class TestBackupDatabasesAdded: + """name: Creates three mongodb resources for oplog, s3 and blockstore and waits until OM resource gets to + running state""" + + def test_backup_mdbs_created( + self, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + oplog_user: MongoDBUser, + ): + """Creates mongodb databases all at once""" + oplog_replica_set.load() + oplog_replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + oplog_replica_set.update() + oplog_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + oplog_user.assert_reaches_phase(Phase.Updated) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, oplog_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"][0]["mongodbUserRef"] = {"name": oplog_user.name} + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + + assert ops_manager.backup_status().get_message() is None + + @skip_if_local + def test_om( + self, + s3_bucket: str, + aws_s3_client: AwsS3Client, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + oplog_user: MongoDBUser, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + # Nothing has changed for daemon + + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + + om_tester.assert_block_stores([new_om_data_store(blockstore_replica_set, "blockStore1")]) + # oplog store has authentication enabled + om_tester.assert_oplog_stores( + [ + new_om_data_store( + oplog_replica_set, + "oplog1", + user_name=oplog_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + om_tester.assert_s3_stores([new_om_s3_store(s3_replica_set, "s3Store1", s3_bucket, aws_s3_client)]) + + def test_generations(self, ops_manager: MongoDBOpsManager): + """There have been an update to the OM spec - all observed generations are expected to be updated""" + assert ops_manager.appdb_status().get_observed_generation() == 2 + assert ops_manager.om_status().get_observed_generation() == 2 + assert ops_manager.backup_status().get_observed_generation() == 2 + + def test_security_contexts_backup( + self, + ops_manager: MongoDBOpsManager, + operator_installation_config: Dict[str, str], + ): + managed = operator_installation_config["managedSecurityContext"] == "true" + pods = ops_manager.read_backup_pods() + for _, pod in pods: + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + +@mark.e2e_om_ops_manager_queryable_backup +class TestOpsManagerWatchesBlockStoreUpdates: + def test_om_running(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=40) + + def test_scramsha_enabled_for_blockstore(self, blockstore_replica_set: MongoDB, blockstore_user: MongoDBUser): + """Enables SCRAM for the blockstore replica set. Note that until CLOUDP-67736 is fixed + the order of operations (scram first, MongoDBUser - next) is important""" + blockstore_replica_set["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}} + blockstore_replica_set.update() + + # timeout of 600 is required when enabling SCRAM in mdb5.0.0 + blockstore_replica_set.assert_reaches_phase(Phase.Running, timeout=900) + blockstore_user.assert_reaches_phase(Phase.Updated) + + def test_fix_om(self, ops_manager: MongoDBOpsManager, blockstore_user: MongoDBUser): + ops_manager.load() + ops_manager["spec"]["backup"]["blockStores"][0]["mongodbUserRef"] = {"name": blockstore_user.name} + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=200, + ignore_errors=True, + ) + assert ops_manager.backup_status().get_message() is None + + @skip_if_local + def test_om( + self, + s3_bucket: str, + aws_s3_client: AwsS3Client, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, + blockstore_replica_set: MongoDB, + oplog_user: MongoDBUser, + blockstore_user: MongoDBUser, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + # Nothing has changed for daemon + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + + # block store has authentication enabled + om_tester.assert_block_stores( + [ + new_om_data_store( + blockstore_replica_set, + "blockStore1", + user_name=blockstore_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + + # oplog has authentication enabled + om_tester.assert_oplog_stores( + [ + new_om_data_store( + oplog_replica_set, + "oplog1", + user_name=oplog_user.get_user_name(), + password=USER_PASSWORD, + ) + ] + ) + om_tester.assert_s3_stores([new_om_s3_store(s3_replica_set, "s3Store1", s3_bucket, aws_s3_client)]) + + +@mark.e2e_om_ops_manager_queryable_backup +class TestBackupForMongodb: + """This part ensures that backup for the client works correctly and the snapshot is created. + Both latest and the one before the latest are tested (as the backup process for them may differ significantly)""" + + def test_mdbs_created(self, mdb42: MongoDB): + mdb42.assert_reaches_phase(Phase.Running) + + def test_add_test_data(self, mdb_test_collection): + mdb_test_collection.insert_one(TEST_DATA) + + def test_mdbs_backuped(self, ops_manager: MongoDBOpsManager): + om_tester = ops_manager.get_om_tester(project_name=PROJECT_NAME) + # wait until a first snapshot is ready + om_tester.wait_until_backup_snapshots_are_ready(expected_count=1) + + +@mark.e2e_om_ops_manager_queryable_backup +class TestQueryableBackup: + """This part queryable backup is enabled and we can query the first snapshot.""" + + def test_queryable_backup(self, ops_manager: MongoDBOpsManager): + CONNECTION_TIMEOUT = 600 + om_tester = ops_manager.get_om_tester(project_name=PROJECT_NAME) + records = om_tester.query_backup(TEST_DB, TEST_COLLECTION, CONNECTION_TIMEOUT) + assert len(records) > 0 diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_scale.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_scale.py new file mode 100644 index 000000000..7c0b28f0e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_scale.py @@ -0,0 +1,269 @@ +import pytest +from kubernetes import client +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.omtester import OMBackgroundTester +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import ( + get_member_cluster_api_client, + is_multi_cluster, + verify_pvc_expanded, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +gen_key_resource_version = None +admin_key_resource_version = None + +# Note the strategy for Ops Manager testing: the tests should have more than 1 updates - this is because the initial +# creation of Ops Manager takes too long, so we try to avoid fine-grained test cases and combine different +# updates in one test + +# Current test should contain all kinds of scale operations to Ops Manager as a sequence of tests +RESIZED_STORAGE_SIZE = "2Gi" + + +@fixture(scope="module") +def ops_manager( + namespace, + custom_version: str, + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_scale.yaml"), namespace=namespace) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource, om_cluster_spec_list=[2, 1, 1]) + + try_load(resource) + return resource + + +@fixture(scope="module") +def background_tester(ops_manager: MongoDBOpsManager) -> OMBackgroundTester: + om_background_tester = OMBackgroundTester(ops_manager.get_om_tester()) + om_background_tester.start() + return om_background_tester + + +@pytest.mark.e2e_om_ops_manager_scale +class TestOpsManagerCreation: + """ + Creates an Ops Manager resource of size 2. There are many configuration options passed to the OM created - + which allows to bypass the welcome wizard (see conf-hosted-mms-public-template.properties in mms) and get OM + ready for use + TODO we need to create a MongoDB resource referencing the OM and check that everything is working during scaling + operations + """ + + def test_create_om(self, ops_manager: MongoDBOpsManager): + # Backup is not fully configured so we wait until Pending phase + + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + timeout=900, + msg_regexp="Oplog Store configuration is required for backup.*", + ) + + def test_number_of_replicas(self, ops_manager: MongoDBOpsManager): + for member_cluster_name in ops_manager.get_om_member_cluster_names(): + statefulset = ops_manager.read_statefulset(member_cluster_name=member_cluster_name) + replicas = ops_manager.get_om_replicas_in_member_cluster(member_cluster_name=member_cluster_name) + assert statefulset.status.ready_replicas == replicas + assert statefulset.status.current_replicas == replicas + + def test_service(self, ops_manager: MongoDBOpsManager): + for _, cluster_spec_item in ops_manager.get_om_indexed_cluster_spec_items(): + internal, external = ops_manager.services(cluster_spec_item["clusterName"]) + assert external is None + assert internal.spec.type == "ClusterIP" + if not ops_manager.is_om_multi_cluster(): + assert internal.spec.cluster_ip == "None" + assert len(internal.spec.ports) == 2 + assert internal.spec.ports[0].target_port == 8080 + assert internal.spec.ports[1].target_port == 25999 + + def test_endpoints(self, ops_manager: MongoDBOpsManager): + """making sure the service points at correct pods""" + for member_cluster_name in ops_manager.get_om_member_cluster_names(): + endpoints = client.CoreV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_endpoints(ops_manager.svc_name(member_cluster_name), ops_manager.namespace) + replicas = ops_manager.get_om_replicas_in_member_cluster(member_cluster_name) + assert len(endpoints.subsets) == 1 + assert len(endpoints.subsets[0].addresses) == replicas + + def test_om_resource(self, ops_manager: MongoDBOpsManager): + assert ops_manager.om_status().get_replicas() == ops_manager.get_total_number_of_om_replicas() + assert ops_manager.om_status().get_url() == "http://om-scale-svc.{}.svc.cluster.local:8080".format( + ops_manager.namespace + ) + + def test_backup_pod(self, ops_manager: MongoDBOpsManager): + """If spec.backup is not specified the backup statefulset is still expected to be created. + Also the number of replicas doesn't depend on OM replicas. The backup daemon pod will become + ready when the web server becomes available. + """ + ops_manager.wait_until_backup_pods_become_ready() + + @skip_if_local + def test_om(self, ops_manager: MongoDBOpsManager): + """Checks that the OM is responsive and test service is available (enabled by 'mms.testUtil.enabled').""" + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_test_service() + + # checking connectivity to each OM instance + # We can only perform this test in the single cluster case because we don't create a service per pod so that + # every OM pod can be addressable by FQDN from a different cluster (the test pod cluster for example). + if not is_multi_cluster(): + om_tester.assert_om_instances_healthiness(ops_manager.pod_urls()) + + +@pytest.mark.e2e_om_ops_manager_scale +class TestOpsManagerPVCExpansion: + + def test_expand_pvc(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["podSpec"]["persistence"]["multiple"]["data"][ + "storage" + ] = RESIZED_STORAGE_SIZE + ops_manager["spec"]["applicationDatabase"]["podSpec"]["persistence"]["multiple"]["journal"][ + "storage" + ] = RESIZED_STORAGE_SIZE + ops_manager.update() + + def test_appdb_ready_after_expansion(self, ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + def test_appdb_expansion_finished(self, ops_manager: MongoDBOpsManager, namespace: str): + for member_cluster_name in ops_manager.get_appdb_member_cluster_names(): + sts = ops_manager.read_appdb_statefulset(member_cluster_name=member_cluster_name) + assert sts.spec.volume_claim_templates[0].spec.resources.requests["storage"] == RESIZED_STORAGE_SIZE + + +@pytest.mark.e2e_om_ops_manager_scale +class TestOpsManagerVersionUpgrade: + """ + The OM version is upgraded - this means the new image is deployed for both OM, appdb and backup. + The OM upgrade happens in rolling manner, we are checking for OM healthiness in parallel + """ + + def test_upgrade_om( + self, + ops_manager: MongoDBOpsManager, + background_tester: OMBackgroundTester, + custom_version: str, + ): + # Adding fixture just to start background tester + _ = background_tester + ops_manager.load() + + # This updates OM by a major version. For instance: + # If running OM6 tests, this will update from 5.0.9 to latest OM6 + ops_manager.set_version(custom_version) + + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + def test_image_url(self, ops_manager: MongoDBOpsManager): + """All pods in statefulset are referencing the correct image""" + pods = ops_manager.read_om_pods() + assert len(pods) == ops_manager.get_total_number_of_om_replicas() + for _, pod in pods: + assert ops_manager.get_version() in pod.spec.containers[0].image + + @skip_if_local + def test_om_has_been_up_during_upgrade(self, background_tester: OMBackgroundTester): + + # 10% of the requests are allowed to fail + background_tester.assert_healthiness(allowed_rate_of_failure=0.1) + + +@pytest.mark.e2e_om_ops_manager_scale +class TestOpsManagerScaleUp: + """ + The OM statefulset is scaled to 3 nodes + """ + + def test_scale_up_om(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + if is_multi_cluster(): + enable_multi_cluster_deployment(ops_manager, om_cluster_spec_list=[3, 2, 1]) + else: + ops_manager["spec"]["replicas"] = 3 + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=500) + + def test_number_of_replicas(self, ops_manager: MongoDBOpsManager): + for member_cluster_name in ops_manager.get_om_member_cluster_names(): + + statefulset = ops_manager.read_statefulset(member_cluster_name=member_cluster_name) + replicas = ops_manager.get_om_replicas_in_member_cluster(member_cluster_name=member_cluster_name) + assert statefulset.status.ready_replicas == replicas + assert statefulset.status.current_replicas == replicas + + assert ops_manager.om_status().get_replicas() == ops_manager.get_total_number_of_om_replicas() + + @skip_if_local + def test_om(self, ops_manager: MongoDBOpsManager, background_tester: OMBackgroundTester): + """Checks that the OM is responsive and test service is available (enabled by 'mms.testUtil.enabled').""" + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_test_service() + + # checking connectivity to each OM instance. + # We can only perform this test in the single cluster case because we don't create a service per pod so that + # every OM pod can be addressable by FQDN from a different cluster (the test pod cluster for example). + if not is_multi_cluster(): + om_tester.assert_om_instances_healthiness(ops_manager.pod_urls()) + + # checking the background thread to make sure the OM was ok during scale up + background_tester.assert_healthiness(allowed_rate_of_failure=0.1) + + +@pytest.mark.e2e_om_ops_manager_scale +class TestOpsManagerScaleDown: + """ + The OM resource is scaled to 1 node. This is expected to be quite fast and not availability. + TODO somehow we need to check that termination for OM pods happened successfully: CLOUDP-52310 + """ + + def test_scale_down_om(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + if is_multi_cluster(): + enable_multi_cluster_deployment(ops_manager, om_cluster_spec_list=[1, 1, 1]) + else: + ops_manager["spec"]["replicas"] = 1 + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=500) + + def test_number_of_replicas(self, ops_manager: MongoDBOpsManager): + for member_cluster_name in ops_manager.get_om_member_cluster_names(): + + statefulset = ops_manager.read_statefulset(member_cluster_name=member_cluster_name) + replicas = ops_manager.get_om_replicas_in_member_cluster(member_cluster_name=member_cluster_name) + if replicas != 0: + assert statefulset.status.ready_replicas == replicas + assert statefulset.status.current_replicas == replicas + + assert ops_manager.om_status().get_replicas() == ops_manager.get_total_number_of_om_replicas() + + @skip_if_local + def test_om(self, ops_manager: MongoDBOpsManager, background_tester: OMBackgroundTester): + """Checks that the OM is responsive""" + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + + # checking connectivity to a single pod + # We can only perform this test in the single cluster case because we don't create a service per pod so that + # every OM pod can be addressable by FQDN from a different cluster (the test pod cluster for example). + if not is_multi_cluster(): + om_tester.assert_om_instances_healthiness(ops_manager.pod_urls()) + + # OM was ok during scale down + background_tester.assert_healthiness(allowed_rate_of_failure=0.1) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_update_before_reconciliation.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_update_before_reconciliation.py new file mode 100644 index 000000000..c6b5edcd2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_update_before_reconciliation.py @@ -0,0 +1,41 @@ +import time +from datetime import datetime +from typing import Optional + +import pytest +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@pytest.mark.e2e_om_update_before_reconciliation +def test_create_om(ops_manager: MongoDBOpsManager, custom_appdb_version: str): + time.sleep(30) + + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["featureCompatibilityVersion"] = ".".join( + custom_appdb_version.split(".")[:2] + ) + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_upgrade.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_upgrade.py new file mode 100644 index 000000000..16bef51ea --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_upgrade.py @@ -0,0 +1,452 @@ +from time import sleep +from typing import Optional + +import pytest +import semver +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import MongoDB, try_load +from kubetester.awss3client import AwsS3Client +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import ( + is_default_architecture_static, + run_periodically, + skip_if_local, +) +from kubetester.mongodb import Phase, get_pods +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests import test_logger +from tests.conftest import is_multi_cluster +from tests.opsmanager.om_appdb_scram import OM_USER_NAME +from tests.opsmanager.om_ops_manager_backup import ( + OPLOG_RS_NAME, + S3_SECRET_NAME, + create_aws_secret, + create_s3_bucket, + new_om_data_store, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +# Current test focuses on Ops Manager upgrade which involves upgrade for both OpsManager and AppDB. +# MongoDBs are also upgraded. In case of major OM version upgrade (5.x -> 6.x) agents are expected to be upgraded +# for the existing MongoDBs. + +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_om_prev_version: str, + custom_mdb_prev_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_upgrade.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.allow_mdb_rc_versions() + resource.set_version(custom_om_prev_version) + resource.set_appdb_version(ensure_ent_version(custom_mdb_prev_version)) + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + try_load(resource) + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_prev_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development-oplog") + resource.set_version(custom_mdb_prev_version) + + try_load(resource) + return resource + + +@fixture(scope="module") +def mdb(ops_manager: MongoDBOpsManager, custom_mdb_prev_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name="my-replica-set", + ) + resource.set_version(custom_mdb_prev_version) + resource.configure(ops_manager, "development") + try_load(resource) + return resource + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestOpsManagerCreation: + """ + Creates an Ops Manager instance with AppDB of size 3. + """ + + def test_create_om(self, ops_manager: MongoDBOpsManager): + logger.info(f"Creating OM with version {ops_manager.get_version()}") + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + def test_gen_key_secret(self, ops_manager: MongoDBOpsManager): + for member_cluster_name in ops_manager.get_om_member_cluster_names(): + secret = ops_manager.read_gen_key_secret(member_cluster_name=member_cluster_name) + data = secret.data + assert "gen.key" in data + + def test_admin_key_secret(self, ops_manager: MongoDBOpsManager): + secret = ops_manager.read_api_key_secret() + data = secret.data + assert "publicKey" in data + assert "privateKey" in data + + @skip_if_local + def test_om(self, ops_manager: MongoDBOpsManager, custom_om_prev_version: str): + """Checks that the OM is responsive and test service is available (enabled by 'mms.testUtil.enabled').""" + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_version(custom_om_prev_version) + + om_tester.assert_test_service() + try: + om_tester.assert_support_page_enabled() + pytest.xfail("mms.helpAndSupportPage.enabled is expected to be false") + except AssertionError: + pass + + def test_appdb_scram_sha(self, ops_manager: MongoDBOpsManager): + auto_generated_password = ops_manager.read_appdb_generated_password() + automation_config_tester = ops_manager.get_automation_config_tester() + automation_config_tester.assert_authentication_mechanism_enabled("MONGODB-CR", False) + automation_config_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + ops_manager.get_appdb_tester().assert_scram_sha_authentication( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-1" + ) + ops_manager.get_appdb_tester().assert_scram_sha_authentication( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-256" + ) + + def test_generations(self, ops_manager: MongoDBOpsManager): + ops_manager.reload() + # The below two should be a no-op but in order to make this test rock solid, we need to ensure that everything + # is running without any interruptions. + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + assert ops_manager.appdb_status().get_observed_generation() == 1 + assert ops_manager.om_status().get_observed_generation() == 1 + assert ops_manager.backup_status().get_observed_generation() == 1 + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestBackupCreation: + def test_oplog_mdb_created( + self, + oplog_replica_set: MongoDB, + ): + oplog_replica_set.update() + oplog_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + def test_add_oplog_config(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"] = [ + {"name": "oplog1", "mongodbResourceRef": {"name": "my-mongodb-oplog"}} + ] + ops_manager.update() + + def test_backup_is_enabled(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + ignore_errors=True, + ) + + def test_generations(self, ops_manager: MongoDBOpsManager): + ops_manager.reload() + + # The below two should be a no-op but in order to make this test rock solid, we need to ensure that everything + # is running without any interruptions. + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + assert ops_manager.appdb_status().get_observed_generation() == 2 + assert ops_manager.om_status().get_observed_generation() == 2 + assert ops_manager.backup_status().get_observed_generation() == 2 + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestOpsManagerWithMongoDB: + def test_mongodb_create(self, mdb: MongoDB, custom_mdb_prev_version: str): + mdb.update() + + mdb.assert_reaches_phase(Phase.Running, timeout=600) + mdb.assert_connectivity() + mdb.tester().assert_version(custom_mdb_prev_version) + + def test_mongodb_upgrade(self, mdb: MongoDB): + """Scales up the mongodb. Note, that we are not upgrading the Mongodb version at this stage as it can be + the major update (e.g. 4.2 -> 4.4) and this requires OM upgrade as well - this happens later.""" + mdb.reload() + mdb["spec"]["members"] = 4 + + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=900) + mdb.assert_connectivity() + assert mdb.get_status_members() == 4 + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestOpsManagerConfigurationChange: + """ + The OM configuration changes: one property is removed, another is added. + Note, that this is quite an artificial change to make it testable, these properties affect the behaviour of different + endpoints in Ops Manager, so we can then check if the changes were propagated to OM + """ + + def test_restart_om(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["configuration"]["mms.testUtil.enabled"] = "" + ops_manager["spec"]["configuration"]["mms.helpAndSupportPage.enabled"] = "true" + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + def test_keys_not_modified( + self, + ops_manager: MongoDBOpsManager, + gen_key_resource_version: str, + admin_key_resource_version: str, + ): + """Making sure that the new reconciliation hasn't tried to generate new gen and api keys""" + gen_key_secret = ops_manager.read_gen_key_secret() + api_key_secret = ops_manager.read_api_key_secret() + + assert gen_key_secret.metadata.resource_version == gen_key_resource_version + assert api_key_secret.metadata.resource_version == admin_key_resource_version + + def test_om(self, ops_manager: MongoDBOpsManager): + """Checks that the OM is responsive and test service is not available""" + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_support_page_enabled() + try: + om_tester.assert_test_service() + pytest.xfail("mms.testUtil.enabled is expected to be false") + except AssertionError: + pass + + def test_generations(self, ops_manager: MongoDBOpsManager): + ops_manager.reload() + + # The below two should be a no-op but in order to make this test rock solid, we need to ensure that everything + # is running without any interruptions. + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.backup_status().assert_reaches_phase(Phase.Running) + + assert ops_manager.appdb_status().get_observed_generation() == 3 + assert ops_manager.om_status().get_observed_generation() == 3 + assert ops_manager.backup_status().get_observed_generation() == 3 + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestOpsManagerVersionUpgrade: + """ + The OM version is upgraded - this means the new image is deployed for both OM and appdb. + """ + + agent_version = None + + def test_agent_version(self, mdb: MongoDB): + if is_default_architecture_static: + # Containers will not call the upgrade endpoint. Therefore, agent_version is not part of AC + pod = client.CoreV1Api().read_namespaced_pod(mdb.name + "-0", mdb.namespace) + image_tag = pod.spec.containers[0].image.split(":")[-1] + TestOpsManagerVersionUpgrade.agent_version = image_tag + + else: + TestOpsManagerVersionUpgrade.agent_version = mdb.get_automation_config_tester().get_agent_version() + + def test_upgrade_om_version( + self, + ops_manager: MongoDBOpsManager, + custom_version: Optional[str], + custom_appdb_version: str, + ): + logger.info(f"Upgrading OM from {ops_manager.get_version()} to {custom_version}") + ops_manager.load() + # custom_version fixture loads CUSTOM_OM_VERSION env variable, which is set in context files with one of the + # values ops_manager_60_latest or ops_manager_70_latest in .evergreen.yml + ops_manager.set_version(custom_version) + ops_manager.set_appdb_version(custom_appdb_version) + + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + def test_image_url(self, ops_manager: MongoDBOpsManager): + pods = ops_manager.read_om_pods() + assert len(pods) == ops_manager.get_total_number_of_om_replicas() + for _, pod in pods: + assert ops_manager.get_version() in pod.spec.containers[0].image + + def test_om(self, ops_manager: MongoDBOpsManager): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_version(ops_manager.get_version()) + + def test_appdb(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + mdb_tester = ops_manager.get_appdb_tester() + mdb_tester.assert_connectivity() + mdb_tester.assert_version(custom_appdb_version) + + def test_appdb_scram_sha(self, ops_manager: MongoDBOpsManager): + auto_generated_password = ops_manager.read_appdb_generated_password() + automation_config_tester = ops_manager.get_automation_config_tester() + automation_config_tester.assert_authentication_mechanism_enabled("MONGODB-CR", False) + automation_config_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + ops_manager.get_appdb_tester().assert_scram_sha_authentication( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-1" + ) + ops_manager.get_appdb_tester().assert_scram_sha_authentication( + OM_USER_NAME, auto_generated_password, auth_mechanism="SCRAM-SHA-256" + ) + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestMongoDbsVersionUpgrade: + def test_mongodb_upgrade(self, mdb: MongoDB, custom_mdb_version: str): + """Ensures that the existing MongoDB works fine with the new Ops Manager (scales up one member) + Some details: + - in case of patch upgrade of OM the existing agent is guaranteed to work with the new OM - we don't require + the upgrade of all the agents + - in case of major/minor OM upgrade the agents MUST be upgraded before reconciling - so that's why the agents upgrade + is enforced before MongoDB reconciliation (the OM reconciliation happened above will drop the 'agents.nextScheduledTime' + counter and schedule an upgrade in the operator opsmanager reconcile) + """ + + mdb.assert_reaches_phase(Phase.Running, timeout=1200) + # Because OM was not in running phase, this resource, mdb, was also not in + # running phase. We will wait for it to come back before applying any changes. + mdb.reload() + + # At this point all the agent versions should be up to date and we can perform the database version upgrade. + mdb["spec"]["version"] = custom_mdb_version + mdb.update() + + # After the Ops Manager Upgrade, there's no time guarantees when a new manifest will be downloaded. + # Therefore, we may occasionally get "Invalid config: MongoDB version 8.0.0 is not available." + # This shouldn't happen very often at our customers as upgrading OM and MDB is usually separate processes. + mdb.assert_reaches_phase(Phase.Running, timeout=1200, ignore_errors=True) + mdb.assert_connectivity() + mdb.tester().assert_version(custom_mdb_version) + + def test_agents_upgraded(self, mdb: MongoDB, ops_manager: MongoDBOpsManager, custom_om_prev_version: str): + # The agents were requested to get upgraded immediately after Ops Manager upgrade. + # Note, that this happens only for OM major/minor upgrade, so we need to check only this case + prev_version = semver.VersionInfo.parse(custom_om_prev_version) + new_version = semver.VersionInfo.parse(ops_manager.get_version()) + if is_default_architecture_static(): + pod = client.CoreV1Api().read_namespaced_pod(mdb.name + "-0", mdb.namespace) + image_tag = pod.spec.containers[0].image.split(":")[-1] + if prev_version.major != new_version.major: + assert TestOpsManagerVersionUpgrade.agent_version != image_tag + else: + if prev_version.major != new_version.major or prev_version.minor != new_version.minor: + assert ( + TestOpsManagerVersionUpgrade.agent_version != mdb.get_automation_config_tester().get_agent_version() + ) + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestAppDBScramShaUpdated: + def test_appdb_reconcile(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["logLevel"] = "DEBUG" + ops_manager.update() + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + @pytest.mark.skip(reason="re-enable when only SCRAM-SHA-256 is supported for the AppDB") + def test_appdb_scram_sha_(self, ops_manager: MongoDBOpsManager): + automation_config_tester = ops_manager.get_automation_config_tester() + automation_config_tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + automation_config_tester.assert_authentication_mechanism_disabled("MONGODB-CR", False) + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestBackupDaemonVersionUpgrade: + def test_upgrade_backup_daemon( + self, + ops_manager: MongoDBOpsManager, + ): + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + ignore_errors=True, + ) + + def test_backup_daemon_image_url( + self, + ops_manager: MongoDBOpsManager, + ): + for _, pod in ops_manager.read_backup_pods(): + assert ops_manager.get_version() in pod.spec.containers[0].image + + +@pytest.mark.e2e_om_ops_manager_upgrade +class TestOpsManagerRemoved: + """ + Deletes an Ops Manager Custom resource and verifies that some of the dependant objects are removed + """ + + def test_opsmanager_deleted(self, ops_manager: MongoDBOpsManager): + ops_manager.delete() + + def om_is_clean(): + try: + ops_manager.load() + return False + except ApiException: + return True + + run_periodically(om_is_clean, timeout=180) + # Some strange race conditions/caching - the api key secret is still queryable right after OM removal + # (in openshift mainly) + sleep(20) + + def test_api_key_not_removed(self, ops_manager: MongoDBOpsManager): + """The API key must not be removed - this is for situations when the appdb is persistent - + so PVs may survive removal""" + ops_manager.read_api_key_secret() + + def test_gen_key_not_removed(self, ops_manager: MongoDBOpsManager): + """The gen key must not be removed - this is for situations when the appdb is persistent - + so PVs may survive removal""" + ops_manager.read_gen_key_secret() + + def test_om_sts_removed(self, ops_manager: MongoDBOpsManager): + for member_cluster_name in ops_manager.get_om_member_cluster_names(): + with pytest.raises(ApiException): + ops_manager.read_statefulset(member_cluster_name=member_cluster_name) + + def test_om_appdb_removed(self, ops_manager: MongoDBOpsManager): + for member_cluster_name in ops_manager.get_appdb_member_cluster_names(): + with pytest.raises(ApiException): + ops_manager.read_appdb_statefulset(member_cluster_name=member_cluster_name) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_weak_password.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_weak_password.py new file mode 100644 index 000000000..e1dbc94ca --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_weak_password.py @@ -0,0 +1,50 @@ +from typing import Optional + +import pytest +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@pytest.mark.e2e_om_weak_password +def test_update_secret_weak_password(ops_manager: MongoDBOpsManager): + data = KubernetesTester.read_secret(ops_manager.namespace, ops_manager.get_admin_secret_name()) + data["Password"] = "weak" + KubernetesTester.update_secret(ops_manager.namespace, ops_manager.get_admin_secret_name(), data) + + +@pytest.mark.e2e_om_weak_password +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Failed, msg_regexp=".*WEAK_PASSWORD.*", timeout=900) + + +@pytest.mark.e2e_om_weak_password +def test_fix_password(ops_manager: MongoDBOpsManager): + data = KubernetesTester.read_secret(ops_manager.namespace, ops_manager.get_admin_secret_name()) + data["Password"] = "Passw0rd." + KubernetesTester.update_secret(ops_manager.namespace, ops_manager.get_admin_secret_name(), data) + + +@pytest.mark.e2e_om_weak_password +def test_om_reaches_running(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_abandons_phase(Phase.Failed) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=150) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_remotemode.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_remotemode.py new file mode 100644 index 000000000..c196ffc3f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_remotemode.py @@ -0,0 +1,220 @@ +import time +from typing import Any, Dict, Optional + +import yaml +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +VERSION_NOT_IN_WEB_SERVER = "4.2.1" + + +def add_mdb_version_to_deployment(deployment: Dict[str, Any], version: str): + """ + Adds a new initContainer to `deployment` to download a particular MongoDB version. + + Please note that the initContainers will never fail, so it is fine to add version that don't + exist for older distributions (like mdb5.0 in ubuntu1604). + """ + mount_path = "/mongodb-ops-manager/mongodb-releases/linux" + distros = ("rhel8", "rhel80", "ubuntu1604", "ubuntu1804") + + base_url_community = "https://fastdl.mongodb.org/linux/mongodb-linux-x86_64" + base_url_enterprise = "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise" + base_url = base_url_community + if version.endswith("-ent"): + # If version is enterprise, the base_url changes slightly + base_url = base_url_enterprise + version = version.replace("-ent", "") + + if "initContainers" not in deployment["spec"]["template"]["spec"]: + deployment["spec"]["template"]["spec"]["initContainers"] = [] + + for distro in distros: + url = f"{base_url}-{distro}-{version}.tgz" + curl_command = f"curl -LO {url} --output-dir {mount_path}" + + container = { + "name": KubernetesTester.random_k8s_name(prefix="mdb-download"), + "image": "curlimages/curl:latest", + "command": ["sh", "-c", f"{curl_command} && true"], + "volumeMounts": [ + { + "name": "mongodb-versions", + "mountPath": mount_path, + } + ], + } + deployment["spec"]["template"]["spec"]["initContainers"].append(container) + + +@fixture(scope="module") +def nginx(namespace: str, custom_mdb_version: str, custom_appdb_version: str): + with open(yaml_fixture("remote_fixtures/nginx-config.yaml"), "r") as f: + config_body = yaml.safe_load(f.read()) + KubernetesTester.clients("corev1").create_namespaced_config_map(namespace, config_body) + + with open(yaml_fixture("remote_fixtures/nginx.yaml"), "r") as f: + nginx_body = yaml.safe_load(f.read()) + + # Adds versions to Nginx deployment. + new_versions = set() + new_versions.add(custom_mdb_version) + new_versions.add(custom_mdb_version + "-ent") + new_versions.add(custom_appdb_version) + + for version in new_versions: + add_mdb_version_to_deployment(nginx_body, version) + + KubernetesTester.create_deployment(namespace, body=nginx_body) + + with open(yaml_fixture("remote_fixtures/nginx-svc.yaml"), "r") as f: + service_body = yaml.safe_load(f.read()) + KubernetesTester.create_service(namespace, body=service_body) + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str, nginx) -> MongoDBOpsManager: + """The fixture for Ops Manager to be created.""" + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("remote_fixtures/om_remotemode.yaml"), + namespace=namespace, + ) + om["spec"]["configuration"]["automation.versions.source"] = "remote" + om["spec"]["configuration"][ + "automation.versions.download.baseUrl" + ] = f"http://nginx-svc.{namespace}.svc.cluster.local:80" + + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + om.allow_mdb_rc_versions() + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + om.update() + return om + + +@fixture(scope="module") +def replica_set(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + ).configure(ops_manager, "my-replica-set") + resource.set_version(custom_mdb_version) + yield resource.create() + + +@fixture(scope="module") +def replica_set_ent(ops_manager: MongoDBOpsManager, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name="the-replica-set-ent", + ).configure(ops_manager, "my-other-replica-set") + resource.set_version(ensure_ent_version(custom_mdb_version)) + yield resource.create() + + +@mark.e2e_om_remotemode +def test_appdb(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + assert ops_manager.appdb_status().get_members() == 3 + + +@skip_if_local +@mark.e2e_om_remotemode +def test_appdb_mongod(ops_manager: MongoDBOpsManager): + mdb_tester = ops_manager.get_appdb_tester() + mdb_tester.assert_connectivity() + + +@mark.e2e_om_remotemode +def test_ops_manager_reaches_running_phase(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + # CLOUDP-83792: some insight: OM has a number of Cron jobs and one of them is responsible for filtering the builds + # returned in the automation config to include only the available ones (in remote/local modes). + # Somehow though as of OM 4.4.9 this filtering didn't work fine and some Enterprise builds were not returned so + # the replica sets using enterprise versions didn't reach the goal. + # We need to sleep for some time to let the cron get into the game and this allowed to reproduce the issue + # (got fixed by switching off the cron by 'automation.versions.download.baseUrl.allowOnlyAvailableBuilds: false') + print("Sleeping for one minute to let Ops Manager Cron jobs kick in") + time.sleep(60) + + +@mark.e2e_om_remotemode +def test_replica_sets_reaches_running_phase(replica_set: MongoDB, replica_set_ent: MongoDB): + """Doing this in parallel for faster success""" + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + replica_set_ent.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_om_remotemode +def test_replica_set_reaches_failed_phase(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["version"] = VERSION_NOT_IN_WEB_SERVER + replica_set.update() + + # ReplicaSet times out attempting to fetch version from web server + replica_set.assert_reaches_phase(Phase.Failed, timeout=200) + + +@mark.e2e_om_remotemode +def test_replica_set_recovers(replica_set: MongoDB, custom_mdb_version: str): + replica_set["spec"]["version"] = custom_mdb_version + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@skip_if_local +@mark.e2e_om_remotemode +def test_client_can_connect_to_mongodb(replica_set: MongoDB): + replica_set.assert_connectivity() + + +@skip_if_local +@mark.e2e_om_remotemode +def test_client_can_connect_to_mongodb_ent(replica_set_ent: MongoDB): + replica_set_ent.assert_connectivity() + + +@skip_if_local +@mark.e2e_om_remotemode +def test_client_can_connect_to_mongodb_ent(replica_set_ent: MongoDB): + replica_set_ent.assert_connectivity() + + +@mark.e2e_om_remotemode +def test_restart_ops_manager_pod(ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["configuration"]["mms.testUtil.enabled"] = "false" + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_remotemode +def test_can_scale_replica_set(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["members"] = 5 + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@skip_if_local +@mark.e2e_om_remotemode +def test_client_can_still_connect(replica_set: MongoDB): + replica_set.assert_connectivity() + + +@skip_if_local +@mark.e2e_om_remotemode +def test_client_can_still_connect_to_ent(replica_set_ent: MongoDB): + replica_set_ent.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/om_validation_webhook.py b/docker/mongodb-enterprise-tests/tests/opsmanager/om_validation_webhook.py new file mode 100644 index 000000000..258442063 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/om_validation_webhook.py @@ -0,0 +1,73 @@ +""" +Ensures that validation warnings for ops manager reflect its current state +""" + +from typing import Optional + +import pytest +from kubernetes.client.rest import ApiException +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark + +APPDB_SHARD_COUNT_WARNING = "ShardCount field is not configurable for application databases as it is for sharded clusters and appdbs are replica sets" + + +@mark.e2e_om_validation_webhook +def test_wait_for_webhook(namespace: str, default_operator: Operator): + default_operator.wait_for_webhook() + + +def om_validation(namespace: str) -> MongoDBOpsManager: + return MongoDBOpsManager.from_yaml(yaml_fixture("om_validation.yaml"), namespace=namespace) + + +@mark.e2e_om_validation_webhook +def test_connectivity_not_allowed_in_appdb(namespace: str): + om = om_validation(namespace) + + om["spec"]["applicationDatabase"]["connectivity"] = {"replicaSetHorizons": [{"test-horizon": "dfdfdf"}]} + + with pytest.raises( + ApiException, + match=r"connectivity field is not configurable for application databases", + ): + om.create() + + +@mark.e2e_om_validation_webhook +def test_opsmanager_version(namespace: str): + om = om_validation(namespace) + om["spec"]["version"] = "4.4.4.4" + + with pytest.raises(ApiException, match=r"is an invalid value for spec.version"): + om.create() + + +@mark.e2e_om_validation_webhook +def test_appdb_version(namespace: str): + om = om_validation(namespace) + om["spec"]["applicationDatabase"]["version"] = "4.4.10.10" + + # this exception is raised by CRD regexp validation for the version, not our internal one + with pytest.raises(ApiException, match=r"spec.applicationDatabase.version in body should match"): + om.create() + + om["spec"]["applicationDatabase"]["version"] = "3.6.12" + with pytest.raises(ApiException, match=r"the version of Application Database must be \\u003e= 4.0"): + om.create() + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + return om.create() + + +@mark.e2e_om_validation_webhook +class TestOpsManagerValidationWarnings: + def test_disable_webhook(self, default_operator: Operator): + default_operator.disable_webhook() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/__init__.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/conftest.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/conftest.py new file mode 100644 index 000000000..d00610675 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/conftest.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +from itertools import zip_longest +from typing import Optional + +import kubernetes +from kubetester.opsmanager import MongoDBOpsManager +from tests.conftest import get_central_cluster_client, is_multi_cluster +from tests.multicluster.conftest import cluster_spec_list + + +def pytest_runtest_setup(item): + """This allows to automatically install the Operator and enable AppDB monitoring before running any test""" + if is_multi_cluster(): + if "multi_cluster_operator_with_monitored_appdb" not in item.fixturenames: + print("\nAdding operator installation fixture: multi_cluster_operator_with_monitored_appdb") + item.fixturenames.insert(0, "multi_cluster_operator_with_monitored_appdb") + else: + if "operator_with_monitored_appdb" not in item.fixturenames: + print("\nAdding operator installation fixture: operator_with_monitored_appdb") + item.fixturenames.insert(0, "operator_with_monitored_appdb") + + +# TODO move to conftest up the test hierarchy? +def get_appdb_member_cluster_names(): + return ["kind-e2e-cluster-2", "kind-e2e-cluster-3"] + + +def get_om_member_cluster_names(): + return ["kind-e2e-cluster-1", "kind-e2e-cluster-2", "kind-e2e-cluster-3"] + + +def enable_multi_cluster_deployment( + resource: MongoDBOpsManager, + om_cluster_spec_list: Optional[list[int]] = None, + appdb_cluster_spec_list: Optional[list[int]] = None, + appdb_member_configs: Optional[list[list[dict]]] = None, +): + resource["spec"]["topology"] = "MultiCluster" + backup_configs = None + + if om_cluster_spec_list is None: + om_cluster_spec_list = [1, 1, 1] + + if appdb_cluster_spec_list is None: + appdb_cluster_spec_list = [1, 2] + + # The operator defaults to enabling backup with 1 member if not specified in the CR so we simulate + # the behavior in the test here + backup = resource["spec"].get("backup") + if backup is None: + resource["spec"]["backup"] = {"enabled": True, "members": 1} + + if resource["spec"].get("backup", {}).get("enabled", False): + desired_members = resource["spec"].get("backup", {}).get("members", 1) + # Here we divide the desired backup members evenly on the member clusters. + # We add 1 extra backup member per cluster for the first *remainder* member clusters + # so that we get exactly as many members were requested in the single cluster case. + # Example (5 total members and 3 member clusters): + # 5 // 3 = 1 members per member cluster + # 5 % 3 = 2 extra members that get assigned to the first and second cluster + members_per_cluster = desired_members // len(get_om_member_cluster_names()) + remainder_members = [1] * (desired_members % len(get_om_member_cluster_names())) + # Using the above example, here we are zipping [1, 1] and [1, 1, 1] with a fill value of 0, + # so we end up with the pairs [(1, 1), (1, 1), (1, 0)] and the backup configs: + # [{"members": 2}, {"members": 2}, {"members": 1}] + backup_configs = [ + {"members": sum(count)} + for count in zip_longest( + remainder_members, [members_per_cluster] * len(get_om_member_cluster_names()), fillvalue=0 + ) + ] + + resource["spec"]["clusterSpecList"] = cluster_spec_list( + get_om_member_cluster_names(), om_cluster_spec_list, backup_configs=backup_configs + ) + resource["spec"]["applicationDatabase"]["topology"] = "MultiCluster" + resource["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + get_appdb_member_cluster_names(), appdb_cluster_spec_list, appdb_member_configs + ) + resource.api = kubernetes.client.CustomObjectsApi(api_client=get_central_cluster_client()) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_external_connectivity.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_external_connectivity.py new file mode 100644 index 000000000..e4a4dc064 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_external_connectivity.py @@ -0,0 +1,151 @@ +import os +from typing import Optional + +from kubetester import create_or_update_secret, get_service +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.common.placeholders import placeholders +from tests.conftest import ( + create_appdb_certs, + default_external_domain, + external_domain_fqdns, + update_coredns_hosts, +) + +OM_NAME = "om-appdb-external" +APPDB_NAME = f"{OM_NAME}-db" +APPDB_EXTERNAL_DOMAINS = external_domain_fqdns(APPDB_NAME, 3) + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + return create_ops_manager_tls_certs(issuer, namespace, OM_NAME) + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, APPDB_NAME, additional_domains=APPDB_EXTERNAL_DOMAINS) + + +@fixture(scope="module") +@mark.usefixtures("appdb_certs", "ops_manager_certs", "issuer_ca_configmap") +def ops_manager( + namespace: str, + issuer_ca_configmap: str, + appdb_certs: str, + ops_manager_certs: str, + custom_version: Optional[str], + custom_appdb_version: str, + issuer_ca_filepath: str, +) -> MongoDBOpsManager: + create_or_update_secret(namespace, "appdb-secret", {"password": "Hello-World!"}) + + print("Creating OM object") + om = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_appdb_monitoring_tls.yaml"), name=OM_NAME, namespace=namespace + ) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + om["spec"]["applicationDatabase"]["externalAccess"] = { + "externalDomain": default_external_domain(), + "externalService": { + "spec": { + "type": "LoadBalancer", + "publishNotReadyAddresses": False, + "ports": [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing2", + "port": 27019, + }, + ], + }, + }, + } + + # ensure the requests library will use this CA when communicating with Ops Manager + os.environ["REQUESTS_CA_BUNDLE"] = issuer_ca_filepath + + return om + + +@mark.e2e_om_appdb_external_connectivity +def test_configure_dns(): + host_mappings = [ + ( + "172.18.255.200", + APPDB_EXTERNAL_DOMAINS[0], + ), + ( + "172.18.255.201", + APPDB_EXTERNAL_DOMAINS[1], + ), + ( + "172.18.255.202", + APPDB_EXTERNAL_DOMAINS[2], + ), + ] + + update_coredns_hosts( + host_mappings=host_mappings, + ) + + +@mark.e2e_om_appdb_external_connectivity +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.assert_appdb_preferred_hostnames_are_added() + ops_manager.assert_appdb_hostnames_are_correct() + + +@mark.e2e_om_appdb_external_connectivity +def test_appdb_group_is_monitored(ops_manager: MongoDBOpsManager): + ops_manager.assert_appdb_monitoring_group_was_created() + ops_manager.assert_monitoring_data_exists() + + +@mark.e2e_om_appdb_external_connectivity +def test_service_exists(namespace: str): + for i in range(3): + service = get_service( + namespace, + f"{APPDB_NAME}-{i}-svc-external", + ) + assert service.spec.type == "LoadBalancer" + assert service.spec.ports[0].port == 27017 + assert service.spec.ports[1].port == 27018 + assert service.spec.ports[2].port == 27019 + + +@mark.e2e_om_appdb_external_connectivity +def test_placeholders_in_external_services(ops_manager: MongoDBOpsManager, namespace: str): + ops_manager.load() + + ops_manager["spec"]["applicationDatabase"]["externalAccess"]["externalService"][ + "annotations" + ] = placeholders.get_annotations_with_placeholders_for_single_cluster() + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=300) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=300) + + for pod_idx in range(3): + service = get_service(namespace, f"{APPDB_NAME}-{pod_idx}-svc-external") + assert ( + service.metadata.annotations + == placeholders.get_expected_annotations_single_cluster_with_external_domain( + APPDB_NAME, namespace, pod_idx, default_external_domain() + ) + ) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_flags_and_config.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_flags_and_config.py new file mode 100644 index 000000000..d61944c42 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_flags_and_config.py @@ -0,0 +1,403 @@ +from typing import Optional + +from kubetester import find_fixture +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml( + find_fixture("om_validation.yaml"), namespace=namespace, name="om-agent-flags" + ) + + # both monitoring and automation agent should see these changes + resource["spec"]["applicationDatabase"]["agent"] = { + "startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFile"} + } + member1_config = { + "votes": 1, + "priority": "0.5", + "tags": { + "tag1": "value1", + "environment": "prod", + }, + } + member2_config = { + "votes": 1, + "priority": "1.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + member3_config = { + "votes": 1, + "priority": "0.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource, + appdb_cluster_spec_list=[1, 2], + appdb_member_configs=[[member1_config], [member2_config, member3_config]], + ) + else: + resource["spec"]["applicationDatabase"]["memberConfig"] = [member1_config, member2_config, member3_config] + + resource.update() + return resource + + +@mark.e2e_om_appdb_flags_and_config +def test_appdb(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_appdb_flags_and_config +def test_om(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_om_appdb_flags_and_config +def test_monitoring_is_configured(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_appdb_flags_and_config +def test_appdb_has_agent_flags(ops_manager: MongoDBOpsManager): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFile* | wc -l", + ] + for api_client, pod in ops_manager.read_appdb_pods(): + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + ops_manager.namespace, + cmd, + container="mongodb-agent", + api_client=api_client, + ) + assert result != "0" + + +@mark.e2e_om_appdb_flags_and_config +def test_appdb_monitoring_agent_flags_inherit_automation_agent_flags( + ops_manager: MongoDBOpsManager, +): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFileMonitoring* | wc -l", + ] + for api_client, pod in ops_manager.read_appdb_pods(): + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + ops_manager.namespace, + cmd, + container="mongodb-agent-monitoring", + api_client=api_client, + ) + assert "No such file or directory" in result + + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFile* | wc -l", + ] + for api_client, pod in ops_manager.read_appdb_pods(): + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + ops_manager.namespace, + cmd, + container="mongodb-agent-monitoring", + api_client=api_client, + ) + assert result != "0" + + +@mark.e2e_om_appdb_flags_and_config +def test_appdb_flags_changed(ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["agent"]["startupOptions"]["dialTimeoutSeconds"] = "70" + + ops_manager["spec"]["applicationDatabase"]["monitoringAgent"] = { + "startupOptions": { + "logFile": "/var/log/mongodb-mms-automation/customLogFileMonitoring", + "dialTimeoutSeconds": "80", + } + } + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_om_appdb_flags_and_config +def test_appdb_has_changed_agent_flags(ops_manager: MongoDBOpsManager, namespace: str): + MMS_AUTOMATION_AGENT_PGREP = [ + "/bin/sh", + "-c", + "pgrep -f -a agent/mongodb-agent", + ] + for api_client, pod in ops_manager.read_appdb_pods(): + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + namespace, + MMS_AUTOMATION_AGENT_PGREP, + container="mongodb-agent", + api_client=api_client, + ) + assert "-logFile=/var/log/mongodb-mms-automation/customLogFile" in result + assert "-dialTimeoutSeconds=70" in result + + result = KubernetesTester.run_command_in_pod_container( + pod.metadata.name, + namespace, + MMS_AUTOMATION_AGENT_PGREP, + container="mongodb-agent-monitoring", + api_client=api_client, + ) + assert "-logFile=/var/log/mongodb-mms-automation/customLogFileMonitoring" in result + assert "-dialTimeoutSeconds=80" in result + + +@mark.e2e_om_appdb_flags_and_config +def test_automation_config_secret_member_options(ops_manager: MongoDBOpsManager): + members = ops_manager.get_automation_config_tester().get_replica_set_members(ops_manager.app_db_name()) + + assert members[0]["votes"] == 1 + assert members[0]["priority"] == 0.5 + assert members[0]["tags"] == {"environment": "prod", "tag1": "value1"} + + assert members[1]["votes"] == 1 + assert members[1]["priority"] == 1.5 + assert members[1]["tags"] == {"environment": "prod", "tag2": "value2"} + + assert members[2]["votes"] == 1 + assert members[2]["priority"] == 0.5 + assert members[2]["tags"] == {"environment": "prod", "tag2": "value2"} + + +@mark.e2e_om_appdb_flags_and_config +def test_update_appdb_member_options(ops_manager: MongoDBOpsManager): + member1_config = { + "votes": 1, + "priority": "0.5", + "tags": { + "tag1": "value1", + "environment": "prod", + }, + } + member2_config = { + "votes": 1, + "priority": "1.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + member3_config = { + "votes": 0, + "priority": "0", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + ops_manager.load() + if is_multi_cluster(): + enable_multi_cluster_deployment( + ops_manager, + appdb_cluster_spec_list=[1, 2], + appdb_member_configs=[[member1_config], [member2_config, member3_config]], + ) + else: + ops_manager["spec"]["applicationDatabase"]["memberConfig"] = [member1_config, member2_config, member3_config] + ops_manager.update() + + ops_manager.appdb_status().assert_abandons_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_appdb_flags_and_config +def test_automation_config_secret_updated_member_options(ops_manager: MongoDBOpsManager): + members = ops_manager.get_automation_config_tester().get_replica_set_members(ops_manager.app_db_name()) + + assert members[0]["votes"] == 1 + assert members[0]["priority"] == 0.5 + assert members[0]["tags"] == {"environment": "prod", "tag1": "value1"} + + assert members[1]["votes"] == 1 + assert members[1]["priority"] == 1.5 + assert members[1]["tags"] == {"environment": "prod", "tag2": "value2"} + + assert members[2]["votes"] == 0 + assert members[2]["priority"] == 0.0 + assert members[2]["tags"] == {"environment": "prod", "tag2": "value2"} + + +@mark.e2e_om_appdb_flags_and_config +def test_scale_up_appdb_with_member_options(ops_manager: MongoDBOpsManager): + member1_config = { + "votes": 1, + "priority": "0.5", + "tags": { + "tag1": "value1", + "environment": "prod", + }, + } + member2_config = { + "votes": 1, + "priority": "1.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + member3_config = { + "votes": 0, + "priority": "0", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + member4_config = { + "votes": 1, + "priority": "2.0", + "tags": { + "tag2": "value4", + "environment": "superprod", + }, + } + member5_config = { + "votes": 1, + "priority": "3.0", + "tags": { + "tag2": "value5", + "environment": "superprod", + }, + } + + ops_manager.load() + if is_multi_cluster(): + enable_multi_cluster_deployment( + ops_manager, + appdb_cluster_spec_list=[2, 3], + appdb_member_configs=[[member1_config, member4_config], [member2_config, member3_config, member5_config]], + ) + else: + ops_manager["spec"]["applicationDatabase"]["memberConfig"] = [ + member1_config, + member2_config, + member3_config, + member4_config, + member5_config, + ] + ops_manager["spec"]["applicationDatabase"]["members"] = 5 + ops_manager.update() + + ops_manager.appdb_status().assert_abandons_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_appdb_flags_and_config +def test_automation_config_secret_scale_up_updated_member_options(ops_manager: MongoDBOpsManager): + members = ops_manager.get_automation_config_tester().get_replica_set_members(ops_manager.app_db_name()) + + assert members[0]["votes"] == 1 + assert members[0]["priority"] == 0.5 + assert members[0]["tags"] == {"environment": "prod", "tag1": "value1"} + + assert members[1]["votes"] == 1 + assert members[1]["priority"] == 1.5 + assert members[1]["tags"] == {"environment": "prod", "tag2": "value2"} + + assert members[2]["votes"] == 0 + assert members[2]["priority"] == 0.0 + assert members[2]["tags"] == {"environment": "prod", "tag2": "value2"} + + assert members[3]["votes"] == 1 + assert members[3]["priority"] == 2.0 + assert members[3]["tags"] == {"environment": "superprod", "tag2": "value4"} + + assert members[4]["votes"] == 1 + assert members[4]["priority"] == 3.0 + assert members[4]["tags"] == {"environment": "superprod", "tag2": "value5"} + + +@mark.e2e_om_appdb_flags_and_config +def test_scale_down_appdb__with_member_options(ops_manager: MongoDBOpsManager): + member1_config = { + "votes": 1, + "priority": "0.5", + "tags": { + "tag1": "value1", + "environment": "prod", + }, + } + member2_config = { + "votes": 1, + "priority": "1.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + } + member4_config = { + "votes": 1, + "priority": "1.7", + "tags": { + "tag2": "value4", + "environment": "superprod", + }, + } + + ops_manager.load() + if is_multi_cluster(): + enable_multi_cluster_deployment( + ops_manager, + appdb_cluster_spec_list=[2, 1], + appdb_member_configs=[[member1_config, member4_config], [member2_config]], + ) + else: + ops_manager["spec"]["applicationDatabase"]["memberConfig"] = [ + member1_config, + member2_config, + member4_config, + ] + ops_manager["spec"]["applicationDatabase"]["members"] = 3 + ops_manager.update() + + ops_manager.appdb_status().assert_abandons_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@mark.e2e_om_appdb_flags_and_config +def test_automation_config_secret_scale_down_updated_member_options(ops_manager: MongoDBOpsManager): + members = ops_manager.get_automation_config_tester().get_replica_set_members(ops_manager.app_db_name()) + + assert members[0]["votes"] == 1 + assert members[0]["priority"] == 0.5 + assert members[0]["tags"] == {"environment": "prod", "tag1": "value1"} + + assert members[1]["votes"] == 1 + assert members[1]["priority"] == 1.5 + assert members[1]["tags"] == {"environment": "prod", "tag2": "value2"} + + assert members[2]["votes"] == 1 + assert members[2]["priority"] == 1.7 + assert members[2]["tags"] == {"environment": "superprod", "tag2": "value4"} diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_scale_up_down.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_scale_up_down.py new file mode 100644 index 000000000..c490508a9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_scale_up_down.py @@ -0,0 +1,144 @@ +from typing import Optional + +import pytest +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import is_multi_cluster + +# Important - you need to ensure that OM and Appdb images are build and pushed into your current docker registry before +# running tests locally - use "make om-image" and "make appdb" to do this + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_appdb_scale_up_down.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + resource.update() + return resource + + +@pytest.mark.e2e_om_appdb_scale_up_down +class TestOpsManagerCreation: + """ + Creates an Ops Manager instance with AppDB of size 3. Note, that the initial creation usually takes ~500 seconds + """ + + def test_create_om(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1200) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + def test_gen_key_secret(self, ops_manager: MongoDBOpsManager): + secret = ops_manager.read_gen_key_secret() + data = secret.data + assert "gen.key" in data + + def test_admin_key_secret(self, ops_manager: MongoDBOpsManager): + secret = ops_manager.read_api_key_secret() + data = secret.data + assert "publicKey" in data + assert "privateKey" in data + + def test_appdb_connection_url_secret(self, ops_manager: MongoDBOpsManager): + assert len(ops_manager.read_appdb_members_from_connection_url_secret()) == 3 + + def test_appdb(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + assert ops_manager.appdb_status().get_version() == custom_appdb_version + + assert ops_manager.appdb_status().get_members() == 3 + if not is_multi_cluster(): + for _, cluster_spec_item in ops_manager.get_appdb_indexed_cluster_spec_items(): + member_cluster_name = cluster_spec_item["clusterName"] + statefulset = ops_manager.read_appdb_statefulset(member_cluster_name) + assert statefulset.status.ready_replicas == 3 + assert statefulset.status.current_replicas == 3 + + def test_appdb_monitoring_group_was_created(self, ops_manager: MongoDBOpsManager): + ops_manager.assert_appdb_monitoring_group_was_created() + + def test_admin_config_map(self, ops_manager: MongoDBOpsManager): + ops_manager.get_automation_config_tester().reached_version(1) + + @skip_if_local + def test_om_connectivity(self, ops_manager: MongoDBOpsManager): + ops_manager.get_om_tester().assert_healthiness() + # todo check the backing db group, automation config and data integrity + + +@pytest.mark.e2e_om_appdb_scale_up_down +class TestOpsManagerAppDbScaleUp: + """ + Scales appdb up to 5 members + """ + + def test_scale_app_db_up(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["members"] = 5 + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + def test_keys_not_touched( + self, + ops_manager: MongoDBOpsManager, + gen_key_resource_version: str, + admin_key_resource_version: str, + ): + """Making sure that the new reconciliation hasn't tried to generate new gen and api keys""" + gen_key_secret = ops_manager.read_gen_key_secret() + api_key_secret = ops_manager.read_api_key_secret() + + assert gen_key_secret.metadata.resource_version == gen_key_resource_version + assert api_key_secret.metadata.resource_version == admin_key_resource_version + + def test_appdb_connection_url_secret(self, ops_manager: MongoDBOpsManager): + assert len(ops_manager.read_appdb_members_from_connection_url_secret()) == 5 + + def test_appdb(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + assert ops_manager.appdb_status().get_members() == 5 + assert ops_manager.appdb_status().get_version() == custom_appdb_version + + statefulset = ops_manager.read_appdb_statefulset() + assert statefulset.status.ready_replicas == 5 + assert statefulset.status.current_replicas == 5 + + def test_admin_config_map(self, ops_manager: MongoDBOpsManager): + ops_manager.get_automation_config_tester().reached_version(2) + + @skip_if_local + def test_om_connectivity(self, ops_manager: MongoDBOpsManager): + ops_manager.get_om_tester().assert_healthiness() + + +@pytest.mark.e2e_om_appdb_scale_up_down +class TestOpsManagerAppDbScaleDown: + """ + name: Ops Manager successful appdb scale down + """ + + def test_scale_app_db_down(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["members"] = 3 + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=1000) + + def test_appdb(self, ops_manager: MongoDBOpsManager): + assert ops_manager.appdb_status().get_members() == 3 + if not is_multi_cluster(): + statefulset = ops_manager.read_appdb_statefulset() + assert statefulset.status.ready_replicas == 3 + assert statefulset.status.current_replicas == 3 + + def test_admin_config_map(self, ops_manager: MongoDBOpsManager): + ops_manager.get_automation_config_tester().reached_version(3) + + @skip_if_local + def test_om_connectivity(self, ops_manager: MongoDBOpsManager): + ops_manager.get_om_tester().assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_upgrade.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_upgrade.py new file mode 100644 index 000000000..5a971e173 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_upgrade.py @@ -0,0 +1,176 @@ +from typing import Optional + +import pytest +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +gen_key_resource_version = None +admin_key_resource_version = None + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_mdb_prev_version: str) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_appdb_upgrade.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(ensure_ent_version(custom_mdb_prev_version)) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@pytest.mark.e2e_om_appdb_upgrade +class TestOpsManagerCreation: + """ + Creates an Ops Manager instance with AppDB of size 3. The test waits until the AppDB is ready, not the OM resource + """ + + def test_appdb(self, ops_manager: MongoDBOpsManager, custom_mdb_prev_version: str): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + assert ops_manager.appdb_status().get_members() == 3 + + assert ops_manager.appdb_status().get_version() == ensure_ent_version(custom_mdb_prev_version) + db_pods = ops_manager.read_appdb_pods() + for _, pod in db_pods: + # the appdb pod container 'mongodb' by default has 500M + assert pod.spec.containers[1].resources.requests["memory"] == "500M" + + def test_admin_config_map(self, ops_manager: MongoDBOpsManager): + ops_manager.get_automation_config_tester().reached_version(1) + + @skip_if_local + def test_mongod(self, ops_manager: MongoDBOpsManager, custom_mdb_prev_version: str): + mdb_tester = ops_manager.get_appdb_tester() + mdb_tester.assert_connectivity() + mdb_tester.assert_version(custom_mdb_prev_version) + + def test_appdb_automation_config(self, ops_manager: MongoDBOpsManager): + expected_roles = { + ("admin", "readWriteAnyDatabase"), + ("admin", "dbAdminAnyDatabase"), + ("admin", "clusterMonitor"), + ("admin", "hostManager"), + ("admin", "backup"), + ("admin", "restore"), + } + + # only user should be the Ops Manager user + tester = ops_manager.get_automation_config_tester() + tester.assert_authentication_mechanism_enabled("SCRAM-SHA-256", False) + tester.assert_has_user("mongodb-ops-manager") + tester.assert_user_has_roles("mongodb-ops-manager", expected_roles) + tester.assert_expected_users(1) + tester.assert_authoritative_set(False) + + @skip_if_local + def test_appdb_scram_sha(self, ops_manager: MongoDBOpsManager): + app_db_tester = ops_manager.get_appdb_tester() + app_db_tester.assert_scram_sha_authentication( + "mongodb-ops-manager", + ops_manager.read_appdb_generated_password(), + auth_mechanism="SCRAM-SHA-256", + ) + + def test_appdb_mongodb_options(self, ops_manager: MongoDBOpsManager): + automation_config_tester = ops_manager.get_automation_config_tester() + for process in automation_config_tester.get_replica_set_processes(ops_manager.app_db_name()): + assert process["args2_6"]["operationProfiling"]["mode"] == "slowOp" + + def test_om_reaches_running(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + def test_appdb_reaches_running(self, ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + def test_appdb_monitoring_is_configured(self, ops_manager: MongoDBOpsManager): + ops_manager.assert_appdb_monitoring_group_was_created() + + def test_om_running(self, ops_manager: MongoDBOpsManager): + ops_manager.get_om_tester().assert_healthiness() + + # TODO check the persistent volumes created + + +@pytest.mark.e2e_om_appdb_upgrade +class TestOpsManagerAppDbUpdateMemory: + """ + Changes memory limits requirements for the AppDB + """ + + def test_appdb_updated(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["podSpec"] = { + "podTemplate": { + "spec": { + "containers": [ + { + "name": "mongodb-agent", + "resources": { + "requests": { + "memory": "350M", + }, + }, + } + ] + } + } + } + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + def test_appdb(self, ops_manager: MongoDBOpsManager): + db_pods = ops_manager.read_appdb_pods() + for _, pod in db_pods: + assert pod.spec.containers[1].resources.requests["memory"] == "350M" + + def test_admin_config_map(self, ops_manager: MongoDBOpsManager): + # The version hasn't changed as there were no changes to the automation config + ops_manager.get_automation_config_tester().reached_version(2) + + def test_om_is_running(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.get_om_tester().assert_healthiness() + + +@pytest.mark.e2e_om_appdb_upgrade +class TestOpsManagerMixed: + """ + Performs changes to both AppDB and Ops Manager spec + """ + + def test_appdb_and_om_updated(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + ops_manager.load() + ops_manager.set_appdb_version(custom_appdb_version) + ops_manager["spec"]["configuration"] = {"mms.helpAndSupportPage.enabled": "true"} + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + ops_manager.backup_status().assert_reaches_phase(Phase.Disabled) + + def test_appdb(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + assert ops_manager.appdb_status().get_members() == 3 + + assert ops_manager.appdb_status().get_version() == custom_appdb_version + + def test_mongod(self, ops_manager: MongoDBOpsManager, custom_appdb_version: str): + mdb_tester = ops_manager.get_appdb_tester() + mdb_tester.assert_connectivity() + mdb_tester.assert_version(custom_appdb_version) + + def test_om_connectivity(self, ops_manager: MongoDBOpsManager): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + om_tester.assert_support_page_enabled() diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_appdb_monitoring_tls.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_appdb_monitoring_tls.py new file mode 100644 index 000000000..480e7fc52 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_appdb_monitoring_tls.py @@ -0,0 +1,99 @@ +import os +from typing import Optional + +import pymongo +from kubetester import create_or_update_secret +from kubetester.certs import create_ops_manager_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import create_appdb_certs, is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +OM_NAME = "om-tls-monitored-appdb" +APPDB_NAME = f"{OM_NAME}-db" + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + return create_ops_manager_tls_certs(issuer, namespace, OM_NAME) + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, APPDB_NAME) + + +@fixture(scope="module") +@mark.usefixtures("appdb_certs", "ops_manager_certs", "issuer_ca_configmap") +def ops_manager( + namespace: str, + issuer_ca_configmap: str, + appdb_certs: str, + ops_manager_certs: str, + custom_version: Optional[str], + custom_appdb_version: str, + issuer_ca_filepath: str, +) -> MongoDBOpsManager: + create_or_update_secret(namespace, "appdb-secret", {"password": "Hello-World!"}) + + print("Creating OM object") + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_appdb_monitoring_tls.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + # ensure the requests library will use this CA when communicating with Ops Manager + os.environ["REQUESTS_CA_BUNDLE"] = issuer_ca_filepath + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + om.update() + return om + + +@mark.e2e_om_appdb_monitoring_tls +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.assert_appdb_preferred_hostnames_are_added() + ops_manager.assert_appdb_hostnames_are_correct() + + +@mark.e2e_om_appdb_monitoring_tls +def test_appdb_group_is_monitored(ops_manager: MongoDBOpsManager): + ops_manager.assert_appdb_monitoring_group_was_created() + ops_manager.assert_monitoring_data_exists() + + +@mark.e2e_om_appdb_monitoring_tls +def test_appdb_password_can_be_changed(ops_manager: MongoDBOpsManager): + # Change the Secret containing the password + data = {"password": "Hello-World!-new"} + create_or_update_secret( + ops_manager.namespace, + ops_manager["spec"]["applicationDatabase"]["passwordSecretKeyRef"]["name"], + data, + ) + + # We know that Ops Manager will detect the changes and be restarted + ops_manager.appdb_status().assert_reaches_phase(Phase.Pending, timeout=120) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.om_status().assert_reaches_phase(Phase.Pending, timeout=120) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_om_appdb_monitoring_tls +def test_new_database_is_monitored_after_restart(ops_manager: MongoDBOpsManager): + # Connect with the new connection string + connection_string = ops_manager.read_appdb_connection_url() + client = pymongo.MongoClient(connection_string, tlsAllowInvalidCertificates=True) + database_name = "new_database" + database = client[database_name] + collection = database["new_collection"] + collection.insert_one({"witness": "database and collection should be created"}) + + # We want to retrieve measurements from "new_database" which will indicate + # that the monitoring agents are working with the new credentials. + ops_manager.assert_monitoring_data_exists(database_name=database_name, timeout=1200, all_hosts=False) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_backup_light.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_backup_light.py new file mode 100644 index 000000000..38d03d168 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_backup_light.py @@ -0,0 +1,269 @@ +from typing import Dict, Optional + +import jsonpatch +import kubernetes.client +from kubernetes import client +from kubetester import MongoDB, try_load, wait_until +from kubetester.awss3client import AwsS3Client +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + get_central_cluster_client, + get_member_cluster_api_client, + is_multi_cluster, +) +from tests.opsmanager.om_ops_manager_backup import ( + HEAD_PATH, + OPLOG_RS_NAME, + S3_SECRET_NAME, + create_aws_secret, + create_s3_bucket, + new_om_data_store, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +DEFAULT_APPDB_USER_NAME = "mongodb-ops-manager" + +""" +This test checks the backup if no separate S3 Metadata database is created and AppDB is used for this. +Note, that it doesn't check for mongodb backup as it's done in 'e2e_om_ops_manager_backup_restore'" +""" + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_light.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + + resource.set_version(custom_mdb_version) + + try_load(resource) + + return resource + + +def service_exists(service_name: str, namespace: str, api_client: Optional[kubernetes.client.ApiClient] = None) -> bool: + try: + client.CoreV1Api(api_client=api_client).read_namespaced_service(service_name, namespace) + except client.rest.ApiException: + return False + return True + + +@mark.e2e_om_ops_manager_backup_light +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + """creates a s3 bucket and an OM resource, the S3 configs get created using AppDB. Oplog store is still required.""" + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="Oplog Store configuration is required for backup", + timeout=600, + ) + + def test_oplog_mdb_created( + self, + oplog_replica_set: MongoDB, + ): + oplog_replica_set.update() + oplog_replica_set.assert_reaches_phase(Phase.Running) + + def test_add_oplog_config(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"] = [ + {"name": "oplog1", "mongodbResourceRef": {"name": "my-mongodb-oplog"}} + ] + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=500, + ignore_errors=True, + ) + + @skip_if_local + def test_om( + self, + ops_manager: MongoDBOpsManager, + s3_bucket: str, + aws_s3_client: AwsS3Client, + oplog_replica_set: MongoDB, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + om_tester.assert_oplog_stores([new_om_data_store(oplog_replica_set, "oplog1")]) + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.assert_appdb_monitoring_group_was_created() + + # TODO uncomment when CLOUDP-70468 is fixed and AppDB supports scram-sha-256 + # making sure the s3 config pushed to OM references the appdb + # appdb_replica_set = ops_manager.get_appdb_resource() + # appdb_password = KubernetesTester.read_secret( + # ops_manager.namespace, ops_manager.app_db_password_secret_name() + # )["password"] + # om_tester.assert_s3_stores( + # [ + # new_om_s3_store( + # appdb_replica_set, + # "s3Store1", + # s3_bucket, + # aws_s3_client, + # user_name=DEFAULT_APPDB_USER_NAME, + # password=appdb_password, + # ) + # ] + # ) + + def test_enable_external_connectivity(self, ops_manager: MongoDBOpsManager, namespace: str): + ops_manager.load() + ops_manager["spec"]["externalConnectivity"] = {"type": "LoadBalancer"} + ops_manager.update() + + for member_cluster_name, _ in ops_manager.get_om_sts_names_in_member_clusters(): + om_external_service_name = f"{ops_manager.name}-svc-ext" + wait_until( + lambda: service_exists( + om_external_service_name, namespace, api_client=get_member_cluster_api_client(member_cluster_name) + ), + timeout=90, + sleep_time=5, + ) + + service = client.CoreV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_service(om_external_service_name, namespace) + + # Tests that the service is created with both externalConnectivity and backup + # and that it contains the correct ports + assert service.spec.type == "LoadBalancer" + assert len(service.spec.ports) == 2 + assert service.spec.ports[0].port == 8080 + assert service.spec.ports[1].port == 25999 + + def test_disable_external_connectivity(self, ops_manager: MongoDBOpsManager, namespace: str): + # We don't have a nice way to delete fields from a resource specification + # in our test env, so we need to achieve it with specific uses of patches + body = {"op": "remove", "path": "/spec/externalConnectivity"} + patch = jsonpatch.JsonPatch([body]) + + om = client.CustomObjectsApi().get_namespaced_custom_object( + ops_manager.group, + ops_manager.version, + namespace, + ops_manager.plural, + ops_manager.name, + ) + client.CustomObjectsApi().replace_namespaced_custom_object( + ops_manager.group, + ops_manager.version, + namespace, + ops_manager.plural, + ops_manager.name, + jsonpatch.apply_patch(om, patch), + ) + + for member_cluster_name, _ in ops_manager.get_om_sts_names_in_member_clusters(): + om_external_service_name = f"{ops_manager.name}-svc-ext" + + wait_until( + lambda: not ( + service_exists( + om_external_service_name, + namespace, + api_client=get_member_cluster_api_client(member_cluster_name), + ) + ), + timeout=90, + sleep_time=5, + ) + + +@mark.e2e_om_ops_manager_backup_light +def test_backup_statefulset_remains_after_disabling_backup( + ops_manager: MongoDBOpsManager, +): + ops_manager.load() + ops_manager["spec"]["backup"]["enabled"] = False + ops_manager.update() + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=200) + + # the backup statefulset should still exist even after we disable backup + ops_manager.read_backup_statefulset() + + +def check_sts_labels(sts, labels: Dict[str, str]): + sts_labels = sts.metadata.labels + + for k in labels: + assert k in sts_labels and sts_labels[k] == labels[k] + + +@mark.e2e_om_ops_manager_backup_light +def test_labels_on_om_and_backup_daemon_and_appdb_sts(ops_manager: MongoDBOpsManager, namespace: str): + labels = {"label1": "val1", "label2": "val2"} + + check_sts_labels(ops_manager.read_statefulset(), labels) + check_sts_labels(ops_manager.read_backup_statefulset(), labels) + check_sts_labels(ops_manager.read_appdb_statefulset(), labels) + + +def check_pvc_labels(pvc_name: str, labels: Dict[str, str], namespace: str, api_client: kubernetes.client.ApiClient): + pvc = client.CoreV1Api(api_client=api_client).read_namespaced_persistent_volume_claim(pvc_name, namespace) + pvc_labels = pvc.metadata.labels + + for k in labels: + assert k in pvc_labels and pvc_labels[k] == labels[k] + + +@mark.e2e_om_ops_manager_backup_light +def test_labels_on_backup_daemon_and_appdb_pvc(ops_manager: MongoDBOpsManager, namespace: str): + labels = {"label1": "val1", "label2": "val2"} + + appdb_pvc_name = "data-{}-0".format(ops_manager.read_appdb_statefulset().metadata.name) + + member_cluster_name = ops_manager.pick_one_appdb_member_cluster_name() + check_pvc_labels(appdb_pvc_name, labels, namespace, api_client=get_member_cluster_api_client(member_cluster_name)) + backupdaemon_pvc_name = "head-{}-0".format(ops_manager.read_backup_statefulset().metadata.name) + + check_pvc_labels(backupdaemon_pvc_name, labels, namespace, api_client=get_central_cluster_client()) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_backup_liveness_probe.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_backup_liveness_probe.py new file mode 100644 index 000000000..589e1ac90 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_backup_liveness_probe.py @@ -0,0 +1,207 @@ +from typing import Optional + +from kubernetes import client +from kubetester import MongoDB, try_load +from kubetester.awss3client import AwsS3Client +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import get_member_cluster_api_client, is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import ( + HEAD_PATH, + OPLOG_RS_NAME, + S3_SECRET_NAME, + create_aws_secret, + create_s3_bucket, + new_om_data_store, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +DEFAULT_APPDB_USER_NAME = "mongodb-ops-manager" + +""" +This test checks the backup if no separate S3 Metadata database is created and AppDB is used for this. +Note, that it doesn't check for mongodb backup as it's done in 'e2e_om_ops_manager_backup_restore'" +""" + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, namespace) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-") + + +@fixture(scope="module") +def ops_manager( + namespace: str, + s3_bucket: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_backup_light.yaml"), namespace=namespace + ) + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + resource["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource.update() + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + + resource.set_version(custom_mdb_version) + + try_load(resource) + + return resource + + +@mark.e2e_om_ops_manager_backup_liveness_probe +class TestOpsManagerCreation: + def test_create_om(self, ops_manager: MongoDBOpsManager): + """creates a s3 bucket and an OM resource, the S3 configs get created using AppDB. Oplog store is still required.""" + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="Oplog Store configuration is required for backup", + timeout=600, + ) + + def test_readiness_probe_is_configured(self, ops_manager: MongoDBOpsManager): + backup_sts = ops_manager.read_backup_statefulset() + readiness_probe = backup_sts.spec.template.spec.containers[0].readiness_probe + assert readiness_probe.period_seconds == 3 + assert readiness_probe.initial_delay_seconds == 1 + assert readiness_probe.success_threshold == 1 + assert readiness_probe.failure_threshold == 3 + + def test_liveness_probe_is_configured(self, ops_manager: MongoDBOpsManager): + backup_sts = ops_manager.read_backup_statefulset() + liveness_probe = backup_sts.spec.template.spec.containers[0].liveness_probe + assert liveness_probe.period_seconds == 30 + assert liveness_probe.initial_delay_seconds == 10 + assert liveness_probe.success_threshold == 1 + assert liveness_probe.failure_threshold == 10 + + def test_oplog_mdb_created( + self, + oplog_replica_set: MongoDB, + ): + oplog_replica_set.update() + oplog_replica_set.assert_reaches_phase(Phase.Running) + + def test_add_oplog_config(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["backup"]["opLogStores"] = [ + {"name": "oplog1", "mongodbResourceRef": {"name": "my-mongodb-oplog"}} + ] + ops_manager.update() + + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + timeout=500, + ignore_errors=True, + ) + + def test_om( + self, + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + ): + om_tester = ops_manager.get_om_tester() + om_tester.assert_healthiness() + for pod_fqdn in ops_manager.backup_daemon_pods_headless_fqdns(): + om_tester.assert_daemon_enabled(pod_fqdn, HEAD_PATH) + + om_tester.assert_oplog_stores([new_om_data_store(oplog_replica_set, "oplog1")]) + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.assert_appdb_monitoring_group_was_created() + + +@mark.e2e_om_ops_manager_backup_liveness_probe +def test_backup_daemon_pod_restarts_when_process_is_killed( + ops_manager: MongoDBOpsManager, +): + + for member_cluster_name, pod_name in ops_manager.backup_daemon_pod_names(): + member_api_client = get_member_cluster_api_client(member_cluster_name) + backup_daemon_pod = client.CoreV1Api(api_client=member_api_client).read_namespaced_pod( + pod_name, ops_manager.namespace + ) + + # ensure the pod has not yet been restarted. + assert MongoDBOpsManager.get_backup_daemon_container_status(backup_daemon_pod).restart_count == 0 + + # get the process id of the Backup Daemon. + cmd = ["/opt/scripts/backup-daemon-liveness-probe.sh"] + process_id = KubernetesTester.run_command_in_pod_container( + pod_name=pod_name, + namespace=ops_manager.namespace, + cmd=cmd, + container="mongodb-backup-daemon", + api_client=member_api_client, + ) + + kill_cmd = ["/bin/sh", "-c", f"kill -9 {process_id}"] + + print(f"kill_cmd in cluster {member_cluster_name}, in pod {pod_name}: {kill_cmd}") + + # kill the process, resulting in the liveness probe terminating the backup daemon. + result = KubernetesTester.run_command_in_pod_container( + pod_name=pod_name, + namespace=ops_manager.namespace, + cmd=kill_cmd, + container="mongodb-backup-daemon", + api_client=member_api_client, + ) + + # ensure the process was existed and was terminated successfully. + assert "No such process" not in result + + for member_cluster_name, pod_name in ops_manager.backup_daemon_pod_names(): + member_api_client = get_member_cluster_api_client(member_cluster_name) + + def backup_daemon_container_has_restarted(): + try: + pod = client.CoreV1Api(api_client=member_api_client).read_namespaced_pod( + pod_name, ops_manager.namespace + ) + return MongoDBOpsManager.get_backup_daemon_container_status(pod).restart_count > 0 + except Exception as e: + print("Error reading pod state: " + str(e)) + return False + + KubernetesTester.wait_until(backup_daemon_container_has_restarted, timeout=3500) + + +@mark.e2e_om_ops_manager_backup_liveness_probe +def test_backup_daemon_reaches_ready_state(ops_manager: MongoDBOpsManager): + + for member_cluster_name, pod_name in ops_manager.backup_daemon_pod_names(): + + def backup_daemon_is_ready(): + try: + pod = client.CoreV1Api( + api_client=get_member_cluster_api_client(member_cluster_name) + ).read_namespaced_pod(pod_name, ops_manager.namespace) + return MongoDBOpsManager.get_backup_daemon_container_status(pod).ready + except Exception as e: + print("Error checking if pod is ready: " + str(e)) + return False + + KubernetesTester.wait_until(backup_daemon_is_ready, timeout=300) diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_pod_spec.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_pod_spec.py new file mode 100644 index 000000000..252921be2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_pod_spec.py @@ -0,0 +1,380 @@ +""" +The fist stage of an Operator-upgrade test. +It creates an OM instance with maximum features (backup, scram etc). +Also it creates a MongoDB referencing the OM. +""" + +from typing import Optional + +from kubernetes import client +from kubetester import try_load +from kubetester.custom_podspec import assert_volume_mounts_are_equal +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + """The fixture for Ops Manager to be created.""" + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_pod_spec.yaml"), namespace=namespace + ) + + if try_load(om): + return om + + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(om) + + return om + + +@mark.e2e_om_ops_manager_pod_spec +class TestOpsManagerCreation: + def test_appdb_0_sts_agents_havent_reached_running_state(self, ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="Application Database Agents haven't reached Running state yet", + timeout=300, + ) + + def test_om_status_0_sts_not_ready(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Pending, msg_regexp="StatefulSet not ready", timeout=600) + + def test_om_status_0_pods_not_ready(self, ops_manager: MongoDBOpsManager): + for _, cluster_spec_item in ops_manager.get_om_indexed_cluster_spec_items(): + ops_manager.om_status().assert_status_resource_not_ready( + ops_manager.om_sts_name(cluster_spec_item["clusterName"]), + msg_regexp=f"Not all the Pods are ready \(wanted: {cluster_spec_item['members']}.*\)", + ) + # we don't run this check for multi-cluster mode + break + + def test_om_status_1_reaches_running_phase(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_empty_status_resources_not_ready() + + def test_appdb_1_reaches_running_phase_1(self, ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_empty_status_resources_not_ready() + + def test_backup_0_reaches_pending_phase(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, msg_regexp=".*is required for backup.*", timeout=900 + ) + ops_manager.backup_status().assert_empty_status_resources_not_ready() + + def test_backup_1_pod_becomes_ready(self, ops_manager: MongoDBOpsManager): + """backup web server is up and running""" + ops_manager.wait_until_backup_pods_become_ready() + + def test_appdb_pod_template_containers(self, ops_manager: MongoDBOpsManager): + appdb_sts = ops_manager.read_appdb_statefulset() + assert len(appdb_sts.spec.template.spec.containers) == 4 + + assert appdb_sts.spec.template.spec.service_account_name == "mongodb-enterprise-appdb" + + appdb_agent_container = appdb_sts.spec.template.spec.containers[2] + assert appdb_agent_container.name == "mongodb-agent" + assert appdb_agent_container.resources.limits["cpu"] == "750m" + assert appdb_agent_container.resources.limits["memory"] == "850M" + + assert appdb_sts.spec.template.spec.containers[0].name == "appdb-sidecar" + assert appdb_sts.spec.template.spec.containers[0].image == "busybox" + assert appdb_sts.spec.template.spec.containers[0].command == ["sleep"] + assert appdb_sts.spec.template.spec.containers[0].args == ["infinity"] + + def test_appdb_persistence(self, ops_manager: MongoDBOpsManager, namespace: str): + # appdb pod volume claim template + appdb_sts = ops_manager.read_appdb_statefulset() + assert len(appdb_sts.spec.volume_claim_templates) == 1 + assert appdb_sts.spec.volume_claim_templates[0].metadata.name == "data" + assert appdb_sts.spec.volume_claim_templates[0].spec.resources.requests["storage"] == "1G" + + for api_client, pod in ops_manager.read_appdb_pods(): + # pod volume claim + expected_claim_name = f"data-{pod.metadata.name}" + claims = [volume for volume in pod.spec.volumes if getattr(volume, "persistent_volume_claim")] + assert len(claims) == 1 + assert claims[0].name == "data" + assert claims[0].persistent_volume_claim.claim_name == expected_claim_name + + # volume claim created + pvc = client.CoreV1Api(api_client=api_client).read_namespaced_persistent_volume_claim( + expected_claim_name, namespace + ) + assert pvc.status.phase == "Bound" + assert pvc.spec.resources.requests["storage"] == "1G" + + def test_om_pod_spec(self, ops_manager: MongoDBOpsManager): + sts = ops_manager.read_statefulset() + assert sts.spec.template.spec.service_account_name == "mongodb-enterprise-ops-manager" + + assert len(sts.spec.template.spec.containers) == 1 + om_container = sts.spec.template.spec.containers[0] + assert om_container.resources.limits["cpu"] == "700m" + assert om_container.resources.limits["memory"] == "6G" + + assert sts.spec.template.metadata.annotations["key1"] == "value1" + assert len(sts.spec.template.spec.tolerations) == 1 + assert sts.spec.template.spec.tolerations[0].key == "key" + assert sts.spec.template.spec.tolerations[0].operator == "Exists" + assert sts.spec.template.spec.tolerations[0].effect == "NoSchedule" + + def test_om_container_override(self, ops_manager: MongoDBOpsManager): + sts = ops_manager.read_statefulset() + om_container = sts.spec.template.spec.containers[0].to_dict() + # Readiness probe got 'failure_threshold' overridden, everything else is the same + # New volume mount was added + expected_spec = { + "name": "mongodb-ops-manager", + "readiness_probe": { + "http_get": { + "host": None, + "http_headers": None, + "path": "/monitor/health", + "port": 8080, + "scheme": "HTTP", + }, + "failure_threshold": 20, + "timeout_seconds": 5, + "period_seconds": 5, + "success_threshold": 1, + "initial_delay_seconds": 5, + "_exec": None, + "tcp_socket": None, + }, + "startup_probe": { + "http_get": { + "host": None, + "http_headers": None, + "path": "/monitor/health", + "port": 8080, + "scheme": "HTTP", + }, + "failure_threshold": 30, + "timeout_seconds": 10, + "period_seconds": 25, + "success_threshold": 1, + "initial_delay_seconds": 1, + "_exec": None, + "tcp_socket": None, + }, + "volume_mounts": [ + { + "name": "gen-key", + "mount_path": "/mongodb-ops-manager/.mongodb-mms", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": True, + }, + { + "name": "mongodb-uri", + "mount_path": "/mongodb-ops-manager/.mongodb-mms-connection-string", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": True, + }, + { + "name": "test-volume", + "mount_path": "/somewhere", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/mongodb-ops-manager/logs", + "sub_path": "logs", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/mongodb-ops-manager/tmp", + "sub_path": "tmp-ops-manager", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/tmp", + "sub_path": "tmp", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/mongodb-ops-manager/conf", + "sub_path": "conf", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/etc/mongodb-mms", + "sub_path": "etc-ops-manager", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/mongodb-ops-manager/mongodb-releases", + "sub_path": "mongodb-releases", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + ], + } + + for k in expected_spec: + if k == "volume_mounts": + continue + assert om_container[k] == expected_spec[k] + + if not is_default_architecture_static(): + expected_spec["volume_mounts"].append( + { + "name": "ops-manager-scripts", + "mount_path": "/opt/scripts", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": True, + }, + ) + + assert_volume_mounts_are_equal(om_container["volume_mounts"], expected_spec["volume_mounts"]) + + # new volume was added and the old ones ('gen-key' and 'ops-manager-scripts') stayed there + if is_default_architecture_static(): + # static containers will not use the ops-manager-scripts volume + assert len(sts.spec.template.spec.volumes) == 4 + else: + assert len(sts.spec.template.spec.volumes) == 5 + + def test_backup_pod_spec(self, ops_manager: MongoDBOpsManager): + backup_sts = ops_manager.read_backup_statefulset() + assert backup_sts.spec.template.spec.service_account_name == "mongodb-enterprise-ops-manager" + + assert len(backup_sts.spec.template.spec.containers) == 1 + om_container = backup_sts.spec.template.spec.containers[0] + assert om_container.resources.requests["cpu"] == "500m" + assert om_container.resources.requests["memory"] == "4500M" + + assert len(backup_sts.spec.template.spec.host_aliases) == 1 + assert backup_sts.spec.template.spec.host_aliases[0].ip == "1.2.3.4" + + +@mark.e2e_om_ops_manager_pod_spec +class TestOpsManagerUpdate: + def test_om_updated(self, ops_manager: MongoDBOpsManager): + ops_manager.load() + # adding annotations + ops_manager["spec"]["applicationDatabase"]["podSpec"]["podTemplate"]["metadata"] = { + "annotations": {"annotation1": "val"} + } + + # changing memory and adding labels for OM + ops_manager["spec"]["statefulSet"]["spec"]["template"]["spec"]["containers"][0]["resources"]["limits"][ + "memory" + ] = "5G" + ops_manager["spec"]["statefulSet"]["spec"]["template"]["metadata"]["labels"] = {"additional": "foo"} + + # termination_grace_period_seconds for Backup + ops_manager["spec"]["backup"]["statefulSet"]["spec"]["template"]["spec"]["terminationGracePeriodSeconds"] = 10 + + ops_manager.update() + + def test_appdb_0_sts_not_ready(self, ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Pending, msg_regexp="StatefulSet not ready", timeout=1200) + + def test_appdb_0_pods_not_ready(self, ops_manager: MongoDBOpsManager): + for _, cluster_spec_item in ops_manager.get_appdb_indexed_cluster_spec_items(): + ops_manager.appdb_status().assert_status_resource_not_ready( + ops_manager.app_db_sts_name(cluster_spec_item["clusterName"]), + msg_regexp=f"Not all the Pods are ready \(wanted: {cluster_spec_item['members']}.*\)", + ) + # we don't run this check for multi-cluster mode + break + + def test_om_status_0_sts_not_ready(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Pending, msg_regexp="StatefulSet not ready", timeout=600) + + def test_om_status_0_pods_not_ready(self, ops_manager: MongoDBOpsManager): + for _, cluster_spec_item in ops_manager.get_om_indexed_cluster_spec_items(): + ops_manager.om_status().assert_status_resource_not_ready( + ops_manager.om_sts_name(cluster_spec_item["clusterName"]), + msg_regexp=f"Not all the Pods are ready \(wanted: {cluster_spec_item['members']}.*\)", + ) + # we don't run this check for multi-cluster mode + break + + def test_om_status_1_reaches_running_phase(self, ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_empty_status_resources_not_ready() + + def test_appdb_1_reaches_running_phase_1(self, ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_empty_status_resources_not_ready() + + def test_backup_0_reaches_pending_phase(self, ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, msg_regexp=".*is required for backup.*", timeout=900 + ) + ops_manager.backup_status().assert_empty_status_resources_not_ready() + + def test_backup_1_pod_becomes_ready(self, ops_manager: MongoDBOpsManager): + """backup web server is up and running""" + ops_manager.wait_until_backup_pods_become_ready() + + def test_appdb_pod_template(self, ops_manager: MongoDBOpsManager): + appdb_sts = ops_manager.read_appdb_statefulset() + assert len(appdb_sts.spec.template.spec.containers) == 4 + + appdb_mongod_container = appdb_sts.spec.template.spec.containers[1] + assert appdb_mongod_container.name == "mongod" + + appdb_agent_container = appdb_sts.spec.template.spec.containers[2] + assert appdb_agent_container.name == "mongodb-agent" + + appdb_agent_monitoring_container = appdb_sts.spec.template.spec.containers[3] + assert appdb_agent_monitoring_container.name == "mongodb-agent-monitoring" + + assert appdb_sts.spec.template.metadata.annotations == {"annotation1": "val"} + + def test_om_pod_spec(self, ops_manager: MongoDBOpsManager): + sts = ops_manager.read_statefulset() + assert len(sts.spec.template.spec.containers) == 1 + om_container = sts.spec.template.spec.containers[0] + assert om_container.resources.limits["cpu"] == "700m" + assert om_container.resources.limits["memory"] == "5G" + + assert sts.spec.template.metadata.annotations["key1"] == "value1" + assert len(sts.spec.template.metadata.labels) == 4 + assert sts.spec.template.metadata.labels["additional"] == "foo" + assert len(sts.spec.template.spec.tolerations) == 1 + + def test_backup_pod_spec(self, ops_manager: MongoDBOpsManager): + backup_sts = ops_manager.read_backup_statefulset() + + assert len(backup_sts.spec.template.spec.host_aliases) == 1 + assert backup_sts.spec.template.spec.termination_grace_period_seconds == 10 diff --git a/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_secure_config.py b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_secure_config.py new file mode 100644 index 000000000..8402f0cb4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_secure_config.py @@ -0,0 +1,216 @@ +import time +from typing import Optional + +import pytest +from kubernetes import client +from kubetester import create_or_update_secret, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture +from tests.conftest import get_central_cluster_client, is_multi_cluster +from tests.opsmanager.om_ops_manager_backup import BLOCKSTORE_RS_NAME, OPLOG_RS_NAME +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +MONGO_URI_VOLUME_MOUNT_NAME = "mongodb-uri" +MONGO_URI_VOLUME_MOUNT_PATH = "/mongodb-ops-manager/.mongodb-mms-connection-string" + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: Optional[str], custom_appdb_version: str) -> MongoDBOpsManager: + resource = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_secure_config.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + create_or_update_secret( + namespace, + "my-password", + {"password": "password"}, + api_client=get_central_cluster_client(), + ) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + resource["spec"]["applicationDatabase"]["passwordSecretKeyRef"] = { + "name": "my-password", + } + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + resource.update() + + return resource + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "oplog") + + resource.set_version(custom_mdb_version) + + try_load(resource) + return resource + + +@fixture(scope="module") +def blockstore_replica_set(ops_manager, custom_mdb_version) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name=BLOCKSTORE_RS_NAME, + ).configure(ops_manager, "blockstore") + + resource.set_version(custom_mdb_version) + + try_load(resource) + return resource + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_om_creation(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_appdb_monitoring_configured(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.assert_appdb_monitoring_group_was_created() + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_backing_dbs_created(oplog_replica_set: MongoDB, blockstore_replica_set: MongoDB): + oplog_replica_set.update() + blockstore_replica_set.update() + + oplog_replica_set.assert_reaches_phase(Phase.Running) + blockstore_replica_set.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_backup_enabled(ops_manager: MongoDBOpsManager): + ops_manager.load() + ops_manager["spec"]["backup"]["enabled"] = True + + if is_multi_cluster(): + enable_multi_cluster_deployment(ops_manager) + + ops_manager.update() + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_connection_string_secret_was_created(ops_manager: MongoDBOpsManager): + secret_data = KubernetesTester.read_secret( + ops_manager.namespace, ops_manager.get_appdb_connection_url_secret_name() + ) + assert "connectionString" in secret_data + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_ops_manager_pod_template_was_annotated(ops_manager: MongoDBOpsManager): + sts = ops_manager.read_statefulset() + pod_template = sts.spec.template + assert "connectionStringHash" in pod_template.metadata.annotations + assert pod_template.metadata.annotations["connectionStringHash"] != "" + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_backup_pod_template_was_annotated(ops_manager: MongoDBOpsManager): + sts = ops_manager.read_backup_statefulset() + pod_template = sts.spec.template + assert "connectionStringHash" in pod_template.metadata.annotations + assert pod_template.metadata.annotations["connectionStringHash"] != "" + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_connection_string_is_configured_securely(ops_manager: MongoDBOpsManager): + sts = ops_manager.read_statefulset() + om_container = sts.spec.template.spec.containers[0] + volume_mounts: list[client.V1VolumeMount] = om_container.volume_mounts + + connection_string_volume_mount = [vm for vm in volume_mounts if vm.name == MONGO_URI_VOLUME_MOUNT_NAME][0] + assert connection_string_volume_mount.mount_path == MONGO_URI_VOLUME_MOUNT_PATH + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_changing_app_db_password_triggers_rolling_restart( + ops_manager: MongoDBOpsManager, +): + KubernetesTester.update_secret( + ops_manager.namespace, + "my-password", + { + "password": "new-password", + }, + ) + # unfortunately changing the external secret doesn't change the metadata.generation so we cannot track + # when the Operator started working on the spec - let's just wait for a bit + time.sleep(5) + ops_manager.appdb_status().assert_abandons_phase(Phase.Running, timeout=100) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=900) + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_no_unnecessary_rolling_upgrades_happen( + ops_manager: MongoDBOpsManager, + custom_appdb_version: str, +): + sts = ops_manager.read_statefulset() + old_generation = sts.metadata.generation + old_hash = sts.spec.template.metadata.annotations["connectionStringHash"] + + backup_sts = ops_manager.read_backup_statefulset() + old_backup_generation = backup_sts.metadata.generation + old_backup_hash = backup_sts.spec.template.metadata.annotations["connectionStringHash"] + + assert old_backup_hash == old_hash + + ops_manager.load() + ops_manager["spec"]["applicationDatabase"]["version"] = custom_appdb_version + ops_manager.update() + + time.sleep(10) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + + sts = ops_manager.read_statefulset() + assert sts.metadata.generation == old_generation, "no change should have happened to the Ops Manager stateful set" + assert ( + sts.spec.template.metadata.annotations["connectionStringHash"] == old_hash + ), "connection string hash should have remained the same" + + backup_sts = ops_manager.read_backup_statefulset() + assert ( + backup_sts.metadata.generation == old_backup_generation + ), "no change should have happened to the backup stateful set" + assert ( + backup_sts.spec.template.metadata.annotations["connectionStringHash"] == old_backup_hash + ), "connection string hash should have remained the same" + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_connection_string_secret_was_updated(ops_manager: MongoDBOpsManager): + connection_string = ops_manager.read_appdb_connection_url() + + assert "new-password" in connection_string + + +@pytest.mark.e2e_om_ops_manager_secure_config +def test_appdb_project_has_correct_auth_tls(ops_manager: MongoDBOpsManager): + automation_config_tester = ops_manager.get_automation_config_tester() + auth = automation_config_tester.automation_config["auth"] + + automation_config_tester.assert_agent_user("mms-automation-agent") + assert auth["keyfile"] == "/var/lib/mongodb-mms-automation/authentication/keyfile" + + # These should have been set to random values + assert auth["autoPwd"] != "" + assert auth["key"] != "" diff --git a/docker/mongodb-enterprise-tests/tests/pod_logs.py b/docker/mongodb-enterprise-tests/tests/pod_logs.py new file mode 100644 index 000000000..e14e0fe88 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/pod_logs.py @@ -0,0 +1,90 @@ +import json +import logging +from typing import Optional + +import kubernetes +from kubetester.kubetester import KubernetesTester, is_default_architecture_static + + +def parse_json_pod_logs(pod_logs: str) -> list[dict[str, any]]: + """Parses pod logs returned asa string and returns list of lines parsed from structured json.""" + lines = pod_logs.strip().split("\n") + log_lines = [] + for line in lines: + try: + log_lines.append(json.loads(line)) + except json.JSONDecodeError as e: + # Even though throwing may seem like a good idea, this way we don't have any way to debug the + # Agent bootstrap script (which uses set -x). Loosen this up here makes a bit more sense + # as we check what lines we do want and what we can't have. + logging.warning(f"Ignoring the following log line {line} because of {e}") + return log_lines + + +def get_structured_json_pod_logs( + namespace: str, + pod_name: str, + container_name: str, + api_client: kubernetes.client.ApiClient, +) -> dict[str, list[str]]: + """Read logs from pod_name and groups the lines by logType.""" + pod_logs_str = KubernetesTester.read_pod_logs(namespace, pod_name, container_name, api_client=api_client) + log_lines = parse_json_pod_logs(pod_logs_str) + + log_contents_by_type = {} + + for log_line in log_lines: + if "logType" not in log_line or "contents" not in log_line: + raise Exception("Invalid log line structure: {log_line}") + + log_type = log_line["logType"] + if log_type not in log_contents_by_type: + log_contents_by_type[log_type] = [] + + log_contents_by_type[log_type].append(log_line["contents"]) + + return log_contents_by_type + + +def get_all_log_types() -> set[str]: + """Returns all possible log types that can be put to pod logs in agent-launcher.sh script""" + return { + "agent-launcher-script", + "automation-agent", + "automation-agent-stderr", + "automation-agent-verbose", + "backup-agent", + "mongodb", + "monitoring-agent", + "mongodb-audit", + } + + +def get_all_default_log_types() -> set[str]: + """Returns log types that are by default enabled in pod logs in agent-launcher.sh script (normally - all without audit log).""" + return get_all_log_types() - {"mongodb-audit"} + + +def assert_log_types_in_structured_json_pod_log( + namespace: str, + pod_name: str, + expected_log_types: Optional[set[str]], + api_client: Optional[kubernetes.client.ApiClient] = None, + container_name: str = "mongodb-agent", +): + """ + Checks pod logs if all expected_log_types are present in structured json logs. + It fails when there are any unexpected log types in logs. + """ + + if not is_default_architecture_static(): + container_name = "mongodb-enterprise-database" + + pod_logs = get_structured_json_pod_logs(namespace, pod_name, container_name, api_client=api_client) + + unwanted_log_types = pod_logs.keys() - expected_log_types + missing_log_types = expected_log_types - pod_logs.keys() + assert len(unwanted_log_types) == 0, f"pod {namespace}/{pod_name} contains unwanted log types: {unwanted_log_types}" + assert ( + len(missing_log_types) == 0 + ), f"pod {namespace}/{pod_name} doesn't contain some log types: {missing_log_types}" diff --git a/docker/mongodb-enterprise-tests/tests/probes/conftest.py b/docker/mongodb-enterprise-tests/tests/probes/conftest.py new file mode 100644 index 000000000..3b9e079a1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/probes/conftest.py @@ -0,0 +1,12 @@ +from tests.conftest import default_operator + + +def pytest_runtest_setup(item): + """Adds the default_operator fixture in case it is not there already. + + If the test has the "no_operator" fixture, the Operator will not be installed + but instead it will rely on the currently installed operator. This is handy to + run local tests.""" + default_operator_name = default_operator.__name__ + if default_operator_name not in item.fixturenames and "no_operator" not in item.fixturenames: + item.fixturenames.insert(0, default_operator_name) diff --git a/docker/mongodb-enterprise-tests/tests/probes/fixtures/deployment_tls.json b/docker/mongodb-enterprise-tests/tests/probes/fixtures/deployment_tls.json new file mode 100644 index 000000000..e051865d2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/probes/fixtures/deployment_tls.json @@ -0,0 +1 @@ +{ "auth": { "authoritativeSet": false, "autoAuthMechanism": "MONGODB-CR", "autoAuthMechanisms": [], "autoAuthRestrictions": [], "disabled": true, "usersDeleted": [], "usersWanted": [] }, "backupVersions": [ { "hostname": "test-tls-additional-domains-0.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "6.6.0.959-1" }, { "hostname": "test-tls-additional-domains-1.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "6.6.0.959-1" }, { "hostname": "test-tls-additional-domains-2.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "6.6.0.959-1" } ], "balancer": {}, "cpsModules": [], "indexConfigs": [], "kerberos": { "serviceName": "mongodb" }, "ldap": {}, "mongosqlds": [], "mongots": [], "monitoringVersions": [ { "hostname": "test-tls-additional-domains-0.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "6.4.0.433-1" } ], "onlineArchiveModules": [], "options": { "downloadBase": "/var/lib/mongodb-mms-automation", "downloadBaseWindows": "%SystemDrive%\\MMSAutomation\\versions" }, "processes": [ { "args2_6": { "net": { "port": 27017, "tls": { "PEMKeyFile": "/mongodb-automation/server.pem", "mode": "requireSSL" } }, "replication": { "replSetName": "test-tls-additional-domains" }, "storage": { "dbPath": "/data" }, "systemLog": { "destination": "file", "path": "/var/log/mongodb-mms-automation/mongodb.log" } }, "authSchemaVersion": 5, "featureCompatibilityVersion": "3.6", "hostname": "test-tls-additional-domains-0.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "test-tls-additional-domains-0", "processType": "mongod", "version": "3.6.8" }, { "args2_6": { "net": { "port": 27017, "tls": { "PEMKeyFile": "/mongodb-automation/server.pem", "mode": "requireSSL" } }, "replication": { "replSetName": "test-tls-additional-domains" }, "storage": { "dbPath": "/data" }, "systemLog": { "destination": "file", "path": "/var/log/mongodb-mms-automation/mongodb.log" } }, "authSchemaVersion": 5, "featureCompatibilityVersion": "3.6", "hostname": "test-tls-additional-domains-1.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "test-tls-additional-domains-1", "processType": "mongod", "version": "3.6.8" }, { "args2_6": { "net": { "port": 27017, "tls": { "PEMKeyFile": "/mongodb-automation/server.pem", "mode": "requireSSL" } }, "replication": { "replSetName": "test-tls-additional-domains" }, "storage": { "dbPath": "/data" }, "systemLog": { "destination": "file", "path": "/var/log/mongodb-mms-automation/mongodb.log" } }, "authSchemaVersion": 5, "featureCompatibilityVersion": "3.6", "hostname": "test-tls-additional-domains-2.test-tls-additional-domains-svc.dev.svc.cluster.local", "name": "test-tls-additional-domains-2", "processType": "mongod", "version": "3.6.8" } ], "replicaSets": [ { "_id": "test-tls-additional-domains", "members": [ { "_id": 0, "host": "test-tls-additional-domains-0", "priority": 1, "votes": 1 }, { "_id": 1, "host": "test-tls-additional-domains-1", "priority": 1, "votes": 1 }, { "_id": 2, "host": "test-tls-additional-domains-2", "priority": 1, "votes": 1 } ], "protocolVersion": "1" } ], "roles": [], "sharding": [], "tls": { "CAFilePath": "/mongodb-automation/ca.pem", "clientCertificateMode": "OPTIONAL" }, "version": 138 } diff --git a/docker/mongodb-enterprise-tests/tests/probes/replication_state_awareness.py b/docker/mongodb-enterprise-tests/tests/probes/replication_state_awareness.py new file mode 100644 index 000000000..0989fe632 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/probes/replication_state_awareness.py @@ -0,0 +1,128 @@ +""" +replication_state_awareness tests that the agent on each Pod is aware of +replication_state and that the entry is writen to '{statuses[0].ReplicaStatus}' +in the file /var/log/mongodb-mms-automation/agent-health-status.json +""" + +import asyncio +import functools +import logging +import random +import string +import time +from typing import Callable, Dict, List + +import pymongo +import yaml +from kubernetes.client.rest import ApiException +from kubetester import find_fixture, wait_until +from kubetester.mongodb import MongoDB, Phase, generic_replicaset +from kubetester.mongotester import upload_random_data +from pytest import fixture, mark + + +def large_json_generator() -> Callable[[], Dict]: + """ + Returns a function that generates a dictionary with a random attribute. + """ + _doc = yaml.safe_load(open(find_fixture("deployment_tls.json"))) + rand_generator = random.SystemRandom() + + def inner() -> Dict: + doc = _doc.copy() + random_id = "".join(rand_generator.choice(string.ascii_uppercase + string.digits) for _ in range(30)) + doc["json_generator_id"] = random_id + + return doc + + return inner + + +async def upload_random_data_async(client: pymongo.MongoClient, task_name: str = None, count: int = 50_000): + fn = functools.partial( + upload_random_data, + client=client, + generation_function=large_json_generator(), + count=count, + task_name=task_name, + ) + return await asyncio.get_event_loop().run_in_executor(None, fn) + + +def create_writing_task(client: pymongo.MongoClient, name: str, count: int) -> asyncio.Task: + """ + Creates an async Task that uploads documents to a MongoDB database. + Async tasks in Python start right away after they are created. + """ + return asyncio.create_task(upload_random_data_async(client, name, count)) + + +def create_writing_tasks(client: pymongo.MongoClient, prefix: str, task_sizes: List[int] = None) -> asyncio: + """ + Creates many async tasks to upload documents to a MongoDB database. + """ + return [create_writing_task(client, prefix + str(task), task) for task in task_sizes] + + +@fixture(scope="module") +def replica_set(namespace: str) -> MongoDB: + rs = generic_replicaset(namespace, version="4.4.2") + + rs["spec"]["persistent"] = True + return rs.create() + + +@mark.e2e_replication_state_awareness +def test_replicaset_reaches_running_state(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replication_state_awareness +def test_inserts_50k_documents(replica_set: MongoDB): + client = replica_set.tester().client + + upload_random_data( + client, + 50_000, + generation_function=large_json_generator(), + task_name="uploader_50k", + ) + + +@mark.e2e_replication_state_awareness +@mark.asyncio +async def test_fill_up_database(replica_set: MongoDB): + """ + Writes 1 million documents to the database. + """ + client: pymongo.MongoClient = replica_set.tester().client + + tasks = create_writing_tasks(client, "uploader_", [500_000, 400_000, 100_000]) + for task in tasks: + await task + + logging.info("All uploaders have finished.") + + +@mark.e2e_replication_state_awareness +def test_kill_pod_while_writing(replica_set: MongoDB): + """Keeps writing documents to the database while it is being + restarted.""" + logging.info("Restarting StatefulSet holding the MongoDBs: sts/{}".format(replica_set.name)) + replica_set["spec"]["podSpec"] = { + "podTemplate": { + "spec": { + "containers": [ + { + "name": "mongodb-enterprise-database", + "resources": {"limits": {"cpu": "2", "memory": "2Gi"}}, + } + ] + } + } + } + current_gen = replica_set.get_generation() + replica_set.update() + wait_until(lambda: replica_set.get_generation() >= current_gen) + + replica_set.assert_reaches_phase(Phase.Running, timeout=800) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/__init__.py b/docker/mongodb-enterprise-tests/tests/replicaset/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/conftest.py b/docker/mongodb-enterprise-tests/tests/replicaset/conftest.py new file mode 100644 index 000000000..cb0664057 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/conftest.py @@ -0,0 +1,4 @@ +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if "default_operator" not in item.fixturenames: + item.fixturenames.insert(0, "default_operator") diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/invalid_replica_set_wrong_auth_mode.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/invalid_replica_set_wrong_auth_mode.yaml new file mode 100644 index 000000000..3f958fe6c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/invalid_replica_set_wrong_auth_mode.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + authentication: + enabled: true + modes: ["SHA"] diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-8-members.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-8-members.yaml new file mode 100644 index 000000000..dea8f4920 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-8-members.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: big-replica-set +spec: + members: 8 + version: 4.0.17 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false + diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-custom-podspec.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-custom-podspec.yaml new file mode 100644 index 000000000..2f38cfaaa --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-custom-podspec.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-custom-podspec +spec: + members: 1 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: true + podSpec: + podTemplate: + metadata: + annotations: + key1: "val1" + spec: + volumes: + - name: test-volume + emptyDir: {} + containers: + - name: side-car + image: busybox:latest + command: ["/bin/sh"] + args: ["-c", "echo ok > /somewhere/busybox_file && sleep 86400"] + volumeMounts: + - mountPath: /somewhere + name: test-volume + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + requests: + cpu: "1" + volumeMounts: + - mountPath: /somewhere + name: test-volume + hostAliases: + - ip: "1.2.3.4" + hostnames: ["hostname"] + terminationGracePeriodSeconds: 30 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: "mykey-rs" + weight: 50 diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-double.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-double.yaml new file mode 100644 index 000000000..205d6516f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-double.yaml @@ -0,0 +1,12 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-double +spec: + members: 2 + version: 4.4.0-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-downgrade.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-downgrade.yaml new file mode 100644 index 000000000..8726915c8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-downgrade.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-downgrade +spec: + members: 3 + version: 4.4.2 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-ent.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-ent.yaml new file mode 100644 index 000000000..d8e306ef6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-ent.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: rs001-ent +spec: + members: 3 + version: 5.0.15-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-externally-exposed.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-externally-exposed.yaml new file mode 100644 index 000000000..614db2689 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-externally-exposed.yaml @@ -0,0 +1,14 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-externally-exposed +spec: + members: 1 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false + externalAccess: {} diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-invalid.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-invalid.yaml new file mode 100644 index 000000000..c8f44c470 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-invalid.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-invalid-replica-set +spec: + members: 3 + version: 6.0.5-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentialsd + logLevel: DEBUG + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-liveness.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-liveness.yaml new file mode 100644 index 000000000..8fb76b625 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-liveness.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: true + # Use a version that does not exist + # This way mongod will never start + version: 4.4.100 diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-mongod-options.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-mongod-options.yaml new file mode 100644 index 000000000..116c440b7 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-mongod-options.yaml @@ -0,0 +1,21 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-options +spec: + members: 3 + version: 4.4.0-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false + additionalMongodConfig: + systemLog: + logAppend: true + verbosity: 4 + operationProfiling: + mode: slowOp + net: + port: 30000 diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-override-agent-launcher-script.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-override-agent-launcher-script.yaml new file mode 100644 index 000000000..29d9b8111 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-override-agent-launcher-script.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: replica-set +spec: + members: 1 + type: ReplicaSet + credentials: my-credentials + logLevel: DEBUG + persistent: true + opsManager: + configMapRef: + name: my-project + podSpec: + podTemplate: + spec: + initContainers: + - name: override-agent-launcher + image: busybox:latest + command: ["/bin/sh"] + volumeMounts: + - mountPath: /opt/scripts + name: database-scripts diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-perf.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-perf.yaml new file mode 100644 index 000000000..38b9c373c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-perf.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 1 + version: 5.0.15 + type: ReplicaSet + opsManager: + configMapRef: + name: om-rs-configmap + credentials: my-credentials + logLevel: DEBUG + podSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 50m + memory: 50Mi \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv-multiple.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv-multiple.yaml new file mode 100644 index 000000000..9782e1d2a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv-multiple.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: rs001-pv-multiple + labels: + label1: val1 + label2: val2 +spec: + members: 2 + version: 4.4.0 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: ReplicaSet + persistent: true + podSpec: + persistence: + multiple: + data: + storage: 2Gi + storageClass: gp2 + journal: + storage: 1Gi + logs: + storage: 1G # 1G is the minimum PV possible in EBS diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv-resize.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv-resize.yaml new file mode 100644 index 000000000..e22a65afd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv-resize.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: rs001-pv +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + podSpec: + persistence: + multiple: + data: + storage: 1Gi + storageClass: csi-hostpath-sc + journal: + storage: 1Gi + storageClass: csi-hostpath-sc + logs: + storage: 1Gi + storageClass: csi-hostpath-sc \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv.yaml new file mode 100644 index 000000000..b756dca84 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-pv.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: rs001-pv +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + podSpec: + storage: 2G + storageClass: gp2 diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-single.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-single.yaml new file mode 100644 index 000000000..09901fcaa --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-single.yaml @@ -0,0 +1,16 @@ +# This is the simplest replica set possible (1 member only) - use it for the tests not focused on replica set +# behaviour (e.g. listening to configmaps changes), this will result in minimal running time and lowest resource +# consumption +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-single +spec: + members: 1 + version: 6.0.5 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-upgrade.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-upgrade.yaml new file mode 100644 index 000000000..f03e72a0e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set-upgrade.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.4.15 + type: ReplicaSet + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set.yaml b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set.yaml new file mode 100644 index 000000000..9c82b76d2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/fixtures/replica-set.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 5.0.15 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: true \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/manual/replica_set_override_agent_launcher_script.py b/docker/mongodb-enterprise-tests/tests/replicaset/manual/replica_set_override_agent_launcher_script.py new file mode 100644 index 000000000..53e3ef108 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/manual/replica_set_override_agent_launcher_script.py @@ -0,0 +1,83 @@ +import base64 + +from kubetester import find_fixture, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture + +# This test is intended for manual run only. +# +# It's for quick iteration on changes to agent-launcher.sh. +# It's deploying a replica set with 1 member with the local copy of agent-launcher.sh and agent-launcher-lib.sh scripts +# from docker/mongodb-enterprise-init-database/content. +# Scripts are injected (mounted) into their standard location in init image and scripts from init-database image are overwritten. +# +# Thanks to this, it is possible to quickly iterate on the script without the need to build and push init-database image. + + +@fixture(scope="module") +def ops_manager(namespace: str, custom_version: str, custom_appdb_version) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + + resource["spec"]["topology"] = "SingleCluster" + resource["spec"]["applicationDatabase"]["topology"] = "SingleCluster" + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if try_load(resource): + return resource + + resource.update() + return resource + + +@fixture(scope="function") +def replica_set(ops_manager: str, namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("replica-set-override-agent-launcher-script.yaml"), + namespace=namespace, + ).configure(ops_manager, "replica-set") + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["logLevel"] = "INFO" + resource["spec"]["additionalMongodConfig"] = { + "auditLog": { + "destination": "file", + "format": "JSON", + "path": "/var/log/mongodb-mms-automation/mongodb-audit-changed.log", + }, + } + resource["spec"]["agent"] = { + "startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFileWithoutExt"} + } + + return resource + + +def test_replica_set(replica_set: MongoDB): + with open("../mongodb-enterprise-init-database/content/agent-launcher.sh", "rb") as f: + agent_launcher = base64.b64encode(f.read()).decode("utf-8") + + with open("../mongodb-enterprise-init-database/content/agent-launcher-lib.sh", "rb") as f: + agent_launcher_lib = base64.b64encode(f.read()).decode("utf-8") + + command = f""" +echo -n "{agent_launcher}" | base64 -d > /opt/scripts/agent-launcher.sh +echo -n "{agent_launcher_lib}" | base64 -d > /opt/scripts/agent-launcher-lib.sh + """ + + replica_set["spec"]["podSpec"]["podTemplate"]["spec"]["initContainers"][0]["args"] = ["-c", command] + + replica_set.update() + + +def test_om_running(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + +def test_replica_set_running(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set.py new file mode 100644 index 000000000..39f79363f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set.py @@ -0,0 +1,568 @@ +import time +from typing import Dict + +import pytest +from kubernetes import client +from kubetester import ( + assert_pod_container_security_context, + assert_pod_security_context, + try_load, +) +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.kubetester import KubernetesTester, fcv_from_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester +from pytest import fixture +from tests.conftest import ( + assert_log_rotation_backup_monitoring, + assert_log_rotation_process, + setup_log_rotate_for_agents, +) + +DEFAULT_BACKUP_VERSION = "11.12.0.7388-1" +DEFAULT_MONITORING_AGENT_VERSION = "11.12.0.7388-1" +RESOURCE_NAME = "my-replica-set" + + +def _get_group_id(envs) -> str: + for e in envs: + if e.name == "GROUP_ID": + return e.value + return "" + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str, cluster_domain: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), "my-replica-set", namespace) + + if try_load(resource): + return resource + + resource.set_version(custom_mdb_version) + resource["spec"]["clusterDomain"] = cluster_domain + + # Setting podSpec shortcut values here to test they are still + # added as resources when needed. + if is_default_architecture_static(): + resource["spec"]["podSpec"] = { + "podTemplate": { + "spec": { + "containers": [ + { + "name": "mongodb-agent", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + }, + "requests": {"cpu": "0.2", "memory": "300M"}, + }, + } + ] + } + } + } + else: + resource["spec"]["podSpec"] = { + "podTemplate": { + "spec": { + "containers": [ + { + "name": "mongodb-enterprise-database", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + }, + "requests": {"cpu": "0.2", "memory": "300M"}, + }, + } + ] + } + } + } + + setup_log_rotate_for_agents(resource) + resource.update() + + return resource + + +@pytest.fixture(scope="class") +def config_version(): + class ConfigVersion: + def __init__(self): + self.version = 0 + + return ConfigVersion() + + +@pytest.mark.e2e_replica_set +class TestReplicaSetCreation(KubernetesTester): + def test_initialize_config_version(self, config_version): + self.ensure_group(self.get_om_org_id(), self.namespace) + config = self.get_automation_config() + config_version.version = config["version"] + + def test_mdb_created(self, replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + def test_replica_set_sts_exists(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + assert sts + + def test_sts_creation(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + assert sts.api_version == "apps/v1" + assert sts.kind == "StatefulSet" + assert sts.status.current_replicas == 3 + assert sts.status.ready_replicas == 3 + + def test_sts_metadata(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + assert sts.metadata.name == RESOURCE_NAME + assert sts.metadata.labels["app"] == "my-replica-set-svc" + assert sts.metadata.namespace == self.namespace + owner_ref0 = sts.metadata.owner_references[0] + assert owner_ref0.api_version == "mongodb.com/v1" + assert owner_ref0.kind == "MongoDB" + assert owner_ref0.name == RESOURCE_NAME + + def test_sts_replicas(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + assert sts.spec.replicas == 3 + + def test_sts_template(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + tmpl = sts.spec.template + assert tmpl.metadata.labels["app"] == "my-replica-set-svc" + assert tmpl.metadata.labels["controller"] == "mongodb-enterprise-operator" + assert tmpl.spec.service_account_name == "mongodb-enterprise-database-pods" + assert tmpl.spec.affinity.node_affinity is None + assert tmpl.spec.affinity.pod_affinity is None + assert tmpl.spec.affinity.pod_anti_affinity is not None + + def _get_pods(self, podname, qty=3): + return [podname.format(i) for i in range(qty)] + + def test_replica_set_pods_exists(self): + for podname in self._get_pods("my-replica-set-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + assert pod.metadata.name == podname + + def test_pods_are_running(self): + for podname in self._get_pods("my-replica-set-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + assert pod.status.phase == "Running" + + def test_pods_containers(self): + for podname in self._get_pods("my-replica-set-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + c0 = pod.spec.containers[0] + if is_default_architecture_static(): + assert c0.name == "mongodb-agent" + else: + assert c0.name == "mongodb-enterprise-database" + + def test_pods_containers_ports(self): + for podname in self._get_pods("my-replica-set-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + c0 = pod.spec.containers[0] + assert c0.ports[0].container_port == 27017 + assert c0.ports[0].host_ip is None + assert c0.ports[0].host_port is None + assert c0.ports[0].protocol == "TCP" + + def test_pods_resources(self): + for podname in self._get_pods("my-replica-set-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + c0 = pod.spec.containers[0] + assert c0.resources.limits["cpu"] == "1" + assert c0.resources.limits["memory"] == "1Gi" + assert c0.resources.requests["cpu"] == "200m" + assert c0.resources.requests["memory"] == "300M" + + def test_pods_container_envvars(self): + for pod_name in self._get_pods("my-replica-set-{}", 3): + assert_container_env_vars(self.namespace, pod_name) + + def test_service_is_created(self): + svc = self.corev1.read_namespaced_service("my-replica-set-svc", self.namespace) + assert svc + + def test_clusterip_service_exists(self): + """Test that replica set is not exposed externally.""" + services = self.clients("corev1").list_namespaced_service( + self.get_namespace(), + label_selector="controller=mongodb-enterprise-operator", + ) + + # 1 for replica set + assert len(services.items) == 1 + assert services.items[0].spec.type == "ClusterIP" + + def test_security_context_pods(self, operator_installation_config: Dict[str, str]): + + managed = operator_installation_config["managedSecurityContext"] == "true" + for podname in self._get_pods("my-replica-set-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + assert_pod_security_context(pod, managed) + assert_pod_container_security_context(pod.spec.containers[0], managed) + + @skip_if_local + def test_security_context_operator(self, operator_installation_config: Dict[str, str]): + # todo there should be a better way to find the pods for deployment + response = self.corev1.list_namespaced_pod(self.namespace) + operator_pod = [pod for pod in response.items if pod.metadata.name.startswith("mongodb-enterprise-operator-")][ + 0 + ] + security_context = operator_pod.spec.security_context + if operator_installation_config["managedSecurityContext"] == "false": + assert security_context.run_as_user == 2000 + assert security_context.run_as_non_root + assert security_context.fs_group is None + else: + # Note, that this code is a bit fragile as may depend on the version of Openshift, but we need to verify + # that this is not "our" security context but the one generated by Openshift + assert security_context.run_as_user is None + assert security_context.run_as_non_root is None + assert security_context.se_linux_options is not None + assert security_context.fs_group is not None + assert security_context.fs_group != 2000 + + def test_om_processes_are_created(self): + config = self.get_automation_config() + assert len(config["processes"]) == 3 + + def test_om_replica_set_is_created(self): + config = self.get_automation_config() + assert len(config["replicaSets"]) == 1 + + def test_om_processes(self, custom_mdb_version: str, cluster_domain: str): + config = self.get_automation_config() + processes = config["processes"] + + for idx in range(0, 2): + name = f"my-replica-set-{idx}" + p = processes[idx] + assert p["name"] == name + assert p["processType"] == "mongod" + assert custom_mdb_version in p["version"] + assert p["authSchemaVersion"] == 5 + assert p["featureCompatibilityVersion"] == fcv_from_version(custom_mdb_version) + assert p["hostname"] == "{}.my-replica-set-svc.{}.svc.{}".format(name, self.namespace, cluster_domain) + assert p["args2_6"]["net"]["port"] == 27017 + assert p["args2_6"]["replication"]["replSetName"] == RESOURCE_NAME + assert p["args2_6"]["storage"]["dbPath"] == "/data" + assert p["args2_6"]["systemLog"]["destination"] == "file" + assert p["args2_6"]["systemLog"]["path"] == "/var/log/mongodb-mms-automation/mongodb.log" + assert_log_rotation_process(p) + + def test_om_replica_set(self): + config = self.get_automation_config() + rs = config["replicaSets"] + assert rs[0]["_id"] == RESOURCE_NAME + + for idx in range(0, 2): + m = rs[0]["members"][idx] + assert m["_id"] == idx + assert m["arbiterOnly"] is False + assert m["hidden"] is False + assert m["buildIndexes"] is True + assert m["host"] == f"my-replica-set-{idx}" + assert m["votes"] == 1 + assert m["priority"] == 1.0 + + def test_monitoring_versions(self, cluster_domain: str): + config = self.get_automation_config() + mv = config["monitoringVersions"] + assert len(mv) == 3 + + # Monitoring agent is installed on all hosts + for i in range(0, 3): + # baseUrl is not present in Cloud Manager response + if "baseUrl" in mv[i]: + assert mv[i]["baseUrl"] is None + hostname = "my-replica-set-{}.my-replica-set-svc.{}.svc.{}".format(i, self.namespace, cluster_domain) + assert mv[i]["hostname"] == hostname + assert mv[i]["name"] == DEFAULT_MONITORING_AGENT_VERSION + + def test_monitoring_log_rotation(self, cluster_domain: str): + mv = self.get_monitoring_config() + assert_log_rotation_backup_monitoring(mv) + + def test_backup(self, cluster_domain): + config = self.get_automation_config() + # 1 backup agent per host + bkp = config["backupVersions"] + assert len(bkp) == 3 + + # Backup agent is installed on all hosts + for i in range(0, 3): + hostname = "my-replica-set-{}.my-replica-set-svc.{}.svc.{}".format(i, self.namespace, cluster_domain) + assert bkp[i]["hostname"] == hostname + assert bkp[i]["name"] == DEFAULT_BACKUP_VERSION + + def test_backup_log_rotation(self): + bvk = self.get_backup_config() + assert_log_rotation_backup_monitoring(bvk) + + def test_proper_automation_config_version(self, config_version): + config = self.get_automation_config() + # We create 3 members of the replicaset here, so there will be 2 changes. + # Anything more than 2 + 4 (logRotation has 4 changes) changes + # indicates that we're sending more things to the Ops/Cloud Manager than we should. + if is_default_architecture_static(): + assert (config["version"] - config_version.version) == 5 + else: + assert (config["version"] - config_version.version) == 6 + + @skip_if_local + def test_replica_set_was_configured(self, cluster_domain: str): + ReplicaSetTester(RESOURCE_NAME, 3, ssl=False, cluster_domain=cluster_domain).assert_connectivity() + + def test_replica_set_was_configured_with_srv(self, cluster_domain: str): + ReplicaSetTester(RESOURCE_NAME, 3, ssl=False, srv=True, cluster_domain=cluster_domain).assert_connectivity() + + +@pytest.mark.e2e_replica_set +def test_replica_set_can_be_scaled_to_single_member(replica_set: MongoDB): + """Scaling to 1 member somehow changes the way the Replica Set is represented and there + will be no more a "Primary" or "Secondaries" in the client, so the test does not check + Replica Set state. An additional test `test_replica_set_can_be_scaled_down_and_connectable` + scales down to 3 (from 5) and makes sure the Replica is connectable with "Primary" and + "Secondaries" set.""" + replica_set["spec"]["members"] = 1 + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=1200) + + actester = AutomationConfigTester(KubernetesTester.get_automation_config()) + + # we should have only 1 process on the replica-set + assert len(actester.get_replica_set_processes(replica_set.name)) == 1 + + assert replica_set["status"]["members"] == 1 + + replica_set.assert_connectivity() + + +@pytest.mark.e2e_replica_set +class TestReplicaSetScaleUp(KubernetesTester): + def test_mdb_updated(self, replica_set: MongoDB): + replica_set["spec"]["members"] = 5 + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=500) + + def test_replica_set_sts_should_exist(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + assert sts + + def test_sts_update(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + assert sts.api_version == "apps/v1" + assert sts.kind == "StatefulSet" + assert sts.status.current_replicas == 5 + assert sts.status.ready_replicas == 5 + + def test_sts_metadata(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + assert sts.metadata.name == RESOURCE_NAME + assert sts.metadata.labels["app"] == "my-replica-set-svc" + assert sts.metadata.namespace == self.namespace + owner_ref0 = sts.metadata.owner_references[0] + assert owner_ref0.api_version == "mongodb.com/v1" + assert owner_ref0.kind == "MongoDB" + assert owner_ref0.name == RESOURCE_NAME + + def test_sts_replicas(self): + sts = self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + assert sts.spec.replicas == 5 + + def _get_pods(self, podname, qty): + return [podname.format(i) for i in range(qty)] + + def test_replica_set_pods_exists(self): + for podname in self._get_pods("my-replica-set-{}", 5): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + assert pod.metadata.name == podname + + def test_pods_are_running(self): + for podname in self._get_pods("my-replica-set-{}", 5): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + assert pod.status.phase == "Running" + + def test_pods_containers(self): + for podname in self._get_pods("my-replica-set-{}", 5): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + c0 = pod.spec.containers[0] + if is_default_architecture_static(): + assert c0.name == "mongodb-agent" + else: + assert c0.name == "mongodb-enterprise-database" + + def test_pods_containers_ports(self): + for podname in self._get_pods("my-replica-set-{}", 5): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + c0 = pod.spec.containers[0] + assert c0.ports[0].container_port == 27017 + assert c0.ports[0].host_ip is None + assert c0.ports[0].host_port is None + assert c0.ports[0].protocol == "TCP" + + def test_pods_container_envvars(self): + for pod_name in self._get_pods("my-replica-set-{}", 5): + assert_container_env_vars(self.namespace, pod_name) + + def test_service_is_created(self): + svc = self.corev1.read_namespaced_service("my-replica-set-svc", self.namespace) + assert svc + + def test_om_processes_are_created(self): + config = self.get_automation_config() + assert len(config["processes"]) == 5 + + def test_om_replica_set_is_created(self): + config = self.get_automation_config() + assert len(config["replicaSets"]) == 1 + + def test_om_processes(self, custom_mdb_version: str, cluster_domain: str): + config = self.get_automation_config() + processes = config["processes"] + for idx in range(0, 4): + name = f"my-replica-set-{idx}" + p = processes[idx] + assert p["name"] == name + assert p["processType"] == "mongod" + assert custom_mdb_version in p["version"] + assert p["authSchemaVersion"] == 5 + assert p["featureCompatibilityVersion"] == fcv_from_version(custom_mdb_version) + assert p["hostname"] == "{}.my-replica-set-svc.{}.svc.{}".format(name, self.namespace, cluster_domain) + assert p["args2_6"]["net"]["port"] == 27017 + assert p["args2_6"]["replication"]["replSetName"] == RESOURCE_NAME + assert p["args2_6"]["storage"]["dbPath"] == "/data" + assert p["args2_6"]["systemLog"]["destination"] == "file" + assert p["args2_6"]["systemLog"]["path"] == "/var/log/mongodb-mms-automation/mongodb.log" + assert p["logRotate"]["sizeThresholdMB"] == 100 + assert p["logRotate"]["timeThresholdHrs"] == 1 + + def test_om_replica_set(self): + config = self.get_automation_config() + rs = config["replicaSets"] + assert rs[0]["_id"] == RESOURCE_NAME + + for idx in range(0, 4): + m = rs[0]["members"][idx] + assert m["_id"] == idx + assert m["arbiterOnly"] is False + assert m["hidden"] is False + assert m["priority"] == 1.0 + assert m["votes"] == 1 + assert m["buildIndexes"] is True + assert m["host"] == f"my-replica-set-{idx}" + + def test_monitoring_versions(self, cluster_domain: str): + config = self.get_automation_config() + mv = config["monitoringVersions"] + assert len(mv) == 5 + + # Monitoring agent is installed on all hosts + for i in range(0, 5): + if "baseUrl" in mv[i]: + assert mv[i]["baseUrl"] is None + hostname = "my-replica-set-{}.my-replica-set-svc.{}.svc.{}".format(i, self.namespace, cluster_domain) + assert mv[i]["hostname"] == hostname + assert mv[i]["name"] == DEFAULT_MONITORING_AGENT_VERSION + + def test_backup(self, cluster_domain: str): + config = self.get_automation_config() + # 1 backup agent per host + bkp = config["backupVersions"] + assert len(bkp) == 5 + + # Backup agent is installed on all hosts + for i in range(0, 5): + hostname = "{resource_name}-{idx}.{resource_name}-svc.{namespace}.svc.{cluster_domain}".format( + resource_name=RESOURCE_NAME, idx=i, namespace=self.namespace, cluster_domain=cluster_domain + ) + assert bkp[i]["hostname"] == hostname + assert bkp[i]["name"] == DEFAULT_BACKUP_VERSION + + +@pytest.mark.e2e_replica_set +def test_replica_set_can_be_scaled_down_and_connectable(replica_set: MongoDB): + """Makes sure that scaling down 5->3 members still reaches a Running & connectable state.""" + replica_set["spec"]["members"] = 3 + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + + actester = AutomationConfigTester(KubernetesTester.get_automation_config()) + + assert len(actester.get_replica_set_processes(RESOURCE_NAME)) == 3 + + assert replica_set["status"]["members"] == 3 + + replica_set.assert_connectivity() + + +@pytest.mark.e2e_replica_set +class TestReplicaSetDelete(KubernetesTester): + """ + name: Replica Set Deletion + tags: replica-set, removal + description: | + Deletes a Replica Set. + delete: + file: replica-set.yaml + wait_until: mongo_resource_deleted + """ + + def test_replica_set_sts_doesnt_exist(self): + """The StatefulSet must be removed by Kubernetes as soon as the MongoDB resource is removed. + Note, that this may lag sometimes (caching or whatever?) and it's more safe to wait a bit""" + time.sleep(15) + with pytest.raises(client.rest.ApiException): + self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + def test_service_does_not_exist(self): + with pytest.raises(client.rest.ApiException): + self.corev1.read_namespaced_service(RESOURCE_NAME + "-svc", self.namespace) + + +def assert_container_env_vars(namespace: str, pod_name: str): + pod = client.CoreV1Api().read_namespaced_pod(pod_name, namespace) + c0 = pod.spec.containers[0] + for envvar in c0.env: + if envvar.name == "AGENT_API_KEY": + assert envvar.value is None, "cannot configure value and value_from" + assert envvar.value_from.secret_key_ref.name == f"{_get_group_id(c0.env)}-group-secret" + assert envvar.value_from.secret_key_ref.key == "agentApiKey" + continue + + assert envvar.name in [ + "AGENT_FLAGS", + "BASE_URL", + "GROUP_ID", + "USER_LOGIN", + "LOG_LEVEL", + "SSL_TRUSTED_MMS_SERVER_CERTIFICATE", + "SSL_REQUIRE_VALID_MMS_CERTIFICATES", + "MULTI_CLUSTER_MODE", + "MDB_LOG_FILE_AUTOMATION_AGENT_VERBOSE", + "MDB_LOG_FILE_AUTOMATION_AGENT_STDERR", + "MDB_LOG_FILE_AUTOMATION_AGENT", + "MDB_LOG_FILE_MONITORING_AGENT", + "MDB_LOG_FILE_BACKUP_AGENT", + "MDB_LOG_FILE_MONGODB", + "MDB_LOG_FILE_MONGODB_AUDIT", + "MDB_STATIC_CONTAINERS_ARCHITECTURE", + ] + assert envvar.value is not None or envvar.name == "AGENT_FLAGS" diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_agent_flags.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_agent_flags.py new file mode 100644 index 000000000..cf95eb81f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_agent_flags.py @@ -0,0 +1,176 @@ +from typing import Optional + +from kubetester import ( + create_or_update_configmap, + find_fixture, + random_k8s_name, + read_configmap, +) +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark +from tests.pod_logs import ( + assert_log_types_in_structured_json_pod_log, + get_all_default_log_types, + get_all_log_types, +) + +custom_agent_log_path = "/var/log/mongodb-mms-automation/customLogFile" +custom_readiness_log_path = "/var/log/mongodb-mms-automation/customReadinessLogFile" + + +@fixture(scope="module") +def project_name_prefix(namespace: str) -> str: + return random_k8s_name(f"{namespace}-project") + + +@fixture(scope="module") +def first_project(namespace: str, project_name_prefix: str) -> str: + cm = read_configmap(namespace=namespace, name="my-project") + project_name = f"{project_name_prefix}-first" + return create_or_update_configmap( + namespace=namespace, + name=project_name, + data={ + "baseUrl": cm["baseUrl"], + "projectName": project_name, + "orgId": cm["orgId"], + }, + ) + + +@fixture(scope="module") +def second_project(namespace: str, project_name_prefix: str) -> str: + cm = read_configmap(namespace=namespace, name="my-project") + project_name = project_name_prefix + return create_or_update_configmap( + namespace=namespace, + name=project_name, + data={ + "baseUrl": cm["baseUrl"], + "projectName": project_name, + "orgId": cm["orgId"], + }, + ) + + +@fixture(scope="module") +def replica_set(namespace: str, first_project: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("replica-set-basic.yaml"), namespace=namespace, name="replica-set") + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + resource["spec"]["opsManager"]["configMapRef"]["name"] = first_project + + resource.update() + return resource + + +@fixture(scope="module") +def second_replica_set(namespace: str, second_project: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("replica-set-basic.yaml"), namespace=namespace, name="replica-set-2") + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["opsManager"]["configMapRef"]["name"] = second_project + + resource.update() + return resource + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_replica_set(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_second_replica_set(second_replica_set: MongoDB): + second_replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_log_types_with_default_automation_log_file(replica_set: MongoDB): + assert_pod_log_types(replica_set, get_all_default_log_types()) + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_set_custom_log_file(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["agent"] = { + "startupOptions": { + "logFile": custom_agent_log_path, + "maxLogFileSize": "10485760", + "maxLogFiles": "5", + "maxLogFileDurationHrs": "16", + "logFile": "/var/log/mongodb-mms-automation/customLogFile", + } + } + replica_set["spec"]["agent"].setdefault("readinessProbe", {}) + # LOG_FILE_PATH is an env var used by the readinessProbe to configure where we log to + replica_set["spec"]["agent"]["readinessProbe"] = { + "environmentVariables": {"LOG_FILE_PATH": custom_readiness_log_path} + } + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_replica_set_has_agent_flags(replica_set: MongoDB, namespace: str): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFile* | wc -l", + ] + for i in range(3): + result = KubernetesTester.run_command_in_pod_container( + f"replica-set-{i}", + namespace, + cmd, + ) + assert result != "0" + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_log_readiness_probe_path_set_via_env_var(replica_set: MongoDB, namespace: str): + cmd = [ + "/bin/sh", + "-c", + f"ls {custom_readiness_log_path}* | wc -l", + ] + for i in range(3): + result = KubernetesTester.run_command_in_pod_container( + f"replica-set-{i}", + namespace, + cmd, + ) + assert result != "0" + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_log_types_with_custom_automation_log_file(replica_set: MongoDB): + assert_pod_log_types(replica_set, get_all_default_log_types()) + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_enable_audit_log(replica_set: MongoDB): + additional_mongod_config = { + "auditLog": { + "destination": "file", + "format": "JSON", + "path": "/var/log/mongodb-mms-automation/mongodb-audit-changed.log", + } + } + replica_set["spec"]["additionalMongodConfig"] = additional_mongod_config + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replica_set_agent_flags_and_readinessProbe +def test_log_types_with_audit_enabled(replica_set: MongoDB): + assert_pod_log_types(replica_set, get_all_log_types()) + + +def assert_pod_log_types(replica_set: MongoDB, expected_log_types: Optional[set[str]]): + for i in range(3): + assert_log_types_in_structured_json_pod_log( + replica_set.namespace, f"{replica_set.name}-{i}", expected_log_types + ) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_config_map.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_config_map.py new file mode 100644 index 000000000..b02bba69a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_config_map.py @@ -0,0 +1,25 @@ +import pytest +from kubernetes.client import V1ConfigMap +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase + + +@pytest.fixture(scope="module") +def mdb(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("replica-set-single.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + return resource.update() + + +@pytest.mark.e2e_replica_set_config_map +def test_create_replica_set(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_replica_set_config_map +def test_patch_config_map(namespace: str, mdb: MongoDB): + config_map = V1ConfigMap(data={"orgId": "wrongId"}) + KubernetesTester.clients("corev1").patch_namespaced_config_map("my-project", namespace, config_map) + print('Patched the ConfigMap - changed orgId to "wrongId"') + mdb.assert_reaches_phase(Phase.Failed, timeout=20) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_custom_podspec.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_custom_podspec.py new file mode 100644 index 000000000..5fb71483d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_custom_podspec.py @@ -0,0 +1,149 @@ +from kubetester import try_load +from kubetester.custom_podspec import assert_stateful_set_podspec +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_static_containers +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set-custom-podspec.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + try_load(resource) + return resource + + +@skip_if_static_containers +@mark.e2e_replica_set_custom_podspec +def test_replica_set_reaches_running_phase(replica_set): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@skip_if_static_containers +@mark.e2e_replica_set_custom_podspec +def test_stateful_set_spec_updated(replica_set, namespace): + appsv1 = KubernetesTester.clients("appsv1") + sts = appsv1.read_namespaced_stateful_set(replica_set.name, namespace) + containers_spec = [ + { + "name": "mongodb-enterprise-database", + "resources": { + "limits": { + "cpu": "2", + }, + "requests": { + "cpu": "1", + }, + }, + "volume_mounts": [ + { + "name": "database-scripts", + "mount_path": "/opt/scripts", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": True, + }, + { + "name": "test-volume", + "mount_path": "/somewhere", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "agent-api-key", + "mount_path": "/mongodb-automation/agent-api-key", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/data", + "sub_path": "data", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/journal", + "sub_path": "journal", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "data", + "mount_path": "/var/log/mongodb-mms-automation", + "sub_path": "logs", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "agent", + "mount_path": "/tmp", + "sub_path": "tmp", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "agent", + "mount_path": "/var/lib/mongodb-mms-automation", + "sub_path": "mongodb-mms-automation", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + { + "name": "agent", + "mount_path": "/mongodb-automation", + "sub_path": "mongodb-automation", + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + }, + ], + }, + { + "name": "side-car", + "image": "busybox:latest", + "volume_mounts": [ + { + "mount_path": "/somewhere", + "name": "test-volume", + "sub_path": None, + "sub_path_expr": None, + "mount_propagation": None, + "read_only": None, + } + ], + "command": ["/bin/sh"], + "args": ["-c", "echo ok > /somewhere/busybox_file && sleep 86400"], + }, + ] + + assert_stateful_set_podspec( + sts.spec.template.spec, + weight=50, + topology_key="mykey-rs", + grace_period_seconds=30, + containers_spec=containers_spec, + ) + + host_aliases = sts.spec.template.spec.host_aliases + alias = host_aliases[0] + + assert len(host_aliases) == 1 + assert alias.ip == "1.2.3.4" + assert alias.hostnames[0] == "hostname" + assert len(sts.spec.template.metadata.annotations) == 1 + assert sts.spec.template.metadata.annotations["key1"] == "val1" diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_custom_sa.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_custom_sa.py new file mode 100644 index 000000000..947e16adc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_custom_sa.py @@ -0,0 +1,42 @@ +from kubetester import create_service_account, delete_service_account +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def create_custom_sa(namespace: str) -> str: + return create_service_account(namespace=namespace, name="test-sa") + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str, create_custom_sa: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), namespace=namespace) + + resource["spec"]["podSpec"] = {"podTemplate": {"spec": {"serviceAccountName": "test-sa"}}} + resource["spec"]["statefulSet"] = {"spec": {"serviceName": "rs-svc"}} + resource.set_version(custom_mdb_version) + yield resource.create() + # teardown, delete the custom service-account + delete_service_account(namespace=namespace, name="test-sa") + + +@mark.e2e_replica_set_custom_sa +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replica_set_custom_sa +def test_stateful_set_spec_service_account(replica_set: MongoDB, namespace: str): + appsv1 = KubernetesTester.clients("appsv1") + sts = appsv1.read_namespaced_stateful_set(replica_set.name, namespace) + + assert sts.spec.template.spec.service_account_name == "test-sa" + + +@mark.e2e_replica_set_custom_sa +def test_service_is_created(namespace: str): + corev1 = KubernetesTester.clients("corev1") + svc = corev1.read_namespaced_service("rs-svc", namespace) + assert svc diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_exposed_externally.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_exposed_externally.py new file mode 100644 index 000000000..43258e074 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_exposed_externally.py @@ -0,0 +1,84 @@ +import pytest +from kubernetes import client +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture +from tests.common.placeholders import placeholders + + +@fixture(scope="module") +def replica_set(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-externally-exposed.yaml"), + "my-replica-set-externally-exposed", + namespace, + ) + try_load(resource) + return resource + + +@pytest.mark.e2e_replica_set_exposed_externally +def test_replica_set_created(replica_set: MongoDB, custom_mdb_version: str): + replica_set["spec"]["members"] = 2 + replica_set.set_version(custom_mdb_version) + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + +def test_service_exists(namespace: str): + for i in range(2): + service = client.CoreV1Api().read_namespaced_service( + f"my-replica-set-externally-exposed-{i}-svc-external", namespace + ) + assert service.spec.type == "LoadBalancer" + assert service.spec.ports[0].port == 27017 + + +@pytest.mark.e2e_replica_set_exposed_externally +def test_service_node_port_stays_the_same(namespace: str, replica_set: MongoDB): + service = client.CoreV1Api().read_namespaced_service("my-replica-set-externally-exposed-0-svc-external", namespace) + node_port = service.spec.ports[0].node_port + + replica_set.load() + replica_set["spec"]["members"] = 3 + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + service = client.CoreV1Api().read_namespaced_service("my-replica-set-externally-exposed-0-svc-external", namespace) + assert service.spec.type == "LoadBalancer" + assert service.spec.ports[0].node_port == node_port + + +@pytest.mark.e2e_replica_set_exposed_externally +def test_placeholders_in_external_services(namespace: str, replica_set: MongoDB): + external_access = replica_set["spec"].get("externalAccess", {}) + external_service = external_access.get("externalService", {}) + external_service["annotations"] = placeholders.get_annotations_with_placeholders_for_single_cluster() + external_access["externalService"] = external_service + replica_set["spec"]["externalAccess"] = external_access + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + name = replica_set["metadata"]["name"] + for pod_idx in range(0, replica_set.get_members()): + service = client.CoreV1Api().read_namespaced_service(f"{name}-{pod_idx}-svc-external", namespace) + assert service.metadata.annotations == placeholders.get_expected_annotations_single_cluster( + name, namespace, pod_idx + ) + + +@pytest.mark.e2e_replica_set_exposed_externally +def test_service_gets_deleted(replica_set: MongoDB, namespace: str): + replica_set.load() + last_transition = replica_set.get_status_last_transition_time() + replica_set["spec"]["externalAccess"] = None + replica_set.update() + + replica_set.assert_state_transition_happens(last_transition) + replica_set.assert_reaches_phase(Phase.Running, timeout=300) + for i in range(replica_set["spec"]["members"]): + with pytest.raises(client.rest.ApiException): + client.CoreV1Api().read_namespaced_service(f"my-replica-set-externally-exposed-{i}-svc-external", namespace) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_groups.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_groups.py new file mode 100644 index 000000000..2a9cb68fa --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_groups.py @@ -0,0 +1,111 @@ +import pytest +from kubetester.kubetester import ( + EXTERNALLY_MANAGED_TAG, + MAX_TAG_LEN, + KubernetesTester, + fixture, +) +from kubetester.omtester import should_include_tag + + +@pytest.mark.e2e_replica_set_groups +class TestReplicaSetOrganizationsPagination(KubernetesTester): + """ + name: Test for configuration when organization id is not specified but the organization exists already + description: | + Both organization and group already exist. + Two things are tested: + 1. organization id is not specified in config map but the organization with the same name already exists + so the Operator will find it by name + 2. the group already exists so no new group will be created + The test is skipped for cloud manager as we cannot create organizations there ("API_KEY_CANNOT_CREATE_ORG") + """ + + all_orgs_ids = [] + all_groups_ids = [] + org_id = None + group_name = None + + @classmethod + def setup_env(cls): + # Create 5 organizations + cls.all_orgs_ids = cls.create_organizations(5) + + # Create another organization with the same name as group + cls.group_name = KubernetesTester.random_k8s_name("replica-set-group-test-") + cls.org_id = cls.create_organization(cls.group_name) + + # Create 5 groups inside the organization + cls.all_groups_ids = cls.create_groups(cls.org_id, 5) + + # Create the group manually (btw no tag will be set - this must be fixed by the Operator) + cls.create_group(cls.org_id, cls.group_name) + + # Update the config map - change the group name, no orgId + cls.patch_config_map( + cls.get_namespace(), + "my-project", + {"projectName": cls.group_name, "orgId": ""}, + ) + + print('Patched config map, now it has the projectName "{}"'.format(cls.group_name)) + + def test_standalone_created_organization_found(self): + groups_in_org = self.get_groups_in_organization_first_page(self.__class__.org_id)["totalCount"] + + # Create a replica set - both the organization and the group will be found (after traversing pages) + self.create_custom_resource_from_file(self.get_namespace(), fixture("replica-set-single.yaml")) + KubernetesTester.wait_until("in_running_state", 150) + + # Making sure no more groups and organizations were created, but the tag was fixed by the Operator + assert len(self.find_organizations(self.__class__.group_name)) == 1 + print('Only one organization with name "{}" exists (as expected)'.format(self.__class__.group_name)) + + assert self.get_groups_in_organization_first_page(self.__class__.org_id)["totalCount"] == groups_in_org + group = self.query_group(self.__class__.group_name) + assert group is not None + assert group["orgId"] == self.__class__.org_id + + version = KubernetesTester.om_version() + expected_tags = [self.namespace[:MAX_TAG_LEN].upper()] + + if should_include_tag(version): + expected_tags.append(EXTERNALLY_MANAGED_TAG) + + assert sorted(group["tags"]) == sorted(expected_tags) + + print('Only one group with name "{}" exists (as expected)'.format(self.__class__.group_name)) + + @staticmethod + def create_organizations(count): + ids = [] + for i in range(0, count): + ids.append(KubernetesTester.create_organization(KubernetesTester.random_k8s_name("fake-{}-".format(i)))) + if (i + 1) % 100 == 0: + print("Created {} fake organizations".format(i + 1)) + return ids + + @staticmethod + def create_groups(org_id, count): + ids = [] + for i in range(0, count): + ids.append( + KubernetesTester.create_group(org_id, KubernetesTester.random_k8s_name("fake-group-{}-".format(i))) + ) + if (i + 1) % 100 == 0: + print("Created {} fake groups inside organization {}".format(i + 1, org_id)) + return ids + + @classmethod + def teardown_env(cls): + # Remove fake organizations from Ops Manager, except for the organization that was used by the standalone + for i, id in enumerate(cls.all_orgs_ids): + cls.remove_organization(id) + if (i + 1) % 100 == 0: + print("Removed {} fake organizations".format(i)) + + # Remove fake groups from Ops Manager + for i, id in enumerate(cls.all_groups_ids): + cls.remove_group(id) + if (i + 1) % 100 == 0: + print("Removed {} fake groups inside organizationn {}".format(i, cls.org_id)) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_liveness_probe.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_liveness_probe.py new file mode 100644 index 000000000..591d1240a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_liveness_probe.py @@ -0,0 +1,107 @@ +import time +from typing import Set + +import pytest +from kubernetes import client +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_static_containers +from kubetester.mongodb import MongoDB +from pytest import fixture + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set-liveness.yaml"), "my-replica-set", namespace) + + resource.update() + + return resource + + +def _get_pods(podname_template: str, qty: int = 3): + return [podname_template.format(i) for i in range(qty)] + + +@skip_if_static_containers +@pytest.mark.e2e_replica_set_liveness_probe +def test_pods_are_running(replica_set: MongoDB, namespace: str): + corev1_client = client.CoreV1Api() + running_pods: Set[str] = set() + tries = 10 + # Wait for all the pods to be running + # We can't wait for the replica set to be running + # as it will never get to it (mongod is not starting) + while tries: + if len(running_pods) == 3: + break + for podname in _get_pods("my-replica-set-{}", 3): + try: + pod = corev1_client.read_namespaced_pod(podname, namespace) + if pod.status.phase == "Running": + running_pods.add(podname) + except: + # Pod not found, will retry + pass + tries -= 1 + time.sleep(30) + assert len(running_pods) == 3 + + +@skip_if_static_containers +@pytest.mark.e2e_replica_set_liveness_probe +def test_no_pods_get_restarted(replica_set: MongoDB, namespace: str): + corev1_client = client.CoreV1Api() + statefulset_liveness_probe = replica_set.read_statefulset().spec.template.spec.containers[0].liveness_probe + failure_threshold = statefulset_liveness_probe.failure_threshold + period_seconds = statefulset_liveness_probe.period_seconds + + # Leave some extra time after the failure threshold just to be sure + time.sleep(failure_threshold * period_seconds + 20) + for podname in _get_pods("my-replica-set-{}", 3): + pod = corev1_client.read_namespaced_pod(podname, namespace) + + # Pods should not restart because of a missing mongod process + assert pod.status.container_statuses[0].restart_count == 0 + + +@skip_if_static_containers +@pytest.mark.e2e_replica_set_liveness_probe +@pytest.mark.skip( + reason="Liveness probe checks for mongod process to be up so killing the agent alone won't trigger a pod restart" +) +def test_pods_are_restarted_if_agent_process_is_terminated(replica_set: MongoDB, namespace: str): + corev1_client = client.CoreV1Api() + + agent_pid_file = "/mongodb-automation/mongodb-mms-automation-agent.pid" + pid_cmd = ["cat", agent_pid_file] + # Get the agent's PID + agent_pid = KubernetesTester.run_command_in_pod_container("my-replica-set-0", namespace, pid_cmd) + + # Kill the agent using its PID + kill_cmd = ["kill", "-s", "SIGTERM", agent_pid.strip()] + KubernetesTester.run_command_in_pod_container("my-replica-set-0", namespace, kill_cmd) + + # Ensure agent's pid file still exists. + # This is to simulate not graceful kill, e.g. by OOM killer + agent_pid_2 = KubernetesTester.run_command_in_pod_container("my-replica-set-0", namespace, pid_cmd) + + assert agent_pid == agent_pid_2 + + statefulset_liveness_probe = replica_set.read_statefulset().spec.template.spec.containers[0].liveness_probe + failure_threshold = statefulset_liveness_probe.failure_threshold + period_seconds = statefulset_liveness_probe.period_seconds + time.sleep(failure_threshold * period_seconds + 20) + + # Pod zero should have restarted, because the agent was killed + assert ( + corev1_client.read_namespaced_pod("my-replica-set-0", namespace).status.container_statuses[0].restart_count > 0 + ) + + # Pods 1 and 2 should not have restarted, because the agent is intact + assert ( + corev1_client.read_namespaced_pod("my-replica-set-1", namespace).status.container_statuses[0].restart_count == 0 + ) + assert ( + corev1_client.read_namespaced_pod("my-replica-set-2", namespace).status.container_statuses[0].restart_count == 0 + ) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_member_options.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_member_options.py new file mode 100644 index 000000000..7815db55d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_member_options.py @@ -0,0 +1,155 @@ +import pytest +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture + +RESOURCE_NAME = "my-replica-set" + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), RESOURCE_NAME, namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["memberConfig"] = [ + { + "votes": 1, + "priority": "0.5", + "tags": { + "tag1": "value1", + "environment": "prod", + }, + }, + { + "votes": 1, + "priority": "1.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + }, + { + "votes": 1, + "priority": "0.5", + "tags": { + "tag2": "value2", + "environment": "prod", + }, + }, + ] + resource.update() + + return resource + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_created(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_member_options_ac(replica_set: MongoDB): + replica_set.load() + + config = replica_set.get_automation_config_tester().automation_config + rs = config["replicaSets"] + + member1 = rs[0]["members"][0] + member2 = rs[0]["members"][1] + member3 = rs[0]["members"][2] + + assert member1["votes"] == 1 + assert member1["priority"] == 0.5 + assert member1["tags"] == {"tag1": "value1", "environment": "prod"} + + assert member2["votes"] == 1 + assert member2["priority"] == 1.5 + assert member2["tags"] == {"tag2": "value2", "environment": "prod"} + + assert member3["votes"] == 1 + assert member3["priority"] == 0.5 + assert member3["tags"] == {"tag2": "value2", "environment": "prod"} + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_update_member_options(replica_set: MongoDB): + replica_set.load() + + replica_set["spec"]["memberConfig"][0] = { + "votes": 1, + "priority": "2.5", + "tags": { + "tag1": "value1", + "tag2": "value2", + "environment": "prod", + }, + } + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running) + + config = replica_set.get_automation_config_tester().automation_config + rs = config["replicaSets"] + + updated_member = rs[0]["members"][0] + assert updated_member["votes"] == 1 + assert updated_member["priority"] == 2.5 + assert updated_member["tags"] == { + "tag1": "value1", + "tag2": "value2", + "environment": "prod", + } + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_member_votes_to_0(replica_set: MongoDB): + replica_set.load() + + # A non-voting member must also have priority set to 0 + replica_set["spec"]["memberConfig"][1]["votes"] = 0 + replica_set["spec"]["memberConfig"][1]["priority"] = "0.0" + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running) + + config = replica_set.get_automation_config_tester().automation_config + rs = config["replicaSets"] + + updated_member = rs[0]["members"][1] + assert updated_member["votes"] == 0 + assert updated_member["priority"] == 0.0 + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_invalid_votes_and_priority(replica_set: MongoDB): + replica_set.load() + # A member with 0 votes must also have priority 0.0 + replica_set["spec"]["memberConfig"][1]["votes"] = 0 + replica_set["spec"]["memberConfig"][1]["priority"] = "1.2" + replica_set.update() + replica_set.assert_reaches_phase( + Phase.Failed, + msg_regexp="Failed to create/update \(Ops Manager reconciliation phase\).*cannot have 0 votes when priority is greater than 0", + ) + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_recover_valid_member_options(replica_set: MongoDB): + replica_set.load() + # A member with priority 0.0 could still be a voting member. It cannot become primary and cannot trigger elections. + # https://www.mongodb.com/docs/v5.0/core/replica-set-priority-0-member/#priority-0-replica-set-members + replica_set["spec"]["memberConfig"][1]["votes"] = 1 + replica_set["spec"]["memberConfig"][1]["priority"] = "0.0" + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_replica_set_member_options +def test_replica_set_only_one_vote_per_member(replica_set: MongoDB): + replica_set.load() + # A single voting member can only have 1 vote + replica_set["spec"]["memberConfig"][2]["votes"] = 5 + replica_set["spec"]["memberConfig"][2]["priority"] = "5.8" + replica_set.update() + replica_set.assert_reaches_phase( + Phase.Failed, + msg_regexp="Failed to create/update \(Ops Manager reconciliation phase\).*cannot have greater than 1 vote", + ) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_migration.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_migration.py new file mode 100644 index 000000000..e8e022830 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_migration.py @@ -0,0 +1,79 @@ +import pymongo +import pytest +from kubetester import MongoDB, try_load +from kubetester.kubetester import assert_statefulset_architecture, ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import get_default_architecture +from kubetester.mongodb import Phase +from kubetester.mongotester import MongoDBBackgroundTester, MongoTester +from pytest import fixture + +MDB_RESOURCE_NAME = "replica-set-migration" + + +@fixture(scope="module") +def mdb(namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + load_fixture("replica-set.yaml"), + namespace=namespace, + name=MDB_RESOURCE_NAME, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + return resource + + +@fixture(scope="module") +def mongo_tester(mdb: MongoDB): + return mdb.tester() + + +@fixture(scope="module") +def mdb_health_checker(mongo_tester: MongoTester) -> MongoDBBackgroundTester: + return MongoDBBackgroundTester( + mongo_tester, + allowed_sequential_failures=1, + health_function_params={ + "attempts": 1, + "write_concern": pymongo.WriteConcern(w="majority"), + }, + ) + + +@pytest.mark.e2e_replica_set_migration +class TestReplicaSetMigrationStatic: + + def test_create_cluster(self, mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running) + + def test_start_health_checker(self, mdb_health_checker): + mdb_health_checker.start() + + def test_migrate_architecture(self, mdb: MongoDB): + """ + If the E2E is running with default architecture as non-static, + then the test will migrate to static and vice versa. + """ + original_default_architecture = get_default_architecture() + target_architecture = "non-static" if original_default_architecture == "static" else "static" + + mdb.trigger_architecture_migration() + + mdb.load() + assert mdb["metadata"]["annotations"]["mongodb.com/v1.architecture"] == target_architecture + + mdb.assert_abandons_phase(Phase.Running, timeout=1000) + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + + # Read StatefulSet after successful reconciliation + sts = mdb.read_statefulset() + assert_statefulset_architecture(sts, target_architecture) + + def test_mdb_healthy_throughout_change_version(self, mdb_health_checker): + mdb_health_checker.assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_mongod_options.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_mongod_options.py new file mode 100644 index 000000000..1c7e3e05a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_mongod_options.py @@ -0,0 +1,93 @@ +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-mongod-options.yaml"), + namespace=namespace, + ) + resource["spec"]["persistent"] = True + return resource.update() + + +@mark.e2e_replica_set_mongod_options +def test_replica_set_created(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) + + +@mark.e2e_replica_set_mongod_options +def test_replica_set_mongodb_options(replica_set: MongoDB): + automation_config_tester = replica_set.get_automation_config_tester() + for process in automation_config_tester.get_replica_set_processes(replica_set.name): + assert process["args2_6"]["systemLog"]["verbosity"] == 4 + assert process["args2_6"]["systemLog"]["logAppend"] + assert process["args2_6"]["operationProfiling"]["mode"] == "slowOp" + assert process["args2_6"]["net"]["port"] == 30000 + + +@mark.e2e_replica_set_mongod_options +def test_replica_set_feature_controls(replica_set: MongoDB): + fc = replica_set.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + assert len(fc["policies"]) == 3 + # unfortunately OM uses a HashSet for policies... + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert policies[0]["policy"] == "DISABLE_SET_MONGOD_CONFIG" + assert policies[1]["policy"] == "DISABLE_SET_MONGOD_VERSION" + assert policies[2]["policy"] == "EXTERNALLY_MANAGED_LOCK" + + # OM stores the params into a set - we need to sort to compare + disabled_params = sorted(policies[0]["disabledParams"]) + assert disabled_params == [ + "net.port", + "operationProfiling.mode", + "systemLog.logAppend", + "systemLog.verbosity", + ] + + +@mark.e2e_replica_set_mongod_options +def test_replica_set_updated(replica_set: MongoDB): + replica_set["spec"]["additionalMongodConfig"]["systemLog"]["verbosity"] = 2 + replica_set["spec"]["additionalMongodConfig"]["net"]["maxIncomingConnections"] = 100 + + # update uses json merge+patch which means that deleting keys is done by setting them to None + replica_set["spec"]["additionalMongodConfig"]["operationProfiling"] = None + + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running) + + +@mark.e2e_replica_set_mongod_options +def test_replica_set_mongodb_options_were_updated(replica_set: MongoDB): + automation_config_tester = replica_set.get_automation_config_tester() + for process in automation_config_tester.get_replica_set_processes(replica_set.name): + assert process["args2_6"]["systemLog"]["verbosity"] == 2 + assert process["args2_6"]["systemLog"]["logAppend"] + assert process["args2_6"]["net"]["maxIncomingConnections"] == 100 + assert process["args2_6"]["net"]["port"] == 30000 + # the mode setting has been removed + assert "mode" not in process["args2_6"]["operationProfiling"] + + +@mark.e2e_replica_set_mongod_options +def test_replica_set_feature_controls_were_updated(replica_set: MongoDB): + fc = replica_set.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + assert len(fc["policies"]) == 3 + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert policies[0]["policy"] == "DISABLE_SET_MONGOD_CONFIG" + assert policies[1]["policy"] == "DISABLE_SET_MONGOD_VERSION" + assert policies[2]["policy"] == "EXTERNALLY_MANAGED_LOCK" + + disabled_params = sorted(policies[0]["disabledParams"]) + assert disabled_params == [ + "net.maxIncomingConnections", + "net.port", + "systemLog.logAppend", + "systemLog.verbosity", + ] diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_perf.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_perf.py new file mode 100644 index 000000000..86f30b9a8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_perf.py @@ -0,0 +1,89 @@ +# * MDB_MAX_CONCURRENT_RECONCILES is set in context +# * prepare_operator_deployment sets helm flag with it in operator-installation-config cm before the test is run +# * default_operator fixture is using it, therefore passing that var into helm chart installing the operator +# In the future we should move MDB_MAX_CONCURRENT_RECONCILES into a env var as well and set the operator accordingly +import os +from typing import Optional + +import pytest +from kubetester import find_fixture, try_load +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.opsmanager import MongoDBOpsManager +from tests.conftest import get_custom_mdb_version + + +@pytest.fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + # We require using the fixture with maxGroups setting increased. + # Otherwise, we will run into the limit of 250 per user + resource = MongoDBOpsManager.from_yaml(find_fixture("om_more_orgs.yaml"), namespace=namespace, name="om") + + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + try_load(resource) + return resource + + +def get_replica_set(ops_manager, namespace: str, idx: int) -> MongoDB: + name = f"mdb-{idx}-rs" + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-perf.yaml"), + namespace=namespace, + name=name, + ).configure(ops_manager, name) + + replicas = int(os.getenv("PERF_TASK_REPLICAS", "3")) + resource["spec"]["members"] = replicas + resource.set_version(get_custom_mdb_version()) + + try_load(resource) + return resource + + +def get_all_rs(ops_manager, namespace) -> list[MongoDB]: + deployments = int(os.getenv("PERF_TASK_DEPLOYMENTS", "100")) + return [get_replica_set(ops_manager, namespace, idx) for idx in range(0, deployments)] + + +@pytest.mark.e2e_om_reconcile_perf +def test_create_om(ops_manager: MongoDBOpsManager): + ops_manager.update() + ops_manager.om_status().assert_reaches_phase(Phase.Running) + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_om_reconcile_perf +def test_create_mdb(ops_manager, namespace: str): + for resource in get_all_rs(ops_manager, namespace): + resource["spec"]["security"] = { + "authentication": {"agents": {"mode": "SCRAM"}, "enabled": True, "modes": ["SCRAM"]} + } + resource.set_version(get_custom_mdb_version()) + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running, timeout=2000) + + +@pytest.mark.e2e_om_reconcile_perf +def test_update_mdb(ops_manager, namespace: str): + for resource in get_all_rs(ops_manager, namespace): + additional_mongod_config = { + "auditLog": { + "destination": "file", + "format": "JSON", + "path": "/var/log/mongodb-mms-automation/mongodb-audit-changed.log", + } + } + resource["spec"]["additionalMongodConfig"] = additional_mongod_config + resource.update() + + for r in get_all_rs(ops_manager, namespace): + r.assert_reaches_phase(Phase.Running, timeout=2000) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_process_hostnames.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_process_hostnames.py new file mode 100644 index 000000000..c0b015852 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_process_hostnames.py @@ -0,0 +1,107 @@ +# This test currently relies on MetalLB IP assignment that is configured for kind in scripts/dev/recreate_kind_cluster.sh +# Each service of type LoadBalancer will get IP starting from 172.18.255.200 +# scripts/dev/coredns_single_cluster.yaml configures that my-replica-set-0.mongodb.interconnected starts at 172.18.255.200 + +# Tests checking externalDomain (consider updating all of them when changing test logic here): +# tls/tls_replica_set_process_hostnames.py +# replicaset/replica_set_process_hostnames.py +# om_ops_manager_backup_tls_custom_ca.py + +import pytest +from kubernetes import client +from kubetester import try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture +from tests.common.placeholders import placeholders +from tests.conftest import ( + default_external_domain, + external_domain_fqdns, + update_coredns_hosts, +) + + +@fixture +def replica_set_name() -> str: + return "my-replica-set" + + +@fixture +def replica_set_members() -> int: + return 3 + + +@fixture(scope="function") +def replica_set( + namespace: str, + replica_set_name: str, + replica_set_members: int, + custom_mdb_version: str, +) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), replica_set_name, namespace) + try_load(resource) + + resource["spec"]["members"] = replica_set_members + resource["spec"]["externalAccess"] = {} + resource["spec"]["externalAccess"]["externalDomain"] = default_external_domain() + resource.set_version(custom_mdb_version) + + return resource + + +@pytest.mark.e2e_replica_set_process_hostnames +def test_update_coredns(): + hosts = [ + ("172.18.255.200", "my-replica-set-0.mongodb.interconnected"), + ("172.18.255.201", "my-replica-set-1.mongodb.interconnected"), + ("172.18.255.202", "my-replica-set-2.mongodb.interconnected"), + ("172.18.255.203", "my-replica-set-3.mongodb.interconnected"), + ] + + update_coredns_hosts(hosts) + + +@pytest.mark.e2e_replica_set_process_hostnames +def test_create_replica_set(replica_set: MongoDB): + replica_set.update() + + +@pytest.mark.e2e_replica_set_process_hostnames +def test_replica_set_in_running_state(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + + +@pytest.mark.e2e_replica_set_process_hostnames +def test_replica_check_automation_config(replica_set: MongoDB): + processes = replica_set.get_automation_config_tester().get_replica_set_processes(replica_set.name) + hostnames = [process["hostname"] for process in processes] + assert hostnames == external_domain_fqdns(replica_set.name, replica_set.get_members(), default_external_domain()) + + +@pytest.mark.e2e_replica_set_process_hostnames +def test_placeholders_in_external_services(namespace: str, replica_set: MongoDB): + # we do it this way to only add annotations and not overwrite anything + external_access = replica_set["spec"].get("externalAccess", {}) + external_service = external_access.get("externalService", {}) + external_service["annotations"] = placeholders.get_annotations_with_placeholders_for_single_cluster() + external_access["externalService"] = external_service + + replica_set["spec"]["externalAccess"] = external_access + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + name = replica_set["metadata"]["name"] + for pod_idx in range(0, replica_set.get_members()): + service = client.CoreV1Api().read_namespaced_service(f"{name}-{pod_idx}-svc-external", namespace) + assert ( + service.metadata.annotations + == placeholders.get_expected_annotations_single_cluster_with_external_domain( + name, namespace, pod_idx, default_external_domain() + ) + ) + + +@pytest.mark.e2e_replica_set_process_hostnames +def test_connectivity(replica_set: MongoDB): + tester = replica_set.tester() + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv.py new file mode 100644 index 000000000..2bb6f94a3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv.py @@ -0,0 +1,125 @@ +import time + +import pytest +from kubernetes import client +from kubetester.kubetester import KubernetesTester, fcv_from_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase + + +@pytest.mark.e2e_replica_set_pv +class TestReplicaSetPersistentVolumeCreation(KubernetesTester): + def test_create_replicaset(self, custom_mdb_version: str): + resource = MongoDB.from_yaml(load_fixture("replica-set-pv.yaml"), namespace=self.namespace) + resource.set_version(custom_mdb_version) + resource.update() + + resource.assert_reaches_phase(Phase.Running) + + def test_replica_set_sts_exists(self): + sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) + assert sts + + def test_sts_creation(self): + sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) + + assert sts.api_version == "apps/v1" + assert sts.kind == "StatefulSet" + assert sts.status.current_replicas == 3 + assert sts.status.ready_replicas == 3 + + def test_sts_metadata(self): + sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) + + assert sts.metadata.name == "rs001-pv" + assert sts.metadata.labels["app"] == "rs001-pv-svc" + assert sts.metadata.namespace == self.namespace + owner_ref0 = sts.metadata.owner_references[0] + assert owner_ref0.api_version == "mongodb.com/v1" + assert owner_ref0.kind == "MongoDB" + assert owner_ref0.name == "rs001-pv" + + def test_sts_replicas(self): + sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) + assert sts.spec.replicas == 3 + + def test_sts_template(self): + sts = self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) + + tmpl = sts.spec.template + assert tmpl.metadata.labels["app"] == "rs001-pv-svc" + assert tmpl.metadata.labels["controller"] == "mongodb-enterprise-operator" + assert tmpl.spec.affinity.node_affinity is None + assert tmpl.spec.affinity.pod_affinity is None + assert tmpl.spec.affinity.pod_anti_affinity is not None + + def _get_pods(self, podname, qty=3): + return [podname.format(i) for i in range(qty)] + + def test_pvc_are_created_and_bound(self): + "PersistentVolumeClaims should be created with the correct attributes." + bound_pvc_names = [] + for podname in self._get_pods("rs001-pv-{}", 3): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + bound_pvc_names.append(pod.spec.volumes[0].persistent_volume_claim.claim_name) + + for pvc_name in bound_pvc_names: + pvc_status = self.corev1.read_namespaced_persistent_volume_claim_status(pvc_name, self.namespace) + assert pvc_status.status.phase == "Bound" + + def test_om_processes(self, custom_mdb_version: str): + config = self.get_automation_config() + processes = config["processes"] + for idx, p in enumerate(processes): + assert custom_mdb_version in p["version"] + assert p["name"] == f"rs001-pv-{idx}" + assert p["processType"] == "mongod" + assert p["authSchemaVersion"] == 5 + assert p["featureCompatibilityVersion"] == fcv_from_version(custom_mdb_version) + assert p["hostname"] == "rs001-pv-" + f"{idx}" + ".rs001-pv-svc.{}.svc.cluster.local".format(self.namespace) + assert p["args2_6"]["net"]["port"] == 27017 + assert p["args2_6"]["replication"]["replSetName"] == "rs001-pv" + assert p["args2_6"]["storage"]["dbPath"] == "/data" + assert p["args2_6"]["systemLog"]["destination"] == "file" + assert p["args2_6"]["systemLog"]["path"] == "/var/log/mongodb-mms-automation/mongodb.log" + assert p["logRotate"]["sizeThresholdMB"] == 1000 + assert p["logRotate"]["timeThresholdHrs"] == 24 + + def test_replica_set_was_configured(self): + "Should connect to one of the mongods and check the replica set was correctly configured." + hosts = ["rs001-pv-{}.rs001-pv-svc.{}.svc.cluster.local:27017".format(i, self.namespace) for i in range(3)] + + primary, secondaries = self.wait_for_rs_is_ready(hosts) + + assert primary is not None + assert len(secondaries) == 2 + + +@pytest.mark.e2e_replica_set_pv +class TestReplicaSetPersistentVolumeDelete(KubernetesTester): + """ + name: Replica Set Deletion + tags: replica-set, persistent-volumes, removal + description: | + Deletes a Replica Set. + delete: + file: replica-set-pv.yaml + wait_until: mongo_resource_deleted + timeout: 200 + """ + + def test_replica_set_sts_doesnt_exist(self): + """The StatefulSet must be removed by Kubernetes as soon as the MongoDB resource is removed. + Note, that this may lag sometimes (caching or whatever?) and it's more safe to wait a bit""" + time.sleep(15) + with pytest.raises(client.rest.ApiException): + self.appsv1.read_namespaced_stateful_set("rs001-pv", self.namespace) + + def test_service_does_not_exist(self): + "Services should not exist" + with pytest.raises(client.rest.ApiException): + self.corev1.read_namespaced_service("rs001-pv-svc", self.namespace) + + def test_pvc_are_unbound(self): + "Should check the used PVC are still there in the expected status." + pass diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv_multiple.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv_multiple.py new file mode 100644 index 000000000..0db87808a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv_multiple.py @@ -0,0 +1,112 @@ +from operator import attrgetter + +import pytest +from kubetester import get_default_storage_class +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase + + +@pytest.mark.e2e_replica_set_pv_multiple +class TestCreateStorageClass(KubernetesTester): + """ + description: | + Creates a gp2 storage class if it does not exist already. + This is required as it seems that this storage class exists in + Kops and Openshift, but not on kind. This type of StorageClass is + based on the rancher.io/local-path provider, so it only works + on Kind. + """ + + def test_setup_gp2_storage_class(self): + KubernetesTester.make_default_gp2_storage_class() + + +@pytest.mark.e2e_replica_set_pv_multiple +class TestReplicaSetMultiplePersistentVolumeCreation(KubernetesTester): + + RESOURCE_NAME = "rs001-pv-multiple" + custom_labels = {"label1": "val1", "label2": "val2"} + + def test_create_replicaset(self, custom_mdb_version: str): + resource = MongoDB.from_yaml(load_fixture("replica-set-pv-multiple.yaml"), namespace=self.namespace) + resource.set_version(custom_mdb_version) + resource.update() + + resource.assert_reaches_phase(Phase.Running) + + def test_sts_creation(self): + sts = self.appsv1.read_namespaced_stateful_set(self.RESOURCE_NAME, self.namespace) + + assert sts.api_version == "apps/v1" + assert sts.kind == "StatefulSet" + assert sts.status.current_replicas == 2 + assert sts.status.ready_replicas == 2 + sts_labels = sts.metadata.labels + for k in self.custom_labels: + assert k in sts_labels and sts_labels[k] == self.custom_labels[k] + + def test_pvc_are_created_and_bound(self): + """3 mount points must be mounted to 3 pvc.""" + for idx, podname in enumerate(self._get_pods(self.RESOURCE_NAME + "-{}", 2)): + pod = self.corev1.read_namespaced_pod(podname, self.namespace) + self.check_pvc_for_pod(idx, pod) + + def check_pvc_for_pod(self, idx, pod): + claims = [volume for volume in pod.spec.volumes if getattr(volume, "persistent_volume_claim")] + assert len(claims) == 3 + + claims.sort(key=attrgetter("name")) + + default_sc = get_default_storage_class() + KubernetesTester.check_single_pvc( + self.namespace, + claims[0], + "data", + "data-{}-{}".format(self.RESOURCE_NAME, idx), + "2Gi", + "gp2", + self.custom_labels, + ) + + # Note that PVC gets the default storage class for cluster even if it wasn't requested initially + KubernetesTester.check_single_pvc( + self.namespace, + claims[1], + "journal", + f"journal-{self.RESOURCE_NAME}-{idx}", + "1Gi", + default_sc, + self.custom_labels, + ) + KubernetesTester.check_single_pvc( + self.namespace, + claims[2], + "logs", + f"logs-{self.RESOURCE_NAME}-{idx}", + "1G", + default_sc, + self.custom_labels, + ) + + +@pytest.mark.e2e_replica_set_pv_multiple +class TestReplicaSetMultiplePersistentVolumeDelete(KubernetesTester): + """ + name: Replica Set Deletion + tags: replica-set, persistent-volumes, removal + description: | + Deletes a Replica Set. + delete: + file: replica-set-pv-multiple.yaml + wait_until: mongo_resource_deleted_no_om + timeout: 240 + """ + + def test_pvc_are_bound(self): + "Should check the used PVC are still there in the Bound status." + all_claims = self.corev1.list_namespaced_persistent_volume_claim(self.namespace) + assert len(all_claims.items) == 6 + + for claim in all_claims.items: + assert claim.status.phase == "Bound" diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv_resize.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv_resize.py new file mode 100644 index 000000000..050bb660e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_pv_resize.py @@ -0,0 +1,91 @@ +from kubernetes import client +from kubetester import get_statefulset, try_load +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark +from tests.conftest import verify_pvc_expanded + +RESIZED_STORAGE_SIZE = "2Gi" + +REPLICA_SET_NAME = "replica-set-resize" + +# Note: This test can only be run in a cluster which uses - by default - a storageClass that is resizable; e.g., GKE +# For kind to work, you need to ensure that the resizable CSI driver has been installed, it will be installed +# Once you re-create your clusters + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + REPLICA_SET_NAME, + f"prefix-{REPLICA_SET_NAME}-cert", + replicas=3, + ) + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-pv-resize.yaml"), + namespace=namespace, + name=REPLICA_SET_NAME, + ) + + resource["spec"]["security"] = {} + resource["spec"]["security"]["tls"] = {"ca": issuer_ca_configmap} + # Setting security.certsSecretPrefix implicitly enables TLS + resource["spec"]["security"]["certsSecretPrefix"] = "prefix" + + resource.set_version(custom_mdb_version) + try_load(resource) + return resource + + +@mark.e2e_replica_set_pv_resize +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_replica_set_pv_resize +def test_replica_set_resize_pvc_state_changes(replica_set: MongoDB): + # Update the resource + replica_set.load() + replica_set["spec"]["podSpec"]["persistence"]["multiple"]["data"]["storage"] = RESIZED_STORAGE_SIZE + replica_set["spec"]["podSpec"]["persistence"]["multiple"]["journal"]["storage"] = RESIZED_STORAGE_SIZE + replica_set.update() + replica_set.assert_reaches_phase(Phase.Pending, timeout=400) + replica_set.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_replica_set_pv_resize +def test_replica_set_resize_finished(replica_set: MongoDB, namespace: str): + sts = get_statefulset(namespace, REPLICA_SET_NAME) + assert sts.spec.volume_claim_templates[0].spec.resources.requests["storage"] == RESIZED_STORAGE_SIZE + + first_data_pvc_name = "data-replica-set-resize-0" + first_journal_pvc_name = "journal-replica-set-resize-0" + first_logs_pvc_name = "logs-replica-set-resize-0" + + initial_storage_size = "1Gi" + verify_pvc_expanded( + first_data_pvc_name, + first_journal_pvc_name, + first_logs_pvc_name, + namespace, + RESIZED_STORAGE_SIZE, + initial_storage_size, + ) + + +@mark.e2e_replica_set_pv_resize +def test_mdb_is_not_reachable_with_no_ssl(replica_set: MongoDB): + replica_set.tester(use_ssl=False).assert_no_connection() + + +@mark.e2e_replica_set_pv_resize +def test_mdb_is_reachable_with_ssl(replica_set: MongoDB, ca_path: str): + replica_set.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_readiness_probe.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_readiness_probe.py new file mode 100644 index 000000000..0d09d57fe --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_readiness_probe.py @@ -0,0 +1,74 @@ +import time + +import pytest +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import get_pods, skip_if_local +from kubetester.mongodb import MongoDB, Phase + +RESOURCE_NAME = "my-replica-set-double" + + +@pytest.fixture(scope="module") +def replica_set(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set-double.yaml"), RESOURCE_NAME, namespace) + resource.set_architecture_annotation() + return resource.update() + + +@pytest.fixture(scope="class") +def config_version(): + class ConfigVersion: + def __init__(self): + self.version = 0 + + return ConfigVersion() + + +@pytest.mark.e2e_replica_set_readiness_probe +class TestReplicaSetNoAgentDeadlock(KubernetesTester): + """ + name: ReplicaSet recovers when all pods are removed + description: | + Creates a 2-members replica set and then removes the pods. The pods are started sequentially (pod-0 waits for + pod-1 to get ready) but the AA in pod-1 needs pod-0 to be running to initialize replica set. The readiness probe + must be clever enough to mark the pod "ready" if the agents is waiting for the other pods. + """ + + def test_mdb_created(self, replica_set: MongoDB, config_version): + replica_set.assert_reaches_phase(Phase.Running) + config_version.version = self.get_automation_config()["version"] + + @skip_if_local() + def test_db_connectable(self, replica_set: MongoDB): + replica_set.assert_connectivity() + + def test_remove_pods_and_wait_for_recovery(self, config_version): + pods = get_pods(RESOURCE_NAME + "-{}", 2) + for podname in pods: + self.corev1.delete_namespaced_pod(podname, self.namespace) + + print("\nRemoved pod {}".format(podname)) + + # sleeping for 5 seconds to let the pods be removed + time.sleep(5) + + # waiting until the pods recover and init the replica set again + KubernetesTester.wait_until(TestReplicaSetNoAgentDeadlock.pods_are_ready, 120) + assert self.get_automation_config()["version"] == config_version.version + + @skip_if_local() + def test_db_connectable_after_recovery(self, replica_set: MongoDB): + replica_set.assert_connectivity() + + def test_replica_set_recovered(self, replica_set: MongoDB, config_version): + replica_set.assert_reaches_phase(Phase.Running) + assert self.get_automation_config()["version"] == config_version.version + + @staticmethod + def pods_are_ready(): + sts = KubernetesTester.clients("appsv1").read_namespaced_stateful_set( + "my-replica-set-double", KubernetesTester.get_namespace() + ) + + return sts.status.ready_replicas == 2 diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_recovery.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_recovery.py new file mode 100644 index 000000000..4f780f97c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_recovery.py @@ -0,0 +1,39 @@ +import pytest +from kubetester.kubetester import KubernetesTester + + +@pytest.mark.e2e_replica_set_recovery +class TestReplicaSetBadStateCreation(KubernetesTester): + """ + name: Replica Set Bad State Creation + tags: replica-set, creation + description: | + Creates a Replica set with a bad configuration (wrong credentials) and ensures it enters a failed state + create: + file: replica-set-invalid.yaml + wait_until: in_error_state + timeout: 180 + """ + + def test_in_error_state(self): + mrs = KubernetesTester.get_resource() + assert mrs["status"]["phase"] == "Failed" + + +@pytest.mark.e2e_replica_set_recovery +class TestReplicaSetRecoversFromBadState(KubernetesTester): + """ + name: Replica Set Bad State Recovery + tags: replica-set, creation + description: | + Updates spec of replica set in a bad state and ensures it is updated to the running state correctly + update: + file: replica-set-invalid.yaml + patch: '[{"op":"replace","path":"/spec/credentials","value":"my-credentials"}]' + wait_until: in_running_state + timeout: 400 + """ + + def test_in_running_state(self): + mrs = KubernetesTester.get_resource() + assert mrs["status"]["phase"] == "Running" diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_report_pending_pods.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_report_pending_pods.py new file mode 100644 index 000000000..2e51d9ecb --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_report_pending_pods.py @@ -0,0 +1,27 @@ +from kubetester import delete_pod, delete_pvc +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["persistent"] = True + return resource.create() + + +@mark.e2e_replica_set_report_pending_pods +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_replica_set_report_pending_pods +def test_cr_reports_pod_pending_status(replica_set: MongoDB, namespace: str): + """delete the 0th pod and it's corresponding pvc to make sure the + pod fails to enter ready state. Another way would be to delete the nodes in the cluster + which makes the pod to be unschedulable. The former process is just easier to reproduce.""" + delete_pod(namespace, "my-replica-set-0") + delete_pvc(namespace, "data-my-replica-set-0") + replica_set.assert_reaches_phase(Phase.Pending, timeout=600) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_schema_validation.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_schema_validation.py new file mode 100644 index 000000000..ad1e90360 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_schema_validation.py @@ -0,0 +1,185 @@ +import time + +import pytest +from kubernetes.client import ApiException +from kubetester import MongoDB +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture + + +def mdb_resource(namespace: str) -> MongoDB: + return MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), namespace=namespace) + + +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetMembersMissing(KubernetesTester): + """ + name: Replica Set Validation (members missing) + tags: replica-set + description: | + Creates a Replica Set with required fields missing. + create: + file: replica-set.yaml + patch: '[{"op":"remove","path":"/spec/members"}]' + exception: 'Unprocessable Entity' + """ + + @pytest.mark.skip(reason="TODO: validation for members must be done in webhook depending on the resource type") + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetInvalidType(KubernetesTester): + """ + name: Replica Set Validation (invalid type) + tags: replica-set + description: | + Creates a Replica Set with an invalid field. + create: + file: replica-set.yaml + patch: '[{"op":"replace","path":"/spec/type","value":"InvalidReplicaSetType"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetInvalidLogLevel(KubernetesTester): + """ + name: Replica Set Validation (invalid type) + tags: replica-set + description: | + Creates a Replica Set with an invalid logLevel. + create: + file: replica-set.yaml + patch: '[{"op":"replace","path":"/spec/logLevel","value":"NotWARNING"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetSchemaShardedClusterMongodConfig(KubernetesTester): + """ + name: Replica Set Validation (sharded cluster additional mongod config) + create: + file: replica-set.yaml + patch: '[{"op":"add","path":"/spec/mongos","value":{"additionalMongodConfig":{"net":{"ssl":{"mode": "AllowSSL"}}}}}]' + exception: 'cannot be specified if type of MongoDB is ReplicaSet' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.skip(reason="OpenAPIv2 does not support this validation.") +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetInvalidWithOpsAndCloudManager(KubernetesTester): + """Creates a Replica Set with both a cloud manager and an ops manager + configuration specified.""" + + init = { + "create": { + "file": "replica-set.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/cloudManager", + "value": {"configMapRef": {"name": "something"}}, + }, + { + "op": "add", + "path": "/spec/opsManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetInvalidWithProjectAndCloudManager(KubernetesTester): + """Creates a Replica Set with both a cloud manager and an ops manager + configuration specified.""" + + init = { + "create": { + "file": "replica-set.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/cloudManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_replica_set_schema_validation +class TestReplicaSetInvalidWithCloudAndOpsManagerAndProject(KubernetesTester): + init = { + "create": { + "file": "replica-set.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/cloudManager", + "value": {"configMapRef": {"name": "something"}}, + }, + { + "op": "add", + "path": "/spec/opsManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_replica_set_schema_validation +def test_resource_type_immutable(namespace: str): + mdb = mdb_resource(namespace).create() + # no need to wait for the resource get to Running state - we can update right away + time.sleep(5) + mdb = mdb_resource(namespace).load() + + with pytest.raises( + ApiException, + match=r"'resourceType' cannot be changed once created", + ): + mdb["spec"]["type"] = "ShardedCluster" + # Setting mandatory fields to avoid getting another validation error + mdb["spec"]["shardCount"] = 1 + mdb["spec"]["mongodsPerShardCount"] = 3 + mdb["spec"]["mongosCount"] = 1 + mdb["spec"]["configServerCount"] = 1 + mdb.update() + + +@pytest.mark.e2e_replica_set_schema_validation +def test_unrecognized_auth_mode(namespace: str): + mdb = MongoDB.from_yaml(yaml_fixture("invalid_replica_set_wrong_auth_mode.yaml"), namespace=namespace) + + with pytest.raises( + ApiException, + match=r".*Unsupported value.*SHA.*supported values.*X509.*SCRAM.*SCRAM-SHA-1.*MONGODB-CR.*SCRAM-SHA-256.*LDAP.*", + ): + mdb.create() diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_statefulset_status.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_statefulset_status.py new file mode 100644 index 000000000..4f29b42ab --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_statefulset_status.py @@ -0,0 +1,37 @@ +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-double.yaml"), + namespace=namespace, + name="replica-set-status", + ) + return resource.create() + + +@mark.e2e_replica_set_statefulset_status +def test_replica_set_reaches_pending_phase(replica_set: MongoDB): + replica_set.wait_for( + lambda s: s.get_status_resources_not_ready() is not None, + timeout=150, + should_raise=True, + ) + # the StatefulSet name is equal to replica set name + replica_set.assert_status_resource_not_ready( + name=replica_set.name, + msg_regexp="Not all the Pods are ready \(wanted: 2.*\)", + idx=0, + ) + replica_set.assert_reaches_phase(Phase.Pending, timeout=120) + assert replica_set.get_status_message() == "StatefulSet not ready" + + +@mark.e2e_replica_set_statefulset_status +def test_replica_set_reaches_running_phase(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + assert replica_set.get_status_resources_not_ready() is None + assert replica_set.get_status_message() is None diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_update_delete_parallel.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_update_delete_parallel.py new file mode 100644 index 000000000..6268d1c47 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_update_delete_parallel.py @@ -0,0 +1,48 @@ +""" +This is a test which makes sure that update and delete calls issued together don't mess up +OM group state. Internal locks are used to ensure OM requests for update/delete operations +don't intersect. Note that K8s objects are removed right after the delete call is made(no +serialization happens) so update operation doesn't succeed as internal locks don't affect +K8s CR and dependent resources removal. +""" + +from time import sleep + +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set-single.yaml"), "my-replica-set", namespace) + resource.set_version(custom_mdb_version) + resource.create() + + return resource + + +@mark.e2e_replica_set_update_delete_parallel +def test_reaches_running_phase(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running) + replica_set.get_automation_config_tester().assert_replica_sets_size(1) + + +@mark.e2e_replica_set_update_delete_parallel +def test_update_delete_in_parallel(replica_set: MongoDB): + replica_set["spec"]["members"] = 2 + replica_set.update() + sleep(5) + replica_set.delete() + + om_tester = replica_set.get_om_tester() + + def om_is_clean(): + try: + om_tester.assert_hosts_empty() + return True + except AssertionError: + return False + + run_periodically(om_is_clean, timeout=180) diff --git a/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_upgrade_downgrade.py b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_upgrade_downgrade.py new file mode 100644 index 000000000..821fc8cd3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/replicaset/replica_set_upgrade_downgrade.py @@ -0,0 +1,114 @@ +import pymongo +from kubetester import try_load +from kubetester.kubetester import KubernetesTester, fcv_from_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ( + MongoDBBackgroundTester, + MongoTester, + ReplicaSetTester, +) +from pytest import fixture, mark + +TEST_DATA = {"foo": "bar"} +TEST_DB = "testdb" +TEST_COLLECTION = "testcollection" + + +@fixture(scope="module") +def mongod_tester(): + return ReplicaSetTester("my-replica-set-downgrade", 3) + + +@fixture(scope="module") +def mdb_health_checker(mongod_tester: MongoTester) -> MongoDBBackgroundTester: + return MongoDBBackgroundTester( + mongod_tester, + allowed_sequential_failures=2, + health_function_params={ + "attempts": 1, + "write_concern": pymongo.WriteConcern(w="majority"), + }, + ) + + +@fixture +def mdb_test_collection(mongod_tester): + collection = mongod_tester.client[TEST_DB] + return collection[TEST_COLLECTION] + + +@fixture(scope="module") +def replica_set(namespace: str, custom_mdb_prev_version: str, cluster_domain: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set-downgrade.yaml"), namespace=namespace) + resource.set_version(custom_mdb_prev_version) + if try_load(resource): + return resource + return resource.update() + + +@mark.e2e_replica_set_upgrade_downgrade +class TestReplicaSetUpgradeDowngradeCreate(KubernetesTester): + + def test_mdb_created(self, replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_start_mongod_background_tester(self, mdb_health_checker): + mdb_health_checker.start() + + def test_db_connectable(self, mongod_tester, custom_mdb_prev_version: str): + mongod_tester.assert_version(custom_mdb_prev_version) + + def test_insert_test_data(self, mdb_test_collection): + mdb_test_collection.insert_one(TEST_DATA) + + +@mark.e2e_replica_set_upgrade_downgrade +class TestReplicaSetUpgradeDowngradeUpdate(KubernetesTester): + + def test_mongodb_upgrade(self, replica_set: MongoDB, custom_mdb_version: str, custom_mdb_prev_version: str): + replica_set.load() + replica_set.set_version(custom_mdb_version) + fcv = fcv_from_version(custom_mdb_prev_version) + replica_set["spec"]["featureCompatibilityVersion"] = fcv + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=700) + replica_set.tester().assert_version(custom_mdb_version) + + def test_mongodb_version_fcv(self, replica_set: MongoDB, custom_mdb_prev_version: str): + # no fcv is set; that means we will use the smaller one between custom_mdb_version and custom_mdb_prev_version + # 5 -> 6, fcv: 5-> 5 + major_minor_prev = custom_mdb_prev_version.split(".") + + assert replica_set.get_status_fcv() == f"{major_minor_prev[0]}.{major_minor_prev[1]}" + + def test_db_connectable(self, mongod_tester, custom_mdb_version: str): + mongod_tester.assert_version(custom_mdb_version) + + +@mark.e2e_replica_set_upgrade_downgrade +class TestReplicaSetUpgradeDowngradeRevert(KubernetesTester): + + def test_mongodb_downgrade(self, replica_set: MongoDB, custom_mdb_prev_version: str, custom_mdb_version: str): + replica_set.load() + replica_set.set_version(custom_mdb_prev_version) + replica_set.update() + + replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + replica_set.tester().assert_version(custom_mdb_prev_version) + + def test_mongodb_version_fcv(self, replica_set: MongoDB, custom_mdb_prev_version: str): + # no fcv is set; that means we will use the smaller one between custom_mdb_version and custom_mdb_prev_version + # 6 -> 5, fcv: 5-> 5 + major_minor = custom_mdb_prev_version.split(".") + + assert replica_set.get_status_fcv() == f"{major_minor[0]}.{major_minor[1]}" + + def test_db_connectable(self, mongod_tester, custom_mdb_prev_version: str): + mongod_tester.assert_version(custom_mdb_prev_version) + + def test_mdb_healthy_throughout_change_version(self, mdb_health_checker): + mdb_health_checker.assert_healthiness() + + def test_data_exists(self, mdb_test_collection): + assert mdb_test_collection.estimated_document_count() == 1 diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/__init__.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py new file mode 100644 index 000000000..116070e79 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py @@ -0,0 +1,289 @@ +import json +from ipaddress import IPv4Address +from typing import Any, List + +import kubernetes +from _pytest.fixtures import fixture +from kubetester import MongoDB, read_configmap +from kubetester.mongodb_multi import MultiClusterClient +from kubetester.operator import Operator +from tests.conftest import ( + LEGACY_CENTRAL_CLUSTER_NAME, + get_central_cluster_client, + get_central_cluster_name, + get_default_operator, + get_member_cluster_clients, + get_member_cluster_names, + get_multi_cluster_operator, + get_multi_cluster_operator_installation_config, + get_operator_installation_config, + is_multi_cluster, +) +from tests.multicluster.conftest import cluster_spec_list + + +@fixture(scope="module") +def operator(namespace: str) -> Operator: + if is_multi_cluster(): + return get_multi_cluster_operator( + namespace, + get_central_cluster_name(), + get_multi_cluster_operator_installation_config(namespace), + get_central_cluster_client(), + get_member_cluster_clients(), + get_member_cluster_names(), + ) + else: + return get_default_operator(namespace, get_operator_installation_config(namespace)) + + +def enable_multi_cluster_deployment( + resource: MongoDB, + shard_members_array: list[int] = None, + mongos_members_array: list[int] = None, + configsrv_members_array: list[int] = None, +): + resource["spec"]["topology"] = "MultiCluster" + resource["spec"]["mongodsPerShardCount"] = None + resource["spec"]["mongosCount"] = None + resource["spec"]["configServerCount"] = None + + # Members and MemberConfig fields should be empty in overrides for MultiCluster + for idx, _ in enumerate(resource["spec"].get("shardOverrides", [])): + if "members" in resource["spec"]["shards"][idx]: + resource["spec"]["shardOverrides"][idx]["members"] = 0 + if "memberConfig" in resource["spec"]["shards"][idx]: + resource["spec"]["shardOverrides"][idx]["memberConfig"] = None + + setup_cluster_spec_list(resource, "shard", shard_members_array or [1, 1, 1]) + setup_cluster_spec_list(resource, "configSrv", configsrv_members_array or [1, 1, 1]) + setup_cluster_spec_list(resource, "mongos", mongos_members_array or [1, 1, 1]) + + resource.api = kubernetes.client.CustomObjectsApi(api_client=get_central_cluster_client()) + + +class ClusterInfo: + def __init__(self, cluster_name: str, cidr: IPv4Address, external_domain: str): + self.cluster_name = cluster_name + self.cidr = cidr + self.external_domain = external_domain + + +KIND_SINGLE_CLUSTER = ClusterInfo("kind-kind", IPv4Address("172.18.255.200"), "kind-kind.interconnected") +KIND_E2E_CLUSTER_1 = ClusterInfo( + "kind-e2e-cluster-1", IPv4Address("172.18.255.210"), "kind-e2e-cluster-1.interconnected" +) +KIND_E2E_CLUSTER_2 = ClusterInfo( + "kind-e2e-cluster-2", IPv4Address("172.18.255.220"), "kind-e2e-cluster-2.interconnected" +) +KIND_E2E_CLUSTER_3 = ClusterInfo( + "kind-e2e-cluster-3", IPv4Address("172.18.255.230"), "kind-e2e-cluster-3.interconnected" +) +KIND_E2E_OPERATOR = ClusterInfo("kind-e2e-operator", IPv4Address("172.18.255.200"), "kind-e2e-operator.interconnected") + +cluster_map = { + KIND_E2E_CLUSTER_1.cluster_name: KIND_E2E_CLUSTER_1, + KIND_E2E_CLUSTER_2.cluster_name: KIND_E2E_CLUSTER_2, + KIND_E2E_CLUSTER_3.cluster_name: KIND_E2E_CLUSTER_3, + KIND_E2E_OPERATOR.cluster_name: KIND_E2E_OPERATOR, + LEGACY_CENTRAL_CLUSTER_NAME: KIND_SINGLE_CLUSTER, +} + + +def get_cluster_info(cluster_name: str) -> ClusterInfo: + val = cluster_map[cluster_name] + if val is None: + raise Exception(f"The {cluster_name} is not defined") + return val + + +def _setup_external_access( + resource: MongoDB, cluster_spec_type: str, cluster_member_list: List[str], enable_external_domain=True +): + ports = [ + { + "name": "mongodb", + "port": 27017, + }, + ] + if cluster_spec_type in ["shard", ""]: + ports = [ + { + "name": "mongodb", + "port": 27017, + }, + { + "name": "backup", + "port": 27018, + }, + { + "name": "testing0", + "port": 27019, + }, + ] + + if "topology" in resource["spec"] and resource["spec"]["topology"] == "MultiCluster": + resource["spec"]["externalAccess"] = {} + for index, cluster_member_name in enumerate(cluster_member_list): + resource["spec"][cluster_spec_type]["clusterSpecList"][index]["externalAccess"] = { + "externalService": { + "spec": { + "type": "LoadBalancer", + "ports": ports, + } + }, + } + if enable_external_domain: + resource["spec"][cluster_spec_type]["clusterSpecList"][index]["externalAccess"]["externalDomain"] = ( + get_cluster_info(cluster_member_name).external_domain + ) + else: + resource["spec"]["externalAccess"] = {} + if enable_external_domain: + resource["spec"]["externalAccess"]["externalDomain"] = get_cluster_info( + cluster_member_list[0] + ).external_domain + + +def setup_external_access(resource: MongoDB, enable_external_domain=True): + if "topology" in resource["spec"] and resource["spec"]["topology"] == "MultiCluster": + _setup_external_access( + resource=resource, + cluster_spec_type="mongos", + cluster_member_list=get_member_cluster_names(), + enable_external_domain=enable_external_domain, + ) + _setup_external_access( + resource=resource, + cluster_spec_type="configSrv", + cluster_member_list=get_member_cluster_names(), + enable_external_domain=enable_external_domain, + ) + _setup_external_access( + resource=resource, + cluster_spec_type="shard", + cluster_member_list=get_member_cluster_names(), + enable_external_domain=enable_external_domain, + ) + else: + _setup_external_access( + resource=resource, + cluster_spec_type="", + cluster_member_list=[get_central_cluster_name()], + enable_external_domain=enable_external_domain, + ) + + +def get_dns_hosts_for_external_access(resource: MongoDB, cluster_member_list: List[str]) -> list[tuple[str, str]]: + hosts = [] + if "topology" in resource["spec"] and resource["spec"]["topology"] == "MultiCluster": + for cluster_index, cluster_member_name in enumerate(cluster_member_list): + cluster_info = get_cluster_info(cluster_member_name) + ip = cluster_info.cidr + # We skip the first IP as Istio Gateway takes it. + ip_iterator = 1 + for i in range(resource["spec"]["mongos"]["clusterSpecList"][cluster_index]["members"]): + fqdn = f"{resource.name}-mongos-{cluster_index}-{i}.{cluster_info.external_domain}" + ip_for_fqdn = str(ip + ip_iterator) + ip_iterator = ip_iterator + 1 + hosts.append((ip_for_fqdn, fqdn)) + for i in range(resource["spec"]["configSrv"]["clusterSpecList"][cluster_index]["members"]): + fqdn = f"{resource.name}-config-{cluster_index}-{i}.{cluster_info.external_domain}" + ip_for_fqdn = str(ip + ip_iterator) + ip_iterator = ip_iterator + 1 + hosts.append((ip_for_fqdn, fqdn)) + for i in range(resource["spec"]["shard"]["clusterSpecList"][cluster_index]["members"]): + fqdn = f"{resource.name}-0-{cluster_index}-{i}.{cluster_info.external_domain}" + ip_for_fqdn = str(ip + ip_iterator) + ip_iterator = ip_iterator + 1 + hosts.append((ip_for_fqdn, fqdn)) + else: + cluster_info = get_cluster_info(cluster_member_list[0]) + ip = cluster_info.cidr + ip_iterator = 0 + for i in range(resource["spec"]["mongosCount"]): + fqdn = f"{resource.name}-mongos-{i}.{cluster_info.external_domain}" + ip_for_fqdn = str(ip + ip_iterator) + ip_iterator = ip_iterator + 1 + hosts.append((ip_for_fqdn, fqdn)) + return hosts + + +def setup_cluster_spec_list(resource: MongoDB, cluster_spec_type: str, members_array: list[int]): + if cluster_spec_type not in resource["spec"]: + resource["spec"][cluster_spec_type] = {} + + if "clusterSpecList" not in resource["spec"][cluster_spec_type]: + resource["spec"][cluster_spec_type]["clusterSpecList"] = cluster_spec_list( + get_member_cluster_names(), members_array + ) + + +def get_member_cluster_clients_using_cluster_mapping(resource_name: str, namespace: str) -> List[MultiClusterClient]: + cluster_mapping = read_deployment_state(resource_name, namespace)["clusterMapping"] + return get_member_cluster_clients(cluster_mapping) + + +def get_member_cluster_client_using_cluster_mapping( + resource_name: str, namespace: str, cluster_name: str +) -> MultiClusterClient: + cluster_mapping = read_deployment_state(resource_name, namespace)["clusterMapping"] + for m in get_member_cluster_clients(cluster_mapping): + if m.cluster_name == cluster_name: + return m + raise Exception(f"cluster {cluster_name} not found in deployment state mapping {cluster_mapping}") + + +def get_mongos_service_names(resource: MongoDB): + service_names = [] + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(resource.name, resource.namespace): + for member_idx in range(resource.mongos_members_in_cluster(cluster_member_client.cluster_name)): + service_name = resource.mongos_service_name(member_idx, cluster_member_client.cluster_index) + service_names.append(service_name) + + return service_names + + +def get_all_sharded_cluster_pod_names(resource: MongoDB): + return get_mongos_pod_names(resource) + get_config_server_pod_names(resource) + get_all_shards_pod_names(resource) + + +def get_mongos_pod_names(resource: MongoDB): + pod_names = [] + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(resource.name, resource.namespace): + for member_idx in range(resource.mongos_members_in_cluster(cluster_member_client.cluster_name)): + pod_name = resource.mongos_pod_name(member_idx, cluster_member_client.cluster_index) + pod_names.append(pod_name) + + return pod_names + + +def get_config_server_pod_names(resource: MongoDB): + pod_names = [] + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(resource.name, resource.namespace): + for member_idx in range(resource.config_srv_members_in_cluster(cluster_member_client.cluster_name)): + pod_name = resource.config_srv_pod_name(member_idx, cluster_member_client.cluster_index) + pod_names.append(pod_name) + + return pod_names + + +def get_all_shards_pod_names(resource: MongoDB): + pod_names = [] + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(resource.name, resource.namespace): + for shard_idx in range(resource["spec"]["shardCount"]): + for member_idx in range(resource.shard_members_in_cluster(cluster_member_client.cluster_name)): + pod_name = resource.shard_pod_name(shard_idx, member_idx, cluster_member_client.cluster_index) + pod_names.append(pod_name) + + return pod_names + + +def read_deployment_state(resource_name: str, namespace: str) -> dict[str, Any]: + deployment_state_cm = read_configmap( + namespace, + f"{resource_name}-state", + get_central_cluster_client(), + ) + state = json.loads(deployment_state_cm["state"]) + return state diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-custom-podspec.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-custom-podspec.yaml new file mode 100644 index 000000000..91a94bc30 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-custom-podspec.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-custom-podspec +spec: + shardCount: 3 + mongodsPerShardCount: 1 + mongosCount: 1 + configServerCount: 1 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: true + configSrvPodSpec: + podTemplate: + spec: + terminationGracePeriodSeconds: 50 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: "config" + weight: 30 + mongosPodSpec: + podTemplate: + spec: + terminationGracePeriodSeconds: 20 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: "mongos" + weight: 40 + shardPodSpec: + cpu: "2" # ignored as podTemplate takes precedence if provided + podTemplate: + spec: + containers: + - name: sharded-cluster-sidecar + image: busybox + command: ["sleep"] + args: [ "infinity" ] + resources: + limits: + cpu: "1" + requests: + cpu: 500m + terminationGracePeriodSeconds: 30 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: "shard" + weight: 50 + shardSpecificPodSpec: + - podTemplate: + spec: + containers: + - name: sharded-cluster-sidecar-override + image: busybox + command: ["sleep"] + args: [ "infinity" ] + resources: + limits: + cpu: "1" + requests: + cpu: 500m + terminationGracePeriodSeconds: 60 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: "shardoverride" + weight: 100 diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-downgrade.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-downgrade.yaml new file mode 100644 index 000000000..c21cb9a51 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-downgrade.yaml @@ -0,0 +1,17 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-downgrade +spec: + type: ShardedCluster + shardCount: 3 + mongodsPerShardCount: 3 + mongosCount: 3 + configServerCount: 3 + version: 4.4.2 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-mongod-options.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-mongod-options.yaml new file mode 100644 index 000000000..84085ccca --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-mongod-options.yaml @@ -0,0 +1,37 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster-options +spec: + members: 3 + version: 4.4.0-ent + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 1 + mongos: + additionalMongodConfig: + net: + port: 30003 + systemLog: + logAppend: true + verbosity: 4 + configSrv: + additionalMongodConfig: + operationProfiling: + mode: slowOp + net: + port: 30002 + shard: + additionalMongodConfig: + storage: + journal: + commitIntervalMs: 50 + net: + port: 30001 diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-pv-resize.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-pv-resize.yaml new file mode 100644 index 000000000..a6cada8c1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-pv-resize.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-pv-resize + labels: + label1: val1 + label2: val2 +spec: + shardCount: 2 + mongodsPerShardCount: 2 + mongosCount: 2 + configServerCount: 2 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + + shardPodSpec: + persistence: + multiple: + data: + storage: 1Gi + storageClass: csi-hostpath-sc + journal: + storage: 1Gi + storageClass: csi-hostpath-sc + logs: + storage: 1Gi + storageClass: csi-hostpath-sc + + configSrvPodSpec: + persistence: + multiple: + data: + storage: 1Gi + storageClass: csi-hostpath-sc + journal: + storage: 1Gi + storageClass: csi-hostpath-sc + logs: + storage: 1Gi + storageClass: csi-hostpath-sc \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-pv.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-pv.yaml new file mode 100644 index 000000000..58991e9ab --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-pv.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-pv + labels: + label1: val1 + label2: val2 +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + shardPodSpec: + persistence: + single: + storage: 1G + + configSrvPodSpec: + persistence: + single: + storage: 1G diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-scale-down-shards.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-scale-down-shards.yaml new file mode 100644 index 000000000..86457e22f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-scale-down-shards.yaml @@ -0,0 +1,16 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-scale-down-shards +spec: + type: ShardedCluster + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 2 + version: 4.4.0 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-scale-shards.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-scale-shards.yaml new file mode 100644 index 000000000..d34330c06 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-scale-shards.yaml @@ -0,0 +1,16 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-scale-down-shards +spec: + type: ShardedCluster + shardCount: 2 + mongodsPerShardCount: 1 + mongosCount: 1 + configServerCount: 1 + version: 4.4.0 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-shard-overrides-multi-cluster.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-shard-overrides-multi-cluster.yaml new file mode 100644 index 000000000..b00bb5dff --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-shard-overrides-multi-cluster.yaml @@ -0,0 +1,64 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-sh-shard-overrides +spec: + shardCount: 4 + topology: MultiCluster + type: ShardedCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + shard: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + configSrv: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + - clusterName: kind-e2e-cluster-2 + members: 2 + mongos: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + + # Shard #2 has no override + shardOverrides: + - shardNames: [ "mdb-sh-shard-overrides-0", "mdb-sh-shard-overrides-1" ] # this override will apply to shards #0 and #1 + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + memberConfig: # we prefer to have primaries in this cluster + - votes: 1 + priority: "5" + - clusterName: kind-e2e-cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "0" + + - shardNames: ["mdb-sh-shard-overrides-3"] # this override will apply to only shard #3 + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + memberConfig: + - votes: 0 + priority: "0" + - clusterName: kind-e2e-cluster-2 + members: 1 + memberConfig: + - votes: 1 + priority: "0" + - clusterName: kind-e2e-cluster-3 # Cluster 3 is used only by this shard + members: 1 + memberConfig: # we prefer to have primaries in this cluster + - votes: 1 + priority: "10" + diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-shard-overrides.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-shard-overrides.yaml new file mode 100644 index 000000000..bdcb0634b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-shard-overrides.yaml @@ -0,0 +1,36 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: mdb-sh-shard-overrides +spec: + shardCount: 4 + mongosCount: 1 + configServerCount: 3 + mongodsPerShardCount: 2 + topology: SingleCluster + type: ShardedCluster + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + + # Shard #2 has no override + shardOverrides: + - shardNames: [ "mdb-sh-shard-overrides-0", "mdb-sh-shard-overrides-1" ] # this override will apply to shards #0 and #1 + members: 3 + memberConfig: + - votes: 0 + priority: "0" + - votes: 1 + priority: "1" + - votes: 1 + priority: "10" + + - shardNames: ["mdb-sh-shard-overrides-3"] # this override will apply to only shard #3 + memberConfig: + - votes: 1 + priority: "5" + - votes: 1 + priority: "0" diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-single.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-single.yaml new file mode 100644 index 000000000..e3a0183b9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster-single.yaml @@ -0,0 +1,20 @@ +# This is the simplest sharded cluster possible (1 member for everything) - use it for the tests not focused on sharded +# cluster behaviour (e.g. listening to secrets changes), this will result in minimal running time and lowest resource +# consumption +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-single +spec: + type: ShardedCluster + shardCount: 1 + mongodsPerShardCount: 1 + mongosCount: 1 + configServerCount: 1 + version: 6.0.5 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster.yaml b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster.yaml new file mode 100644 index 000000000..31db687f5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/fixtures/sharded-cluster.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh001-base +spec: + shardCount: 1 + type: ShardedCluster + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 5.0.15 + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: WARN + persistent: true diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster.py new file mode 100644 index 000000000..926dd2638 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster.py @@ -0,0 +1,201 @@ +import kubernetes +from kubetester import try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import run_periodically, skip_if_local, skip_if_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, + get_mongos_service_names, +) + +SCALED_SHARD_COUNT = 2 +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="module") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("sharded-cluster.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[1, 1, 1], + ) + + return resource + + +@mark.e2e_sharded_cluster +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster +class TestShardedClusterCreation: + def test_create_sharded_cluster(self, sc: MongoDB): + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=800) + + def test_sharded_cluster_sts(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.shard_members_in_cluster(cluster_member_client.cluster_name) > 0: + # Single shard exists in this test case + sts_name = sc.shard_statefulset_name(0, cluster_member_client.cluster_index) + sts = cluster_member_client.read_namespaced_stateful_set(sts_name, sc.namespace) + assert sts + + def test_config_srv_sts(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.config_srv_members_in_cluster(cluster_member_client.cluster_name) > 0: + sts_name = sc.config_srv_statefulset_name(cluster_member_client.cluster_index) + sts = cluster_member_client.read_namespaced_stateful_set(sts_name, sc.namespace) + assert sts + + def test_mongos_sts(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.mongos_members_in_cluster(cluster_member_client.cluster_name) > 0: + sts_name = sc.mongos_statefulset_name(cluster_member_client.cluster_index) + sts = cluster_member_client.read_namespaced_stateful_set(sts_name, sc.namespace) + assert sts + + def test_mongod_sharded_cluster_service(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.shard_members_in_cluster(cluster_member_client.cluster_name) > 0: + svc_name = sc.shard_service_name() + svc = cluster_member_client.read_namespaced_service(svc_name, sc.namespace) + assert svc + + # When testing locally make sure you have kubefwd forwarding all cluster hostnames + # kubefwd does not contain fix for multiple cluster, use https://github.com/lsierant/kubefwd fork instead + def test_shards_were_configured_and_accessible(self, sc: MongoDB): + for service_name in get_mongos_service_names(sc): + tester = sc.tester(service_names=[service_name]) + tester.assert_connectivity() + + @skip_if_local() # Local machine DNS don't contain K8s CoreDNS SRV records which are required + @skip_if_multi_cluster() # srv option does not work for multi-cluster tests as each cluster DNS contains entries + # related only to that cluster. Additionally, we don't pass srv option when building multi-cluster conn string + def test_shards_were_configured_with_srv_and_accessible(self, sc: MongoDB): + for service_name in get_mongos_service_names(sc): + tester = sc.tester(service_names=[service_name], srv=True) + tester.assert_connectivity() + + def test_monitoring_versions(self, sc: MongoDB): + """Verifies that monitoring agent is configured for each process in the deployment""" + config = KubernetesTester.get_automation_config() + mv = config["monitoringVersions"] + assert len(mv) == 8 + + for process in config["processes"]: + assert any(agent for agent in mv if agent["hostname"] == process["hostname"]) + + def test_backup_versions(self, sc: MongoDB): + """Verifies that backup agent is configured for each process in the deployment""" + config = KubernetesTester.get_automation_config() + mv = config["backupVersions"] + assert len(mv) == 8 + + for process in config["processes"]: + assert any(agent for agent in mv if agent["hostname"] == process["hostname"]) + + +@mark.e2e_sharded_cluster +class TestShardedClusterUpdate: + def test_scale_up_sharded_cluster(self, sc: MongoDB): + sc.load() + sc["spec"]["shardCount"] = SCALED_SHARD_COUNT + sc.update() + + sc.assert_reaches_phase(Phase.Running) + + def test_both_shards_are_configured(self, sc: MongoDB): + for shard_idx in range(SCALED_SHARD_COUNT): + hosts = [] + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + for member_idx in range(sc.shard_members_in_cluster(cluster_member_client.cluster_name)): + hostname = sc.shard_hostname(shard_idx, member_idx, cluster_member_client.cluster_index) + hosts.append(hostname) + + logger.debug(f"Checking for connectivity of hosts: {hosts}") + primary, secondaries = KubernetesTester.wait_for_rs_is_ready(hosts) + assert primary is not None + assert len(secondaries) == 2 + + def test_monitoring_versions(self, sc: MongoDB): + """Verifies that monitoring agent is configured for each process in the deployment""" + config = KubernetesTester.get_automation_config() + mv = config["monitoringVersions"] + assert len(mv) == 11 + + for process in config["processes"]: + assert any(agent for agent in mv if agent["hostname"] == process["hostname"]) + + def test_backup_versions(self, sc: MongoDB): + """Verifies that backup agent is configured for each process in the deployment""" + config = KubernetesTester.get_automation_config() + mv = config["backupVersions"] + assert len(mv) == 11 + + for process in config["processes"]: + assert any(agent for agent in mv if agent["hostname"] == process["hostname"]) + + +@mark.e2e_sharded_cluster +class TestShardedClusterDeletion: + + # We need to store cluster_member_clients somehow after deleting the MongoDB resource. + # Cluster mapping from deployment state is needed to compute cluster_member_clients. + @fixture(scope="class") + def cluster_member_clients(self, sc: MongoDB): + return get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace) + + def test_delete_sharded_cluster_resource(self, sc: MongoDB, cluster_member_clients): + sc.delete() + + def resource_is_deleted() -> bool: + try: + sc.load() + return False + except kubernetes.client.ApiException as e: + return e.status == 404 + + run_periodically(resource_is_deleted, timeout=240) + + def test_sharded_cluster_doesnt_exist(self, sc: MongoDB, cluster_member_clients): + def sts_are_deleted() -> bool: + for cluster_member_client in cluster_member_clients: + sts = cluster_member_client.list_namespaced_stateful_set(sc.namespace) + if len(sts.items) != 0: + return False + + return True + + run_periodically(sts_are_deleted, timeout=60) + + def test_service_does_not_exist(self, sc: MongoDB, cluster_member_clients): + def svc_are_deleted() -> bool: + for cluster_member_client in cluster_member_clients: + try: + cluster_member_client.read_namespaced_service(sc.shard_service_name(), sc.namespace) + return False + except kubernetes.client.ApiException as e: + if e.status != 404: + return False + + return True + + run_periodically(svc_are_deleted, timeout=60) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_agent_flags.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_agent_flags.py new file mode 100644 index 000000000..6e9a5d172 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_agent_flags.py @@ -0,0 +1,147 @@ +from kubetester import find_fixture, try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.pod_logs import ( + assert_log_types_in_structured_json_pod_log, + get_all_default_log_types, + get_all_log_types, +) +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, +) + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("sharded-cluster.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + resource["spec"]["configSrv"] = { + "agent": {"startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFileSrv"}} + } + resource["spec"]["mongos"] = { + "agent": {"startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFileMongos"}} + } + resource["spec"]["shard"] = { + "agent": {"startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFileShard"}} + } + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource.update() + + +@mark.e2e_sharded_cluster_agent_flags +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_agent_flags +def test_create_sharded_cluster(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_sharded_cluster_agent_flags +def test_sharded_cluster_has_agent_flags(sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + for member_idx in range(sc.shard_members_in_cluster(cluster_member_client.cluster_name)): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFileShard* | wc -l", + ] + result = KubernetesTester.run_command_in_pod_container( + sc.shard_pod_name(0, member_idx, cluster_idx), + sc.namespace, + cmd, + api_client=cluster_member_client.api_client, + ) + assert result != "0" + + for member_idx in range(sc.config_srv_members_in_cluster(cluster_member_client.cluster_name)): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFileSrv* | wc -l", + ] + result = KubernetesTester.run_command_in_pod_container( + sc.config_srv_pod_name(member_idx, cluster_idx), + sc.namespace, + cmd, + api_client=cluster_member_client.api_client, + ) + assert result != "0" + + for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFileMongos* | wc -l", + ] + result = KubernetesTester.run_command_in_pod_container( + sc.mongos_pod_name(member_idx, cluster_idx), + sc.namespace, + cmd, + api_client=cluster_member_client.api_client, + ) + assert result != "0" + + +@mark.e2e_sharded_cluster_agent_flags +def test_log_types_without_audit_enabled(sc: MongoDB): + _assert_log_types_in_pods(sc, get_all_default_log_types()) + + +@mark.e2e_sharded_cluster_agent_flags +def test_enable_audit_log(sc: MongoDB): + additional_mongod_config = { + "auditLog": { + "destination": "file", + "format": "JSON", + "path": "/var/log/mongodb-mms-automation/mongodb-audit-changed.log", + } + } + sc["spec"]["configSrv"]["additionalMongodConfig"] = additional_mongod_config + sc["spec"]["mongos"]["additionalMongodConfig"] = additional_mongod_config + sc["spec"]["shard"]["additionalMongodConfig"] = additional_mongod_config + sc.update() + + sc.assert_reaches_phase(Phase.Running, timeout=2500) + + +@mark.e2e_sharded_cluster_agent_flags +def test_log_types_with_audit_enabled(sc: MongoDB): + _assert_log_types_in_pods(sc, get_all_log_types()) + + +def _assert_log_types_in_pods(sc: MongoDB, expected_log_types: set[str]): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + api_client = cluster_member_client.api_client + + for member_idx in range(sc.shard_members_in_cluster(cluster_member_client.cluster_name)): + assert_log_types_in_structured_json_pod_log( + sc.namespace, sc.shard_pod_name(0, member_idx, cluster_idx), expected_log_types, api_client=api_client + ) + + for member_idx in range(sc.config_srv_members_in_cluster(cluster_member_client.cluster_name)): + assert_log_types_in_structured_json_pod_log( + sc.namespace, sc.config_srv_pod_name(member_idx, cluster_idx), expected_log_types, api_client=api_client + ) + + for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)): + assert_log_types_in_structured_json_pod_log( + sc.namespace, sc.mongos_pod_name(member_idx, cluster_idx), expected_log_types, api_client=api_client + ) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_custom_podspec.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_custom_podspec.py new file mode 100644 index 000000000..421004450 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_custom_podspec.py @@ -0,0 +1,128 @@ +from kubetester import try_load +from kubetester.custom_podspec import assert_stateful_set_podspec +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, +) + +SHARD_WEIGHT = 50 +MONGOS_WEIGHT = 40 +CONFIG_WEIGHT = 30 +SHARD0_WEIGHT = 100 + +SHARD_GRACE_PERIOD = 30 +MONGOS_GRACE_PERIOD = 20 +CONFIG_GRACE_PERIOD = 50 +SHARD0_GRACE_PERIOD = 60 + +SHARD_TOPOLOGY_KEY = "shard" +MONGOS_TOPOLOGY_KEY = "mongos" +CONFIG_TOPOLOGY_KEY = "config" +SHARD0_TOPLOGY_KEY = "shardoverride" + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("sharded-cluster-custom-podspec.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + resource["spec"]["persistent"] = True + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource.update() + + +@mark.e2e_sharded_cluster_custom_podspec +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_custom_podspec +def test_create_sharded_cluster(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_sharded_cluster_custom_podspec +def test_stateful_sets_spec_updated(sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + shard0_sts_name = sc.shard_statefulset_name(0, cluster_idx) + shard0_sts = cluster_member_client.read_namespaced_stateful_set(shard0_sts_name, sc.namespace) + + shard1_sts_name = sc.shard_statefulset_name(1, cluster_idx) + shard1_sts = cluster_member_client.read_namespaced_stateful_set(shard1_sts_name, sc.namespace) + + mongos_sts_name = sc.mongos_statefulset_name(cluster_idx) + mongos_sts = cluster_member_client.read_namespaced_stateful_set(mongos_sts_name, sc.namespace) + + config_sts_name = sc.config_srv_statefulset_name(cluster_idx) + config_sts = cluster_member_client.read_namespaced_stateful_set(config_sts_name, sc.namespace) + + assert_stateful_set_podspec( + shard0_sts.spec.template.spec, + weight=SHARD0_WEIGHT, + grace_period_seconds=SHARD0_GRACE_PERIOD, + topology_key=SHARD0_TOPLOGY_KEY, + ) + assert_stateful_set_podspec( + shard1_sts.spec.template.spec, + weight=SHARD_WEIGHT, + grace_period_seconds=SHARD_GRACE_PERIOD, + topology_key=SHARD_TOPOLOGY_KEY, + ) + assert_stateful_set_podspec( + config_sts.spec.template.spec, + weight=CONFIG_WEIGHT, + grace_period_seconds=CONFIG_GRACE_PERIOD, + topology_key=CONFIG_TOPOLOGY_KEY, + ) + assert_stateful_set_podspec( + mongos_sts.spec.template.spec, + weight=MONGOS_WEIGHT, + grace_period_seconds=MONGOS_GRACE_PERIOD, + topology_key=MONGOS_TOPOLOGY_KEY, + ) + + if is_default_architecture_static(): + containers = shard0_sts.spec.template.spec.containers + assert len(containers) == 3 + assert containers[0].name == "mongodb-agent" + assert containers[1].name == "mongodb-enterprise-database" + assert containers[2].name == "sharded-cluster-sidecar-override" + + containers = shard1_sts.spec.template.spec.containers + assert len(containers) == 3 + assert containers[0].name == "mongodb-agent" + assert containers[1].name == "mongodb-enterprise-database" + assert containers[2].name == "sharded-cluster-sidecar" + + resources = containers[2].resources + else: + containers = shard1_sts.spec.template.spec.containers + assert len(containers) == 2 + assert containers[0].name == "mongodb-enterprise-database" + assert containers[1].name == "sharded-cluster-sidecar" + + containers = shard0_sts.spec.template.spec.containers + assert len(containers) == 2 + assert containers[0].name == "mongodb-enterprise-database" + assert containers[1].name == "sharded-cluster-sidecar-override" + resources = containers[1].resources + + assert resources.limits["cpu"] == "1" + assert resources.requests["cpu"] == "500m" diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_migration.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_migration.py new file mode 100644 index 000000000..c19571e89 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_migration.py @@ -0,0 +1,113 @@ +import pymongo +import pytest +from kubernetes import client +from kubetester import MongoDB, try_load +from kubetester.kubetester import assert_statefulset_architecture, ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import ( + get_default_architecture, + is_multi_cluster, + skip_if_multi_cluster, +) +from kubetester.mongodb import Phase +from kubetester.mongotester import MongoDBBackgroundTester, MongoTester +from kubetester.operator import Operator +from pytest import fixture +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, + get_mongos_service_names, +) + +MDB_RESOURCE_NAME = "sharded-cluster-migration" + + +@fixture(scope="module") +def mdb(namespace, custom_mdb_version: str): + resource = MongoDB.from_yaml( + load_fixture("sharded-cluster.yaml"), + namespace=namespace, + name=MDB_RESOURCE_NAME, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource + + +@fixture(scope="module") +def mongo_tester(mdb: MongoDB): + service_names = get_mongos_service_names(mdb) + return mdb.tester(use_ssl=False, service_names=service_names) + + +@fixture(scope="module") +def mdb_health_checker(mongo_tester: MongoTester) -> MongoDBBackgroundTester: + return MongoDBBackgroundTester( + mongo_tester, + allowed_sequential_failures=1, + health_function_params={ + "attempts": 1, + "write_concern": pymongo.WriteConcern(w="majority"), + }, + ) + + +@pytest.mark.e2e_sharded_cluster_migration +class TestShardedClusterMigrationStatic: + + def test_install_operator(self, operator: Operator): + operator.assert_is_running() + + def test_create_cluster(self, mdb: MongoDB): + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_start_health_checker(self, mdb_health_checker): + mdb_health_checker.start() + + def test_migrate_architecture(self, mdb: MongoDB): + """ + If the E2E is running with default architecture as non-static, + then the test will migrate to static and vice versa. + """ + original_default_architecture = get_default_architecture() + target_architecture = "non-static" if original_default_architecture == "static" else "static" + + mdb.trigger_architecture_migration() + + mdb.load() + assert mdb["metadata"]["annotations"]["mongodb.com/v1.architecture"] == target_architecture + + mdb.assert_abandons_phase(Phase.Running, timeout=1200) + mdb.assert_reaches_phase(Phase.Running, timeout=1200) + + # Read StatefulSet after successful reconciliation + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(mdb.name, mdb.namespace): + cluster_idx = cluster_member_client.cluster_index + + for shard_idx in range(mdb["spec"]["shardCount"]): + shard_sts_name = mdb.shard_statefulset_name(shard_idx, cluster_idx) + shard_sts = cluster_member_client.read_namespaced_stateful_set(shard_sts_name, mdb.namespace) + + assert_statefulset_architecture(shard_sts, target_architecture) + + mongos_sts_name = mdb.mongos_statefulset_name(cluster_idx) + mongos_sts = cluster_member_client.read_namespaced_stateful_set(mongos_sts_name, mdb.namespace) + assert_statefulset_architecture(mongos_sts, target_architecture) + + config_sts_name = mdb.config_srv_statefulset_name(cluster_idx) + config_sts = cluster_member_client.read_namespaced_stateful_set(config_sts_name, mdb.namespace) + assert_statefulset_architecture(config_sts, target_architecture) + + @skip_if_multi_cluster() # Currently we are experiencing more than single failure during migration. More info + # in the ticket -> https://jira.mongodb.org/browse/CLOUDP-286686 + def test_mdb_healthy_throughout_change_version(self, mdb_health_checker): + mdb_health_checker.assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_mongod_options.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_mongod_options.py new file mode 100644 index 000000000..5b35ec570 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_mongod_options.py @@ -0,0 +1,178 @@ +from kubetester import try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + assert_log_rotation_backup_monitoring, + assert_log_rotation_process, + is_multi_cluster, + setup_log_rotate_for_agents, +) +from tests.shardedcluster.conftest import enable_multi_cluster_deployment + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-mongod-options.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + setup_log_rotate_for_agents(resource) + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1], + configsrv_members_array=[1], + ) + + return resource.update() + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_sharded_cluster_created(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_sharded_cluster_mongodb_options_mongos(sc: MongoDB): + automation_config_tester = sc.get_automation_config_tester() + for process in automation_config_tester.get_mongos_processes(): + assert process["args2_6"]["systemLog"]["verbosity"] == 4 + assert process["args2_6"]["systemLog"]["logAppend"] + assert "operationProfiling" not in process["args2_6"] + assert "storage" not in process["args2_6"] + assert process["args2_6"]["net"]["port"] == 30003 + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_sharded_cluster_mongodb_options_config_srv(sc: MongoDB): + automation_config_tester = sc.get_automation_config_tester() + config_srv_replicaset_name = sc.config_srv_replicaset_name() + for process in automation_config_tester.get_replica_set_processes(config_srv_replicaset_name): + assert process["args2_6"]["operationProfiling"]["mode"] == "slowOp" + assert "verbosity" not in process["args2_6"]["systemLog"] + assert "logAppend" not in process["args2_6"]["systemLog"] + assert "journal" not in process["args2_6"]["storage"] + assert process["args2_6"]["net"]["port"] == 30002 + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_sharded_cluster_mongodb_options_shards(sc: MongoDB): + automation_config_tester = sc.get_automation_config_tester() + for shard_replicaset_name in sc.shard_replicaset_names(): + for process in automation_config_tester.get_replica_set_processes(shard_replicaset_name): + assert process["args2_6"]["storage"]["journal"]["commitIntervalMs"] == 50 + assert "verbosity" not in process["args2_6"]["systemLog"] + assert "logAppend" not in process["args2_6"]["systemLog"] + assert "operationProfiling" not in process["args2_6"] + assert process["args2_6"]["net"]["port"] == 30001 + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_sharded_cluster_feature_controls(sc: MongoDB): + fc = sc.get_om_tester().get_feature_controls() + assert fc["externalManagementSystem"]["name"] == "mongodb-enterprise-operator" + + assert len(fc["policies"]) == 3 + # unfortunately OM uses a HashSet for policies... + policies = sorted(fc["policies"], key=lambda policy: policy["policy"]) + assert policies[0]["policy"] == "DISABLE_SET_MONGOD_CONFIG" + assert policies[1]["policy"] == "DISABLE_SET_MONGOD_VERSION" + assert policies[2]["policy"] == "EXTERNALLY_MANAGED_LOCK" + # OM stores the params into a set - we need to sort to compare + disabled_params = sorted(policies[0]["disabledParams"]) + assert disabled_params == [ + "net.port", + "operationProfiling.mode", + "storage.journal.commitIntervalMs", + "systemLog.logAppend", + "systemLog.verbosity", + ] + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_remove_fields(sc: MongoDB): + # delete a field from each component + sc["spec"]["mongos"]["additionalMongodConfig"]["systemLog"]["verbosity"] = None + sc["spec"]["shard"]["additionalMongodConfig"]["storage"]["journal"]["commitIntervalMs"] = None + sc["spec"]["configSrv"]["additionalMongodConfig"]["operationProfiling"]["mode"] = None + + sc.update() + + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_fields_are_successfully_removed_from_mongos(sc: MongoDB): + automation_config_tester = sc.get_automation_config_tester() + for process in automation_config_tester.get_mongos_processes(): + assert "verbosity" not in process["args2_6"]["systemLog"] + + # other fields are still there + assert process["args2_6"]["systemLog"]["logAppend"] + assert "operationProfiling" not in process["args2_6"] + assert "storage" not in process["args2_6"] + assert process["args2_6"]["net"]["port"] == 30003 + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_fields_are_successfully_removed_from_config_srv(sc: MongoDB): + automation_config_tester = sc.get_automation_config_tester() + config_srv_replicaset_name = sc.config_srv_replicaset_name() + for process in automation_config_tester.get_replica_set_processes(config_srv_replicaset_name): + assert "mode" not in process["args2_6"]["operationProfiling"] + + # other fields are still there + assert "verbosity" not in process["args2_6"]["systemLog"] + assert "logAppend" not in process["args2_6"]["systemLog"] + assert "journal" not in process["args2_6"]["storage"] + assert process["args2_6"]["net"]["port"] == 30002 + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_fields_are_successfully_removed_from_shards(sc: MongoDB): + automation_config_tester = sc.get_automation_config_tester() + for shard_replicaset_name in sc.shard_replicaset_names(): + for process in automation_config_tester.get_replica_set_processes(shard_replicaset_name): + assert "commitIntervalMs" not in process["args2_6"]["storage"]["journal"] + + # other fields are still there + assert "verbosity" not in process["args2_6"]["systemLog"] + assert "logAppend" not in process["args2_6"]["systemLog"] + assert "operationProfiling" not in process["args2_6"] + assert process["args2_6"]["net"]["port"] == 30001 + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_process_log_rotation(): + config = KubernetesTester.get_automation_config() + for process in config["processes"]: + assert_log_rotation_process(process) + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_backup_log_rotation(): + bvk = KubernetesTester.get_backup_config() + assert_log_rotation_backup_monitoring(bvk) + + +@mark.e2e_sharded_cluster_mongod_options_and_log_rotation +def test_backup_log_rotation(): + mc = KubernetesTester.get_monitoring_config() + assert_log_rotation_backup_monitoring(mc) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv.py new file mode 100644 index 000000000..b3feea015 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv.py @@ -0,0 +1,172 @@ +import kubernetes +from kubetester import MongoDB, try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, + get_mongos_service_names, +) + + +@fixture(scope="module") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-pv.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource + + +@mark.e2e_sharded_cluster_pv +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_pv +class TestShardedClusterCreation: + custom_labels = {"label1": "val1", "label2": "val2"} + + def test_sharded_cluster_created(self, sc: MongoDB): + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def check_sts_labels(self, sts): + sts_labels = sts.metadata.labels + for k in self.custom_labels: + assert k in sts_labels and sts_labels[k] == self.custom_labels[k] + + def check_pvc_labels(self, pvc): + pvc_labels = pvc.metadata.labels + for k in self.custom_labels: + assert k in pvc_labels and pvc_labels[k] == self.custom_labels[k] + + def test_sharded_cluster_sts(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + shard_sts_name = sc.shard_statefulset_name(0, cluster_idx) + shard_sts = cluster_member_client.read_namespaced_stateful_set(shard_sts_name, sc.namespace) + assert shard_sts + self.check_sts_labels(shard_sts) + + def test_config_sts(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + config_srv_sts_name = sc.config_srv_statefulset_name(cluster_idx) + config_srv_sts = cluster_member_client.read_namespaced_stateful_set(config_srv_sts_name, sc.namespace) + assert config_srv_sts + self.check_sts_labels(config_srv_sts) + + def test_mongos_sts(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + mongos_sts_name = sc.config_srv_statefulset_name(cluster_idx) + mongos_sts = cluster_member_client.read_namespaced_stateful_set(mongos_sts_name, sc.namespace) + assert mongos_sts + self.check_sts_labels(mongos_sts) + + def test_mongod_sharded_cluster_service(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + shard_service_name = sc.shard_service_name() + shard_service = cluster_member_client.read_namespaced_service(shard_service_name, sc.namespace) + assert shard_service + + def test_shard0_was_configured(self, sc: MongoDB): + hosts = [] + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + for member_idx in range(sc.shard_members_in_cluster(cluster_member_client.cluster_name)): + hostname = sc.shard_hostname(0, member_idx, cluster_member_client.cluster_index) + hosts.append(hostname) + + primary, secondaries = KubernetesTester.wait_for_rs_is_ready(hosts) + + assert primary is not None + assert len(secondaries) == 2 + + def test_pvc_are_bound(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + for member_idx in range(sc.shard_members_in_cluster(cluster_member_client.cluster_name)): + pvc_name = sc.shard_pvc_name(0, member_idx, cluster_idx) + pvc = cluster_member_client.read_namespaced_persistent_volume_claim(pvc_name, sc.namespace) + assert pvc.status.phase == "Bound" + assert pvc.spec.resources.requests["storage"] == "1G" + self.check_pvc_labels(pvc) + + for member_idx in range(sc.config_srv_members_in_cluster(cluster_member_client.cluster_name)): + pvc_name = sc.config_srv_pvc_name(member_idx, cluster_idx) + pvc = cluster_member_client.read_namespaced_persistent_volume_claim(pvc_name, sc.namespace) + assert pvc.status.phase == "Bound" + assert pvc.spec.resources.requests["storage"] == "1G" + self.check_pvc_labels(pvc) + + def test_mongos_are_reachable(self, sc: MongoDB): + for service_name in get_mongos_service_names(sc): + tester = sc.tester(service_names=[service_name]) + tester.assert_connectivity() + + +@mark.e2e_sharded_cluster_pv +class TestShardedClusterDeletion: + + # We need to store cluster_member_clients somehow after deleting the MongoDB resource. + # Cluster mapping from deployment state is needed to compute cluster_member_clients. + @fixture(scope="class") + def cluster_member_clients(self, sc: MongoDB): + return get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace) + + def test_delete_sharded_cluster_resource(self, sc: MongoDB, cluster_member_clients): + sc.delete() + + def resource_is_deleted() -> bool: + try: + sc.load() + return False + except kubernetes.client.ApiException as e: + return e.status == 404 + + run_periodically(resource_is_deleted, timeout=240) + + def test_sharded_cluster_doesnt_exist(self, sc: MongoDB, cluster_member_clients): + def sts_are_deleted() -> bool: + for cluster_member_client in cluster_member_clients: + sts = cluster_member_client.list_namespaced_stateful_set(sc.namespace) + if len(sts.items) != 0: + return False + + return True + + run_periodically(sts_are_deleted, timeout=60) + + def test_service_does_not_exist(self, sc: MongoDB, cluster_member_clients): + def svc_are_deleted() -> bool: + for cluster_member_client in cluster_member_clients: + try: + cluster_member_client.read_namespaced_service(sc.shard_service_name(), sc.namespace) + return False + except kubernetes.client.ApiException as e: + if e.status != 404: + return False + + return True + + run_periodically(svc_are_deleted, timeout=60) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv_resize.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv_resize.py new file mode 100644 index 000000000..fe066b0ae --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv_resize.py @@ -0,0 +1,91 @@ +from kubetester import MongoDB, try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, +) + +SHARD_COUNT = 2 +RESIZED_STORAGE_SIZE = "2Gi" + + +# Note: This test can only be run in a cluster which uses - by default - a storageClass that is resizable. +# In Kind cluster you need to ensure that the resizable CSI driver has been installed. It should be automatically +# installed for new clusters + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-pv-resize.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource.update() + + +@mark.e2e_sharded_cluster_pv_resize +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_pv_resize +def test_create_sharded_cluster(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@mark.e2e_sharded_cluster_pv_resize +def test_sharded_cluster_resize_pvc_state_changes(sc: MongoDB): + # Mongos do not support persistent storage + sc["spec"]["shardPodSpec"]["persistence"]["multiple"]["journal"]["storage"] = RESIZED_STORAGE_SIZE + sc["spec"]["shardPodSpec"]["persistence"]["multiple"]["data"]["storage"] = RESIZED_STORAGE_SIZE + sc["spec"]["configSrvPodSpec"]["persistence"]["multiple"]["data"]["storage"] = RESIZED_STORAGE_SIZE + sc["spec"]["configSrvPodSpec"]["persistence"]["multiple"]["journal"]["storage"] = RESIZED_STORAGE_SIZE + + sc.update() + + sc.assert_reaches_phase(Phase.Pending, timeout=400) + sc.assert_reaches_phase(Phase.Running, timeout=2000) + + +@mark.e2e_sharded_cluster_pv_resize +def test_sharded_cluster_resize_finished(sc: MongoDB, namespace: str): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + shards_sts = [] + for shard_idx in range(SHARD_COUNT): + shard_sts_name = sc.shard_statefulset_name(shard_idx, cluster_idx) + shard_sts = cluster_member_client.read_namespaced_stateful_set(shard_sts_name, sc.namespace) + shards_sts.append(shard_sts) + + config_sts_name = sc.config_srv_statefulset_name(cluster_idx) + config_sts = cluster_member_client.read_namespaced_stateful_set(config_sts_name, sc.namespace) + + for sts in (config_sts, *shards_sts): + assert sts.spec.volume_claim_templates[0].spec.resources.requests["storage"] == RESIZED_STORAGE_SIZE + pvc_name = f"data-{sts.metadata.name}-0" + pvc_data = cluster_member_client.read_namespaced_persistent_volume_claim(pvc_name, namespace) + assert pvc_data.status.capacity["storage"] == RESIZED_STORAGE_SIZE + + pvc_name = f"journal-{sts.metadata.name}-0" + pvc_data = cluster_member_client.read_namespaced_persistent_volume_claim(pvc_name, namespace) + assert pvc_data.status.capacity["storage"] == RESIZED_STORAGE_SIZE + + initial_storage_size = "1Gi" + pvc_name = f"logs-{sts.metadata.name}-0" + pvc_data = cluster_member_client.read_namespaced_persistent_volume_claim(pvc_name, namespace) + assert pvc_data.status.capacity["storage"] == initial_storage_size diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_recovery.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_recovery.py new file mode 100644 index 000000000..5637b0dd1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_recovery.py @@ -0,0 +1,63 @@ +from kubetester import try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.shardedcluster.conftest import enable_multi_cluster_deployment + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("sharded-cluster-single.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, None, None], + configsrv_members_array=[None, 1, None], + ) + + return resource.update() + + +@mark.e2e_sharded_cluster_recovery +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_recovery +class TestShardedClusterRecoversBadOmConfiguration: + """ + name: Sharded cluster broken OM connection + description: | + Creates a sharded cluster with a bad OM connection (public key is broken) and ensures it enters a failed state | + Then the secret is fixed and the standalone is expected to reach good state eventually + """ + + def test_create_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_sharded_cluster_reaches_failed_state(self, sc: MongoDB): + secret_data = {"publicApiKey": "wrongKey"} + KubernetesTester.update_secret(sc.namespace, "my-credentials", secret_data) + + sc.assert_reaches_phase(Phase.Failed, timeout=20) + + sc.load() + assert "You are not authorized for this resource" in sc["status"]["message"] + + def test_recovery(self, sc: MongoDB): + secret_data = {"publicApiKey": KubernetesTester.get_om_api_key()} + KubernetesTester.update_secret(sc.namespace, "my-credentials", secret_data) + + # We need to ignore errors here because the CM change can be faster than the check + sc.assert_reaches_phase(Phase.Running, ignore_errors=True) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_scale_shards.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_scale_shards.py new file mode 100644 index 000000000..3d172eee0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_scale_shards.py @@ -0,0 +1,135 @@ +import kubernetes +import pytest +from kubetester import try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, + get_mongos_service_names, +) + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-scale-shards.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + # this test requires the volumes to not be persistent as we first scale shard down and then scale up without clearing PV + # in order to get rid of persistent: False we should add removing PV here + resource["spec"]["persistent"] = False + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, None, None], + configsrv_members_array=[None, 1, None], + ) + + return resource.update() + + +@mark.e2e_sharded_cluster_scale_shards +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_scale_shards +class TestShardedClusterScaleShardsCreate: + """ + name: ShardedCluster scale of shards (create) + description: | + Creates a sharded cluster with 2 shards + """ + + def test_sharded_cluster_running(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_db_connectable(self, sc: MongoDB): + service_names = get_mongos_service_names(sc) + mongod_tester = sc.tester(service_names=service_names) + + mongod_tester.shard_collection(f"{sc.name}-{{}}", 2, "type") + mongod_tester.upload_random_data(50_000) + mongod_tester.prepare_for_shard_removal(f"{sc.name}-{{}}", 2) + mongod_tester.assert_number_of_shards(2) + # todo would be great to verify that chunks are distributed over shards, but I didn't manage to get the same + # results as from CMD sh.status()(alisovenko) + # self.client.config.command('printShardingStatus') --> doesn't work + + +@mark.e2e_sharded_cluster_scale_shards +class TestShardedClusterScaleDownShards: + """ + name: ShardedCluster scale down of shards (update) + description: | + Updates the sharded cluster, scaling down its shards count to 1. Makes sure no data is lost. + (alisovenko) Implementation notes: I tried to get long rebalancing to make sure it's covered with multiple reconciliations, + but in fact rebalancing is almost immediate (insertion is way longer) so the single reconciliation manages to get + agents + """ + + def test_scale_down_sharded_cluster(self, sc: MongoDB): + sc["spec"]["shardCount"] = 1 + sc.update() + + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_db_data_the_same_count(self, sc: MongoDB): + service_names = get_mongos_service_names(sc) + mongod_tester = sc.tester(service_names=service_names) + + mongod_tester.assert_number_of_shards(1) + mongod_tester.assert_data_size(50_000) + + def test_statefulset_for_shard_removed(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + with pytest.raises(kubernetes.client.ApiException) as api_exception: + shard_sts_name = sc.shard_statefulset_name(1, cluster_idx) + cluster_member_client.read_namespaced_stateful_set(shard_sts_name, sc.namespace) + assert api_exception.value.status == 404 + + +@mark.e2e_sharded_cluster_scale_shards +class TestShardedClusterScaleUpShards: + """ + name: ShardedCluster scale up of shards (sc) + description: | + Updates the sharded cluster, scaling up its shards count to 2. Makes sure no data is lost. + """ + + def test_scale_up_sharded_cluster(self, sc: MongoDB): + sc.load() + sc["spec"]["shardCount"] = 2 + sc.update() + + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_db_data_the_same_count(self, sc: MongoDB): + service_names = get_mongos_service_names(sc) + mongod_tester = sc.tester(service_names=service_names) + + mongod_tester.assert_number_of_shards(2) + mongod_tester.assert_data_size(50_000) + + def test_statefulset_for_shard_added(self, sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + + shard_sts_name = sc.shard_statefulset_name(1, cluster_idx) + shard_sts = cluster_member_client.read_namespaced_stateful_set(shard_sts_name, sc.namespace) + assert shard_sts is not None diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_schema_validation.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_schema_validation.py new file mode 100644 index 000000000..a7bb079cc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_schema_validation.py @@ -0,0 +1,155 @@ +from kubetester.kubetester import KubernetesTester +from kubetester.operator import Operator +from pytest import mark + + +@mark.e2e_sharded_cluster_schema_validation +def test_install_operator(default_operator: Operator): + default_operator.assert_is_running() + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterValidationMongosMissing(KubernetesTester): + """ + name: Sharded Cluster Validation (mongos missing) + create: + file: sharded-cluster.yaml + patch: '[{"op":"remove","path":"/spec/mongosCount"}]' + exception: 'Unprocessable Entity' + """ + + @mark.skip + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterValidationShardCountMissing(KubernetesTester): + """ + name: Sharded Cluster Validation (shardCount missing) + create: + file: sharded-cluster.yaml + patch: '[{"op":"remove","path":"/spec/shardCount"}]' + exception: 'Unprocessable Entity' + """ + + @mark.skip + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterValidationConfigServerCount(KubernetesTester): + """ + name: Sharded Cluster Validation (configServerCount missing) + create: + file: sharded-cluster.yaml + patch: '[{"op":"remove","path":"/spec/configServerCount"}]' + exception: 'Unprocessable Entity' + """ + + @mark.skip + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterValidationMongoDsPerShard(KubernetesTester): + """ + name: Sharded Cluster Validation (mongodsPerShardCount missing) + create: + file: sharded-cluster.yaml + patch: '[{"op":"remove","path":"/spec/mongodsPerShardCount"}]' + exception: 'Unprocessable Entity' + """ + + @mark.skip + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterValidationInvalidType(KubernetesTester): + """ + name: Sharded Cluster Validation (invalid type) + create: + file: sharded-cluster.yaml + patch: '[{"op":"replace","path":"/spec/type","value":"InvalidShardedClusterType"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterValidationInvalidLogLevel(KubernetesTester): + """ + name: Sharded Cluster Validation (invalid logLevel) + create: + file: sharded-cluster.yaml + patch: '[{"op":"replace","path":"/spec/logLevel","value":"NotDEBUG"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterSchemaAdditionalMongodConfigNotAllowed(KubernetesTester): + """ + name: Sharded Cluster Validation (additional Mongod Config not allowed) + create: + file: sharded-cluster.yaml + patch: '[{"op":"add","path":"/spec/additionalMongodConfig","value":{"net":{"ssl":{"mode": "disabled"}}}}]' + exception: 'cannot be specified if type of MongoDB is ShardedCluster' + """ + + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterInvalidWithProjectAndOpsManager(KubernetesTester): + init = { + "create": { + "file": "sharded-cluster.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/opsManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True + + +@mark.e2e_sharded_cluster_schema_validation +class TestShardedClusterInvalidWithCloudAndOpsManagerAndProject(KubernetesTester): + init = { + "create": { + "file": "sharded-cluster.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/cloudManager", + "value": {"configMapRef": {"name": "something"}}, + }, + { + "op": "add", + "path": "/spec/opsManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_secret.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_secret.py new file mode 100644 index 000000000..546fc5e51 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_secret.py @@ -0,0 +1,52 @@ +from kubetester import try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import enable_multi_cluster_deployment + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-single.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource) + + return resource.update() + + +@mark.e2e_sharded_cluster_secret +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_secret +class TestShardedClusterListensSecret: + """ + name: ShardedCluster tracks configmap changes + description: | + Creates a sharded cluster, then changes secret - breaks the api key and checks that the reconciliation for the | + standalone happened and it got into Failed state. Note, that this test cannot be run with 'make e2e .. light=true' | + flag locally as secret must be recreated + """ + + def test_create_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_patch_config_map(self, sc: MongoDB): + KubernetesTester.update_secret(sc.namespace, "my-credentials", {"publicApiKey": "wrongKey"}) + + print('Patched the Secret - changed publicApiKey to "wrongKey"') + sc.assert_reaches_phase(Phase.Failed, timeout=20) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_shard_overrides.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_shard_overrides.py new file mode 100644 index 000000000..35f04cd6a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_shard_overrides.py @@ -0,0 +1,136 @@ +from kubernetes.client import ApiClient +from kubetester import try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import get_member_cluster_names +from tests.multicluster.conftest import cluster_spec_list +from tests.multicluster_shardedcluster import ( + build_expected_statefulsets, + build_expected_statefulsets_multi, + validate_correct_sts_in_cluster, + validate_correct_sts_in_cluster_multi, + validate_member_count_in_ac, + validate_shard_configurations_in_ac, + validate_shard_configurations_in_ac_multi, +) +from tests.shardedcluster.conftest import read_deployment_state + +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + fixture_name = ( + "sharded-cluster-shard-overrides-multi-cluster.yaml" + if is_multi_cluster() + else "sharded-cluster-shard-overrides.yaml" + ) + resource = MongoDB.from_yaml( + yaml_fixture(fixture_name), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + return resource.update() + + +@mark.e2e_sharded_cluster_shard_overrides +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_shard_overrides +class TestShardedClusterShardOverrides: + """ + Creates a sharded cluster configured with shard overrides. Verify deployed stateful sets and automation config. + """ + + def test_sharded_cluster_running(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000 if is_multi_cluster() else 800) + + def test_assert_correct_automation_config(self, sc: MongoDB): + config = KubernetesTester.get_automation_config() + logger.info("Validating automation config correctness") + validate_member_count_in_ac(sc, config) + if is_multi_cluster(): + validate_shard_configurations_in_ac_multi(sc, config) + else: + validate_shard_configurations_in_ac(sc, config) + + def test_assert_stateful_sets( + self, sc: MongoDB, namespace: str, central_cluster_client: ApiClient, member_cluster_clients + ): + logger.info("Validating statefulsets in cluster(s)") + if is_multi_cluster(): + # We need the unique cluster index, stored in the state configmap, for computing expected sts names + cluster_mapping = read_deployment_state(sc.name, namespace)["clusterMapping"] + logger.debug(f"Cluster mapping in state: {cluster_mapping}") + expected_statefulsets = build_expected_statefulsets_multi(sc, cluster_mapping) + validate_correct_sts_in_cluster_multi(expected_statefulsets, namespace, member_cluster_clients) + else: + expected_statefulsets = build_expected_statefulsets(sc) + validate_correct_sts_in_cluster(expected_statefulsets, namespace, "__default", central_cluster_client) + + def test_scale_shard_overrides(self, sc: MongoDB): + if is_multi_cluster(): + # Override for shards 0 and 1 + sc["spec"]["shardOverrides"][0]["clusterSpecList"] = cluster_spec_list( + get_member_cluster_names()[:2], [1, 2] + ) # cluster2: 1->2 + + # Override for shard 3 + sc["spec"]["shardOverrides"][1]["clusterSpecList"] = cluster_spec_list( + get_member_cluster_names(), [1, 1, 3] + ) # cluster3: 1->3 + + # This replica initially had 0 votes, we need to restore the setting after using 'cluster_spec_list' above + sc["spec"]["shardOverrides"][1]["clusterSpecList"][0]["memberConfig"] = [{"votes": 0, "priority": "0"}] + + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=1000) + else: + # In the single cluster case, we first scale up, and then down + # We cannot scale in both ways at the same time + logger.info("Scaling up shard 3 with override") + sc["spec"]["shardOverrides"][1]["members"] = 4 # no member count specified (2) -> 4 members (shard 3) + sc["spec"]["shardOverrides"][1]["memberConfig"] = None + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=400) + + logger.info("Scaling down shards 0 and 1 with override") + sc["spec"]["shardOverrides"][0]["members"] = 2 # Override for shards 0 and 1: 3-> 2 + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=400) + + def test_assert_correct_automation_config_after_scaling(self, sc: MongoDB): + resource = sc.load() + config = KubernetesTester.get_automation_config() + logger.info("Validating automation config correctness") + validate_member_count_in_ac(resource, config) + if is_multi_cluster(): + validate_shard_configurations_in_ac_multi(resource, config) + else: + validate_shard_configurations_in_ac(resource, config) + + def test_assert_stateful_sets_after_scaling( + self, sc: MongoDB, namespace: str, central_cluster_client: ApiClient, member_cluster_clients + ): + logger.info("Validating statefulsets in cluster(s)") + if is_multi_cluster(): + # We need the unique cluster index, stored in the state configmap, for computing expected sts names + cluster_mapping = read_deployment_state(sc.name, namespace)["clusterMapping"] + logger.debug(f"Cluster mapping in state: {cluster_mapping}") + expected_statefulsets = build_expected_statefulsets_multi(sc, cluster_mapping) + validate_correct_sts_in_cluster_multi(expected_statefulsets, namespace, member_cluster_clients) + else: + expected_statefulsets = build_expected_statefulsets(sc) + validate_correct_sts_in_cluster(expected_statefulsets, namespace, "__default", central_cluster_client) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_single_cluster_external_access.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_single_cluster_external_access.py new file mode 100644 index 000000000..7de9bbe0e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_single_cluster_external_access.py @@ -0,0 +1,72 @@ +import logging + +import kubernetes +from kubetester import try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests import test_logger +from tests.conftest import ( + get_central_cluster_name, + is_multi_cluster, + update_coredns_hosts, +) +from tests.shardedcluster.conftest import ( + get_dns_hosts_for_external_access, + setup_external_access, +) + +SCALED_SHARD_COUNT = 2 +logger = test_logger.get_test_logger(__name__) + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("sharded-cluster.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + setup_external_access(resource) + + if is_multi_cluster(): + raise Exception("This test has been designed to run only in Single Cluster mode") + + return resource.update() + + +# Even though this is theoretically not needed, it is useful for testing with Multi Cluster EVG hosts. +@mark.e2e_sharded_cluster_external_access +def test_disable_istio(disable_istio): + logging.info("Istio disabled") + + +@mark.e2e_sharded_cluster_external_access +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_external_access +def test_update_coredns(cluster_clients: dict[str, kubernetes.client.ApiClient], sc: MongoDB): + hosts = get_dns_hosts_for_external_access(resource=sc, cluster_member_list=[get_central_cluster_name()]) + for cluster_name, cluster_api in cluster_clients.items(): + update_coredns_hosts(hosts, cluster_name, api_client=cluster_api) + + +@mark.e2e_sharded_cluster_external_access +class TestShardedClusterCreation: + + def test_create_sharded_cluster(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + # Testing connectivity with External Access requires using the same DNS as deployed in Kube within + # test_update_coredns. There's no easy way to set it up locally. + @skip_if_local() + def test_shards_were_configured_and_accessible(self, sc: MongoDB): + tester = sc.tester() + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_statefulset_status.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_statefulset_status.py new file mode 100644 index 000000000..b09f0db94 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_statefulset_status.py @@ -0,0 +1,115 @@ +import re + +from kubetester import try_load +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, +) + +""" +This test checks the 'status.resourcesNotReady' element during sharded cluster reconciliation. It's expected to +be populated with the information about current StatefulSet pending in the following order: config server, shard 0, +shard 1, mongos. +""" + + +@fixture(scope="function") +def sc(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster-single.yaml"), + namespace=namespace, + name="sharded-cluster-status", + ) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + resource["spec"]["shardCount"] = 2 + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[None, 1, 1], + ) + + return resource.update() + + +@mark.e2e_sharded_cluster_statefulset_status +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_statefulset_status +def test_config_srv_reaches_pending_phase(sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.config_srv_members_in_cluster(cluster_member_client.cluster_name) > 0: + cluster_idx = cluster_member_client.cluster_index + configsrv_sts_name = sc.config_srv_statefulset_name(cluster_idx) + cluster_reaches_not_ready(sc, configsrv_sts_name) + + +@mark.e2e_sharded_cluster_statefulset_status +def test_first_shard_reaches_pending_phase(sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.shard_members_in_cluster(cluster_member_client.cluster_name) > 0: + cluster_idx = cluster_member_client.cluster_index + shard0_sts_name = sc.shard_statefulset_name(0, cluster_idx) + cluster_reaches_not_ready(sc, shard0_sts_name) + + +@mark.e2e_sharded_cluster_statefulset_status +def test_second_shard_reaches_pending_phase(sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.shard_members_in_cluster(cluster_member_client.cluster_name) > 0: + cluster_idx = cluster_member_client.cluster_index + shard1_sts_name = sc.shard_statefulset_name(1, cluster_idx) + cluster_reaches_not_ready(sc, shard1_sts_name) + + +@mark.e2e_sharded_cluster_statefulset_status +def test_mongos_reaches_pending_phase(sc: MongoDB): + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + if sc.mongos_members_in_cluster(cluster_member_client.cluster_name) > 0: + cluster_idx = cluster_member_client.cluster_index + mongos_sts_name = sc.mongos_statefulset_name(cluster_idx) + cluster_reaches_not_ready(sc, mongos_sts_name) + + +@mark.e2e_sharded_cluster_statefulset_status +def test_sharded_cluster_reaches_running_phase(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + assert sc.get_status_resources_not_ready() is None + + +def cluster_reaches_not_ready(sc: MongoDB, sts_name: str): + """This function waits until the sharded cluster status gets 'resource_not_ready' element for the specified + StatefulSet""" + + def resource_not_ready(s: MongoDB) -> bool: + if s.get_status_resources_not_ready() is None: + return False + + for idx, resource in enumerate(s.get_status_resources_not_ready()): + if resource["name"] == sts_name: + assert resource["kind"] == "StatefulSet" + assert re.search("Not all the Pods are ready \(wanted: 1.*\)", resource["message"]) is not None + + return True + + return False + + sc.wait_for(resource_not_ready, timeout=150, should_raise=True) + + assert sc.get_status_phase() == Phase.Pending diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_upgrade_downgrade.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_upgrade_downgrade.py new file mode 100644 index 000000000..8c6d17b34 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_upgrade_downgrade.py @@ -0,0 +1,105 @@ +import pymongo +from kubetester import MongoDB, try_load +from kubetester.kubetester import KubernetesTester, ensure_ent_version, fcv_from_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import Phase +from kubetester.mongotester import MongoDBBackgroundTester, MongoTester +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import is_multi_cluster +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, + get_mongos_service_names, +) + + +@fixture(scope="module") +def sc(namespace: str, custom_mdb_prev_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("sharded-cluster-downgrade.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_version(ensure_ent_version(custom_mdb_prev_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, 1], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@fixture(scope="module") +def mongod_tester(sc: MongoDB) -> MongoTester: + service_names = get_mongos_service_names(sc) + + return sc.tester(service_names=service_names) + + +@fixture(scope="module") +def mdb_health_checker(mongod_tester: MongoTester) -> MongoDBBackgroundTester: + return MongoDBBackgroundTester( + mongod_tester, + # After running multiple tests, it seems that on sharded_cluster version changes we have more sequential errors. + allowed_sequential_failures=5, + health_function_params={ + "attempts": 1, + "write_concern": pymongo.WriteConcern(w="majority"), + }, + ) + + +@mark.e2e_sharded_cluster_upgrade_downgrade +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_sharded_cluster_upgrade_downgrade +class TestShardedClusterUpgradeDowngradeCreate(KubernetesTester): + + def test_mdb_created(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1000) + + def test_start_mongod_background_tester(self, mdb_health_checker): + mdb_health_checker.start() + + def test_db_connectable(self, mongod_tester: MongoTester, custom_mdb_prev_version: str): + mongod_tester.assert_connectivity() + mongod_tester.assert_version(custom_mdb_prev_version) + + +@mark.e2e_sharded_cluster_upgrade_downgrade +class TestShardedClusterUpgradeDowngradeUpdate: + + def test_mongodb_upgrade(self, sc: MongoDB, custom_mdb_version: str, custom_mdb_prev_version: str): + sc.set_version(custom_mdb_version) + fcv = fcv_from_version(custom_mdb_prev_version) + sc["spec"]["featureCompatibilityVersion"] = fcv + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=2400) + + def test_db_connectable(self, mongod_tester: MongoTester, custom_mdb_version: str): + mongod_tester.assert_connectivity() + mongod_tester.assert_version(custom_mdb_version) + + +@mark.e2e_sharded_cluster_upgrade_downgrade +class TestShardedClusterUpgradeDowngradeRevert: + + def test_mongodb_downgrade(self, sc: MongoDB, custom_mdb_prev_version: str): + sc.set_version(custom_mdb_prev_version) + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=2400) + + def test_db_connectable(self, mongod_tester: MongoTester, custom_mdb_prev_version): + mongod_tester.assert_connectivity() + mongod_tester.assert_version(custom_mdb_prev_version) + + def test_mdb_healthy_throughout_change_version(self, mdb_health_checker): + mdb_health_checker.assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/standalone/__init__.py b/docker/mongodb-enterprise-tests/tests/standalone/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/standalone/conftest.py b/docker/mongodb-enterprise-tests/tests/standalone/conftest.py new file mode 100644 index 000000000..cb0664057 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/conftest.py @@ -0,0 +1,4 @@ +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if "default_operator" not in item.fixturenames: + item.fixturenames.insert(0, "default_operator") diff --git a/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone-custom-podspec.yaml b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone-custom-podspec.yaml new file mode 100644 index 000000000..871d2191a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone-custom-podspec.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone-custom-podspec +spec: + type: Standalone + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + podSpec: + podTemplate: + metadata: + labels: + label1: "value1" + spec: + containers: + - name: standalone-sidecar + image: busybox + command: ["sleep"] + args: [ "infinity" ] + terminationGracePeriodSeconds: 10 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + topologyKey: "mykey" + weight: 50 diff --git a/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone-downgrade.yaml b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone-downgrade.yaml new file mode 100644 index 000000000..456d02793 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone-downgrade.yaml @@ -0,0 +1,12 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone-downgrade +spec: + version: 4.4.2 + type: Standalone + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone.yaml b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone.yaml new file mode 100644 index 000000000..2bf5a77cf --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone.yaml @@ -0,0 +1,13 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone +spec: + version: 6.0.5-ent + type: Standalone + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: false + logLevel: INFO diff --git a/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone_pv_invalid.yaml b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone_pv_invalid.yaml new file mode 100644 index 000000000..be1c3f78d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/standalone_pv_invalid.yaml @@ -0,0 +1,17 @@ +# This is a standalone that references the non-existent storageClass "foo" which is created eventually +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-vol-broken +spec: + type: Standalone + version: 5.0.5-ent + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + podSpec: + persistence: + single: + storage: 2G diff --git a/docker/mongodb-enterprise-tests/tests/standalone/fixtures/test_storage_class.yaml b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/test_storage_class.yaml new file mode 100644 index 000000000..6c9596751 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/fixtures/test_storage_class.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + k8s-addon: storage-aws.addons.k8s.io + name: foo +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_config_map.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_config_map.py new file mode 100644 index 000000000..3cf9a94b3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_config_map.py @@ -0,0 +1,46 @@ +import pytest +from kubernetes import client +from kubernetes.client import V1ConfigMap +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture + + +@fixture(scope="module") +def standalone(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("standalone.yaml"), namespace=namespace) + return resource.create() + + +@fixture(scope="module") +def new_project_name(standalone: MongoDB) -> str: + yield KubernetesTester.random_om_project_name() + # Cleaning the new group in any case - the updated config map will be used to get the new name + print("\nRemoving the generated group from Ops Manager/Cloud Manager") + print(standalone.get_om_tester().api_remove_group()) + + +@pytest.mark.e2e_standalone_config_map +class TestStandaloneListensConfigMap: + group_id = "" + + def test_create_standalone(self, standalone: MongoDB): + standalone.assert_reaches_phase(Phase.Running, timeout=150) + + def test_patch_config_map(self, standalone: MongoDB, new_project_name: str): + # saving the group id for later check + TestStandaloneListensConfigMap.group_id = standalone.get_om_tester().find_group_id() + + config_map = V1ConfigMap(data={"projectName": new_project_name}) + client.CoreV1Api().patch_namespaced_config_map(standalone.config_map_name, standalone.namespace, config_map) + + print('\nPatched the ConfigMap - changed group name to "{}"'.format(new_project_name)) + + def test_standalone_handles_changes(self, standalone: MongoDB): + standalone.assert_abandons_phase(phase=Phase.Running) + standalone.assert_reaches_phase(Phase.Running, timeout=200) + + def test_new_group_was_created(self, standalone: MongoDB): + # Checking that the new group was created in OM + assert standalone.get_om_tester().find_group_id() != TestStandaloneListensConfigMap.group_id diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_custom_podspec.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_custom_podspec.py new file mode 100644 index 000000000..5252c9195 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_custom_podspec.py @@ -0,0 +1,40 @@ +from kubetester.custom_podspec import assert_stateful_set_podspec +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def standalone(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("standalone-custom-podspec.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + return resource.create() + + +@mark.e2e_standalone_custom_podspec +def test_replica_set_reaches_running_phase(standalone): + standalone.assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_standalone_custom_podspec +def test_stateful_set_spec_updated(standalone, namespace): + appsv1 = KubernetesTester.clients("appsv1") + sts = appsv1.read_namespaced_stateful_set(standalone.name, namespace) + assert_stateful_set_podspec(sts.spec.template.spec, weight=50, topology_key="mykey", grace_period_seconds=10) + + containers = sts.spec.template.spec.containers + + if is_default_architecture_static(): + assert len(containers) == 3 + assert containers[0].name == "mongodb-agent" + assert containers[1].name == "mongodb-enterprise-database" + assert containers[2].name == "standalone-sidecar" + else: + assert len(containers) == 2 + assert containers[0].name == "mongodb-enterprise-database" + assert containers[1].name == "standalone-sidecar" + + labels = sts.spec.template.metadata.labels + assert labels["label1"] == "value1" diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_groups.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_groups.py new file mode 100644 index 000000000..467e9849c --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_groups.py @@ -0,0 +1,79 @@ +import pytest +from kubetester.kubetester import ( + EXTERNALLY_MANAGED_TAG, + MAX_TAG_LEN, + KubernetesTester, + fixture, +) +from kubetester.omtester import should_include_tag, skip_if_cloud_manager + + +@pytest.mark.e2e_standalone_groups +class TestStandaloneOrganizationSpecified(KubernetesTester): + """ + name: Test for config map with specified organization id. + description: | + Tests the configuration in config map with organization specified which already exists. The group + must be created automatically. + Skipped in Cloud Manager as programmatic API keys won't allow for organizations to be created. + """ + + org_id = None + org_name = None + + @classmethod + def setup_env(cls): + # Create some organization and update the config map with its organization id + cls.org_name = KubernetesTester.random_k8s_name("standalone-group-test-") + cls.org_id = cls.create_organization(cls.org_name) + cls.patch_config_map(cls.get_namespace(), "my-project", {"orgId": cls.org_id}) + + print("Patched config map, now it references organization " + cls.org_id) + + # todo + @pytest.mark.skip( + reason="project reconciliation adds some flakiness - sometimes it gets on time to create the " + "project in OM before this method is called - should be fixed by one project" + ) + def test_standalone_created_organization_found(self): + groups_in_org = self.get_groups_in_organization_first_page(self.__class__.org_id)["totalCount"] + + # no group is created when organization is created + assert groups_in_org == 0 + + def test_standalone_cr_is_created(self): + # Create a standalone - the organization will be found and new group will be created + self.create_custom_resource_from_file(self.get_namespace(), fixture("standalone.yaml")) + + KubernetesTester.wait_until("in_running_state", 150) + + def test_standalone_organizations_are_found(self): + # Making sure no more organizations were created but the group was created inside the organization + assert len(self.find_organizations(self.__class__.org_name)) == 1 + print('Only one organization with name "{}" exists (as expected)'.format(self.__class__.org_name)) + + def test_standalone_get_groups_in_orgs(self): + page = self.get_groups_in_organization_first_page(self.__class__.org_id) + assert page["totalCount"] == 1 + group = page["results"][0] + assert group is not None + assert group["orgId"] == self.__class__.org_id + + print( + 'The group "{}" has been created by the Operator in organization "{}"'.format( + self.get_om_group_name(), self.__class__.org_name + ), + ) + + @skip_if_cloud_manager() + def test_group_tag_was_set(self): + page = self.get_groups_in_organization_first_page(self.__class__.org_id) + group = page["results"][0] + + version = KubernetesTester.om_version() + expected_tags = [self.namespace[:MAX_TAG_LEN].upper()] + + if should_include_tag(version): + expected_tags.append(EXTERNALLY_MANAGED_TAG) + + assert sorted(group["tags"]) == sorted(expected_tags) diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_recovery.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_recovery.py new file mode 100644 index 000000000..c2b5610d4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_recovery.py @@ -0,0 +1,37 @@ +import pytest +from kubernetes.client import V1ConfigMap +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase + + +@pytest.fixture(scope="module") +def mdb(namespace: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("standalone.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + return resource.update() + + +@pytest.mark.e2e_standalone_recovery +class TestStandaloneRecoversBadOmConfiguration(KubernetesTester): + """ + name: Standalone broken OM connection + description: | + Creates a standalone with a bad OM connection (ConfigMap is broken) and ensures it enters a failed state | + Then the config map is fixed and the standalone is expected to reach good state eventually + """ + + def test_standalone_reaches_failed_state(self, mdb: MongoDB): + config_map = V1ConfigMap(data={"baseUrl": "http://foo.bar"}) + self.clients("corev1").patch_namespaced_config_map("my-project", self.get_namespace(), config_map) + + mdb.assert_reaches_phase(Phase.Failed, timeout=20) + + mdb.load() + assert "Failed to prepare Ops Manager connection" in mdb["status"]["message"] + + def test_recovery(self, mdb: MongoDB): + config_map = V1ConfigMap(data={"baseUrl": KubernetesTester.get_om_base_url()}) + self.clients("corev1").patch_namespaced_config_map("my-project", KubernetesTester.get_namespace(), config_map) + # We need to ignore errors here because the CM change can be faster than the check + mdb.assert_reaches_phase(Phase.Running, ignore_errors=True) diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_schema_validation.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_schema_validation.py new file mode 100644 index 000000000..2709b2ecd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_schema_validation.py @@ -0,0 +1,117 @@ +import pytest +from kubetester.kubetester import KubernetesTester + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneSchemaCredentialsMissing(KubernetesTester): + """ + name: Validation for standalone (credentials missing) + create: + file: standalone.yaml + patch: '[{"op":"remove","path":"/spec/credentials"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneSchemaVersionMissing(KubernetesTester): + """ + name: Validation for standalone (version missing) + create: + file: standalone.yaml + patch: '[{"op":"remove","path":"/spec/version"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneSchemaInvalidType(KubernetesTester): + """ + name: Validation for standalone (invalid type) + create: + file: standalone.yaml + patch: '[{"op":"replace","path":"/spec/type","value":"InvalidStandaloneType"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneSchemaInvalidLogLevel(KubernetesTester): + """ + name: Validation for standalone (invalid logLevel) + create: + file: standalone.yaml + patch: '[{"op":"replace","path":"/spec/logLevel","value":"NotINFO"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneSchemaShardedClusterMongodConfig(KubernetesTester): + """ + name: Validation for standalone (sharded cluster additional mongod config) + create: + file: standalone.yaml + patch: '[{"op":"add","path":"/spec/shard","value":{"operationProfiling":{"mode":true}}}]' + exception: 'cannot be specified if type of MongoDB is Standalone' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneInvalidWithProjectAndCloudManager(KubernetesTester): + init = { + "create": { + "file": "standalone.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/cloudManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_standalone_schema_validation +class TestStandaloneInvalidWithCloudAndOpsManagerAndProject(KubernetesTester): + init = { + "create": { + "file": "standalone.yaml", + "patch": [ + { + "op": "add", + "path": "/spec/cloudManager", + "value": {"configMapRef": {"name": "something"}}, + }, + { + "op": "add", + "path": "/spec/opsManager", + "value": {"configMapRef": {"name": "something"}}, + }, + ], + "exception": "must validate one and only one schema", + }, + } + + def test_validation_ok(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_set_agent_flags.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_set_agent_flags.py new file mode 100644 index 000000000..4c085914f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_set_agent_flags.py @@ -0,0 +1,33 @@ +from kubetester import find_fixture +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import MongoDB, Phase +from pytest import fixture, mark + + +@fixture(scope="module") +def standalone(namespace: str) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("standalone.yaml"), namespace=namespace) + + resource["spec"]["agent"] = {"startupOptions": {"logFile": "/var/log/mongodb-mms-automation/customLogFile"}} + + return resource.create() + + +@mark.e2e_standalone_agent_flags +def test_standalone(standalone: MongoDB): + standalone.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_standalone_agent_flags +def test_standalone_has_agent_flags(standalone: MongoDB, namespace: str): + cmd = [ + "/bin/sh", + "-c", + "ls /var/log/mongodb-mms-automation/customLogFile* | wc -l", + ] + result = KubernetesTester.run_command_in_pod_container( + "my-standalone-0", + namespace, + cmd, + ) + assert result != "0" diff --git a/docker/mongodb-enterprise-tests/tests/standalone/standalone_upgrade_downgrade.py b/docker/mongodb-enterprise-tests/tests/standalone/standalone_upgrade_downgrade.py new file mode 100644 index 000000000..0478a3ad0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/standalone/standalone_upgrade_downgrade.py @@ -0,0 +1,81 @@ +import pytest +from kubetester.kubetester import KubernetesTester, fcv_from_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import StandaloneTester + + +@pytest.fixture(scope="module") +def standalone(namespace: str, custom_mdb_prev_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("standalone-downgrade.yaml"), namespace=namespace) + resource.set_version(custom_mdb_prev_version) + return resource.update() + + +@pytest.mark.e2e_standalone_upgrade_downgrade +class TestStandaloneUpgradeDowngradeCreate(KubernetesTester): + """ + name: Standalone upgrade downgrade (create) + description: | + Creates a standalone, then upgrades it with compatibility version set and then downgrades back + """ + + def test_create_standalone(self, standalone: MongoDB): + standalone.assert_reaches_phase(Phase.Running) + + @skip_if_local + def test_db_connectable(self, custom_mdb_prev_version: str): + mongod_tester = StandaloneTester("my-standalone-downgrade") + mongod_tester.assert_version(custom_mdb_prev_version) + + def test_noop(self): + assert True + + +@pytest.mark.e2e_standalone_upgrade_downgrade +class TestStandaloneUpgradeDowngradeUpdate(KubernetesTester): + """ + name: Standalone upgrade downgrade (update) + description: | + Updates a Standalone to bigger version, leaving feature compatibility version as it was + """ + + def test_upgrade_standalone(self, standalone: MongoDB, custom_mdb_prev_version: str, custom_mdb_version: str): + fcv = fcv_from_version(custom_mdb_prev_version) + + standalone.load() + standalone.set_version(custom_mdb_version) + standalone["spec"]["featureCompatibilityVersion"] = fcv + standalone.update() + + standalone.assert_reaches_phase(Phase.Running) + + @skip_if_local + def test_db_connectable(self, custom_mdb_version): + mongod_tester = StandaloneTester("my-standalone-downgrade") + mongod_tester.assert_version(custom_mdb_version) + + +@pytest.mark.e2e_standalone_upgrade_downgrade +class TestStandaloneUpgradeDowngradeRevert(KubernetesTester): + """ + name: Standalone upgrade downgrade (downgrade) + description: | + Updates a Standalone to the same version it was created initially + """ + + def test_downgrade_standalone(self, standalone: MongoDB, custom_mdb_prev_version: str): + standalone.load() + standalone.set_version(custom_mdb_prev_version) + standalone.update() + + standalone.assert_reaches_phase(Phase.Running) + + @skip_if_local + def test_db_connectable(self, custom_mdb_prev_version): + mongod_tester = StandaloneTester("my-standalone-downgrade") + mongod_tester.assert_version(custom_mdb_prev_version) + + def test_noop(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/test_logger.py b/docker/mongodb-enterprise-tests/tests/test_logger.py new file mode 100644 index 000000000..40b181c4b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/test_logger.py @@ -0,0 +1,27 @@ +import logging +import os +import sys + +LOGLEVEL = os.environ.get("LOGLEVEL", "DEBUG").upper() + +# Create handlers to output Debug and Info logs to stdout, and above to stderr +# They are attached to each logger below +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setLevel(logging.DEBUG) +stdout_handler.addFilter(lambda record: record.levelno <= logging.INFO) +stderr_handler = logging.StreamHandler(sys.stderr) +stderr_handler.setLevel(logging.WARNING) + +# Format the logs +formatter = logging.Formatter("%(levelname)-8s %(asctime)s [%(module)s] %(message)s") +stdout_handler.setFormatter(formatter) +stderr_handler.setFormatter(formatter) + + +def get_test_logger(name: str) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(LOGLEVEL) + logger.propagate = False + logger.addHandler(stdout_handler) + logger.addHandler(stderr_handler) + return logger diff --git a/docker/mongodb-enterprise-tests/tests/tls/__init__.py b/docker/mongodb-enterprise-tests/tests/tls/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/tls/conftest.py b/docker/mongodb-enterprise-tests/tests/tls/conftest.py new file mode 100644 index 000000000..42b099ae1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/conftest.py @@ -0,0 +1,28 @@ +from _pytest.fixtures import fixture +from kubetester.operator import Operator +from tests.conftest import ( + get_central_cluster_client, + get_central_cluster_name, + get_default_operator, + get_member_cluster_clients, + get_member_cluster_names, + get_multi_cluster_operator, + get_multi_cluster_operator_installation_config, + get_operator_installation_config, + is_multi_cluster, +) + + +@fixture(scope="module") +def operator(namespace: str) -> Operator: + if is_multi_cluster(): + return get_multi_cluster_operator( + namespace, + get_central_cluster_name(), + get_multi_cluster_operator_installation_config(namespace), + get_central_cluster_client(), + get_member_cluster_clients(), + get_member_cluster_names(), + ) + else: + return get_default_operator(namespace, get_operator_installation_config(namespace)) diff --git a/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_rs.py b/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_rs.py new file mode 100644 index 000000000..af184f34d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_rs.py @@ -0,0 +1,56 @@ +import pytest +from kubetester.certs import ( + ISSUER_CA_NAME, + create_agent_tls_certs, + create_mongodb_tls_certs, +) +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester +from kubetester.operator import Operator + +MDB_RESOURCE = "my-replica-set" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("replica-set.yaml"), namespace=namespace) + res["spec"]["security"] = {"tls": {"ca": issuer_ca_configmap}} + return res.create() + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_rs +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_rs +def test_mdb_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_rs +def test_connectivity(): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_connectivity() + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_rs +def test_enable_x509(mdb: MongoDB, agent_certs: str): + mdb.load() + mdb["spec"]["security"] = { + "authentication": {"enabled": True}, + "modes": ["X509"], + } + + mdb.assert_reaches_phase(Phase.Running, timeout=400) diff --git a/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_sc.py b/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_sc.py new file mode 100644 index 000000000..7287b22ee --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_sc.py @@ -0,0 +1,90 @@ +import pytest +from kubetester import try_load +from kubetester.certs import create_agent_tls_certs, create_sharded_cluster_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import is_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ShardedClusterTester +from kubetester.operator import Operator +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_mongos_service_names, +) + +MDB_RESOURCE = "test-ssl-with-x509-sc" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + shard_distribution = None + mongos_distribution = None + config_srv_distribution = None + if is_multi_cluster(): + shard_distribution = [1, 1, 1] + mongos_distribution = [1, 1, None] + config_srv_distribution = [1, 1, 1] + + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + shard_distribution=shard_distribution, + mongos_distribution=mongos_distribution, + config_srv_distribution=config_srv_distribution, + ) + + +@pytest.fixture(scope="module") +def sc(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("sharded-cluster.yaml"), namespace=namespace, name=MDB_RESOURCE) + + if try_load(resource): + return resource + + resource.set_architecture_annotation() + resource["spec"]["security"] = {"tls": {"ca": issuer_ca_configmap}} + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_sc +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_sc +def test_standalone_running(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_sc +def test_connectivity_without_ssl(sc: MongoDB): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=False, service_names=service_names) + tester.assert_connectivity() + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_sc +def test_enable_x509(sc: MongoDB, agent_certs: str): + sc["spec"]["security"] = { + "authentication": {"enabled": True}, + "modes": ["X509"], + } + + sc.assert_reaches_phase(Phase.Running, timeout=1200) diff --git a/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_standalone.py b/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_standalone.py new file mode 100644 index 000000000..5b6c4f824 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/e2e_configure_tls_and_x509_simultaneously_standalone.py @@ -0,0 +1,56 @@ +import pytest +from kubetester.certs import ( + ISSUER_CA_NAME, + create_agent_tls_certs, + create_mongodb_tls_certs, +) +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import StandaloneTester +from kubetester.operator import Operator + +MDB_RESOURCE = "my-standalone" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert", replicas=1) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("standalone.yaml"), namespace=namespace) + res["spec"]["security"] = {"tls": {"ca": issuer_ca_configmap}} + return res.create() + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_st +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_st +def test_mdb_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_st +def test_connectivity(): + tester = StandaloneTester(MDB_RESOURCE) + tester.assert_connectivity() + + +@pytest.mark.e2e_configure_tls_and_x509_simultaneously_st +def test_enable_x509(mdb: MongoDB, agent_certs: str): + mdb.load() + mdb["spec"]["security"] = { + "authentication": {"enabled": True}, + "modes": ["X509"], + } + + mdb.assert_reaches_phase(Phase.Running, timeout=400) diff --git a/docker/mongodb-enterprise-tests/tests/tls/e2e_tls_disable_and_scale_up.py b/docker/mongodb-enterprise-tests/tests/tls/e2e_tls_disable_and_scale_up.py new file mode 100644 index 000000000..dc544f154 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/e2e_tls_disable_and_scale_up.py @@ -0,0 +1,54 @@ +import pytest +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + resource_name = "test-tls-base-rs" + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, resource_name, "test-tls-base-rs-cert") + + +@pytest.fixture(scope="module") +def replica_set(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + + # Set this ReplicaSet to allowSSL mode + # this is the only mode that can go to "disabled" state. + res["spec"]["additionalMongodConfig"] = {"net": {"ssl": {"mode": "allowSSL"}}} + + return res.create() + + +@pytest.mark.e2e_disable_tls_scale_up +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_disable_tls_scale_up +def test_rs_is_running(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_disable_tls_scale_up +def test_tls_is_disabled_and_scaled_up(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["members"] = 5 + + replica_set.update() + + +@pytest.mark.e2e_disable_tls_scale_up +def test_tls_is_disabled_and_scaled_up(replica_set: MongoDB): + replica_set.load() + replica_set["spec"]["security"]["tls"]["enabled"] = False + del replica_set["spec"]["additionalMongodConfig"] + + replica_set.update() + + # timeout is longer because the operator first needs to + # disable TLS and then, scale down one by one. + replica_set.assert_reaches_phase(Phase.Running, timeout=800) diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/node-port-service.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/node-port-service.yaml new file mode 100644 index 000000000..11e1db724 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/node-port-service.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + controller: mongodb-enterprise-operator + name: test-tls-base-rs-external-access-svc-external +spec: + ports: + - nodePort: + port: 27017 + protocol: TCP + targetPort: 27017 + type: NodePort + selector: + controller: mongodb-enterprise-operator + statefulset.kubernetes.io/pod-name: -0 diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/server-key.pem b/docker/mongodb-enterprise-tests/tests/tls/fixtures/server-key.pem new file mode 100644 index 000000000..ba2717f65 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/server-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIIpjJQ4k1yWb3qTxO2BUrw8uSGVdyWyaY+PYIc8Ui2mboAoGCCqGSM49 +AwEHoUQDQgAEKKjALmtc9/ZYqTn7ADAqpdiEDWmwTfSPAvQSHWHTRRlefeStVLl2 +XOVT9iy1jhgdI43uNqc2AoQwzmW0Gz+U+g== +-----END EC PRIVATE KEY----- diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-no-tls-no-status.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-no-tls-no-status.yaml new file mode 100644 index 000000000..62b28157b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-no-tls-no-status.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-no-tls-no-status +spec: + members: 3 + version: 4.4.2 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-additional-domains.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-additional-domains.yaml new file mode 100644 index 000000000..7a27a3180 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-additional-domains.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-additional-domains +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + security: + tls: + enabled: true + additionalCertificateDomains: + - "additional-cert-test.com" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-allow-ssl.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-allow-ssl.yaml new file mode 100644 index 000000000..775290297 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-allow-ssl.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs-allow-ssl +spec: + members: 3 + version: 4.4.2 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + security: + tls: + enabled: true + + additionalMongodConfig: + net: + ssl: + mode: "allowSSL" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-external-access.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-external-access.yaml new file mode 100644 index 000000000..67c6419c0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-external-access.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs-external-access +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: true + security: + tls: + enabled: true + + connectivity: + replicaSetHorizons: + - "test-horizon": "mdb0-test.website.com:1337" + - "test-horizon": "mdb1-test.website.com:1337" + - "test-horizon": "mdb2-test.website.com:1337" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-prefer-ssl.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-prefer-ssl.yaml new file mode 100644 index 000000000..75da1b6d4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-prefer-ssl.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs-prefer-ssl +spec: + members: 3 + version: 4.4.2 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: true + security: + tls: + enabled: true + + additionalMongodConfig: + net: + ssl: + mode: "preferSSL" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl-custom-ca.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl-custom-ca.yaml new file mode 100644 index 000000000..cda418439 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl-custom-ca.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs-require-ssl +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + security: + tls: + enabled: true + ca: customer-ca diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl-upgrade.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl-upgrade.yaml new file mode 100644 index 000000000..8bd76266f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl-upgrade.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-upgrade +spec: + members: 3 + version: 4.4.2 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: true + + + additionalMongodConfig: + net: + ssl: + mode: "allowSSL" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl.yaml new file mode 100644 index 000000000..0ddce091f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-require-ssl.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs-require-ssl +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: true + security: + tls: + enabled: true diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-x509.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-x509.yaml new file mode 100644 index 000000000..f0230b69f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs-x509.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs-x509 +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + security: + tls: + enabled: true + authentication: + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs.yaml new file mode 100644 index 000000000..829111566 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-rs.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-rs +spec: + members: 3 + version: 5.0.15 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + security: + tls: + enabled: true diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-sc-require-ssl-custom-ca.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-sc-require-ssl-custom-ca.yaml new file mode 100644 index 000000000..388b59f69 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-sc-require-ssl-custom-ca.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-sc-require-ssl +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + security: + tls: + enabled: true + ca: customer-ca diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-sc-require-ssl.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-sc-require-ssl.yaml new file mode 100644 index 000000000..8b3397f73 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-base-sc-require-ssl.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-base-sc-require-ssl +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: true + security: + tls: + enabled: true diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-rs-external-access-multiple-horizons.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-rs-external-access-multiple-horizons.yaml new file mode 100644 index 000000000..69e418f9d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-rs-external-access-multiple-horizons.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-rs-external-access-multiple-horizons +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + security: + tls: + enabled: true + + connectivity: + replicaSetHorizons: + - "test-horizon-1": "mdb0-test-1.website.com:1337" + "test-horizon-2": "mdb0-test-2.website.com:2337" + - "test-horizon-1": "mdb1-test-1.website.com:1338" + "test-horizon-2": "mdb1-test-2.website.com:2338" + - "test-horizon-1": "mdb2-test-1.website.com:1339" + "test-horizon-2": "mdb2-test-2.website.com:2339" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-sc-additional-domains.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-sc-additional-domains.yaml new file mode 100644 index 000000000..eff1367e0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-tls-sc-additional-domains.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-sc-additional-domains +spec: + shardCount: 1 + mongodsPerShardCount: 1 + mongosCount: 2 + configServerCount: 1 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + + security: + tls: + enabled: true + additionalCertificateDomains: + - "additional-cert-test.com" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-all-options-rs.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-all-options-rs.yaml new file mode 100644 index 000000000..0e9fa41f1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-all-options-rs.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-x509-all-options-rs +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + persistent: false + security: + tls: + enabled: true + authentication: + internalCluster: X509 + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-all-options-sc.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-all-options-sc.yaml new file mode 100644 index 000000000..1789e830a --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-all-options-sc.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-x509-all-options-sc +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 3 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + security: + tls: + enabled: true + authentication: + internalCluster: X509 + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-rs.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-rs.yaml new file mode 100644 index 000000000..9dfa126bf --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-rs.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-x509-rs +spec: + members: 3 + version: 4.4.0 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + security: + tls: + enabled: true + authentication: + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-sc-custom-ca.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-sc-custom-ca.yaml new file mode 100644 index 000000000..2bbc0fae4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-sc-custom-ca.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-x509-sc-custom-ca +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + security: + clusterAuthenticationMode: x509 + tls: + enabled: true + authentication: + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-sc.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-sc.yaml new file mode 100644 index 000000000..c45147c37 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-sc.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-x509-sc +spec: + shardCount: 1 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0 + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + security: + tls: + enabled: true + authentication: + enabled: true + modes: ["X509"] diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-user.yaml b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-user.yaml new file mode 100644 index 000000000..1883ada9f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/test-x509-user.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: test-x509-user +spec: + username: 'CN=x509-testing-user' + db: '$external' + mongodbResourceRef: + name: 'test-tls-base-rs-x509' + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "userAdminAnyDatabase" + - db: "admin" + name: "readWrite" diff --git a/docker/mongodb-enterprise-tests/tests/tls/fixtures/x509-testing-user.csr b/docker/mongodb-enterprise-tests/tests/tls/fixtures/x509-testing-user.csr new file mode 100644 index 000000000..cde042567 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/fixtures/x509-testing-user.csr @@ -0,0 +1,7 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHWMH4CAQAwHDEaMBgGA1UEAxMReDUwOS10ZXN0aW5nLXVzZXIwWTATBgcqhkjO +PQIBBggqhkjOPQMBBwNCAAQoqMAua1z39lipOfsAMCql2IQNabBN9I8C9BIdYdNF +GV595K1UuXZc5VP2LLWOGB0jje42pzYChDDOZbQbP5T6oAAwCgYIKoZIzj0EAwID +SAAwRQIhAPWTGJDQhZu2RgciBYdwP49xpqtQSdWv89Lrqjf2GIElAiBU63o478xt +MmPnkC2ALbKeVY8xXbIIr1pgXikYFQsg0g== +-----END CERTIFICATE REQUEST----- diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_allowssl.py b/docker/mongodb-enterprise-tests/tests/tls/tls_allowssl.py new file mode 100644 index 000000000..f26d27856 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_allowssl.py @@ -0,0 +1,42 @@ +import pytest +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE = "test-tls-base-rs-allow-ssl" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-allow-ssl.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_replica_set_tls_allow +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_replica_set_tls_allow +def test_replica_set_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_replica_set_tls_allow +@skip_if_local() +def test_mdb_is_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_allow +@skip_if_local() +def test_mdb_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_multiple_different_ssl_configs.py b/docker/mongodb-enterprise-tests/tests/tls/tls_multiple_different_ssl_configs.py new file mode 100644 index 000000000..a63460e49 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_multiple_different_ssl_configs.py @@ -0,0 +1,101 @@ +import pytest +from kubetester.kubetester import KubernetesTester, skip_if_local +from kubetester.mongotester import ReplicaSetTester +from kubetester.operator import Operator + +mdb_resources = { + "ssl_enabled": "test-tls-base-rs-require-ssl", + "ssl_disabled": "test-no-tls-no-status", +} + + +def cert_names(namespace, members=3): + return ["{}-{}.{}".format(mdb_resources["ssl_enabled"], i, namespace) for i in range(members)] + + +@pytest.mark.e2e_tls_multiple_different_ssl_configs +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_multiple_different_ssl_configs +class TestMultipleCreation(KubernetesTester): + """ + name: 2 Replica Sets on same project with different SSL configurations + description: | + 2 MDB/ReplicaSet object will be created, one of them will be have TLS enabled (required) + and the other one will not. Both of them should work, and listen on respective SSL/non SSL endpoints. + create: + - file: test-tls-base-rs-require-ssl.yaml + wait_for_message: Not all certificates have been approved by Kubernetes CA + timeout: 120 + + - file: test-no-tls-no-status.yaml + wait_until: in_running_state + timeout: 120 + """ + + def test_mdb_tls_resource_status_is_correct(self): + mdb = self.customv1.get_namespaced_custom_object( + "mongodb.com", "v1", self.namespace, "mongodb", mdb_resources["ssl_enabled"] + ) + assert mdb["status"]["message"] == "Not all certificates have been approved by Kubernetes CA" + + mdb = self.customv1.get_namespaced_custom_object( + "mongodb.com", + "v1", + self.namespace, + "mongodb", + mdb_resources["ssl_disabled"], + ) + assert mdb["status"]["phase"] == "Running" + + +@pytest.mark.e2e_tls_multiple_different_ssl_configs +class TestMultipleApproval(KubernetesTester): + """ + name: Approval of certificates + description: | + Approves the certificates in Kubernetes, the MongoDB resource should move to Successful state. + """ + + def setup_method(self): + super().setup_method() + [self.approve_certificate(cert) for cert in cert_names(self.namespace)] + + def test_noop(self): + assert True + + +@pytest.mark.e2e_tls_multiple_different_ssl_configs +class TestMultipleRunning0(KubernetesTester): + """ + name: Both MDB objects should be in running state. + wait: + - resource: test-tls-base-rs-require-ssl + until: in_running_state + timeout: 200 + - resource: test-no-tls-no-status + until: in_running_state + timeout: 60 + """ + + @skip_if_local() + def test_mdb_ssl_enabled_is_not_reachable_with_no_ssl(self): + mongo_tester = ReplicaSetTester(mdb_resources["ssl_enabled"], 3) + mongo_tester.assert_no_connection() + + @skip_if_local() + def test_mdb_ssl_enabled_is_reachable_with_ssl(self): + mongo_tester = ReplicaSetTester(mdb_resources["ssl_enabled"], 3, ssl=True) + mongo_tester.assert_connectivity() + + @skip_if_local() + def test_mdb_ssl_disabled_is_reachable_with_no_ssl(self): + mongo_tester = ReplicaSetTester(mdb_resources["ssl_disabled"], 3) + mongo_tester.assert_connectivity() + + @skip_if_local() + def test_mdb_ssl_disabled_is_not_reachable_with_ssl(self): + mongo_tester = ReplicaSetTester(mdb_resources["ssl_disabled"], 3, ssl=True) + mongo_tester.assert_no_connection() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_no_status.py b/docker/mongodb-enterprise-tests/tests/tls/tls_no_status.py new file mode 100644 index 000000000..9337b767f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_no_status.py @@ -0,0 +1,45 @@ +import pytest +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE = "test-no-tls-no-status" + + +@pytest.mark.e2e_standalone_no_tls_no_status_is_set +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_standalone_no_tls_no_status_is_set +class TestStandaloneWithNoTLS(KubernetesTester): + """ + name: Standalone with no TLS should not have empty "additionalMongodConfig" attribute set. + """ + + def test_create_standalone(self, custom_mdb_version: str): + resource = MongoDB.from_yaml(load_fixture("test-no-tls-no-status.yaml"), namespace=self.namespace) + resource.set_version(custom_mdb_version) + resource.update() + resource.assert_reaches_phase(Phase.Running) + + def test_mdb_resource_status_is_correct(self): + mdb = self.customv1.get_namespaced_custom_object("mongodb.com", "v1", self.namespace, "mongodb", MDB_RESOURCE) + + assert mdb["status"]["phase"] == "Running" + assert "additionalMongodConfig" not in mdb["spec"] + assert "security" not in mdb + + +@pytest.mark.e2e_standalone_no_tls_no_status_is_set +class TestStandaloneWithNoTLSDeletion(KubernetesTester): + """ + name: Standalone with no TLS Status should be removed + delete: + file: test-no-tls-no-status.yaml + wait_until: mongo_resource_deleted + """ + + def test_deletion(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_permissions_default.py b/docker/mongodb-enterprise-tests/tests/tls/tls_permissions_default.py new file mode 100644 index 000000000..654a813db --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_permissions_default.py @@ -0,0 +1,60 @@ +from kubetester import find_fixture, try_load +from kubetester.certs import create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark + + +@fixture(scope="module") +def certs_secret_prefix(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, "test-tls-base-rs", "certs-test-tls-base-rs-cert") + return "certs" + + +@fixture(scope="module") +def replica_set( + issuer_ca_configmap: str, + namespace: str, + certs_secret_prefix, + custom_mdb_version: str, +) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("test-tls-base-rs.yaml"), namespace=namespace) + resource.configure_custom_tls(issuer_ca_configmap, certs_secret_prefix) + resource.set_version(custom_mdb_version) + + try_load(resource) + return resource + + +@mark.e2e_replica_set_tls_default +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_replica_set_tls_default +def test_replica_set(replica_set: MongoDB): + + replica_set.update() + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_tls_default +def test_file_has_correct_permissions(namespace: str): + # We test that the permissions are as expected by executing the stat + # command on all the pem files in the secrets/certs directory + cmd = [ + "/bin/sh", + "-c", + 'stat -c "%a" /mongodb-automation/tls/..data/*', + ] + for i in range(3): + result = KubernetesTester.run_command_in_pod_container( + f"test-tls-base-rs-{i}", + namespace, + cmd, + ).splitlines() + for res in result: + assert ( + res == "640" + ) # stat has no option for decimal values, so we check for 640, which is the octal representation for 416 diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_permissions_override.py b/docker/mongodb-enterprise-tests/tests/tls/tls_permissions_override.py new file mode 100644 index 000000000..56bb1a5da --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_permissions_override.py @@ -0,0 +1,69 @@ +from kubetester import find_fixture +from kubetester.certs import create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.mongodb import MongoDB, Phase +from kubetester.omtester import get_rs_cert_names +from kubetester.operator import Operator +from pytest import fixture, mark + + +@fixture(scope="module") +def certs_secret_prefix(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, "test-tls-base-rs", "certs-test-tls-base-rs-cert") + return "certs" + + +@fixture(scope="module") +def replica_set(issuer_ca_configmap: str, namespace: str, certs_secret_prefix) -> MongoDB: + resource = MongoDB.from_yaml(find_fixture("test-tls-base-rs.yaml"), namespace=namespace) + + resource["spec"]["podSpec"] = { + "podTemplate": { + "spec": { + "volumes": [ + { + "name": "secret-certs", + "secret": { + "defaultMode": 420, # This is the decimal value corresponding to 0644 permissions, different from the default 0640 (416) + }, + } + ] + } + } + } + resource.configure_custom_tls(issuer_ca_configmap, certs_secret_prefix) + return resource.create() + + +@mark.e2e_replica_set_tls_override +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_replica_set_tls_override +def test_replica_set(replica_set: MongoDB, namespace: str): + + certs = get_rs_cert_names(replica_set["metadata"]["name"], namespace, with_agent_certs=False) + + replica_set.assert_reaches_phase(Phase.Running, timeout=400) + + +@mark.e2e_replica_set_tls_override +def test_file_has_correct_permissions(replica_set: MongoDB, namespace: str): + # We test that the permissions are as expected by executing the stat + # command on all the pem files in the secrets/certs directory + cmd = [ + "/bin/sh", + "-c", + 'stat -c "%a" /mongodb-automation/tls/..data/*', + ] + for i in range(3): + result = KubernetesTester.run_command_in_pod_container( + f"test-tls-base-rs-{i}", + namespace, + cmd, + ).splitlines() + for res in result: + assert ( + res == "644" + ) # stat has no option for decimal values, so we check for 644, which is the octal representation for 420 diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_preferssl.py b/docker/mongodb-enterprise-tests/tests/tls/tls_preferssl.py new file mode 100644 index 000000000..ca5bde00d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_preferssl.py @@ -0,0 +1,43 @@ +import pytest +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE = "test-tls-base-rs-prefer-ssl" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-prefer-ssl.yaml"), namespace=namespace) + + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_replica_set_tls_prefer +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_replica_set_tls_prefer +def test_replica_set_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_replica_set_tls_prefer +@skip_if_local() +def test_mdb_is_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_prefer +@skip_if_local() +def test_mdb_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_replica_set_process_hostnames.py b/docker/mongodb-enterprise-tests/tests/tls/tls_replica_set_process_hostnames.py new file mode 100644 index 000000000..06e0232b5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_replica_set_process_hostnames.py @@ -0,0 +1,137 @@ +# This test currently relies on MetalLB IP assignment that is configured for kind in scripts/dev/recreate_kind_cluster.sh +# Each service of type LoadBalancer will get IP starting from 172.18.255.200 +# scripts/dev/coredns_single_cluster.yaml configures that my-replica-set-0.mongodb.interconnected starts at 172.18.255.200 + +# Other e2e tests checking externalDomain (consider updating all of them when changing test logic here): +# tls/tls_replica_set_process_hostnames.py +# replicaset/replica_set_process_hostnames.py +# om_ops_manager_backup_tls_custom_ca.py + + +from kubetester import try_load +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import assert_statefulset_architecture +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import get_default_architecture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import ( + default_external_domain, + external_domain_fqdns, + update_coredns_hosts, +) + + +@fixture(scope="module") +def replica_set_name() -> str: + return "my-replica-set" + + +@fixture(scope="module") +def replica_set_members() -> int: + return 3 + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str, replica_set_members: int, replica_set_name: str): + """ + Issues certificate containing only custom_service_fqdns in SANs + """ + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + replica_set_name, + f"{replica_set_name}-cert", + process_hostnames=external_domain_fqdns(replica_set_name, replica_set_members), + ) + + +@fixture(scope="function") +def replica_set( + namespace: str, + replica_set_name: str, + replica_set_members: int, + custom_mdb_version: str, + server_certs: str, + issuer_ca_configmap: str, +) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("test-tls-base-rs.yaml"), replica_set_name, namespace) + try_load(resource) + + resource["spec"]["members"] = replica_set_members + resource["spec"]["externalAccess"] = {} + resource["spec"]["externalAccess"]["externalDomain"] = default_external_domain() + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + resource.set_version(custom_mdb_version) + resource.set_architecture_annotation() + + return resource + + +@mark.e2e_replica_set_tls_process_hostnames +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_replica_set_tls_process_hostnames +def test_update_coredns(): + hosts = [ + ("172.18.255.200", "my-replica-set-0.mongodb.interconnected"), + ("172.18.255.201", "my-replica-set-1.mongodb.interconnected"), + ("172.18.255.202", "my-replica-set-2.mongodb.interconnected"), + ("172.18.255.203", "my-replica-set-3.mongodb.interconnected"), + ] + + update_coredns_hosts(hosts) + + +@mark.e2e_replica_set_tls_process_hostnames +def test_create_replica_set(replica_set: MongoDB): + replica_set.update() + + +@mark.e2e_replica_set_tls_process_hostnames +def test_replica_set_in_running_state(replica_set: MongoDB): + replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_replica_set_tls_process_hostnames +def test_automation_config_contains_external_domains_in_hostnames(replica_set: MongoDB): + processes = replica_set.get_automation_config_tester().get_replica_set_processes(replica_set.name) + hostnames = [process["hostname"] for process in processes] + assert hostnames == external_domain_fqdns(replica_set.name, replica_set.get_members()) + + +@mark.e2e_replica_set_tls_process_hostnames +def test_connectivity(replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(ca_path=ca_path) + tester.assert_connectivity() + + +@mark.e2e_replica_set_tls_process_hostnames +def test_migrate_architecture(replica_set: MongoDB): + """ + If the E2E is running with default architecture as non-static, + then the test will migrate to static and vice versa. + """ + original_default_architecture = get_default_architecture() + target_architecture = "non-static" if original_default_architecture == "static" else "static" + + replica_set.trigger_architecture_migration() + + replica_set.load() + assert replica_set["metadata"]["annotations"]["mongodb.com/v1.architecture"] == target_architecture + + replica_set.assert_abandons_phase(Phase.Running, timeout=1000) + replica_set.assert_reaches_phase(Phase.Running, timeout=1000) + + # Read StatefulSet after successful reconciliation + sts = replica_set.read_statefulset() + assert_statefulset_architecture(sts, target_architecture) + + +@mark.e2e_replica_set_tls_process_hostnames +def test_db_connectable_after_architecture_change(replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(ca_path=ca_path) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_replicaset_certsSecretPrefix.py b/docker/mongodb-enterprise-tests/tests/tls/tls_replicaset_certsSecretPrefix.py new file mode 100644 index 000000000..ad0597be3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_replicaset_certsSecretPrefix.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import pytest +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE = "test-tls-base-rs-require-ssl" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + MDB_RESOURCE, + f"prefix-{MDB_RESOURCE}-cert", + replicas=3, + ) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-require-ssl.yaml"), namespace=namespace) + + res["spec"]["security"]["tls"] = {"ca": issuer_ca_configmap} + # Setting security.certsSecretPrefix implicitly enables TLS + res["spec"]["security"]["certsSecretPrefix"] = "prefix" + return res.create() + + +@pytest.mark.e2e_replica_set_tls_certs_secret_prefix +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_replica_set_tls_certs_secret_prefix +def test_replica_set_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_replica_set_tls_certs_secret_prefix +@skip_if_local() +def test_mdb_is_not_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_certs_secret_prefix +@skip_if_local() +def test_mdb_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl.py b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl.py new file mode 100644 index 000000000..636e6a0e2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl.py @@ -0,0 +1,112 @@ +import pytest +from kubetester.certs import ISSUER_CA_NAME, Certificate, create_mongodb_tls_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE = "test-tls-base-rs-require-ssl" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + MDB_RESOURCE, + f"{MDB_RESOURCE}-cert", + replicas=5, + ) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-require-ssl.yaml"), namespace=namespace) + + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_replica_set_tls_require +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_replica_set_tls_require +def test_replica_set_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_is_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require +def test_scale_up_replica_set(mdb: MongoDB): + mdb.load() + mdb["spec"]["members"] = 5 + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_scaled_up_is_not_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_scaled_up_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require +def test_scale_down_replica_set(mdb: MongoDB): + mdb.load() + mdb["spec"]["members"] = 3 + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=1000) + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_scaled_down_is_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_scaled_down_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require +def test_change_certificate_and_wait_for_running(mdb: MongoDB, namespace: str): + """ + Perform certificate rotation by updating dns in certs + """ + cert = Certificate(name=f"{MDB_RESOURCE}-cert", namespace=namespace).load() + cert["spec"]["dnsNames"].append("foo") + cert.update() + mdb.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_renewed_is_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require +@skip_if_local() +def test_mdb_renewed_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_and_disable.py b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_and_disable.py new file mode 100644 index 000000000..0d9c7070b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_and_disable.py @@ -0,0 +1,140 @@ +import pytest +from kubetester import MongoDB, delete_secret, try_load +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.operator import Operator + +MDB_RESOURCE_NAME = "tls-replica-set" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE_NAME, f"prefix-{MDB_RESOURCE_NAME}-cert") + + +@pytest.fixture(scope="module") +def tls_replica_set(namespace: str, custom_mdb_version: str, issuer_ca_configmap: str, server_certs: str) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("test-tls-base-rs-require-ssl.yaml"), MDB_RESOURCE_NAME, namespace) + + # TLS can be enabled implicitly by specifying security.certsSecretPrefix field + resource["spec"]["security"] = { + "certsSecretPrefix": "prefix", + "tls": {"ca": issuer_ca_configmap}, + } + resource.set_version(custom_mdb_version) + try_load(resource) + return resource + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +def test_replica_set_creation(tls_replica_set: MongoDB): + tls_replica_set.update() + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_not_reachable_without_tls(tls_replica_set: MongoDB): + tester = tls_replica_set.tester(use_ssl=False) + tester.assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_reachable_with_tls(tls_replica_set: MongoDB, ca_path: str): + tester = tls_replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +def test_configure_prefer_ssl(tls_replica_set: MongoDB): + """ + Change ssl configuration to preferSSL + """ + tls_replica_set["spec"]["additionalMongodConfig"] = {"net": {"ssl": {"mode": "preferSSL"}}} + + tls_replica_set.update() + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_reachable_without_ssl_prefer_ssl(tls_replica_set: MongoDB): + tester = tls_replica_set.tester(use_ssl=False) + tester.assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_reachable_with_ssl_prefer_ssl(tls_replica_set: MongoDB, ca_path: str): + tester = tls_replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +def test_configure_allow_ssl(tls_replica_set: MongoDB): + """ + Change ssl configuration to allowSSL + """ + tls_replica_set["spec"]["additionalMongodConfig"] = {"net": {"ssl": {"mode": "allowSSL"}}} + + tls_replica_set.update() + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_reachable_without_tls_allow_ssl(tls_replica_set: MongoDB): + tester = tls_replica_set.tester(use_ssl=False) + tester.assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_reachable_with_tls_allow_ssl(tls_replica_set: MongoDB, ca_path: str): + tester = tls_replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +def test_disabled_ssl(tls_replica_set: MongoDB): + """ + Disable ssl explicitly + """ + tls_replica_set.load() + + # TLS can be disabled explicitly by setting security.tls.enabled to false and having + # no configuration for certificate secret + tls_replica_set["spec"]["security"]["certsSecretPrefix"] = None + tls_replica_set["spec"]["security"]["tls"] = {"enabled": False} + + tls_replica_set.update() + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_reachable_with_tls_disabled(tls_replica_set: MongoDB): + tester = tls_replica_set.tester(use_ssl=False) + tester.assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@skip_if_local() +def test_replica_set_is_not_reachable_over_ssl_with_ssl_disabled(tls_replica_set: MongoDB, ca_path: str): + tester = tls_replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require_and_disable +@pytest.mark.xfail(reason="Changing the TLS secret should not cause reconciliations after TLS is disabled") +def test_changes_to_secret_do_not_cause_reconciliation(tls_replica_set: MongoDB, namespace: str): + + delete_secret(namespace, f"{MDB_RESOURCE_NAME}-cert") diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_to_allow.py b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_to_allow.py new file mode 100644 index 000000000..7c337d40f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_to_allow.py @@ -0,0 +1,96 @@ +from kubetester import MongoDB +from kubetester.certs import create_mongodb_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.operator import Operator +from pytest import fixture, mark + +MDB_RESOURCE = "test-tls-base-rs-require-ssl" + + +@fixture(scope="module") +def rs_certs_secret(namespace: str, issuer: str): + create_mongodb_tls_certs(issuer, namespace, MDB_RESOURCE, "certs-test-tls-base-rs-require-ssl-cert") + return "certs" + + +@fixture(scope="module") +def tls_replica_set( + namespace: str, + custom_mdb_version: str, + issuer_ca_configmap: str, + rs_certs_secret: str, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("test-tls-base-rs-require-ssl.yaml"), + name=MDB_RESOURCE, + namespace=namespace, + ) + + resource.set_version(custom_mdb_version) + # no TLS to start with + resource["spec"]["security"] = {} + + yield resource.create() + + resource.delete() + + +@mark.e2e_replica_set_tls_require_to_allow +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_replica_set_tls_require_to_allow +def test_replica_set_creation(tls_replica_set: MongoDB): + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_replica_set_tls_require_to_allow +def test_enable_tls(tls_replica_set: MongoDB, issuer_ca_configmap: str, rs_certs_secret: str): + tls_replica_set.configure_custom_tls( + issuer_ca_configmap_name=issuer_ca_configmap, + tls_cert_secret_name=rs_certs_secret, + ) + tls_replica_set.update() + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_replica_set_tls_require_to_allow +@skip_if_local() +def test_replica_set_is_not_reachable_without_tls(tls_replica_set: MongoDB): + tester = tls_replica_set.tester(use_ssl=False) + tester.assert_no_connection() + + +@mark.e2e_replica_set_tls_require_to_allow +@skip_if_local() +def test_replica_set_is_reachable_with_tls(tls_replica_set: MongoDB, ca_path: str): + tester = tls_replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + +@mark.e2e_replica_set_tls_require_to_allow +def test_configure_allow_ssl(tls_replica_set: MongoDB): + """ + Change ssl configuration to allowSSL + """ + tls_replica_set["spec"]["additionalMongodConfig"] = {"net": {"ssl": {"mode": "allowSSL"}}} + + tls_replica_set.update() + tls_replica_set.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_replica_set_tls_require_to_allow +@skip_if_local() +def test_replica_set_is_reachable_without_tls_allow_ssl(tls_replica_set: MongoDB): + tester = tls_replica_set.tester(use_ssl=False) + tester.assert_connectivity() + + +@mark.e2e_replica_set_tls_require_to_allow +@skip_if_local() +def test_replica_set_is_reachable_with_tls_allow_ssl(tls_replica_set: MongoDB, ca_path: str): + tester = tls_replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_upgrade.py b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_upgrade.py new file mode 100644 index 000000000..027778814 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_requiressl_upgrade.py @@ -0,0 +1,66 @@ +import pytest +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester +from kubetester.operator import Operator + +MDB_RESOURCE = "test-tls-upgrade" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str, custom_mdb_version: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-require-ssl-upgrade.yaml"), namespace=namespace) + res.set_version(ensure_ent_version(custom_mdb_version)) + res["spec"]["security"] = {"tls": {"ca": issuer_ca_configmap}} + return res.create() + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +def test_replica_set_running(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +def test_mdb_is_reachable_with_no_ssl(mdb: MongoDB): + mdb.tester(use_ssl=False).assert_connectivity() + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +def test_enables_TLS_replica_set(mdb: MongoDB, server_certs: str, issuer_ca_configmap: str): + mdb.load() + mdb["spec"]["security"] = {"tls": {"enabled": True}, "ca": issuer_ca_configmap} + mdb.update() + mdb.assert_reaches_phase(Phase.Running) + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +def test_require_TLS(mdb: MongoDB): + mdb.load() + mdb["spec"]["additionalMongodConfig"]["net"]["ssl"]["mode"] = "requireSSL" + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +@skip_if_local() +def test_mdb_is_not_reachable_with_no_ssl(): + ReplicaSetTester(MDB_RESOURCE, 3).assert_no_connection() + + +@pytest.mark.e2e_replica_set_tls_require_upgrade +@skip_if_local() +def test_mdb_is_reachable_with_ssl(mdb: MongoDB, ca_path: str): + mdb.tester(use_ssl=True, ca_path=ca_path).assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_rs_additional_certs.py b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_additional_certs.py new file mode 100644 index 000000000..f1c1736f3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_additional_certs.py @@ -0,0 +1,87 @@ +import jsonpatch +import pytest +from kubernetes import client +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE_NAME = "test-tls-additional-domains" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + MDB_RESOURCE_NAME, + f"{MDB_RESOURCE_NAME}-cert", + additional_domains=[ + "test-tls-additional-domains-0.additional-cert-test.com", + "test-tls-additional-domains-1.additional-cert-test.com", + "test-tls-additional-domains-2.additional-cert-test.com", + ], + ) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-additional-domains.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_tls_rs_additional_certs +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_rs_additional_certs +class TestReplicaSetWithAdditionalCertDomains(KubernetesTester): + def test_replica_set_is_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + @skip_if_local + def test_can_still_connect(self, mdb: MongoDB, ca_path: str): + tester = mdb.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + +@pytest.mark.e2e_tls_rs_additional_certs +class TestReplicaSetRemoveAdditionalCertDomains(KubernetesTester): + def test_remove_additional_certs(self, namespace: str, mdb: MongoDB): + # We don't have a nice way to delete fields from a resource specification + # in our test env, so we need to achieve it with specific uses of patches + body = { + "op": "remove", + "path": "/spec/security/tls/additionalCertificateDomains", + } + patch = jsonpatch.JsonPatch([body]) + + last_transition = mdb.get_status_last_transition_time() + + mdb_resource = client.CustomObjectsApi().get_namespaced_custom_object( + mdb.group, + mdb.version, + namespace, + mdb.plural, + mdb.name, + ) + client.CustomObjectsApi().replace_namespaced_custom_object( + mdb.group, + mdb.version, + namespace, + mdb.plural, + mdb.name, + jsonpatch.apply_patch(mdb_resource, patch), + ) + + mdb.assert_state_transition_happens(last_transition) + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + @skip_if_local + def test_can_still_connect(self, mdb: MongoDB, ca_path: str): + tester = mdb.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access.py b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access.py new file mode 100644 index 000000000..37f74607b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access.py @@ -0,0 +1,184 @@ +import pytest +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + mdb_resource = "test-tls-base-rs-external-access" + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + mdb_resource, + f"{mdb_resource}-cert", + replicas=4, + additional_domains=[ + "*.website.com", + ], + ) + + +@pytest.fixture(scope="module") +def server_certs_multiple_horizons(issuer: str, namespace: str): + mdb_resource = "test-tls-rs-external-access-multiple-horizons" + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + mdb_resource, + f"{mdb_resource}-cert", + replicas=4, + additional_domains=[ + "mdb0-test-1.website.com", + "mdb0-test-2.website.com", + "mdb1-test-1.website.com", + "mdb1-test-2.website.com", + "mdb2-test-1.website.com", + "mdb2-test-2.website.com", + ], + ) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str, custom_mdb_version: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-external-access.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + res.set_version(custom_mdb_version) + return res.update() + + +@pytest.fixture(scope="module") +def mdb_multiple_horizons(namespace: str, server_certs_multiple_horizons: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml( + load_fixture("test-tls-rs-external-access-multiple-horizons.yaml"), + namespace=namespace, + ) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_tls_rs_external_access +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_rs_external_access +class TestReplicaSetWithExternalAccess(KubernetesTester): + def test_replica_set_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=240) + + @skip_if_local + def test_can_still_connect(self, mdb: MongoDB, ca_path: str): + tester = mdb.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + def test_automation_config_is_right(self): + ac = self.get_automation_config() + members = ac["replicaSets"][0]["members"] + horizon_names = [m["horizons"] for m in members] + assert horizon_names == [ + {"test-horizon": "mdb0-test.website.com:1337"}, + {"test-horizon": "mdb1-test.website.com:1337"}, + {"test-horizon": "mdb2-test.website.com:1337"}, + ] + + @skip_if_local + def test_has_right_certs(self): + """ + Check that mongod processes behind the replica set service are + serving the right certificates. + """ + host = f"test-tls-base-rs-external-access-svc.{self.namespace}.svc" + assert any(san.endswith(".website.com") for san in self.get_mongo_server_sans(host)) + + +@pytest.mark.e2e_tls_rs_external_access +def test_scale_up_replica_set(mdb: MongoDB): + mdb.load() + mdb["spec"]["members"] = 4 + mdb["spec"]["connectivity"]["replicaSetHorizons"] = [ + {"test-horizon": "mdb0-test.website.com:1337"}, + {"test-horizon": "mdb1-test.website.com:1337"}, + {"test-horizon": "mdb2-test.website.com:1337"}, + {"test-horizon": "mdb3-test.website.com:1337"}, + ] + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=240) + + +@pytest.mark.e2e_tls_rs_external_access +def tests_invalid_cert(mdb: MongoDB): + mdb.load() + mdb["spec"]["connectivity"]["replicaSetHorizons"] = [ + {"test-horizon": "mdb0-test.website.com:1337"}, + {"test-horizon": "mdb1-test.website.com:1337"}, + {"test-horizon": "mdb2-test.website.com:1337"}, + {"test-horizon": "mdb5-test.wrongwebsite.com:1337"}, + ] + mdb.update() + mdb.assert_reaches_phase( + Phase.Failed, + timeout=240, + ) + + +@pytest.mark.e2e_tls_rs_external_access +def test_can_remove_horizons(mdb: MongoDB): + mdb.load() + mdb["spec"]["connectivity"]["replicaSetHorizons"] = [] + mdb.update() + mdb.assert_reaches_phase(Phase.Running, timeout=240) + + +@pytest.mark.e2e_tls_rs_external_access +class TestReplicaSetWithNoTLSDeletion(KubernetesTester): + """ + delete: + file: test-tls-base-rs-external-access.yaml + wait_until: mongo_resource_deleted_no_om + timeout: 240 + """ + + def test_deletion(self): + assert True + + +@pytest.mark.e2e_tls_rs_external_access +class TestReplicaSetWithMultipleHorizons(KubernetesTester): + def test_replica_set_running(self, mdb_multiple_horizons: MongoDB): + mdb_multiple_horizons.assert_reaches_phase(Phase.Running, timeout=240) + + def test_automation_config_is_right(self): + ac = self.get_automation_config() + members = ac["replicaSets"][0]["members"] + horizons = [m["horizons"] for m in members] + assert horizons == [ + { + "test-horizon-1": "mdb0-test-1.website.com:1337", + "test-horizon-2": "mdb0-test-2.website.com:2337", + }, + { + "test-horizon-1": "mdb1-test-1.website.com:1338", + "test-horizon-2": "mdb1-test-2.website.com:2338", + }, + { + "test-horizon-1": "mdb2-test-1.website.com:1339", + "test-horizon-2": "mdb2-test-2.website.com:2339", + }, + ] + + +@pytest.mark.e2e_tls_rs_external_access +class TestReplicaSetDeleteMultipleHorizon(KubernetesTester): + """ + delete: + file: test-tls-rs-external-access-multiple-horizons.yaml + wait_until: mongo_resource_deleted_no_om + timeout: 240 + """ + + def test_deletion(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access_manual_connectivity.py b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access_manual_connectivity.py new file mode 100644 index 000000000..8316a92fd --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access_manual_connectivity.py @@ -0,0 +1,116 @@ +from typing import List + +import pytest +import yaml +from kubetester.certs import ISSUER_CA_NAME, create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +# This test will set up an environment which will configure a resource with split horizon enabled. + +# Steps to run this test. + +# 1. Change the following variable to any worker node in your cluster. (describe output of node) +WORKER_NODE_HOSTNAME = "ec2-18-222-120-63.us-east-2.compute.amazonaws.com" + +# 2. Run the test `make e2e test=e2e_tls_rs_external_access_manual_connectivity` + +# 3. Wait for the test to pass (this means the environment is set up.) + +# 4. Exec into any database pod and note the contents of the files referenced by the fields +# * net.tls.certificateKeyFile +# * net.tlsCAFile +# from the /data/automation-mongod.conf file. + +# 5. Test the connection +# Testing the connection can be done from either the worker node or from your local machine(note accessing traffic from a pod inside the cluster would work irrespective SH is configured correctly or not) +# 1. Acsessing from worker node +# * ssh into any worker node +# * Install the mongo shell +# * Create files from the two files mentioned above. (server.pem and ca.crt) +# * Run "mongo "mongodb://${WORKER_NODE}:30100,${WORKER_NODE}:30101,${WORKER_NODE}:30102/?replicaSet=test-tls-base-rs-external-access" --tls --tlsCertificateKeyFile server.pem --tlsCAFile ca.crt" +# 2. Accessing from local machine +# * Install the mongo shell +# * Create files from the two files mentioned above. (server.pem and ca.crt) +# * Open access to KOPS nodes from your local machine by following these steps(by default KOPS doesn't expose traffic from all ports to the internet) +# : https://stackoverflow.com/questions/45543694/kubernetes-cluster-on-aws-with-kops-nodeport-service-unavailable/45561848#45561848 +# * Run "mongo "mongodb://${WORKER_NODE}:30100,${WORKER_NODE}:30101,${WORKER_NODE}:30102/?replicaSet=test-tls-base-rs-external-access" --tls --tlsCertificateKeyFile server.pem --tlsCAFile ca.crt" +# When split horizon is not configured, specifying the replicaset name should fail. +# When split horizon is configured, it will successfully connect to the primary. + +# 6. Clean the namespace +# * This test creates node ports, which we should delete. + + +@pytest.fixture(scope="module") +def worker_node_hostname() -> str: + return WORKER_NODE_HOSTNAME + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str, worker_node_hostname: str): + mdb_resource = "test-tls-base-rs-external-access" + return create_mongodb_tls_certs( + ISSUER_CA_NAME, + namespace, + mdb_resource, + f"{mdb_resource}-cert", + replicas=3, + additional_domains=[ + worker_node_hostname, + ], + ) + + +@pytest.fixture(scope="module") +def node_ports() -> List[int]: + return [30100, 30101, 30102] + + +@pytest.fixture(scope="module") +def mdb( + namespace: str, + server_certs: str, + issuer_ca_configmap: str, + worker_node_hostname: str, + node_ports: List[int], +) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-tls-base-rs-external-access.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + res["spec"]["connectivity"]["replicaSetHorizons"] = [ + {"test-horizon": f"{worker_node_hostname}:{node_ports[0]}"}, + {"test-horizon": f"{worker_node_hostname}:{node_ports[1]}"}, + {"test-horizon": f"{worker_node_hostname}:{node_ports[2]}"}, + ] + + return res.create() + + +@pytest.mark.e2e_tls_rs_external_access_manual_connectivity +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_rs_external_access_manual_connectivity +def test_mdb_reaches_running_state(mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=900) + + +@pytest.mark.e2e_tls_rs_external_access_manual_connectivity +def test_create_node_ports(mdb: MongoDB, node_ports: List[int]): + for i in range(mdb["spec"]["members"]): + with open( + load_fixture("node-port-service.yaml"), + "r", + ) as f: + service_body = yaml.safe_load(f.read()) + service_body["metadata"]["name"] = f"{mdb.name}-external-access-svc-external-{i}" + service_body["spec"]["ports"][0]["nodePort"] = node_ports[i] + service_body["spec"]["selector"]["statefulset.kubernetes.io/pod-name"] = f"{mdb.name}-{i}" + + KubernetesTester.create_service( + mdb.namespace, + body=service_body, + ) diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access_transitions_without_approval.py b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access_transitions_without_approval.py new file mode 100644 index 000000000..d5e3bc119 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_external_access_transitions_without_approval.py @@ -0,0 +1,87 @@ +import pytest +from kubetester.kubetester import KubernetesTester, skip_if_local +from kubetester.mongotester import ReplicaSetTester +from kubetester.omtester import get_rs_cert_names +from kubetester.operator import Operator + + +@pytest.mark.e2e_tls_rs_external_access_tls_transition_without_approval +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_rs_external_access_tls_transition_without_approval +class TestReplicaSetWithExternalAccess(KubernetesTester): + init = { + "create": { + "file": "test-tls-base-rs-external-access.yaml", + "wait_for_message": "Not all certificates have been approved by Kubernetes CA", + "timeout": 60, + "patch": [{"op": "remove", "path": "/spec/connectivity/replicaSetHorizons"}], + } + } + + def test_tls_certs_approved(self): + pass + + +@pytest.mark.e2e_tls_rs_external_access_tls_transition_without_approval +class TestReplicaSetExternalAccessAddHorizons(KubernetesTester): + init = { + "update": { + "file": "test-tls-base-rs-external-access.yaml", + "wait_for_message": "Please manually remove the CSR in order to proceed.", + "timeout": 60, + } + } + + def test_certs_approved(self): + csr_names = get_rs_cert_names("test-tls-base-rs-external-access", self.namespace) + for csr_name in self.yield_existing_csrs(csr_names): + self.delete_csr(csr_name) + self.wait_for_status_message( + { + "wait_for_message": "Not all certificates have been approved", + "timeout": 30, + } + ) + for csr_name in self.yield_existing_csrs(csr_names): + self.approve_certificate(csr_name) + KubernetesTester.wait_until("in_running_state", 360) + + @skip_if_local + def test_can_still_connect(self): + tester = ReplicaSetTester("test-tls-base-rs-external-access", 3, ssl=True) + tester.assert_connectivity() + + def test_automation_config_is_right(self): + ac = self.get_automation_config() + members = ac["replicaSets"][0]["members"] + horizon_names = [m["horizons"] for m in members] + assert horizon_names == [ + {"test-horizon": "mdb0-test-website.com:1337"}, + {"test-horizon": "mdb1-test-website.com:1337"}, + {"test-horizon": "mdb2-test-website.com:1337"}, + ] + + @skip_if_local + def test_has_right_certs(self): + """ + Check that mongod processes behind the replica set service are + serving the right certificates. + """ + host = f"test-tls-base-rs-external-access-svc.{self.namespace}.svc" + assert any(san.endswith("test-website.com") for san in self.get_mongo_server_sans(host)) + + +@pytest.mark.e2e_tls_rs_external_access_tls_transition_without_approval +class TestExternalAccessDeletion(KubernetesTester): + """ + delete: + file: test-tls-base-rs-external-access.yaml + wait_until: mongo_resource_deleted_no_om + timeout: 240 + """ + + def test_deletion(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_rs_intermediate_ca.py b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_intermediate_ca.py new file mode 100644 index 000000000..68944ef50 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_rs_intermediate_ca.py @@ -0,0 +1,51 @@ +import pytest +from kubetester.certs import create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE_NAME = "test-tls-rs-intermediate-ca" + + +@pytest.fixture(scope="module") +def server_certs(intermediate_issuer: str, namespace: str): + return create_mongodb_tls_certs( + intermediate_issuer, + namespace, + MDB_RESOURCE_NAME, + f"{MDB_RESOURCE_NAME}-cert", + additional_domains=[ + "test-tls-rs-intermediate-ca-0.additional-cert-test.com", + "test-tls-rs-intermediate-ca-1.additional-cert-test.com", + "test-tls-rs-intermediate-ca-2.additional-cert-test.com", + ], + ) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml( + load_fixture("test-tls-additional-domains.yaml"), + namespace=namespace, + name=MDB_RESOURCE_NAME, + ) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_tls_rs_intermediate_ca +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_rs_intermediate_ca +class TestReplicaSetWithAdditionalCertDomains(KubernetesTester): + def test_replica_set_is_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=400) + + @skip_if_local + def test_can_still_connect(self, mdb: MongoDB, ca_path: str): + tester = mdb.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_sc_additional_certs.py b/docker/mongodb-enterprise-tests/tests/tls/tls_sc_additional_certs.py new file mode 100644 index 000000000..eadfec3b8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_sc_additional_certs.py @@ -0,0 +1,108 @@ +import re + +import pytest +from kubetester import try_load +from kubetester.certs import create_sharded_cluster_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import is_multi_cluster, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_member_cluster_clients_using_cluster_mapping, + get_mongos_service_names, +) + +MDB_RESOURCE_NAME = "test-tls-sc-additional-domains" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + shard_distribution = None + mongos_distribution = None + config_srv_distribution = None + if is_multi_cluster(): + shard_distribution = [1, 1, 1] + mongos_distribution = [1, 1, None] + config_srv_distribution = [1, 1, 1] + + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE_NAME, + shards=1, + mongod_per_shard=1, + config_servers=1, + mongos=2, + additional_domains=["additional-cert-test.com"], + shard_distribution=shard_distribution, + mongos_distribution=mongos_distribution, + config_srv_distribution=config_srv_distribution, + ) + + +@pytest.fixture(scope="module") +def sc(namespace: str, server_certs: str, issuer_ca_configmap: str, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("test-tls-sc-additional-domains.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_architecture_annotation() + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@pytest.mark.e2e_tls_sc_additional_certs +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_sc_additional_certs +class TestShardedClusterWithAdditionalCertDomains: + def test_sharded_cluster_running(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + @skip_if_local + def test_has_right_certs(self, sc: MongoDB): + """Check that mongos processes serving the right certificates.""" + for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace): + cluster_idx = cluster_member_client.cluster_index + for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)): + mongos_pod_name = sc.mongos_pod_name(member_idx, cluster_idx) + host = sc.mongos_hostname(member_idx, cluster_idx) + assert any( + re.match(rf"{mongos_pod_name}\.additional-cert-test\.com", san) + for san in KubernetesTester.get_mongo_server_sans(host) + ) + + @skip_if_local + def test_can_still_connect(self, sc: MongoDB, ca_path: str): + service_names = get_mongos_service_names(sc) + tester = sc.tester(ca_path=ca_path, use_ssl=True, service_names=service_names) + tester.assert_connectivity() + + +@pytest.mark.e2e_tls_sc_additional_certs +def test_remove_additional_certificate_domains(sc: MongoDB): + sc["spec"]["security"]["tls"].pop("additionalCertificateDomains") + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=240) + + +@pytest.mark.e2e_tls_sc_additional_certs +@skip_if_local +def test_can_still_connect(sc: MongoDB, ca_path: str): + service_names = get_mongos_service_names(sc) + tester = sc.tester(ca_path=ca_path, use_ssl=True, service_names=service_names) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_sc_requiressl_custom_ca.py b/docker/mongodb-enterprise-tests/tests/tls/tls_sc_requiressl_custom_ca.py new file mode 100644 index 000000000..c8c8157fb --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_sc_requiressl_custom_ca.py @@ -0,0 +1,106 @@ +import pytest +from kubetester import try_load +from kubetester.certs import Certificate, create_sharded_cluster_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import is_multi_cluster, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ShardedClusterTester +from kubetester.operator import Operator +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_mongos_service_names, +) + +MDB_RESOURCE_NAME = "test-tls-base-sc-require-ssl" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + shard_distribution = None + mongos_distribution = None + config_srv_distribution = None + if is_multi_cluster(): + shard_distribution = [1, 1, 1] + mongos_distribution = [1, 1, None] + config_srv_distribution = [1, 1, 1] + + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE_NAME, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + shard_distribution=shard_distribution, + mongos_distribution=mongos_distribution, + config_srv_distribution=config_srv_distribution, + ) + + +@pytest.fixture(scope="module") +def sc(namespace: str, server_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("test-tls-base-sc-require-ssl-custom-ca.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource.set_architecture_annotation() + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@pytest.mark.e2e_sharded_cluster_tls_require_custom_ca +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_sharded_cluster_tls_require_custom_ca +class TestClusterWithTLSCreation: + def test_sharded_cluster_running(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + @skip_if_local + def test_mongos_are_reachable_with_ssl(self, sc: MongoDB, ca_path: str): + service_names = get_mongos_service_names(sc) + tester = sc.tester(ca_path=ca_path, use_ssl=True, service_names=service_names) + tester.assert_connectivity() + + @skip_if_local + def test_mongos_are_not_reachable_with_no_ssl(self, sc: MongoDB): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=False, service_names=service_names) + tester.assert_no_connection() + + +@pytest.mark.e2e_sharded_cluster_tls_require_custom_ca +class TestCertificateIsRenewed: + def test_mdb_reconciles_succesfully(self, sc: MongoDB, namespace: str): + cert = Certificate(name=f"{MDB_RESOURCE_NAME}-0-cert", namespace=namespace).load() + cert["spec"]["dnsNames"].append("foo") + cert.update() + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + @skip_if_local + def test_mongos_are_reachable_with_ssl(self, sc: MongoDB, ca_path: str): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=True, ca_path=ca_path, service_names=service_names) + tester.assert_connectivity() + + @skip_if_local + def test_mongos_are_not_reachable_with_no_ssl( + self, + sc: MongoDB, + ): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=False, service_names=service_names) + tester.assert_no_connection() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_sharded_cluster_certs_prefix.py b/docker/mongodb-enterprise-tests/tests/tls/tls_sharded_cluster_certs_prefix.py new file mode 100644 index 000000000..1b14fd460 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_sharded_cluster_certs_prefix.py @@ -0,0 +1,184 @@ +import kubernetes +from kubetester import try_load +from kubetester.certs import Certificate, create_sharded_cluster_certs +from kubetester.kubetester import KubernetesTester, ensure_ent_version +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import is_multi_cluster, skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture, mark +from tests.conftest import central_cluster_client +from tests.shardedcluster.conftest import ( + enable_multi_cluster_deployment, + get_mongos_service_names, +) + +MDB_RESOURCE = "sharded-cluster-custom-certs" + + +@fixture(scope="module") +def all_certs(central_cluster_client: kubernetes.client.ApiClient, issuer: str, namespace: str) -> None: + """Generates all required TLS certificates: Servers and Client/Member.""" + + shard_distribution = None + mongos_distribution = None + config_srv_distribution = None + if is_multi_cluster(): + shard_distribution = [1, 1, 1] + mongos_distribution = [1, 1, None] + config_srv_distribution = [1, 1, 1] + + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + x509_certs=True, + secret_prefix="prefix-", + shard_distribution=shard_distribution, + mongos_distribution=mongos_distribution, + config_srv_distribution=config_srv_distribution, + ) + + +@fixture(scope="module") +def sc(namespace: str, issuer_ca_configmap: str, custom_mdb_version: str, all_certs) -> MongoDB: + resource = MongoDB.from_yaml( + load_fixture("test-tls-base-sc-require-ssl.yaml"), + name=MDB_RESOURCE, + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource["spec"]["security"] = { + "tls": { + "enabled": True, + "ca": issuer_ca_configmap, + }, + "certsSecretPrefix": "prefix", + } + + resource.set_version(ensure_ent_version(custom_mdb_version)) + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@mark.e2e_tls_sharded_cluster_certs_prefix +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@mark.e2e_tls_sharded_cluster_certs_prefix +def test_sharded_cluster_with_prefix_gets_to_running_state(sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_tls_sharded_cluster_certs_prefix +@skip_if_local +def test_sharded_cluster_has_connectivity_with_tls(sc: MongoDB, ca_path: str): + service_names = get_mongos_service_names(sc) + tester = sc.tester(ca_path=ca_path, use_ssl=True, service_names=service_names) + tester.assert_connectivity() + + +@mark.e2e_tls_sharded_cluster_certs_prefix +@skip_if_local +def test_sharded_cluster_has_no_connectivity_without_tls(sc: MongoDB): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=False, service_names=service_names) + tester.assert_no_connection() + + +@mark.e2e_tls_sharded_cluster_certs_prefix +def test_rotate_tls_certificate(sc: MongoDB, namespace: str): + # update the shard cert + cert = Certificate(name=f"prefix-{MDB_RESOURCE}-0-cert", namespace=namespace).load() + cert["spec"]["dnsNames"].append("foo") + cert.update() + + sc.assert_abandons_phase(Phase.Running) + sc.assert_reaches_phase(Phase.Running, timeout=800) + + +@mark.e2e_tls_sharded_cluster_certs_prefix +def test_disable_tls(sc: MongoDB): + last_transition = sc.get_status_last_transition_time() + sc.load() + sc["spec"]["security"]["tls"]["enabled"] = False + sc.update() + + sc.assert_state_transition_happens(last_transition) + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + +@mark.e2e_tls_sharded_cluster_certs_prefix +@mark.xfail(reason="Disabling security.tls.enabled does not disable TLS when security.tls.secretRef.prefix is set") +def test_sharded_cluster_has_connectivity_without_tls(sc: MongoDB): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=False, service_names=service_names) + tester.assert_connectivity(opts=[{"serverSelectionTimeoutMs": 30000}], attempts=1) + + +@mark.e2e_tls_sharded_cluster_certs_prefix +def test_sharded_cluster_with_allow_tls(sc: MongoDB): + sc["spec"]["security"]["tls"]["enabled"] = True + + additional_mongod_config = { + "additionalMongodConfig": { + "net": { + "tls": { + "mode": "allowTLS", + } + } + } + } + + sc["spec"]["mongos"] = additional_mongod_config + sc["spec"]["shard"] = additional_mongod_config + sc["spec"]["configSrv"] = additional_mongod_config + + sc.update() + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + automation_config = KubernetesTester.get_automation_config() + + tls_modes = [ + process.get("args2_6", {}).get("net", {}).get("tls", {}).get("mode") + for process in automation_config["processes"] + ] + + # 3 mongod + 3 configSrv + 2 mongos = 8 processes + assert len(tls_modes) == 8 + tls_modes_set = set(tls_modes) + # all processes should have the same allowTLS value + assert len(tls_modes_set) == 1 + assert tls_modes_set.pop() == "allowTLS" + + +@mark.e2e_tls_sharded_cluster_certs_prefix +@skip_if_local +def test_sharded_cluster_has_connectivity_with_tls_with_allow_tls_mode(sc: MongoDB, ca_path: str): + service_names = get_mongos_service_names(sc) + tester = sc.tester(ca_path=ca_path, use_ssl=True, service_names=service_names) + tester.assert_connectivity() + + +@mark.e2e_tls_sharded_cluster_certs_prefix +@skip_if_local +def test_sharded_cluster_has_connectivity_without_tls_with_allow_tls_mode(sc: MongoDB): + service_names = get_mongos_service_names(sc) + tester = sc.tester(use_ssl=False, service_names=service_names) + tester.assert_connectivity() diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py new file mode 100644 index 000000000..3151d294d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_rs.py @@ -0,0 +1,66 @@ +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + ISSUER_CA_NAME, + create_x509_agent_tls_certs, + create_x509_mongodb_tls_certs, + rotate_cert, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator + +MDB_RESOURCE = "test-x509-all-options-rs" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + create_x509_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + create_x509_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-clusterfile") + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-x509-all-options-rs.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.update() + + +@pytest.mark.e2e_tls_x509_configure_all_options_rs +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_x509_configure_all_options_rs +class TestReplicaSetEnableAllOptions(KubernetesTester): + def test_gets_to_running_state(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=600) + + def test_ops_manager_state_correctly_updated(self): + ac_tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + ac_tester.assert_internal_cluster_authentication_enabled() + ac_tester.assert_authentication_enabled() + + def test_rotate_certificate(self, mdb: MongoDB, namespace: str): + rotate_cert(namespace, "{}-cert".format(MDB_RESOURCE)) + mdb.assert_abandons_phase(Phase.Running, timeout=900) + mdb.assert_reaches_phase(Phase.Running, timeout=900) + + def test_rotate_certificate_with_sts_restarting(self, mdb: MongoDB, namespace: str): + mdb.trigger_sts_restart() + assert_certificate_rotation(mdb, namespace, "{}-cert".format(MDB_RESOURCE)) + + def test_rotate_clusterfile_with_sts_restarting(self, mdb: MongoDB, namespace: str): + mdb.trigger_sts_restart() + assert_certificate_rotation(mdb, namespace, "{}-clusterfile".format(MDB_RESOURCE)) + + +def assert_certificate_rotation(mdb, namespace, certificate_name): + rotate_cert(namespace, certificate_name) + mdb.assert_reaches_phase(Phase.Running, timeout=900) diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py new file mode 100644 index 000000000..93e107410 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_configure_all_options_sc.py @@ -0,0 +1,143 @@ +import pytest +from kubetester import find_fixture, try_load +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + create_sharded_cluster_certs, + create_x509_agent_tls_certs, + rotate_cert, +) +from kubetester.kubetester import KubernetesTester, ensure_ent_version, is_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from pytest import fixture +from tests.shardedcluster.conftest import enable_multi_cluster_deployment + +MDB_RESOURCE_NAME = "test-x509-all-options-sc" + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE_NAME) + + +@fixture(scope="module") +def server_certs(issuer: str, namespace: str): + shard_distribution = None + mongos_distribution = None + config_srv_distribution = None + if is_multi_cluster(): + shard_distribution = [1, 1, 1] + mongos_distribution = [1, 1, 1] + config_srv_distribution = [1, 1, 1] + + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE_NAME, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=3, + internal_auth=True, + x509_certs=True, + shard_distribution=shard_distribution, + mongos_distribution=mongos_distribution, + config_srv_distribution=config_srv_distribution, + ) + + +@fixture(scope="module") +def sc(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml( + find_fixture("test-x509-all-options-sc.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, 1], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@pytest.mark.e2e_tls_x509_configure_all_options_sc +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_x509_configure_all_options_sc +class TestShardedClusterEnableAllOptions: + + def test_gets_to_running_state(self, sc: MongoDB): + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_ops_manager_state_correctly_updated(self): + ac_tester = AutomationConfigTester(KubernetesTester.get_automation_config()) + ac_tester.assert_internal_cluster_authentication_enabled() + ac_tester.assert_authentication_enabled() + ac_tester.assert_expected_users(0) + + def test_rotate_shard_cert(self, sc: MongoDB, namespace: str): + rotate_cert(namespace, f"{MDB_RESOURCE_NAME}-0-cert") + sc.assert_abandons_phase(Phase.Running, timeout=900) + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_rotate_config_cert(self, sc: MongoDB, namespace: str): + rotate_cert(namespace, f"{MDB_RESOURCE_NAME}-config-cert") + sc.assert_abandons_phase(Phase.Running, timeout=900) + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_rotate_mongos_cert(self, sc: MongoDB, namespace: str): + rotate_cert(namespace, f"{MDB_RESOURCE_NAME}-mongos-cert") + sc.assert_abandons_phase(Phase.Running, timeout=900) + sc.assert_reaches_phase(Phase.Running, timeout=1200) + + def test_rotate_shard_cert_with_sts_restarting(self, sc: MongoDB, namespace: str): + sc.trigger_sts_restart("shard") + assert_certificate_rotation(sc, namespace, f"{MDB_RESOURCE_NAME}-0-cert") + + def test_rotate_config_cert_with_sts_restarting(self, sc: MongoDB, namespace: str): + sc.trigger_sts_restart("config") + assert_certificate_rotation(sc, namespace, f"{MDB_RESOURCE_NAME}-config-cert") + + def test_rotate_mongos_cert_with_sts_restarting(self, sc: MongoDB, namespace: str): + sc.trigger_sts_restart("mongos") + assert_certificate_rotation(sc, namespace, f"{MDB_RESOURCE_NAME}-mongos-cert") + + def test_rotate_shard_certfile_with_sts_restarting(self, sc: MongoDB, namespace: str): + sc.trigger_sts_restart("shard") + assert_certificate_rotation( + sc, + namespace, + f"{MDB_RESOURCE_NAME}-0-clusterfile", + ) + + def test_rotate_config_certfile_with_sts_restarting(self, sc: MongoDB, namespace: str): + sc.trigger_sts_restart("config") + assert_certificate_rotation( + sc, + namespace, + f"{MDB_RESOURCE_NAME}-config-clusterfile", + ) + + def test_rotate_mongos_certfile_with_sts_restarting(self, sc: MongoDB, namespace: str): + sc.trigger_sts_restart("mongos") + assert_certificate_rotation( + sc, + namespace, + f"{MDB_RESOURCE_NAME}-mongos-clusterfile", + ) + + +def assert_certificate_rotation(sharded_cluster, namespace, certificate_name): + rotate_cert(namespace, certificate_name) + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_rs.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_rs.py new file mode 100644 index 000000000..8f8ff29b6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_rs.py @@ -0,0 +1,66 @@ +import pytest +from kubetester.certs import ( + ISSUER_CA_NAME, + create_agent_tls_certs, + create_mongodb_tls_certs, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester +from kubetester.operator import Operator + +MDB_RESOURCE = "test-x509-rs" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-x509-rs.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_tls_x509_rs +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_x509_rs +class TestReplicaSetWithNoTLSCreation(KubernetesTester): + def test_gets_to_running_state(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=1200) + + @skip_if_local + def test_mdb_is_reachable_with_no_ssl(self): + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_no_connection() + + @skip_if_local + def test_mdb_is_reachable_with_ssl(self): + # This one will also fails, as it expects x509 client certs which we are not passing. + tester = ReplicaSetTester(MDB_RESOURCE, 3, ssl=True) + tester.assert_no_connection() + + +@pytest.mark.e2e_tls_x509_rs +class TestReplicaSetWithNoTLSDeletion(KubernetesTester): + """ + delete: + file: test-x509-rs.yaml + wait_until: mongo_resource_deleted_no_om + timeout: 240 + """ + + def test_deletion(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_sc.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_sc.py new file mode 100644 index 000000000..c2ff55cf7 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_sc.py @@ -0,0 +1,70 @@ +import pytest +from kubetester import try_load +from kubetester.certs import create_sharded_cluster_certs, create_x509_agent_tls_certs +from kubetester.kubetester import fixture as load_fixture +from kubetester.kubetester import is_multi_cluster +from kubetester.mongodb import MongoDB, Phase +from kubetester.operator import Operator +from tests.shardedcluster.conftest import enable_multi_cluster_deployment + +MDB_RESOURCE_NAME = "test-x509-sc" + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + shard_distribution = None + mongos_distribution = None + config_srv_distribution = None + if is_multi_cluster(): + shard_distribution = [1, 1, 1] + mongos_distribution = [1, 1, None] + config_srv_distribution = [1, 1, 1] + + create_sharded_cluster_certs( + namespace, + MDB_RESOURCE_NAME, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + shard_distribution=shard_distribution, + mongos_distribution=mongos_distribution, + config_srv_distribution=config_srv_distribution, + ) + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE_NAME) + + +@pytest.fixture(scope="module") +def sharded_cluster(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + resource = MongoDB.from_yaml(load_fixture("test-x509-sc.yaml"), namespace=namespace) + + if try_load(resource): + return resource + + resource["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + resource.set_architecture_annotation() + + if is_multi_cluster(): + enable_multi_cluster_deployment( + resource=resource, + shard_members_array=[1, 1, 1], + mongos_members_array=[1, 1, None], + configsrv_members_array=[1, 1, 1], + ) + + return resource.update() + + +@pytest.mark.e2e_tls_x509_sc +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_x509_sc +class TestClusterWithTLSCreation: + def test_sharded_cluster_running(self, sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=1200) diff --git a/docker/mongodb-enterprise-tests/tests/tls/tls_x509_user_connectivity.py b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_user_connectivity.py new file mode 100644 index 000000000..2f4435ca0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/tls/tls_x509_user_connectivity.py @@ -0,0 +1,111 @@ +import tempfile + +import pytest +from kubetester.automation_config_tester import AutomationConfigTester +from kubetester.certs import ( + create_agent_tls_certs, + create_mongodb_tls_certs, + create_x509_user_cert, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as _fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ReplicaSetTester +from kubetester.operator import Operator + +MDB_RESOURCE = "test-x509-rs" +X509_AGENT_SUBJECT = "CN=automation,OU={namespace},O=cert-manager" + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str) -> str: + return create_mongodb_tls_certs(issuer, namespace, MDB_RESOURCE, MDB_RESOURCE + "-cert") + + +@pytest.fixture(scope="module") +def replica_set(namespace, agent_certs, server_certs, issuer_ca_configmap): + _ = server_certs + res = MongoDB.from_yaml(_fixture("test-x509-rs.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + + return res.create() + + +@pytest.mark.e2e_tls_x509_user_connectivity +def test_install_operator(operator: Operator): + operator.assert_is_running() + + +@pytest.mark.e2e_tls_x509_user_connectivity +class TestReplicaSetWithTLSCreation(KubernetesTester): + def test_users_wanted_is_correct(self, replica_set, namespace): + """At this stage we should have 2 members in the usersWanted list, + monitoring-agent and backup-agent.""" + + replica_set.assert_reaches_phase(Phase.Running, timeout=600) + + automation_config = KubernetesTester.get_automation_config() + users = [u["user"] for u in automation_config["auth"]["usersWanted"]] + + for subject in users: + names = dict(name.split("=") for name in subject.split(",")) + + assert "OU" in names + assert "CN" in names + + +@pytest.mark.e2e_tls_x509_user_connectivity +class TestAddMongoDBUser(KubernetesTester): + """ + create: + file: test-x509-user.yaml + patch: '[{"op":"replace","path":"/spec/mongodbResourceRef/name","value": "test-x509-rs" }]' + wait_until: user_exists + """ + + def test_add_user(self): + assert True + + @staticmethod + def user_exists(): + ac = KubernetesTester.get_automation_config() + users = ac["auth"]["usersWanted"] + + return "CN=x509-testing-user" in [user["user"] for user in users] + + +@pytest.mark.e2e_tls_x509_user_connectivity +class TestX509CertCreationAndApproval(KubernetesTester): + def setup_method(self): + super().setup_method() + self.cert_file = tempfile.NamedTemporaryFile(delete=False, mode="w") + + def test_create_user_and_authenticate(self, issuer: str, namespace: str, ca_path: str): + create_x509_user_cert(issuer, namespace, path=self.cert_file.name) + tester = ReplicaSetTester(MDB_RESOURCE, 3) + tester.assert_x509_authentication(cert_file_name=self.cert_file.name, tlsCAFile=ca_path) + + def teardown(self): + self.cert_file.close() + + +@pytest.mark.e2e_tls_x509_user_connectivity +class TestX509CorrectlyConfigured(KubernetesTester): + def test_om_state_is_correct(self, namespace): + automation_config = KubernetesTester.get_automation_config() + tester = AutomationConfigTester(automation_config) + + tester.assert_authentication_mechanism_enabled("MONGODB-X509") + tester.assert_authoritative_set(True) + tester.assert_expected_users(1) + + user = automation_config["auth"]["autoUser"] + names = dict(name.split("=") for name in user.split(",")) + + assert "OU" in names + assert "CN=mms-automation-agent" in user diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/__init__.py b/docker/mongodb-enterprise-tests/tests/upgrades/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py new file mode 100644 index 000000000..80c56024f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py @@ -0,0 +1,167 @@ +from kubetester import get_statefulset +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import Phase +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark +from tests.conftest import ( + create_appdb_certs, + install_official_operator, + is_multi_cluster, +) +from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment + +APPDB_NAME = "om-appdb-upgrade-tls-db" + +CERT_PREFIX = "prefix" + +# TODO: this test runs in the cloudqa variant but still creates OM. We might want to move that to OM variant instead + + +def appdb_name(namespace: str) -> str: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_appdb_upgrade_tls.yaml"), namespace=namespace + ) + return "{}-db".format(resource["metadata"]["name"]) + + +@fixture(scope="module") +def appdb_certs_secret(namespace: str, issuer: str): + return create_appdb_certs(namespace, issuer, APPDB_NAME, cert_prefix=CERT_PREFIX) + + +# This deployment uses TLS for appdb, which means that monitoring won't work due to missing hostnames in the TLS certs +# This is to test that the operator upgrade will fix monitoring +@fixture(scope="module") +def ops_manager_tls( + namespace, issuer_ca_configmap: str, appdb_certs_secret: str, custom_version: str, custom_appdb_version: str +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_appdb_upgrade_tls.yaml"), namespace=namespace + ) + resource["spec"]["applicationDatabase"]["security"]["certsSecretPrefix"] = CERT_PREFIX + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource, om_cluster_spec_list=[1, 0, 0]) + + return resource + + +# This deployment does not use TLS for appdb, therefore monitoring will work +# This is to test that the operator migrates monitoring seamlessly +@fixture(scope="module") +def ops_manager_non_tls( + namespace, issuer_ca_configmap: str, custom_version: str, custom_appdb_version: str +) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("om_ops_manager_appdb_upgrade_tls.yaml"), namespace=namespace, name="om-appdb-upgrade" + ) + resource["spec"]["applicationDatabase"]["security"] = None + resource.set_version(custom_version) + resource.set_appdb_version(custom_appdb_version) + + if is_multi_cluster(): + enable_multi_cluster_deployment(resource, om_cluster_spec_list=[1, 0, 0]) + + return resource + + +@mark.e2e_operator_upgrade_appdb_tls +def test_install_latest_official_operator( + namespace: str, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, +): + operator = install_official_operator( + namespace, + managed_security_context, + operator_installation_config, + central_cluster_name, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + "1.32.0", # latest operator version before fixing the appdb hostnames + ) + operator.assert_is_running() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_create_om_tls(ops_manager_tls: MongoDBOpsManager): + ops_manager_tls.update() + ops_manager_tls.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_tls.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_tls.get_om_tester().assert_healthiness() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_create_om_non_tls(ops_manager_non_tls: MongoDBOpsManager): + ops_manager_non_tls.update() + ops_manager_non_tls.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_non_tls.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_non_tls.get_om_tester().assert_healthiness() + ops_manager_non_tls.assert_monitoring_data_exists() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_upgrade_operator(default_operator: Operator): + default_operator.assert_is_running() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_om_tls_ok(ops_manager_tls: MongoDBOpsManager): + ops_manager_tls.load() + ops_manager_tls.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_tls.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_tls.get_om_tester().assert_healthiness() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_om_non_tls_ok(ops_manager_non_tls: MongoDBOpsManager): + ops_manager_non_tls.load() + ops_manager_non_tls.appdb_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_non_tls.om_status().assert_reaches_phase(Phase.Running, timeout=900) + ops_manager_non_tls.get_om_tester().assert_healthiness() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_appdb_tls_hostnames(ops_manager_tls: MongoDBOpsManager): + ops_manager_tls.load() + ops_manager_tls.assert_appdb_preferred_hostnames_are_added() + ops_manager_tls.assert_appdb_hostnames_are_correct() + ops_manager_tls.assert_appdb_monitoring_group_was_created() + ops_manager_tls.assert_monitoring_data_exists() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_appdb_non_tls_hostnames(ops_manager_non_tls: MongoDBOpsManager): + ops_manager_non_tls.load() + ops_manager_non_tls.assert_appdb_preferred_hostnames_are_added() + ops_manager_non_tls.assert_appdb_hostnames_are_correct() + ops_manager_non_tls.assert_appdb_monitoring_group_was_created() + ops_manager_non_tls.assert_monitoring_data_exists() + + +@mark.e2e_operator_upgrade_appdb_tls +def test_using_official_images(namespace: str, ops_manager_tls: MongoDBOpsManager): + """ + This test ensures that after upgrading from 1.x to 1.20 that our operator automatically replaces the old appdb + image with the official on + """ + # -> old quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.14-ent + # -> new quay.io/mongodb/mongodb-enterprise-server:5.0.14-ubi8 + ops_manager_tls.load() + sts = ops_manager_tls.read_appdb_statefulset() + found_official_image = any( + [ + "quay.io/mongodb/mongodb-enterprise-server" in container.image + for container in sts.spec.template.spec.containers + ] + ) + assert found_official_image diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py new file mode 100644 index 000000000..e3c1e151f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py @@ -0,0 +1,171 @@ +from typing import Optional + +from kubetester.awss3client import AwsS3Client +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import skip_if_local +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import MongoDBBackgroundTester +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str) -> str: + """creates a s3 bucket and a s3 config""" + + bucket_name = KubernetesTester.random_k8s_name("test-bucket-") + aws_s3_client.create_s3_bucket(bucket_name) + print(f"\nCreated S3 bucket {bucket_name}") + + KubernetesTester.create_secret( + namespace, + "my-s3-secret", + { + "accessKey": aws_s3_client.aws_access_key, + "secretKey": aws_s3_client.aws_secret_access_key, + }, + ) + yield bucket_name + + +@fixture(scope="module") +def ops_manager( + namespace: str, s3_bucket: str, custom_version: Optional[str], custom_appdb_version: str +) -> MongoDBOpsManager: + """The fixture for Ops Manager to be created. Also results in a new s3 bucket + created and used in OM spec""" + om: MongoDBOpsManager = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_full.yaml"), namespace=namespace) + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + om["spec"]["backup"]["s3Stores"][0]["s3BucketName"] = s3_bucket + return om.create() + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name="my-mongodb-oplog", + ).configure(ops_manager, "development") + resource.set_version(custom_mdb_version) + resource["spec"]["members"] = 1 + resource["spec"]["persistent"] = True + + yield resource.create() + + +@fixture(scope="module") +def s3_replica_set(ops_manager: MongoDBOpsManager, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name="my-mongodb-s3", + ).configure(ops_manager, "s3metadata") + + resource.set_version(custom_mdb_version) + resource["spec"]["members"] = 1 + resource["spec"]["persistent"] = True + + yield resource.create() + + +@fixture(scope="module") +def some_mdb(ops_manager: MongoDBOpsManager, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=ops_manager.namespace, + name="some-mdb", + ).configure(ops_manager, "someProject") + resource.set_version(custom_mdb_version) + resource["spec"]["persistent"] = True + + return resource.create() + + +@fixture(scope="module") +def some_mdb_health_checker(some_mdb: MongoDB) -> MongoDBBackgroundTester: + # TODO increasing allowed_sequential_failures to 5 to remove flakiness until CLOUDP-56877 is solved + return MongoDBBackgroundTester(some_mdb.tester(), allowed_sequential_failures=5) + + +# The first stage of the Operator upgrade test. Create Ops Manager with backup enabled, +# creates backup databases and some extra database referencing the OM. +# TODO CLOUDP-54130: this database needs to get enabled for backup and this needs to be verified +# on the second stage + + +@mark.e2e_operator_upgrade_ops_manager +def test_install_latest_official_operator(official_operator: Operator): + official_operator.assert_is_running() + + +@mark.e2e_operator_upgrade_ops_manager +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + +@mark.e2e_operator_upgrade_ops_manager +def test_backup_enabled( + ops_manager: MongoDBOpsManager, + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, +): + oplog_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + # We are ignoring any errors as there could be temporary blips in connectivity to backing + # databases by this time + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=200, ignore_errors=True) + + +@skip_if_local +@mark.e2e_operator_upgrade_ops_manager +def test_om_is_ok(ops_manager: MongoDBOpsManager): + ops_manager.get_om_tester().assert_healthiness() + + +@mark.e2e_operator_upgrade_ops_manager +def test_mdb_created(some_mdb: MongoDB): + some_mdb.assert_reaches_phase(Phase.Running) + # TODO we need to enable backup for the mongodb - it's critical to make sure the backup for + # deployments continue to work correctly after upgrade + + +# This is a part 2 of the Operator upgrade test. Upgrades the Operator the latest development one and checks +# that everything works + + +@mark.e2e_operator_upgrade_ops_manager +def test_upgrade_operator(default_operator: Operator): + default_operator.assert_is_running() + + +@mark.e2e_operator_upgrade_ops_manager +def test_start_mongod_background_tester( + some_mdb_health_checker: MongoDBBackgroundTester, +): + some_mdb_health_checker.start() + + +@mark.e2e_operator_upgrade_ops_manager +def test_om_ok(ops_manager: MongoDBOpsManager): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=1200) + + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=400) + ops_manager.backup_status().assert_reaches_phase(Phase.Running, timeout=200) + + ops_manager.get_om_tester().assert_healthiness() + + +@mark.e2e_operator_upgrade_ops_manager +def test_some_mdb_ok(some_mdb: MongoDB, some_mdb_health_checker: MongoDBBackgroundTester): + # TODO make sure the backup is working when it's implemented + some_mdb.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + # The mongodb was supposed to be healthy all the time + some_mdb_health_checker.assert_healthiness() diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py new file mode 100644 index 000000000..f8fe9be20 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py @@ -0,0 +1,113 @@ +import pytest +from kubetester import MongoDB +from kubetester.certs import create_mongodb_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase +from kubetester.mongodb_user import MongoDBUser +from kubetester.operator import Operator +from pytest import fixture + +RS_NAME = "my-replica-set" +USER_PASSWORD = "/qwerty@!#:" +CERT_PREFIX = "prefix" + + +@fixture(scope="module") +def rs_certs_secret(namespace: str, issuer: str): + return create_mongodb_tls_certs(issuer, namespace, RS_NAME, "{}-{}-cert".format(CERT_PREFIX, RS_NAME)) + + +@fixture(scope="module") +def replica_set( + namespace: str, + issuer_ca_configmap: str, + rs_certs_secret: str, + custom_mdb_version: str, +) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set.yaml"), + namespace=namespace, + name=RS_NAME, + ) + resource.set_version(custom_mdb_version) + + # Make sure we persist in order to be able to upgrade gracefully + # and it is also faster. + resource["spec"]["persistent"] = True + + # TLS + resource.configure_custom_tls( + issuer_ca_configmap, + CERT_PREFIX, + ) + + # SCRAM-SHA + resource["spec"]["security"]["authentication"] = { + "enabled": True, + "modes": ["SCRAM"], + } + + return resource.create() + + +@fixture(scope="module") +def replica_set_user(replica_set: MongoDB) -> MongoDBUser: + """Creates a password secret and then the user referencing it""" + resource = MongoDBUser.from_yaml( + yaml_fixture("scram-sha-user.yaml"), + namespace=replica_set.namespace, + name="rs-user", + ) + resource["spec"]["mongodbResourceRef"]["name"] = replica_set.name + resource["spec"]["passwordSecretKeyRef"]["name"] = "rs-user-password" + resource["spec"]["username"] = "rs-user" + + print(f"\nCreating password for MongoDBUser {resource.name} in secret/{resource.get_secret_name()} ") + KubernetesTester.create_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + { + "password": USER_PASSWORD, + }, + ) + + yield resource.create() + + +@pytest.mark.e2e_operator_upgrade_replica_set +def test_install_latest_official_operator(official_operator: Operator): + official_operator.assert_is_running() + + +@pytest.mark.e2e_operator_upgrade_replica_set +def test_install_replicaset(replica_set: MongoDB): + replica_set.assert_reaches_phase(phase=Phase.Running) + + +@pytest.mark.e2e_operator_upgrade_replica_set +def test_replicaset_user_created(replica_set_user: MongoDBUser): + replica_set_user.assert_reaches_phase(Phase.Updated) + + +@pytest.mark.e2e_operator_upgrade_replica_set +def test_upgrade_operator(default_operator: Operator): + default_operator.assert_is_running() + + +@pytest.mark.e2e_operator_upgrade_replica_set +def test_replicaset_reconciled(replica_set: MongoDB): + replica_set.assert_abandons_phase(phase=Phase.Running, timeout=300) + replica_set.assert_reaches_phase(phase=Phase.Running, timeout=800) + + +@pytest.mark.e2e_operator_upgrade_replica_set +def test_replicaset_connectivity(replica_set: MongoDB, ca_path: str): + tester = replica_set.tester(use_ssl=True, ca_path=ca_path) + tester.assert_connectivity() + + # TODO refactor tester to flexibly test tls + custom CA + scram + # tester.assert_scram_sha_authentication( + # password=USER_PASSWORD, + # username="rs-user", + # auth_mechanism="SCRAM-SHA-256") diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py new file mode 100644 index 000000000..9300d8d72 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py @@ -0,0 +1,183 @@ +from typing import Dict + +import pytest +from kubetester import read_configmap +from kubetester.certs import create_sharded_cluster_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongotester import ShardedClusterTester +from kubetester.operator import Operator +from tests import test_logger +from tests.conftest import ( + LEGACY_DEPLOYMENT_STATE_VERSION, + install_official_operator, + log_deployments_info, +) + +MDB_RESOURCE = "sh001-base" +CERT_PREFIX = "prefix" + +logger = test_logger.get_test_logger(__name__) + +""" +e2e_operator_upgrade_sharded_cluster ensures the correct operation of a single cluster sharded cluster, when +upgrading/downgrading from/to the legacy state management (versions <= 1.27) and the current operator (from master) +while performing scaling operations. +It will always be pinned to version 1.27 (variable LEGACY_DEPLOYMENT_STATE_VERSION) for the initial deployment, so +in the future will test upgrade paths of multiple versions at a time (e.g 1.27 -> currently developed 1.30), even +though we don't officially support these paths. + +The workflow of this test is the following +Install Operator 1.27 -> Deploy Sharded Cluster -> Scale Up Cluster -> Upgrade operator (dev version) -> Scale down +-> Downgrade Operator to 1.27 -> Scale up +If the sharded cluster resource correctly reconciles after upgrade/downgrade and scaling steps, we assume it works +correctly. +""" + + +def log_state_configmap(namespace: str): + configmap_name = f"{MDB_RESOURCE}-state" + try: + state_configmap_data = read_configmap(namespace, configmap_name) + except Exception as e: + logger.error(f"Error when trying to read the configmap {configmap_name} in namespace {namespace}: {e}") + return + logger.debug(f"state_configmap_data: {state_configmap_data}") + + +# Fixtures +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str) -> str: + return create_sharded_cluster_certs( + namespace, + MDB_RESOURCE, + shards=1, + mongod_per_shard=3, + config_servers=3, + mongos=2, + secret_prefix=f"{CERT_PREFIX}-", + ) + + +@pytest.fixture(scope="module") +def sharded_cluster( + issuer_ca_configmap: str, + namespace: str, + server_certs: str, + custom_mdb_version: str, +): + resource = MongoDB.from_yaml( + yaml_fixture("sharded-cluster.yaml"), + namespace=namespace, + name=MDB_RESOURCE, + ) + resource.set_version(custom_mdb_version) + resource["spec"]["mongodsPerShardCount"] = 2 + resource["spec"]["configServerCount"] = 2 + resource["spec"]["mongosCount"] = 1 + resource["spec"]["persistent"] = True + resource.configure_custom_tls(issuer_ca_configmap, CERT_PREFIX) + + return resource.update() + + +@pytest.mark.e2e_operator_upgrade_sharded_cluster +class TestShardedClusterDeployment: + def test_install_latest_official_operator( + self, + namespace: str, + managed_security_context: str, + operator_installation_config: Dict[str, str], + ): + logger.info( + f"Installing the official operator from helm charts, with version {LEGACY_DEPLOYMENT_STATE_VERSION}" + ) + operator = install_official_operator( + namespace, + managed_security_context, + operator_installation_config, + central_cluster_name=None, # These 4 fields apply to multi cluster operator only + central_cluster_client=None, + member_cluster_clients=None, + member_cluster_names=None, + custom_operator_version=LEGACY_DEPLOYMENT_STATE_VERSION, + ) + operator.assert_is_running() + # Dumping deployments in logs ensures we are using the correct operator version + log_deployments_info(namespace) + + def test_create_sharded_cluster(self, sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(phase=Phase.Running, timeout=350) + + def test_scale_up_sharded_cluster(self, sharded_cluster: MongoDB): + sharded_cluster.load() + sharded_cluster["spec"]["mongodsPerShardCount"] = 3 + sharded_cluster["spec"]["configServerCount"] = 3 + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(phase=Phase.Running, timeout=300) + + +@pytest.mark.e2e_operator_upgrade_sharded_cluster +class TestOperatorUpgrade: + def test_upgrade_operator(self, default_operator: Operator, namespace: str): + logger.info("Installing the operator built from master") + default_operator.assert_is_running() + # Dumping deployments in logs ensures we are using the correct operator version + log_deployments_info(namespace) + + def test_sharded_cluster_reconciled(self, sharded_cluster: MongoDB, namespace: str): + sharded_cluster.assert_abandons_phase(phase=Phase.Running, timeout=200) + sharded_cluster.assert_reaches_phase(phase=Phase.Running, timeout=500) + logger.debug("State configmap after upgrade") + log_state_configmap(namespace) + + def test_assert_connectivity(self, ca_path: str): + ShardedClusterTester(MDB_RESOURCE, 1, ssl=True, ca_path=ca_path).assert_connectivity() + + def test_scale_down_sharded_cluster(self, sharded_cluster: MongoDB, namespace: str): + sharded_cluster.load() + # Scale down both by 1 + sharded_cluster["spec"]["mongodsPerShardCount"] = 2 + sharded_cluster["spec"]["configServerCount"] = 2 + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(phase=Phase.Running, timeout=450) + logger.debug("State configmap after upgrade and scaling") + log_state_configmap(namespace) + + +@pytest.mark.e2e_operator_upgrade_sharded_cluster +class TestOperatorDowngrade: + def test_downgrade_operator( + self, + namespace: str, + managed_security_context: str, + operator_installation_config: Dict[str, str], + ): + logger.info(f"Downgrading the operator to version {LEGACY_DEPLOYMENT_STATE_VERSION}, from helm chart release") + operator = install_official_operator( + namespace, + managed_security_context, + operator_installation_config, + central_cluster_name=None, # These 4 fields apply to multi cluster operator only + central_cluster_client=None, + member_cluster_clients=None, + member_cluster_names=None, + custom_operator_version=LEGACY_DEPLOYMENT_STATE_VERSION, + ) + operator.assert_is_running() + # Dumping deployments in logs ensures we are using the correct operator version + log_deployments_info(namespace) + + def test_sharded_cluster_reconciled(self, sharded_cluster: MongoDB): + sharded_cluster.assert_abandons_phase(phase=Phase.Running, timeout=200) + sharded_cluster.assert_reaches_phase(phase=Phase.Running, timeout=850, ignore_errors=True) + + def test_assert_connectivity(self, ca_path: str): + ShardedClusterTester(MDB_RESOURCE, 1, ssl=True, ca_path=ca_path).assert_connectivity() + + def test_scale_up_sharded_cluster(self, sharded_cluster: MongoDB): + sharded_cluster.load() + sharded_cluster["spec"]["mongodsPerShardCount"] = 3 + sharded_cluster["spec"]["configServerCount"] = 3 + sharded_cluster.update() + sharded_cluster.assert_reaches_phase(phase=Phase.Running, timeout=350) diff --git a/docker/mongodb-enterprise-tests/tests/users/__init__.py b/docker/mongodb-enterprise-tests/tests/users/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/users/conftest.py b/docker/mongodb-enterprise-tests/tests/users/conftest.py new file mode 100644 index 000000000..cb0664057 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/conftest.py @@ -0,0 +1,4 @@ +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if "default_operator" not in item.fixturenames: + item.fixturenames.insert(0, "default_operator") diff --git a/docker/mongodb-enterprise-tests/tests/users/fixtures/user_scram.yaml b/docker/mongodb-enterprise-tests/tests/users/fixtures/user_scram.yaml new file mode 100644 index 000000000..174ffef5b --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/fixtures/user_scram.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: scram-user +spec: + passwordSecretKeyRef: + name: secret + # Match to metadata.name of the User Secret + key: password + username: "username" + db: "admin" # + mongodbResourceRef: + name: "my-replica-set" + # Match to MongoDB resource using authenticaiton + roles: + - db: "admin" + name: "clusterAdmin" + - db: "admin" + name: "userAdminAnyDatabase" + - db: "admin" + name: "readWrite" diff --git a/docker/mongodb-enterprise-tests/tests/users/fixtures/user_with_roles.yaml b/docker/mongodb-enterprise-tests/tests/users/fixtures/user_with_roles.yaml new file mode 100644 index 000000000..5322e836f --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/fixtures/user_with_roles.yaml @@ -0,0 +1,13 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: user-with-roles +spec: + username: "CN=mms-user-1,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + opsManager: + configMapRef: + name: my-project + roles: + - db: "admin" + name: "clusterAdmin" diff --git a/docker/mongodb-enterprise-tests/tests/users/fixtures/users_multiple.yaml b/docker/mongodb-enterprise-tests/tests/users/fixtures/users_multiple.yaml new file mode 100644 index 000000000..1c054fdc6 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/fixtures/users_multiple.yaml @@ -0,0 +1,73 @@ +--- +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: mms-user-1 + spec: + username: "CN=mms-user-1,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + mongodbResourceRef: + name: test-x509-rs + roles: + - db: "admin" + name: "clusterAdmin" +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: mms-user-2 + spec: + username: "CN=mms-user-2,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + mongodbResourceRef: + name: test-x509-rs + roles: + - db: "admin" + name: "clusterManager" +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: mms-user-3 + spec: + username: "CN=mms-user-3,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + mongodbResourceRef: + name: test-x509-rs + roles: + - db: "admin" + name: "clusterAdmin" +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: mms-user-4 + spec: + username: "CN=mms-user-4,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + mongodbResourceRef: + name: test-x509-rs + roles: + - db: "admin" + name: "clusterManager" +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: mms-user-5 + spec: + username: "CN=mms-user-5,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + mongodbResourceRef: + name: test-x509-rs + roles: + - db: "admin" + name: "clusterAdmin" +- apiVersion: mongodb.com/v1 + kind: MongoDBUser + metadata: + name: mms-user-6 + spec: + username: "CN=mms-user-6,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" + db: "$external" + mongodbResourceRef: + name: test-x509-rs + roles: + - db: "admin" + name: "clusterManager" diff --git a/docker/mongodb-enterprise-tests/tests/users/users_addition_removal.py b/docker/mongodb-enterprise-tests/tests/users/users_addition_removal.py new file mode 100644 index 000000000..558707c16 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/users_addition_removal.py @@ -0,0 +1,115 @@ +import pytest +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.x509.oid import NameOID +from kubetester import find_fixture +from kubetester.certs import ( + ISSUER_CA_NAME, + Certificate, + create_agent_tls_certs, + create_mongodb_tls_certs, +) +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase + +MDB_RESOURCE = "test-x509-rs" +NUM_AGENTS = 2 + + +def get_subjects(start, end): + return [f"CN=mms-user-{i},OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" for i in range(start, end)] + + +@pytest.fixture(scope="module") +def server_certs(issuer: str, namespace: str): + return create_mongodb_tls_certs(ISSUER_CA_NAME, namespace, MDB_RESOURCE, f"{MDB_RESOURCE}-cert") + + +def get_names_from_certificate_attributes(cert): + names = {} + subject = cert.subject + names["OU"] = subject.get_attributes_for_oid(NameOID.ORGANIZATIONAL_UNIT_NAME)[0].value + names["CN"] = subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + + return names + + +@pytest.fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_agent_tls_certs(issuer, namespace, MDB_RESOURCE) + + +@pytest.fixture(scope="module") +def mdb(namespace: str, server_certs: str, agent_certs: str, issuer_ca_configmap: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("test-x509-rs.yaml"), namespace=namespace) + res["spec"]["security"]["tls"]["ca"] = issuer_ca_configmap + return res.create() + + +@pytest.mark.e2e_tls_x509_users_addition_removal +class TestReplicaSetUpgradeToTLSWithX509Project(KubernetesTester): + def test_mdb_resource_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=300) + + def test_certificates_have_sane_subject(self, namespace): + agent_certs = KubernetesTester.read_secret(namespace, "agent-certs-pem") + + bytecert = bytes(agent_certs["mms-automation-agent-pem"], "utf-8") + cert = x509.load_pem_x509_certificate(bytecert, default_backend()) + names = get_names_from_certificate_attributes(cert) + + assert names["CN"] == "mms-automation-agent" + assert names["OU"] == namespace + + +@pytest.mark.e2e_tls_x509_users_addition_removal +class TestMultipleUsersAreAdded(KubernetesTester): + """ + name: Test users are added correctly + create_many: + file: users_multiple.yaml + wait_until: all_users_ready + """ + + def test_users_ready(self): + pass + + @staticmethod + def all_users_ready(): + ac = KubernetesTester.get_automation_config() + return len(ac["auth"]["usersWanted"]) == 6 # 6 MongoDBUsers + + def test_users_are_added_to_automation_config(self): + ac = KubernetesTester.get_automation_config() + existing_users = sorted(ac["auth"]["usersWanted"], key=lambda user: user["user"]) + expected_users = sorted(get_subjects(1, 7)) + existing_subjects = [u["user"] for u in ac["auth"]["usersWanted"]] + + for expected in expected_users: + assert expected in existing_subjects + + +@pytest.mark.e2e_tls_x509_users_addition_removal +class TestTheCorrectUserIsDeleted(KubernetesTester): + """ + delete: + delete_name: mms-user-4 + file: users_multiple.yaml + wait_until: user_has_been_deleted + """ + + @staticmethod + def user_has_been_deleted(): + ac = KubernetesTester.get_automation_config() + return len(ac["auth"]["usersWanted"]) == 5 # One user has been deleted + + def test_deleted_user_is_gone(self): + ac = KubernetesTester.get_automation_config() + users = ac["auth"]["usersWanted"] + assert "CN=mms-user-4,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US" not in [user["user"] for user in users] + + +def get_user_pkix_names(ac, agent_name: str) -> str: + subject = [u["user"] for u in ac["auth"]["usersWanted"] if agent_name in u["user"]][0] + return dict(name.split("=") for name in subject.split(",")) diff --git a/docker/mongodb-enterprise-tests/tests/users/users_finalizer.py b/docker/mongodb-enterprise-tests/tests/users/users_finalizer.py new file mode 100644 index 000000000..55213e060 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/users_finalizer.py @@ -0,0 +1,100 @@ +import time + +import pytest +from kubernetes import client +from kubernetes.client.exceptions import ApiException +from kubetester import create_or_update_secret, find_fixture +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser + +USER_PASSWORD = "my-password" +RESOURCE_NAME = "my-replica-set" + + +@pytest.fixture(scope="module") +def mdb(namespace: str, custom_mdb_version: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("replica-set-scram-sha-256.yaml"), namespace=namespace) + res.set_version(custom_mdb_version) + return res.update() + + +@pytest.fixture(scope="module") +def scram_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(find_fixture("user_scram.yaml"), namespace=namespace) + + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + {"password": USER_PASSWORD}, + ) + + return resource.update() + + +@pytest.mark.e2e_users_finalizer +class TestReplicaSetIsRunning(KubernetesTester): + + def test_mdb_resource_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=300) + + +@pytest.mark.e2e_users_finalizer +class TestUserIsAdded(KubernetesTester): + + def test_user_is_ready(mdb: MongoDB, scram_user: MongoDBUser): + scram_user.assert_reaches_phase(Phase.Updated) + + ac = KubernetesTester.get_automation_config() + assert len(ac["auth"]["usersWanted"]) == 1 + + def test_users_are_added_to_automation_config(self): + ac = KubernetesTester.get_automation_config() + + assert ac["auth"]["usersWanted"][0]["user"] == "username" + + def test_user_has_finalizer(self, scram_user: MongoDBUser): + scram_user.load() + finalizers = scram_user["metadata"]["finalizers"] + + assert finalizers[0] == "mongodb.com/v1.userRemovalFinalizer" + + +@pytest.mark.e2e_users_finalizer +class TestTheDeletedUserRemainsInCluster(KubernetesTester): + + def test_deleted_user_has_deletion_timestamp(self): + resource = MongoDBUser.from_yaml(find_fixture("user_scram.yaml"), namespace="mongodb-test") + resource.load() + resource.delete() + resource.reload() + + finalizers = resource["metadata"]["finalizers"] + + assert finalizers[0] == "mongodb.com/v1.userRemovalFinalizer" + assert resource["metadata"]["deletionTimestamp"] != None + + +@pytest.mark.e2e_users_finalizer +class TestCleanupIdPerformedBeforeDeletingUser(KubernetesTester): + """ + delete: + file: user_scram.yaml + wait_until: finalizer_is_removed + """ + + @staticmethod + def finalizer_is_removed(): + resource = MongoDBUser.from_yaml(find_fixture("user_scram.yaml"), namespace="mongodb-test") + try: + resource.load() + except ApiException: + return True + + return resource["metadata"]["finalizers"] == [] + + def test_deleted_user_is_removed_from_automation_config(self): + ac = KubernetesTester.get_automation_config() + users = ac["auth"]["usersWanted"] + assert "username" not in [user["user"] for user in users] diff --git a/docker/mongodb-enterprise-tests/tests/users/users_finalizer_removal.py b/docker/mongodb-enterprise-tests/tests/users/users_finalizer_removal.py new file mode 100644 index 000000000..3761d5f0d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/users_finalizer_removal.py @@ -0,0 +1,93 @@ +import time + +import pytest +from kubernetes import client +from kubernetes.client.exceptions import ApiException +from kubetester import create_or_update_secret, find_fixture +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as load_fixture +from kubetester.mongodb import MongoDB, Phase +from kubetester.mongodb_user import MongoDBUser + +USER_PASSWORD = "my-password" +RESOURCE_NAME = "my-replica-set" + + +@pytest.fixture(scope="module") +def mdb(namespace: str, custom_mdb_version: str) -> MongoDB: + res = MongoDB.from_yaml(load_fixture("replica-set-scram-sha-256.yaml"), namespace=namespace) + res.set_version(custom_mdb_version) + return res.update() + + +@pytest.fixture(scope="module") +def scram_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(find_fixture("user_scram.yaml"), namespace=namespace) + + create_or_update_secret( + KubernetesTester.get_namespace(), + resource.get_secret_name(), + {"password": USER_PASSWORD}, + ) + + return resource.update() + + +@pytest.mark.e2e_users_finalizer_removal +class TestReplicaSetIsRunning(KubernetesTester): + + def test_mdb_resource_running(self, mdb: MongoDB): + mdb.assert_reaches_phase(Phase.Running, timeout=300) + + +@pytest.mark.e2e_users_finalizer_removal +class TestUserIsAdded(KubernetesTester): + + def test_user_is_ready(mdb: MongoDB, scram_user: MongoDBUser): + scram_user.assert_reaches_phase(Phase.Updated) + + ac = KubernetesTester.get_automation_config() + assert len(ac["auth"]["usersWanted"]) == 1 + + +@pytest.mark.e2e_users_finalizer_removal +class TestReplicaSetIsDleted(KubernetesTester): + """ + delete: + file: replica-set-scram-sha-256.yaml + wait_until: mongo_resource_deleted + """ + + def test_replica_set_sts_doesnt_exist(self): + """The StatefulSet must be removed by Kubernetes as soon as the MongoDB resource is removed. + Note, that this may lag sometimes (caching or whatever?) and it's more safe to wait a bit""" + time.sleep(15) + with pytest.raises(client.rest.ApiException): + self.appsv1.read_namespaced_stateful_set(RESOURCE_NAME, self.namespace) + + def test_service_does_not_exist(self): + with pytest.raises(client.rest.ApiException): + self.corev1.read_namespaced_service(RESOURCE_NAME + "-svc", self.namespace) + + +@pytest.mark.e2e_users_finalizer_removal +class TestUserIsRemovedAfterMongoDBIsDeleted(KubernetesTester): + """ + delete: + file: user_scram.yaml + wait_until: finalizer_is_removed + """ + + @staticmethod + def finalizer_is_removed(): + resource = MongoDBUser.from_yaml(find_fixture("user_scram.yaml"), namespace="mongodb-test") + try: + resource.load() + except ApiException: + return True + + return resource["metadata"]["finalizers"] == [] + + def test_user_is_deleted(self): + ac = KubernetesTester.get_automation_config() + assert len(ac["auth"]["usersWanted"]) == 0 diff --git a/docker/mongodb-enterprise-tests/tests/users/users_schema_validation.py b/docker/mongodb-enterprise-tests/tests/users/users_schema_validation.py new file mode 100644 index 000000000..0cc5301ed --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/users/users_schema_validation.py @@ -0,0 +1,43 @@ +import pytest +from kubetester.kubetester import KubernetesTester + + +class TestUsersSchemaValidationUserName(KubernetesTester): + """ + name: Validation for mongodbusers (username) + create: + file: user_with_roles.yaml + patch: '[{"op":"remove","path":"/spec/username"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_users_schema_validation +class TestUsersSchemaValidationRoleName(KubernetesTester): + """ + name: Validation for mongodbusers (role name) + create: + file: user_with_roles.yaml + patch: '[{"op":"remove","path":"/spec/roles/0/name"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True + + +@pytest.mark.e2e_users_schema_validation +class TestUsersSchemaValidationRoleDb(KubernetesTester): + """ + name: Validation for mongodbusers (role db) + create: + file: user_with_roles.yaml + patch: '[{"op":"remove","path":"/spec/roles/0/db"}]' + exception: 'Unprocessable Entity' + """ + + def test_validation_ok(self): + assert True diff --git a/docker/mongodb-enterprise-tests/tests/vaultintegration/__init__.py b/docker/mongodb-enterprise-tests/tests/vaultintegration/__init__.py new file mode 100644 index 000000000..501f71972 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/vaultintegration/__init__.py @@ -0,0 +1,61 @@ +from typing import Dict, List, Optional + +from kubetester.kubetester import KubernetesTester + + +def run_command_in_vault( + vault_namespace: str, + vault_name: str, + cmd: List[str], + expected_message: Optional[List[str]] = None, +) -> str: + result = KubernetesTester.run_command_in_pod_container( + pod_name=f"{vault_name}-0", + namespace=vault_namespace, + cmd=cmd, + container="vault", + ) + + if expected_message is None: + expected_message = ["Success!"] + + if not expected_message: + return result + + for message in expected_message: + if message in result: + return result + + raise ValueError(f"Command failed. Got {result}") + + +def vault_sts_name() -> str: + return "vault" + + +def vault_namespace_name() -> str: + return "vault" + + +def store_secret_in_vault(vault_namespace: str, vault_name: str, secretData: Dict[str, str], path: str): + cmd = ["vault", "kv", "put", path] + + for k, v in secretData.items(): + cmd.append(f"{k}={v}") + + run_command_in_vault( + vault_namespace, + vault_name, + cmd, + expected_message=["created_time"], + ) + + +def assert_secret_in_vault(vault_namespace: str, vault_name: str, path: str, expected_message: List[str]): + cmd = [ + "vault", + "kv", + "get", + path, + ] + run_command_in_vault(vault_namespace, vault_name, cmd, expected_message) diff --git a/docker/mongodb-enterprise-tests/tests/vaultintegration/conftest.py b/docker/mongodb-enterprise-tests/tests/vaultintegration/conftest.py new file mode 100644 index 000000000..3f38108a1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/vaultintegration/conftest.py @@ -0,0 +1,155 @@ +import time + +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import get_pod_when_ready, get_pod_when_running +from kubetester.certs import create_vault_certs +from kubetester.helm import helm_install_from_chart +from kubetester.kubetester import KubernetesTester +from pytest import fixture + +from . import run_command_in_vault, vault_namespace_name, vault_sts_name + + +@fixture(scope="module") +def vault(namespace: str, version="v0.17.1", name="vault") -> str: + + try: + KubernetesTester.create_namespace(name) + except ApiException as e: + pass + helm_install_from_chart( + namespace=name, + release=name, + chart=f"hashicorp/vault", + version=version, + custom_repo=("hashicorp", "https://helm.releases.hashicorp.com"), + ) + + # check if vault pod is running + # We need to perform the initialization of the vault in order to the pod + # to be ready. But we need to wait for it to be running in order to run commands in it. + get_pod_when_running( + namespace=name, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name={name}", + ) + perform_vault_initialization(name, name) + + # check if vault pod is ready + get_pod_when_ready( + namespace=name, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name={name}", + ) + + # check if vault agent injector pod is ready + get_pod_when_ready( + namespace=name, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name=vault-agent-injector", + ) + + return name + + +@fixture(scope="module") +def vault_tls(namespace: str, issuer: str, vault_namespace: str, version="v0.17.1", name="vault") -> str: + + try: + KubernetesTester.create_namespace(vault_namespace) + except ApiException as e: + pass + create_vault_certs(namespace, issuer, vault_namespace, name, "vault-tls") + + helm_install_from_chart( + namespace=name, + release=name, + chart=f"hashicorp/vault", + version=version, + custom_repo=("hashicorp", "https://helm.releases.hashicorp.com"), + override_path="/tests/vaultconfig/override.yaml", + ) + + # check if vault pod is running + get_pod_when_running( + namespace=name, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name={name}", + ) + perform_vault_initialization(name, name) + + # check if vault pod is ready + get_pod_when_ready( + namespace=name, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name={name}", + ) + + # check if vault agent injector pod is ready + get_pod_when_ready( + namespace=name, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name=vault-agent-injector", + ) + + +def perform_vault_initialization(namespace: str, name: str): + + # If the pod is already ready, this means we don't have to perform anything + pods = client.CoreV1Api().list_namespaced_pod( + namespace, + label_selector=f"app.kubernetes.io/instance={name},app.kubernetes.io/name={name}", + ) + + pod = pods.items[0] + if pod.status.conditions is not None: + for condition in pod.status.conditions: + if condition.type == "Ready" and condition.status == "True": + return + + run_command_in_vault(name, name, ["mkdir", "-p", "/vault/data"], []) + + response = run_command_in_vault(name, name, ["vault", "operator", "init"], ["Unseal"]) + + response = response.split("\n") + unseal_keys = [] + for i in range(5): + unseal_keys.append(response[i].split(": ")[1]) + + for i in range(3): + run_command_in_vault(name, name, ["vault", "operator", "unseal", unseal_keys[i]], []) + + token = response[6].split(": ")[1] + run_command_in_vault(name, name, ["vault", "login", token]) + + run_command_in_vault(name, name, ["vault", "secrets", "enable", "-path=secret/", "kv-v2"]) + + +@fixture(scope="module") +def vault_url(vault_name: str, vault_namespace: str) -> str: + return f"{vault_name}.{vault_namespace}.svc.cluster.local" + + +@fixture(scope="module") +def vault_namespace() -> str: + return vault_namespace_name() + + +@fixture(scope="module") +def vault_operator_policy_name() -> str: + return "mongodbenterprise" + + +@fixture(scope="module") +def vault_name() -> str: + return vault_sts_name() + + +@fixture(scope="module") +def vault_database_policy_name() -> str: + return "mongodbenterprisedatabase" + + +@fixture(scope="module") +def vault_appdb_policy_name() -> str: + return "mongodbenterpriseappdb" + + +@fixture(scope="module") +def vault_om_policy_name() -> str: + return "mongodbenterpriseopsmanager" diff --git a/docker/mongodb-enterprise-tests/tests/vaultintegration/mongodb_deployment_vault.py b/docker/mongodb-enterprise-tests/tests/vaultintegration/mongodb_deployment_vault.py new file mode 100644 index 000000000..165a0fea4 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/vaultintegration/mongodb_deployment_vault.py @@ -0,0 +1,501 @@ +import time +import uuid + +import kubetester +import pytest +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import ( + MongoDB, + create_configmap, + delete_secret, + get_statefulset, + random_k8s_name, + read_secret, +) +from kubetester.certs import ( + create_mongodb_tls_certs, + create_x509_agent_tls_certs, + create_x509_mongodb_tls_certs, +) +from kubetester.http import https_endpoint_is_reachable +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static +from kubetester.mongodb import Phase, get_pods +from kubetester.mongodb_user import MongoDBUser +from kubetester.operator import Operator +from pytest import fixture, mark + +from . import run_command_in_vault, store_secret_in_vault + +OPERATOR_NAME = "mongodb-enterprise-operator" +MDB_RESOURCE = "my-replica-set" +DATABASE_SA_NAME = "mongodb-enterprise-database-pods" +USER_NAME = "my-user-1" +PASSWORD_SECRET_NAME = "mms-user-1-password" +USER_PASSWORD = "my-password" + + +def certs_for_prometheus(issuer: str, namespace: str, resource_name: str) -> str: + secret_name = random_k8s_name(resource_name + "-") + "-prometheus-cert" + + return create_mongodb_tls_certs( + issuer, + namespace, + resource_name, + secret_name, + secret_backend="Vault", + vault_subpath="database", + ) + + +@fixture(scope="module") +def replica_set( + namespace: str, + custom_mdb_version: str, + server_certs: str, + agent_certs: str, + clusterfile_certs: str, + issuer_ca_configmap: str, + issuer: str, + vault_namespace: str, + vault_name: str, +) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), MDB_RESOURCE, namespace) + resource.set_version(custom_mdb_version) + resource["spec"]["security"] = { + "tls": {"enabled": True, "ca": issuer_ca_configmap}, + "authentication": { + "enabled": True, + "modes": ["X509", "SCRAM"], + "agents": {"mode": "X509"}, + "internalCluster": "X509", + }, + } + + prom_cert_secret = certs_for_prometheus(issuer, namespace, resource.name) + store_secret_in_vault( + vault_namespace, + vault_name, + {"password": "prom-password"}, + f"secret/mongodbenterprise/operator/{namespace}/prom-password", + ) + resource["spec"]["prometheus"] = { + "username": "prom-user", + "passwordSecretRef": { + "name": "prom-password", + }, + "tlsSecretKeyRef": { + "name": prom_cert_secret, + }, + } + resource.create() + + return resource + + +@fixture(scope="module") +def agent_certs(issuer: str, namespace: str) -> str: + return create_x509_agent_tls_certs(issuer, namespace, MDB_RESOURCE, secret_backend="Vault") + + +@fixture(scope="module") +def server_certs(vault_namespace: str, vault_name: str, namespace: str, issuer: str) -> str: + create_x509_mongodb_tls_certs( + issuer, + namespace, + MDB_RESOURCE, + f"{MDB_RESOURCE}-cert", + secret_backend="Vault", + vault_subpath="database", + ) + + +@fixture(scope="module") +def clusterfile_certs(vault_namespace: str, vault_name: str, namespace: str, issuer: str) -> str: + create_x509_mongodb_tls_certs( + issuer, + namespace, + MDB_RESOURCE, + f"{MDB_RESOURCE}-clusterfile", + secret_backend="Vault", + vault_subpath="database", + ) + + +@fixture(scope="module") +def sharded_cluster_configmap(namespace: str) -> str: + cm = KubernetesTester.read_configmap(namespace, "my-project") + epoch_time = int(time.time()) + project_name = "sharded-" + str(epoch_time) + "-" + uuid.uuid4().hex[0:10] + data = { + "baseUrl": cm["baseUrl"], + "projectName": project_name, + "orgId": cm["orgId"], + } + create_configmap(namespace=namespace, name=project_name, data=data) + return project_name + + +@fixture(scope="module") +def sharded_cluster( + namespace: str, + sharded_cluster_configmap: str, + issuer: str, + vault_namespace: str, + vault_name: str, +) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("sharded-cluster.yaml"), namespace=namespace) + resource["spec"]["cloudManager"]["configMapRef"]["name"] = sharded_cluster_configmap + + # Password stored in Prometheus + store_secret_in_vault( + vault_namespace, + vault_name, + {"password": "prom-password"}, + f"secret/mongodbenterprise/operator/{namespace}/prom-password-cluster", + ) + + # A prometheus certificate stored in Vault + prom_cert_secret = certs_for_prometheus(issuer, namespace, resource.name) + resource["spec"]["prometheus"] = { + "username": "prom-user", + "passwordSecretRef": { + "name": "prom-password-cluster", + }, + "tlsSecretKeyRef": { + "name": prom_cert_secret, + }, + } + + return resource.create() + + +@fixture(scope="module") +def mongodb_user(namespace: str) -> MongoDBUser: + resource = MongoDBUser.from_yaml(yaml_fixture("mongodb-user.yaml"), "vault-replica-set-scram-user", namespace) + + resource["spec"]["username"] = USER_NAME + resource["spec"]["passwordSecretKeyRef"] = { + "name": PASSWORD_SECRET_NAME, + "key": "password", + } + + resource["spec"]["mongodbResourceRef"]["name"] = MDB_RESOURCE + resource["spec"]["mongodbResourceRef"]["namespace"] = namespace + return resource.create() + + +@mark.e2e_vault_setup +def test_vault_creation(vault: str, vault_name: str, vault_namespace: str): + vault + + # assert if vault statefulset is ready, this is sort of redundant(we already assert for pod phase) + # but this is basic assertion at the moment, will remove in followup PR + sts = get_statefulset(namespace=vault_namespace, name=vault_name) + assert sts.status.ready_replicas == 1 + + +@mark.e2e_vault_setup +def test_create_vault_operator_policy(vault_name: str, vault_namespace: str): + # copy hcl file from local machine to pod + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/operator-policy.hcl", + "/tmp/operator-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprise", + "/tmp/operator-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup +def test_enable_kubernetes_auth(vault_name: str, vault_namespace: str): + # enable Kubernetes auth for Vault + cmd = [ + "vault", + "auth", + "enable", + "kubernetes", + ] + + run_command_in_vault( + vault_namespace, + vault_name, + cmd, + expected_message=["Success!", "path is already in use at kubernetes"], + ) + + cmd = [ + "cat", + "/var/run/secrets/kubernetes.io/serviceaccount/token", + ] + + token = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + + cmd = ["env"] + + response = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + + response = response.split("\n") + for line in response: + l = line.strip() + if str.startswith(l, "KUBERNETES_PORT_443_TCP_ADDR"): + cluster_ip = l.split("=")[1] + break + + cmd = [ + "vault", + "write", + "auth/kubernetes/config", + f"token_reviewer_jwt={token}", + f"kubernetes_host=https://{cluster_ip}:443", + "kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "disable_iss_validation=true", + ] + + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup +def test_enable_vault_role_for_operator_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_operator_policy_name: str, +): + cmd = [ + "vault", + "write", + "auth/kubernetes/role/mongodbenterprise", + f"bound_service_account_names={OPERATOR_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_operator_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup +def test_operator_install_with_vault_backend(operator_vault_secret_backend: Operator): + operator_vault_secret_backend.assert_is_running() + + +@mark.e2e_vault_setup +def test_vault_config_map_exists(namespace: str): + # no exception should be raised + KubernetesTester.read_configmap(namespace, name="secret-configuration") + + +@mark.e2e_vault_setup +def test_store_om_credentials_in_vault(vault_namespace: str, vault_name: str, namespace: str): + credentials = read_secret(namespace, "my-credentials") + store_secret_in_vault( + vault_namespace, + vault_name, + credentials, + f"secret/mongodbenterprise/operator/{namespace}/my-credentials", + ) + + cmd = [ + "vault", + "kv", + "get", + f"secret/mongodbenterprise/operator/{namespace}/my-credentials", + ] + run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=["publicApiKey"]) + delete_secret(namespace, "my-credentials") + + +@mark.e2e_vault_setup +def test_create_database_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/database-policy.hcl", + "/tmp/database-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprisedatabase", + "/tmp/database-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup +def test_enable_vault_role_for_database_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_database_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_database_policy_name}", + f"bound_service_account_names={DATABASE_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_database_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup +def test_mdb_created(replica_set: MongoDB, namespace: str): + replica_set.assert_reaches_phase(Phase.Running, timeout=500, ignore_errors=True) + for pod_name in get_pods(MDB_RESOURCE + "-{}", 3): + pod = client.CoreV1Api().read_namespaced_pod(pod_name, namespace) + if is_default_architecture_static(): + assert len(pod.spec.containers) == 3 + else: + assert len(pod.spec.containers) == 2 + + +@mark.e2e_vault_setup +def test_rotate_server_certs(replica_set: MongoDB, vault_namespace: str, vault_name: str, namespace: str, issuer: str): + replica_set.load() + old_version = replica_set["metadata"]["annotations"]["agent-certs"] + + create_x509_mongodb_tls_certs( + issuer, + namespace, + MDB_RESOURCE, + f"{MDB_RESOURCE}-cert", + secret_backend="Vault", + vault_subpath="database", + ) + + replica_set.assert_abandons_phase(Phase.Running, timeout=600) + + def wait_for_server_certs() -> bool: + replica_set.load() + return old_version != replica_set["metadata"]["annotations"]["my-replica-set-cert"] + + kubetester.wait_until(wait_for_server_certs, timeout=600, sleep_time=10) + replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + + +@mark.e2e_vault_setup +def test_rotate_server_certs_with_sts_restarting( + replica_set: MongoDB, vault_namespace: str, vault_name: str, namespace: str, issuer: str +): + replica_set.trigger_sts_restart() + create_x509_mongodb_tls_certs( + issuer, + namespace, + MDB_RESOURCE, + f"{MDB_RESOURCE}-cert", + secret_backend="Vault", + vault_subpath="database", + ) + + replica_set.assert_reaches_phase(Phase.Running, timeout=900, ignore_errors=True) + + +@mark.e2e_vault_setup +def test_rotate_agent_certs(replica_set: MongoDB, vault_namespace: str, vault_name: str, namespace: str): + replica_set.load() + old_version = replica_set["metadata"]["annotations"]["agent-certs"] + cmd = [ + "vault", + "kv", + "patch", + f"secret/mongodbenterprise/database/{namespace}/agent-certs", + "foo=bar", + ] + run_command_in_vault(vault_namespace, vault_name, cmd, ["version"]) + replica_set.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + + def wait_for_agent_certs() -> bool: + replica_set.load() + return old_version != replica_set["metadata"]["annotations"]["agent-certs"] + + kubetester.wait_until(wait_for_agent_certs, timeout=600, sleep_time=10) + + +@mark.e2e_vault_setup +def test_no_certs_in_kubernetes(namespace: str): + with pytest.raises(ApiException): + read_secret(namespace, f"{MDB_RESOURCE}-clusterfile") + with pytest.raises(ApiException): + read_secret(namespace, f"{MDB_RESOURCE}-cert") + with pytest.raises(ApiException): + read_secret(namespace, "agent-certs") + + +@mark.e2e_vault_setup +def test_api_key_in_pod(replica_set: MongoDB): + cmd = ["cat", "/mongodb-automation/agent-api-key/agentApiKey"] + + result = KubernetesTester.run_command_in_pod_container( + pod_name=f"{replica_set.name}-0", + namespace=replica_set.namespace, + cmd=cmd, + container="mongodb-enterprise-database", + ) + + assert result != "" + + +@mark.e2e_vault_setup +def test_prometheus_endpoint_on_replica_set(replica_set: MongoDB, namespace: str): + members = replica_set["spec"]["members"] + name = replica_set.name + + auth = ("prom-user", "prom-password") + + for idx in range(members): + member_url = f"https://{name}-{idx}.{name}-svc.{namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(member_url, auth, tls_verify=False) + + +@mark.e2e_vault_setup +def test_sharded_mdb_created(sharded_cluster: MongoDB): + sharded_cluster.assert_reaches_phase(Phase.Running, timeout=600, ignore_errors=True) + + +@mark.e2e_vault_setup +def test_prometheus_endpoint_works_on_every_pod_on_the_cluster(sharded_cluster: MongoDB, namespace: str): + auth = ("prom-user", "prom-password") + name = sharded_cluster.name + + mongos_count = sharded_cluster["spec"]["mongosCount"] + for idx in range(mongos_count): + url = f"https://{name}-mongos-{idx}.{name}-svc.{namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + shard_count = sharded_cluster["spec"]["shardCount"] + mongodbs_per_shard_count = sharded_cluster["spec"]["mongodsPerShardCount"] + for shard in range(shard_count): + for mongodb in range(mongodbs_per_shard_count): + url = f"https://{name}-{shard}-{mongodb}.{name}-sh.{namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + config_server_count = sharded_cluster["spec"]["configServerCount"] + for idx in range(config_server_count): + url = f"https://{name}-config-{idx}.{name}-cs.{namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + +@mark.e2e_vault_setup +def test_create_mongodb_user(mongodb_user: MongoDBUser, vault_name: str, vault_namespace: str, namespace: str): + data = {"password": USER_PASSWORD} + store_secret_in_vault( + vault_namespace, + vault_name, + data, + f"secret/mongodbenterprise/database/{namespace}/{PASSWORD_SECRET_NAME}", + ) + + mongodb_user.assert_reaches_phase(Phase.Updated, timeout=100) diff --git a/docker/mongodb-enterprise-tests/tests/vaultintegration/om_backup_vault.py b/docker/mongodb-enterprise-tests/tests/vaultintegration/om_backup_vault.py new file mode 100644 index 000000000..ac92f7066 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/vaultintegration/om_backup_vault.py @@ -0,0 +1,460 @@ +from typing import Dict, Optional + +import pytest +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import ( + create_configmap, + create_secret, + delete_secret, + get_default_storage_class, + get_statefulset, + random_k8s_name, + read_secret, +) +from kubetester.awss3client import AwsS3Client, s3_endpoint +from kubetester.certs import create_mongodb_tls_certs, create_ops_manager_tls_certs +from kubetester.http import https_endpoint_is_reachable +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB, Phase, get_pods +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark + +from . import assert_secret_in_vault, run_command_in_vault, store_secret_in_vault + +OPERATOR_NAME = "mongodb-enterprise-operator" +APPDB_SA_NAME = "mongodb-enterprise-appdb" +OM_SA_NAME = "mongodb-enterprise-ops-manager" +OM_NAME = "om-basic" +S3_RS_NAME = "my-mongodb-s3" +S3_SECRET_NAME = "my-s3-secret" +OPLOG_RS_NAME = "my-mongodb-oplog" + +DATABASE_SA_NAME = "mongodb-enterprise-database-pods" + + +def certs_for_prometheus(issuer: str, namespace: str, resource_name: str) -> str: + secret_name = random_k8s_name(resource_name + "-") + "-prometheus-cert" + + return create_mongodb_tls_certs( + issuer, + namespace, + resource_name, + secret_name, + secret_backend="Vault", + vault_subpath="appdb", + ) + + +def new_om_s3_store( + mdb: MongoDB, + s3_id: str, + s3_bucket_name: str, + aws_s3_client: AwsS3Client, + assignment_enabled: bool = True, + path_style_access_enabled: bool = True, + user_name: Optional[str] = None, + password: Optional[str] = None, +) -> Dict: + return { + "uri": mdb.mongo_uri(user_name=user_name, password=password), + "id": s3_id, + "pathStyleAccessEnabled": path_style_access_enabled, + "s3BucketEndpoint": s3_endpoint(AWS_REGION), + "s3BucketName": s3_bucket_name, + "awsAccessKey": aws_s3_client.aws_access_key, + "awsSecretKey": aws_s3_client.aws_secret_access_key, + "assignmentEnabled": assignment_enabled, + } + + +@fixture(scope="module") +def s3_bucket(aws_s3_client: AwsS3Client, namespace: str, vault_namespace: str, vault_name: str) -> str: + create_aws_secret(aws_s3_client, S3_SECRET_NAME, vault_namespace, vault_name, namespace) + yield from create_s3_bucket(aws_s3_client, bucket_prefix="test-s3-bucket-") + + +def create_aws_secret( + aws_s3_client, + secret_name: str, + vault_namespace: str, + vault_name: str, + namespace: str, +): + data = { + "accessKey": aws_s3_client.aws_access_key, + "secretKey": aws_s3_client.aws_secret_access_key, + } + path = f"secret/mongodbenterprise/operator/{namespace}/{secret_name}" + store_secret_in_vault(vault_namespace, vault_name, data, path) + + +def create_s3_bucket(aws_s3_client, bucket_prefix: str = "test-bucket-"): + """creates a s3 bucket and a s3 config""" + bucket_prefix = KubernetesTester.random_k8s_name(bucket_prefix) + aws_s3_client.create_s3_bucket(bucket_prefix) + print("Created S3 bucket", bucket_prefix) + + yield bucket_prefix + print("\nRemoving S3 bucket", bucket_prefix) + aws_s3_client.delete_s3_bucket(bucket_prefix) + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, + s3_bucket: str, + issuer_ca_configmap: str, + issuer: str, + vault_namespace: str, + vault_name: str, +) -> MongoDBOpsManager: + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + + om["spec"]["backup"] = { + "enabled": True, + "s3Stores": [ + { + "name": "s3Store1", + "s3BucketName": s3_bucket, + "mongodbResourceRef": {"name": "my-mongodb-s3"}, + "s3SecretRef": {"name": S3_SECRET_NAME}, + "pathStyleAccessEnabled": True, + "s3BucketEndpoint": "s3.us-east-1.amazonaws.com", + }, + ], + "headDB": { + "storage": "500M", + "storageClass": get_default_storage_class(), + }, + "opLogStores": [ + { + "name": "oplog1", + "mongodbResourceRef": {"name": "my-mongodb-oplog"}, + }, + ], + } + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + prom_cert_secret = certs_for_prometheus(issuer, namespace, om.name + "-db") + store_secret_in_vault( + vault_namespace, + vault_name, + {"password": "prom-password"}, + f"secret/mongodbenterprise/operator/{namespace}/prom-password", + ) + + om["spec"]["applicationDatabase"]["prometheus"] = { + "username": "prom-user", + "passwordSecretRef": {"name": "prom-password"}, + "tlsSecretKeyRef": { + "name": prom_cert_secret, + }, + } + + return om.create() + + +@fixture(scope="module") +def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=OPLOG_RS_NAME, + ).configure(ops_manager, "development") + + resource.set_version(custom_mdb_version) + + yield resource.create() + + +@fixture(scope="module") +def s3_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB: + resource = MongoDB.from_yaml( + yaml_fixture("replica-set-for-om.yaml"), + namespace=namespace, + name=S3_RS_NAME, + ).configure(ops_manager, "s3metadata") + + resource.set_version(custom_mdb_version) + yield resource.create() + + +@mark.e2e_vault_setup_om_backup +def test_vault_creation(vault: str, vault_name: str, vault_namespace: str): + vault + sts = get_statefulset(namespace=vault_namespace, name=vault_name) + assert sts.status.ready_replicas == 1 + + +@mark.e2e_vault_setup_om_backup +def test_create_vault_operator_policy(vault_name: str, vault_namespace: str): + # copy hcl file from local machine to pod + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/operator-policy.hcl", + "/tmp/operator-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprise", + "/tmp/operator-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_enable_kubernetes_auth(vault_name: str, vault_namespace: str): + cmd = [ + "vault", + "auth", + "enable", + "kubernetes", + ] + + run_command_in_vault( + vault_namespace, + vault_name, + cmd, + expected_message=["Success!", "path is already in use at kubernetes"], + ) + + cmd = [ + "cat", + "/var/run/secrets/kubernetes.io/serviceaccount/token", + ] + + token = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + cmd = ["env"] + + response = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + + response = response.split("\n") + for line in response: + l = line.strip() + if str.startswith(l, "KUBERNETES_PORT_443_TCP_ADDR"): + cluster_ip = l.split("=")[1] + break + cmd = [ + "vault", + "write", + "auth/kubernetes/config", + f"token_reviewer_jwt={token}", + f"kubernetes_host=https://{cluster_ip}:443", + "kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "disable_iss_validation=true", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_enable_vault_role_for_operator_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_operator_policy_name: str, +): + cmd = [ + "vault", + "write", + "auth/kubernetes/role/mongodbenterprise", + f"bound_service_account_names={OPERATOR_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_operator_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_put_admin_credentials_to_vault(namespace: str, vault_namespace: str, vault_name: str): + admin_credentials_secret_name = "ops-manager-admin-secret" + # read the -admin-secret from namespace and store in vault + data = read_secret(namespace, admin_credentials_secret_name) + path = f"secret/mongodbenterprise/operator/{namespace}/{admin_credentials_secret_name}" + store_secret_in_vault(vault_namespace, vault_name, data, path) + delete_secret(namespace, admin_credentials_secret_name) + + +@mark.e2e_vault_setup_om_backup +def test_operator_install_with_vault_backend(operator_vault_secret_backend: Operator): + operator_vault_secret_backend.assert_is_running() + + +@mark.e2e_vault_setup_om_backup +def test_create_appdb_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/appdb-policy.hcl", + "/tmp/appdb-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterpriseappdb", + "/tmp/appdb-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_create_om_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/opsmanager-policy.hcl", + "/tmp/om-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterpriseopsmanager", + "/tmp/om-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_enable_vault_role_for_appdb_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_appdb_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_appdb_policy_name}", + f"bound_service_account_names={APPDB_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_appdb_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_enable_vault_role_for_om_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_om_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_om_policy_name}", + f"bound_service_account_names={OM_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_om_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_create_database_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/database-policy.hcl", + "/tmp/database-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprisedatabase", + "/tmp/database-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_enable_vault_role_for_database_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_database_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_database_policy_name}", + f"bound_service_account_names={DATABASE_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_database_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om_backup +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase( + Phase.Pending, + msg_regexp="The MongoDB object .+ doesn't exist", + timeout=900, + ) + + +@mark.e2e_vault_setup_om_backup +def test_prometheus_endpoint_works_on_every_pod_on_appdb(ops_manager: MongoDB): + auth = ("prom-user", "prom-password") + name = ops_manager.name + "-db" + + for idx in range(ops_manager["spec"]["applicationDatabase"]["members"]): + url = f"https://{name}-{idx}.{name}-svc.{ops_manager.namespace}.svc.cluster.local:9216/metrics" + assert https_endpoint_is_reachable(url, auth, tls_verify=False) + + +@mark.e2e_vault_setup_om_backup +def test_backup_mdbs_created( + oplog_replica_set: MongoDB, + s3_replica_set: MongoDB, +): + """Creates mongodb databases all at once""" + oplog_replica_set.assert_reaches_phase(Phase.Running) + s3_replica_set.assert_reaches_phase(Phase.Running) + + +@mark.e2e_vault_setup_om_backup +def test_om_backup_running(ops_manager: MongoDBOpsManager): + ops_manager.backup_status().assert_reaches_phase( + Phase.Running, + ) + + +@mark.e2e_vault_setup_om_backup +def test_no_admin_key_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{namespace}-{ops_manager.name}-admin-key") + + +@mark.e2e_vault_setup_om_backup +def test_appdb_reached_running_and_pod_count(ops_manager: MongoDBOpsManager, namespace: str): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + # check AppDB has 4 containers(+1 because of vault-agent) + for pod_name in get_pods(ops_manager.name + "-db-{}", 3): + pod = client.CoreV1Api().read_namespaced_pod(pod_name, namespace) + assert len(pod.spec.containers) == 4 + + +@mark.e2e_vault_setup_om_backup +def test_no_s3_credentials__secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret( + namespace, + S3_SECRET_NAME, + ) diff --git a/docker/mongodb-enterprise-tests/tests/vaultintegration/om_deployment_vault.py b/docker/mongodb-enterprise-tests/tests/vaultintegration/om_deployment_vault.py new file mode 100644 index 000000000..577e950d8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/vaultintegration/om_deployment_vault.py @@ -0,0 +1,361 @@ +import time +from typing import Optional + +import pytest +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester import ( + create_configmap, + create_secret, + delete_secret, + get_statefulset, + read_secret, +) +from kubetester.certs import create_mongodb_tls_certs, create_ops_manager_tls_certs +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import Phase, get_pods +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark + +from . import assert_secret_in_vault, run_command_in_vault, store_secret_in_vault + +OPERATOR_NAME = "mongodb-enterprise-operator" +APPDB_SA_NAME = "mongodb-enterprise-appdb" +OM_SA_NAME = "mongodb-enterprise-ops-manager" +OM_NAME = "om-basic" + + +@fixture(scope="module") +def ops_manager_certs(namespace: str, issuer: str): + prefix = "prefix" + return create_ops_manager_tls_certs( + issuer, + namespace, + om_name=OM_NAME, + secret_name=f"{prefix}-{OM_NAME}-cert", + secret_backend="Vault", + ) + + +@fixture(scope="module") +def appdb_certs(namespace: str, issuer: str): + create_mongodb_tls_certs( + issuer, + namespace, + f"{OM_NAME}-db", + f"appdb-{OM_NAME}-db-cert", + secret_backend="Vault", + vault_subpath="appdb", + ) + return "appdb" + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, + appdb_certs: str, + issuer_ca_configmap: str, + ops_manager_certs: str, +) -> MongoDBOpsManager: + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + om["spec"]["security"] = { + "tls": {"ca": issuer_ca_configmap}, + "certsSecretPrefix": "prefix", + } + om["spec"]["applicationDatabase"]["security"] = { + "tls": { + "ca": issuer_ca_configmap, + }, + "certsSecretPrefix": "appdb", + } + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + return om.create() + + +@mark.e2e_vault_setup_om +def test_vault_creation(vault: str, vault_name: str, vault_namespace: str): + sts = get_statefulset(namespace=vault_namespace, name=vault_name) + assert sts.status.ready_replicas == 1 + + +@mark.e2e_vault_setup_om +def test_create_vault_operator_policy(vault_name: str, vault_namespace: str): + # copy hcl file from local machine to pod + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/operator-policy.hcl", + "/tmp/operator-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprise", + "/tmp/operator-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_enable_kubernetes_auth(vault_name: str, vault_namespace: str): + cmd = [ + "vault", + "auth", + "enable", + "kubernetes", + ] + + run_command_in_vault( + vault_namespace, + vault_name, + cmd, + expected_message=["Success!", "path is already in use at kubernetes"], + ) + + cmd = [ + "cat", + "/var/run/secrets/kubernetes.io/serviceaccount/token", + ] + + token = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + cmd = ["env"] + + response = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + + response = response.split("\n") + for line in response: + l = line.strip() + if str.startswith(l, "KUBERNETES_PORT_443_TCP_ADDR"): + cluster_ip = l.split("=")[1] + break + cmd = [ + "vault", + "write", + "auth/kubernetes/config", + f"token_reviewer_jwt={token}", + f"kubernetes_host=https://{cluster_ip}:443", + "kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "disable_iss_validation=true", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_enable_vault_role_for_operator_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_operator_policy_name: str, +): + cmd = [ + "vault", + "write", + "auth/kubernetes/role/mongodbenterprise", + f"bound_service_account_names={OPERATOR_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_operator_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_put_admin_credentials_to_vault(namespace: str, vault_namespace: str, vault_name: str): + admin_credentials_secret_name = "ops-manager-admin-secret" + # read the -admin-secret from namespace and store in vault + data = read_secret(namespace, admin_credentials_secret_name) + path = f"secret/mongodbenterprise/operator/{namespace}/{admin_credentials_secret_name}" + store_secret_in_vault(vault_namespace, vault_name, data, path) + delete_secret(namespace, admin_credentials_secret_name) + + +@mark.e2e_vault_setup_om +def test_operator_install_with_vault_backend(operator_vault_secret_backend: Operator): + operator_vault_secret_backend.assert_is_running() + + +@mark.e2e_vault_setup_om +def test_create_appdb_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/appdb-policy.hcl", + "/tmp/appdb-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterpriseappdb", + "/tmp/appdb-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_create_om_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/opsmanager-policy.hcl", + "/tmp/om-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterpriseopsmanager", + "/tmp/om-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_enable_vault_role_for_appdb_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_appdb_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_appdb_policy_name}", + f"bound_service_account_names={APPDB_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_appdb_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_enable_vault_role_for_om_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_om_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_om_policy_name}", + f"bound_service_account_names={OM_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_om_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_om +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_vault_setup_om +def test_no_admin_key_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{namespace}-{ops_manager.name}-admin-key") + + +@mark.e2e_vault_setup_om +def test_no_gen_key_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-gen-key") + + +@mark.e2e_vault_setup_om +def test_appdb_reached_running_and_pod_count(ops_manager: MongoDBOpsManager, namespace: str): + ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600) + # check AppDB has 4 containers(+1 because of vault-agent) + for pod_name in get_pods(ops_manager.name + "-db-{}", 3): + pod = client.CoreV1Api().read_namespaced_pod(pod_name, namespace) + assert len(pod.spec.containers) == 4 + + +@mark.e2e_vault_setup_om +def test_no_appdb_connection_string_secret(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-connection-string") + + +@mark.e2e_vault_setup_om +def test_no_db_agent_password_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-agent-password") + + +@mark.e2e_vault_setup_om +def test_no_db_scram_password_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-om-user-scram-credentials") + + +@mark.e2e_vault_setup_om +def test_no_om_password_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-om-password") + + +@mark.e2e_vault_setup_om +def test_no_db_keyfile_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-keyfile") + + +@mark.e2e_vault_setup_om +def test_no_db_automation_config_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-config") + + +@mark.e2e_vault_setup_om +def test_no_db_monitoring_automation_config_secret_in_kubernetes(namespace: str, ops_manager: MongoDBOpsManager): + with pytest.raises(ApiException): + read_secret(namespace, f"{ops_manager.name}-db-monitoring-config") + + +@mark.e2e_vault_setup_om +def test_rotate_appdb_certs( + ops_manager: MongoDBOpsManager, + vault_namespace: str, + vault_name: str, + namespace: str, +): + omTries = 10 + while omTries > 0: + ops_manager.load() + secret_name = f"appdb-{ops_manager.name}-db-cert" + if secret_name not in ops_manager["metadata"]["annotations"]: + omTries -= 1 + time.sleep(30) + continue + old_version = ops_manager["metadata"]["annotations"][secret_name] + break + + cmd = [ + "vault", + "kv", + "patch", + f"secret/mongodbenterprise/appdb/{namespace}/{secret_name}", + "foo=bar", + ] + + run_command_in_vault(vault_namespace, vault_name, cmd, ["version"]) + + tries = 30 + while tries > 0: + ops_manager.load() + if old_version != ops_manager["metadata"]["annotations"][secret_name]: + return + tries -= 1 + time.sleep(30) + pytest.fail("Not reached new annotation") diff --git a/docker/mongodb-enterprise-tests/tests/vaultintegration/vault_tls.py b/docker/mongodb-enterprise-tests/tests/vaultintegration/vault_tls.py new file mode 100644 index 000000000..e05db0387 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/vaultintegration/vault_tls.py @@ -0,0 +1,327 @@ +from typing import Optional + +from kubernetes import client +from kubernetes.client import V1ConfigMap +from kubetester import create_secret, delete_secret, get_statefulset, read_secret +from kubetester.certs import Certificate +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import is_default_architecture_static +from kubetester.mongodb import MongoDB, Phase, get_pods +from kubetester.operator import Operator +from kubetester.opsmanager import MongoDBOpsManager +from pytest import fixture, mark + +from . import assert_secret_in_vault, run_command_in_vault, store_secret_in_vault + +OPERATOR_NAME = "mongodb-enterprise-operator" +DATABASE_SA_NAME = "mongodb-enterprise-database-pods" +MDB_RESOURCE = "my-replica-set" +APPDB_SA_NAME = "mongodb-enterprise-appdb" +OM_SA_NAME = "mongodb-enterprise-ops-manager" +OM_NAME = "om-basic" + + +@fixture(scope="module") +def replica_set( + namespace: str, + custom_mdb_version: str, +) -> MongoDB: + resource = MongoDB.from_yaml(yaml_fixture("replica-set.yaml"), MDB_RESOURCE, namespace) + resource.set_version(custom_mdb_version) + resource.create() + + return resource + + +@fixture(scope="module") +def ops_manager( + namespace: str, + custom_version: Optional[str], + custom_appdb_version: str, +) -> MongoDBOpsManager: + om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_basic.yaml"), namespace=namespace) + om["spec"]["backup"] = { + "enabled": False, + } + om.set_version(custom_version) + om.set_appdb_version(custom_appdb_version) + + return om.create() + + +@mark.e2e_vault_setup_tls +def test_vault_creation(vault_tls: str, vault_name: str, vault_namespace: str, issuer: str): + + # assert if vault statefulset is ready, this is sort of redundant(we already assert for pod phase) + # but this is basic assertion at the moment, will remove in followup PR + sts = get_statefulset(namespace=vault_namespace, name=vault_name) + assert sts.status.ready_replicas == 1 + + +@mark.e2e_vault_setup_tls +def test_create_appdb_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/appdb-policy.hcl", + "/tmp/appdb-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterpriseappdb", + "/tmp/appdb-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_enable_kubernetes_auth(vault_name: str, vault_namespace: str): + # enable Kubernetes auth for Vault + cmd = [ + "vault", + "auth", + "enable", + "kubernetes", + ] + + run_command_in_vault( + vault_namespace, + vault_name, + cmd, + expected_message=["Success!", "path is already in use at kubernetes"], + ) + + cmd = [ + "cat", + "/var/run/secrets/kubernetes.io/serviceaccount/token", + ] + + token = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + + cmd = ["env"] + + response = run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=[]) + + response = response.split("\n") + for line in response: + l = line.strip() + if str.startswith(l, "KUBERNETES_PORT_443_TCP_ADDR"): + cluster_ip = l.split("=")[1] + break + + cmd = [ + "vault", + "write", + "auth/kubernetes/config", + f"token_reviewer_jwt={token}", + f"kubernetes_host=https://{cluster_ip}:443", + "kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + "disable_iss_validation=true", + ] + + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_create_om_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/opsmanager-policy.hcl", + "/tmp/om-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterpriseopsmanager", + "/tmp/om-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_enable_vault_role_for_appdb_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_appdb_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_appdb_policy_name}", + f"bound_service_account_names={APPDB_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_appdb_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_enable_vault_role_for_om_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_om_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_om_policy_name}", + f"bound_service_account_names={OM_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_om_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_create_vault_operator_policy(vault_name: str, vault_namespace: str): + # copy hcl file from local machine to pod + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/operator-policy.hcl", + "/tmp/operator-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprise", + "/tmp/operator-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_enable_vault_role_for_operator_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_operator_policy_name: str, +): + cmd = [ + "vault", + "write", + "auth/kubernetes/role/mongodbenterprise", + f"bound_service_account_names={OPERATOR_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_operator_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_put_admin_credentials_to_vault(namespace: str, vault_namespace: str, vault_name: str): + admin_credentials_secret_name = "ops-manager-admin-secret" + # read the -admin-secret from namespace and store in vault + data = read_secret(namespace, admin_credentials_secret_name) + path = f"secret/mongodbenterprise/operator/{namespace}/{admin_credentials_secret_name}" + store_secret_in_vault(vault_namespace, vault_name, data, path) + delete_secret(namespace, admin_credentials_secret_name) + + +@mark.e2e_vault_setup_tls +def test_remove_cert_and_key_from_secret(namespace: str): + data = read_secret(namespace, "vault-tls") + cert = Certificate(name="vault-tls", namespace=namespace).load() + cert.delete() + del data["tls.crt"] + del data["tls.key"] + delete_secret(namespace, "vault-tls") + create_secret(namespace, "vault-tls", data) + + +@mark.e2e_vault_setup_tls +def test_operator_install_with_vault_backend( + operator_vault_secret_backend_tls: Operator, +): + operator_vault_secret_backend_tls.assert_is_running() + + +@mark.e2e_vault_setup_tls +def test_store_om_credentials_in_vault(vault_namespace: str, vault_name: str, namespace: str): + credentials = read_secret(namespace, "my-credentials") + store_secret_in_vault( + vault_namespace, + vault_name, + credentials, + f"secret/mongodbenterprise/operator/{namespace}/my-credentials", + ) + + cmd = [ + "vault", + "kv", + "get", + f"secret/mongodbenterprise/operator/{namespace}/my-credentials", + ] + run_command_in_vault(vault_namespace, vault_name, cmd, expected_message=["publicApiKey"]) + delete_secret(namespace, "my-credentials") + + +@mark.e2e_vault_setup_tls +def test_create_database_policy(vault_name: str, vault_namespace: str): + KubernetesTester.copy_file_inside_pod( + f"{vault_name}-0", + "vaultpolicies/database-policy.hcl", + "/tmp/database-policy.hcl", + namespace=vault_namespace, + ) + + cmd = [ + "vault", + "policy", + "write", + "mongodbenterprisedatabase", + "/tmp/database-policy.hcl", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_enable_vault_role_for_database_pod( + vault_name: str, + vault_namespace: str, + namespace: str, + vault_database_policy_name: str, +): + cmd = [ + "vault", + "write", + f"auth/kubernetes/role/{vault_database_policy_name}", + f"bound_service_account_names={DATABASE_SA_NAME}", + f"bound_service_account_namespaces={namespace}", + f"policies={vault_database_policy_name}", + ] + run_command_in_vault(vault_namespace, vault_name, cmd) + + +@mark.e2e_vault_setup_tls +def test_om_created(ops_manager: MongoDBOpsManager): + ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=600) + + +@mark.e2e_vault_setup_tls +def test_mdb_created(replica_set: MongoDB, namespace: str): + replica_set.assert_reaches_phase(Phase.Running, timeout=500, ignore_errors=True) + for pod_name in get_pods(MDB_RESOURCE + "-{}", 3): + pod = client.CoreV1Api().read_namespaced_pod(pod_name, namespace) + if is_default_architecture_static(): + assert len(pod.spec.containers) == 3 + else: + assert len(pod.spec.containers) == 2 + + +@mark.e2e_vault_setup_tls +def test_no_cert_in_secret(namespace: str): + data = read_secret(namespace, "vault-tls") + assert "tls.crt" not in data + assert "tls.key" not in data diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/__init__.py b/docker/mongodb-enterprise-tests/tests/webhooks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/conftest.py b/docker/mongodb-enterprise-tests/tests/webhooks/conftest.py new file mode 100644 index 000000000..cb0664057 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/conftest.py @@ -0,0 +1,4 @@ +def pytest_runtest_setup(item): + """This allows to automatically install the default Operator before running any test""" + if "default_operator" not in item.fixturenames: + item.fixturenames.insert(0, "default_operator") diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/e2e_mongodb_roles_validation_webhook.py b/docker/mongodb-enterprise-tests/tests/webhooks/e2e_mongodb_roles_validation_webhook.py new file mode 100644 index 000000000..ffb071af2 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/e2e_mongodb_roles_validation_webhook.py @@ -0,0 +1,210 @@ +import pytest +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb import MongoDB +from kubetester.operator import Operator +from pytest import fixture + + +@fixture(scope="function") +def mdb(namespace: str, custom_mdb_version: str) -> str: + resource = MongoDB.from_yaml(yaml_fixture("role-validation-base.yaml"), namespace=namespace) + resource.set_version(custom_mdb_version) + return resource + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_wait_for_webhook(namespace: str, default_operator: Operator): + default_operator.wait_for_webhook() + + +# Basic testing for invalid empty values +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_empty_role_name(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Cannot create a role with an empty name", + ): + mdb.create() + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_empty_db_name(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Cannot create a role with an empty db", + ): + mdb.create() + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_inherited_role_empty_name(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + "roles": [{"db": "admin", "role": ""}], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Cannot inherit from a role with an empty name", + ): + mdb.create() + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_inherited_role_empty_db(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + "roles": [{"db": "", "role": "role"}], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Cannot inherit from a role with an empty db", + ): + mdb.create() + + +# Testing for invalid authentication Restrictions +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_invalid_client_source(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + "authenticationRestrictions": [{"clientSource": ["355.127.0.1"]}], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - AuthenticationRestriction is invalid - clientSource 355.127.0.1 is neither a valid IP address nor a valid CIDR range", + ): + mdb.create() + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_invalid_server_address(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + "authenticationRestrictions": [{"serverAddress": ["355.127.0.1"]}], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - AuthenticationRestriction is invalid - serverAddress 355.127.0.1 is neither a valid IP address nor a valid CIDR range", + ): + mdb.create() + + +# Testing for invalid privileges +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_invalid_cluster_and_db_collection(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [ + { + "actions": ["insert"], + "resource": {"collection": "foo", "db": "admin", "cluster": True}, + } + ], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Privilege is invalid - Cluster: true is not compatible with setting db/collection", + ): + mdb.create() + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_invalid_cluster_not_true(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [{"actions": ["insert"], "resource": {"cluster": False}}], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Privilege is invalid - The only valid value for privilege.cluster, if set, is true", + ): + mdb.create() + + +@pytest.mark.e2e_mongodb_roles_validation_webhook +def test_invalid_action(mdb: str): + mdb["spec"]["security"]["roles"] = [ + { + "role": "role", + "db": "admin", + "privileges": [ + { + "actions": ["insertFoo"], + "resource": {"collection": "foo", "db": "admin"}, + } + ], + } + ] + with pytest.raises( + client.rest.ApiException, + match="Error validating role - Privilege is invalid - Actions are not valid - insertFoo is not a valid db action", + ): + mdb.create() diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/e2e_mongodb_validation_webhook.py b/docker/mongodb-enterprise-tests/tests/webhooks/e2e_mongodb_validation_webhook.py new file mode 100644 index 000000000..716ebe854 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/e2e_mongodb_validation_webhook.py @@ -0,0 +1,125 @@ +import pytest +import yaml +from kubetester.kubetester import KubernetesTester +from kubetester.kubetester import fixture as yaml_fixture + + +@pytest.mark.e2e_mongodb_validation_webhook +class TestWebhookValidation(KubernetesTester): + def test_horizons_tls_validation(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_horizons_tls.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="TLS must be enabled in order to use replica set horizons", + ) + + def test_horizons_members(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_horizons_members.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="Number of horizons must be equal to number of members in replica set", + ) + + def test_x509_without_tls(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_x509_no_tls.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="Cannot have a non-tls deployment when x509 authentication is enabled", + ) + + def test_auth_without_modes(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_agent_auth_not_in_modes.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="Cannot configure an Agent authentication mechanism that is not specified in authentication modes", + ) + + def test_agent_auth_enabled_with_no_modes(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_auth_no_modes.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="Cannot enable authentication without modes specified", + ) + + def test_ldap_auth_with_mongodb_community(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_ldap_community.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="Cannot enable LDAP authentication with MongoDB Community Builds", + ) + + def test_no_agent_auth_mode_with_multiple_modes_enabled(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_no_agent_mode.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="spec.security.authentication.agents.mode must be specified if more than one entry is present in spec.security.authentication.modes", + ) + + def test_ldap_auth_with_no_ldapgroupdn(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_replica_set_ldapauthz_no_ldapgroupdn.yaml"))) + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="automationLdapGroupDN must be specified if LDAP authorization is used and agent auth mode is $external (x509 or LDAP)", + ) + + def test_replicaset_members_is_specified(self): + resource = yaml.safe_load(open(yaml_fixture("invalid_mdb_member_count.yaml"))) + + self.create_custom_resource_from_object( + self.get_namespace(), + resource, + exception_reason="'spec.members' must be specified if type of MongoDB is ReplicaSet", + ) + + def test_replicaset_members_is_specified_without_webhook(self): + self._assert_validates_without_webhook( + "mdbpolicy.mongodb.com", + "invalid_mdb_member_count.yaml", + "'spec.members' must be specified if type of MongoDB is ReplicaSet", + ) + + def test_horizons_without_tls_validates_without_webhook(self): + self._assert_validates_without_webhook( + "mdbpolicy.mongodb.com", + "invalid_replica_set_horizons_tls.yaml", + "TLS must be enabled", + ) + + def test_incorrect_members_validates_without_webhook(self): + self._assert_validates_without_webhook( + "mdbpolicy.mongodb.com", + "invalid_replica_set_horizons_members.yaml", + "number of members", + ) + + def _assert_validates_without_webhook(self, webhook_name: str, fixture: str, expected_msg: str): + webhook_api = self.client.AdmissionregistrationV1Api() + + # break the existing webhook + webhook = webhook_api.read_validating_webhook_configuration(webhook_name) + old_webhooks = webhook.webhooks + webhook.webhooks[0].client_config.service.name = "a-non-existent-service" + webhook.metadata.uid = "" + webhook_api.replace_validating_webhook_configuration(webhook_name, webhook) + + # check that the webhook doesn't block and that the resource gets into + # the errored state + resource = yaml.safe_load(open(yaml_fixture(fixture))) + self.create_custom_resource_from_object(self.get_namespace(), resource) + KubernetesTester.wait_until("in_error_state", 20) + mrs = KubernetesTester.get_resource() + assert expected_msg in mrs["status"]["message"] + + # fix webhooks + webhook = webhook_api.read_validating_webhook_configuration(webhook_name) + webhook.webhooks = old_webhooks + webhook.metadata.uid = "" + webhook_api.replace_validating_webhook_configuration(webhook_name, webhook) diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_appdb_shard_count.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_appdb_shard_count.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_mdb_member_count.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_mdb_member_count.yaml new file mode 100644 index 000000000..de6eee8f3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_mdb_member_count.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-rs-member-not-specified +spec: + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + security: + tls: + enabled: true + persistent: false \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_agent_auth_not_in_modes.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_agent_auth_not_in_modes.yaml new file mode 100644 index 000000000..404bb3e94 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_agent_auth_not_in_modes.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-rs-invalid-agent-auth-not-in-modes +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + security: + authentication: + agents: + mode: X509 + enabled: true + modes: ["SCRAM"] diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_auth_no_modes.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_auth_no_modes.yaml new file mode 100644 index 000000000..66318698e --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_auth_no_modes.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-rs-invalid-auth +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + security: + authentication: + agents: + mode: SCRAM + enabled: true + modes: [] diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_horizons_members.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_horizons_members.yaml new file mode 100644 index 000000000..bf61090a9 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_horizons_members.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-rs-external-access-multiple-horizons-member-count +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + security: + tls: + enabled: true + + persistent: false + + connectivity: + replicaSetHorizons: + - "test-horizon-1": "mdb0-test-1-website.com:1337" + - "test-horizon-1": "mdb1-test-1-website.com:1338" + - "test-horizon-1": "mdb2-test-1-website.com:1339" + - "test-horizon-2": "mdb0-test-2-website.com:2337" + - "test-horizon-2": "mdb2-test-2-website.com:2339" diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_horizons_tls.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_horizons_tls.yaml new file mode 100644 index 000000000..1b26690cc --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_horizons_tls.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-rs-external-access-multiple-horizons +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + connectivity: + replicaSetHorizons: + - "test-horizon-1": "mdb0-test-1-website.com:1337" + "test-horizon-2": "mdb0-test-2-website.com:2337" + - "test-horizon-1": "mdb1-test-1-website.com:1338" + "test-horizon-2": "mdb1-test-2-website.com:2338" + - "test-horizon-1": "mdb2-test-1-website.com:1339" + "test-horizon-2": "mdb2-test-2-website.com:2339" diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_ldap_community.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_ldap_community.yaml new file mode 100644 index 000000000..f4250184d --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_ldap_community.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-rs-external-access-multiple-horizons +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + modes: ["LDAP"] diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_ldapauthz_no_ldapgroupdn.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_ldapauthz_no_ldapgroupdn.yaml new file mode 100644 index 000000000..d5bd5b9b0 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_ldapauthz_no_ldapgroupdn.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-invalid-replica-set-ldapauthz-no-ldapgroupdn +spec: + members: 3 + version: 4.0.12-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + security: + authentication: + agents: + mode: "LDAP" + enabled: true + modes: ["SCRAM", "LDAP"] + ldap: + authzQueryTemplate: "{USER}?memberOf?base" diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_no_agent_mode.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_no_agent_mode.yaml new file mode 100644 index 000000000..652206623 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_no_agent_mode.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-invalid-replica-set-no-agent-auth-mode +spec: + members: 3 + version: 4.0.12-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + security: + authentication: + enabled: true + modes: ["SCRAM", "LDAP"] diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_x509_no_tls.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_x509_no_tls.yaml new file mode 100644 index 000000000..167a15373 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/invalid_replica_set_x509_no_tls.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-rs-external-access-multiple-horizons +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + logLevel: DEBUG + + persistent: false + + security: + authentication: + enabled: true + modes: ["X509"] + internalCluster: "X509" diff --git a/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/role-validation-base.yaml b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/role-validation-base.yaml new file mode 100644 index 000000000..2af894881 --- /dev/null +++ b/docker/mongodb-enterprise-tests/tests/webhooks/fixtures/role-validation-base.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: test-tls-rs-external-access-invalid-action +spec: + members: 3 + version: 4.0.12 + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + role: "" + db: "" + privileges: + actions: + resource: + db: + collection: + cluster: + authenticationRestrictions: + clientSource: + serverAddress: + roles: + name: + db: + diff --git a/docker/mongodb-enterprise-tests/vaultconfig/override.yaml b/docker/mongodb-enterprise-tests/vaultconfig/override.yaml new file mode 100644 index 000000000..5d85c97b3 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vaultconfig/override.yaml @@ -0,0 +1,29 @@ +# The content of this file has been taken from the Vault tutorial +# at https://vaultproject.io/docs/platform/k8s/helm/examples/standalone-tls + +global: + enabled: true + tlsDisable: false + +server: + extraEnvironmentVars: + VAULT_CACERT: /vault/userconfig/vault-tls/ca.crt + + extraVolumes: + - type: secret + name: vault-tls + + standalone: + enabled: true + config: | + listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + tls_cert_file = "/vault/userconfig/vault-tls/tls.crt" + tls_key_file = "/vault/userconfig/vault-tls/tls.key" + tls_client_ca_file = "/vault/userconfig/vault-tls/ca.crt" + } + + storage "file" { + path = "/vault/data" + } diff --git a/docker/mongodb-enterprise-tests/vaultpolicies/appdb-policy.hcl b/docker/mongodb-enterprise-tests/vaultpolicies/appdb-policy.hcl new file mode 100644 index 000000000..2c13a20d5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vaultpolicies/appdb-policy.hcl @@ -0,0 +1,8 @@ +// NOTE: if you edit this file, make sure to also edit the one under public/vault_policies + +path "secret/data/mongodbenterprise/appdb/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/mongodbenterprise/appdb/*" { + capabilities = ["list"] +} diff --git a/docker/mongodb-enterprise-tests/vaultpolicies/database-policy.hcl b/docker/mongodb-enterprise-tests/vaultpolicies/database-policy.hcl new file mode 100644 index 000000000..1c0d20486 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vaultpolicies/database-policy.hcl @@ -0,0 +1,8 @@ +// NOTE: if you edit this file, make sure to also edit the one under public/vault_policies + +path "secret/data/mongodbenterprise/database/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/mongodbenterprise/database/*" { + capabilities = ["list"] +} diff --git a/docker/mongodb-enterprise-tests/vaultpolicies/operator-policy.hcl b/docker/mongodb-enterprise-tests/vaultpolicies/operator-policy.hcl new file mode 100644 index 000000000..f23e0b5c1 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vaultpolicies/operator-policy.hcl @@ -0,0 +1,8 @@ +// NOTE: if you edit this file, make sure to also edit the one under public/vault_policies + +path "secret/data/mongodbenterprise/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} +path "secret/metadata/mongodbenterprise/*" { + capabilities = ["list", "read"] +} diff --git a/docker/mongodb-enterprise-tests/vaultpolicies/opsmanager-policy.hcl b/docker/mongodb-enterprise-tests/vaultpolicies/opsmanager-policy.hcl new file mode 100644 index 000000000..fb1bae1b5 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vaultpolicies/opsmanager-policy.hcl @@ -0,0 +1,8 @@ +// NOTE: if you edit this file, make sure to also edit the one under public/vault_policies + +path "secret/data/mongodbenterprise/opsmanager/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/mongodbenterprise/opsmanager/*" { + capabilities = ["list"] +} diff --git a/docker/mongodb-enterprise-tests/vendor/README.md b/docker/mongodb-enterprise-tests/vendor/README.md new file mode 100644 index 000000000..f4b578d30 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/README.md @@ -0,0 +1,10 @@ +# Vendored Packages + +In this directory you'll find vendored dependencies of the E2E tests. + +## OpenLDAP + +Originally this [Helm chart](https://github.com/helm/charts/tree/master/stable/openldap). + +- This specifically vendors the 1.2.4 version at [this + commit](https://github.com/helm/charts/tree/b2f720d33515d2308c558d927722e197163efc3e/stable/openldap). \ No newline at end of file diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/.helmignore b/docker/mongodb-enterprise-tests/vendor/openldap/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/Chart.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/Chart.yaml new file mode 100644 index 000000000..d8eee030d --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +name: openldap +home: https://www.openldap.org +version: 1.2.4 +appVersion: 2.4.48 +description: Community developed LDAP software +icon: http://www.openldap.org/images/headers/LDAPworm.gif +keywords: + - ldap + - openldap +sources: + - https://github.com/kubernetes/charts +maintainers: + - name: enis + email: enis@apache.org +engine: gotpl diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/README.md b/docker/mongodb-enterprise-tests/vendor/openldap/README.md new file mode 100644 index 000000000..3496c40eb --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/README.md @@ -0,0 +1,100 @@ +# OpenLDAP Helm Chart + +## Prerequisites Details +* Kubernetes 1.8+ +* PV support on the underlying infrastructure + +## Chart Details +This chart will do the following: + +* Instantiate an instance of OpenLDAP server + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ helm install --name my-release stable/openldap +``` + +## Configuration + +We use the docker images provided by https://github.com/osixia/docker-openldap. The docker image is highly configurable and well documented. Please consult to documentation for the docker image for more information. + +The following table lists the configurable parameters of the openldap chart and their default values. + +| Parameter | Description | Default | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | +| `replicaCount` | Number of replicas | `1` | +| `strategy` | Deployment strategy | `{}` | +| `image.repository` | Container image repository | `osixia/openldap` | +| `image.tag` | Container image tag | `1.1.10` | +| `image.pullPolicy` | Container pull policy | `IfNotPresent` | +| `extraLabels` | Labels to add to the Resources | `{}` | +| `podAnnotations` | Annotations to add to the pod | `{}` | +| `existingSecret` | Use an existing secret for admin and config user passwords | `""` | +| `service.annotations` | Annotations to add to the service | `{}` | +| `service.clusterIP` | IP address to assign to the service | `nil` | +| `service.externalIPs` | Service external IP addresses | `[]` | +| `service.ldapPort` | External service port for LDAP | `389` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""` | +| `service.loadBalancerSourceRanges` | List of IP CIDRs allowed access to load balancer (if supported) | `[]` | +| `service.sslLdapPort` | External service port for SSL+LDAP | `636` | +| `service.type` | Service type | `ClusterIP` | +| `env` | List of key value pairs as env variables to be sent to the docker image. See https://github.com/osixia/docker-openldap for available ones | `[see values.yaml]` | +| `tls.enabled` | Set to enable TLS/LDAPS - should also set `tls.secret` | `false` | +| `tls.secret` | Secret containing TLS cert and key (eg, generated via cert-manager) | `""` | +| `tls.CA.enabled` | Set to enable custom CA crt file - should also set `tls.CA.secret` | `false` | +| `tls.CA.secret` | Secret containing CA certificate (ca.crt) | `""` | +| `adminPassword` | Password for admin user. Unset to auto-generate the password | None | +| `configPassword` | Password for config user. Unset to auto-generate the password | None | +| `customLdifFiles` | Custom ldif files to seed the LDAP server. List of filename -> data pairs | None | +| `persistence.enabled` | Whether to use PersistentVolumes or not | `false` | +| `persistence.storageClass` | Storage class for PersistentVolumes. | `` | +| `persistence.accessMode` | Access mode for PersistentVolumes | `ReadWriteOnce` | +| `persistence.size` | PersistentVolumeClaim storage size | `8Gi` | +| `persistence.existingClaim` | An Existing PVC name for openLDAPA volume | None | +| `resources` | Container resource requests and limits in yaml | `{}` | +| `initResources` | initContainer resource requests and limits in yaml | `{}` | +| `test.enabled` | Conditionally provision test resources | `false` | +| `test.image.repository` | Test container image requires bats framework | `dduportal/bats` | +| `test.image.tag` | Test container tag | `0.4.0` | + + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```bash +$ helm install --name my-release -f values.yaml stable/openldap +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +## Cleanup orphaned Persistent Volumes + +Deleting the Deployment will not delete associated Persistent Volumes if persistence is enabled. + +Do the following after deleting the chart release to clean up orphaned Persistent Volumes. + +```bash +$ kubectl delete pvc -l release=${RELEASE-NAME} +``` + +## Custom Secret + +`existingSecret` can be used to override the default secret.yaml provided + +## Testing + +Helm tests are included and they confirm connection to slapd. + +```bash +helm install . --set test.enabled=true +helm test +RUNNING: foolish-mouse-openldap-service-test-akmms +PASSED: foolish-mouse-openldap-service-test-akmms +``` + +It will confirm that we can do an ldapsearch with the default credentials diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/NOTES.txt b/docker/mongodb-enterprise-tests/vendor/openldap/templates/NOTES.txt new file mode 100644 index 000000000..09cf0edc8 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/NOTES.txt @@ -0,0 +1,20 @@ +OpenLDAP has been installed. You can access the server from within the k8s cluster using: + + {{ template "openldap.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.ldapPort }} + + +You can access the LDAP adminPassword and configPassword using: + + kubectl get secret --namespace {{ .Release.Namespace }} {{ template "openldap.secretName" . }} -o jsonpath="{.data.LDAP_ADMIN_PASSWORD}" | base64 --decode; echo + kubectl get secret --namespace {{ .Release.Namespace }} {{ template "openldap.secretName" . }} -o jsonpath="{.data.LDAP_CONFIG_PASSWORD}" | base64 --decode; echo + + +You can access the LDAP service, from within the cluster (or with kubectl port-forward) with a command like (replace password and domain): + ldapsearch -x -H ldap://{{ template "openldap.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.ldapPort }} -b dc=example,dc=org -D "cn=admin,dc=example,dc=org" -w $LDAP_ADMIN_PASSWORD + + +Test server health using Helm test: + helm test {{ .Release.Name }} + + +You can also consider installing the helm chart for phpldapadmin to manage this instance of OpenLDAP, or install Apache Directory Studio, and connect using kubectl port-forward. diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/_helpers.tpl b/docker/mongodb-enterprise-tests/vendor/openldap/templates/_helpers.tpl new file mode 100644 index 000000000..75a118d0f --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/_helpers.tpl @@ -0,0 +1,40 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "openldap.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "openldap.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "openldap.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Generate chart secret name +*/}} +{{- define "openldap.secretName" -}} +{{ default (include "openldap.fullname" .) .Values.existingSecret }} +{{- end -}} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/configmap-customldif.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/configmap-customldif.yaml new file mode 100644 index 000000000..f060d1d8d --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/configmap-customldif.yaml @@ -0,0 +1,23 @@ +# +# A ConfigMap spec for openldap slapd that map directly to files under +# /container/service/slapd/assets/config/bootstrap/ldif/custom +# +{{- if .Values.customLdifFiles }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "openldap.fullname" . }}-customldif + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +data: +{{- range $key, $val := .Values.customLdifFiles }} + {{ $key }}: |- +{{ $val | indent 4}} +{{- end }} +{{- end }} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/configmap-env.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/configmap-env.yaml new file mode 100644 index 000000000..d8fe9a497 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/configmap-env.yaml @@ -0,0 +1,20 @@ +# +# A ConfigMap spec for openldap slapd that map directly to env variables in the Pod. +# List of environment variables supported is from the docker image: +# https://github.com/osixia/docker-openldap#beginner-guide +# Note that passwords are defined as secrets +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "openldap.fullname" . }}-env + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +data: +{{ toYaml .Values.env | indent 2 }} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/deployment.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/deployment.yaml new file mode 100644 index 000000000..a5847f198 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/deployment.yaml @@ -0,0 +1,174 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "openldap.fullname" . }} + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.replicaCount }} +{{- if .Values.strategy }} + strategy: +{{ toYaml .Values.strategy | indent 4 }} +{{- end }} + selector: + matchLabels: + app: {{ template "openldap.name" . }} + release: {{ .Release.Name }} + template: + metadata: + annotations: + checksum/configmap-env: {{ include (print $.Template.BasePath "/configmap-env.yaml") . | sha256sum }} +{{- if .Values.customLdifFiles}} + checksum/configmap-customldif: {{ include (print $.Template.BasePath "/configmap-customldif.yaml") . | sha256sum }} +{{- end }} +{{- if .Values.podAnnotations}} +{{ toYaml .Values.podAnnotations | indent 8}} +{{- end }} + labels: + app: {{ template "openldap.name" . }} + release: {{ .Release.Name }} + spec: + {{- if or .Values.customLdifFiles .Values.tls.enabled }} + initContainers: + {{- end }} + {{- if .Values.customLdifFiles }} + - name: {{ .Chart.Name }}-init-ldif + image: busybox + command: ['sh', '-c', 'cp /customldif/* /ldifworkingdir'] + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: customldif + mountPath: /customldif + - name: ldifworkingdir + mountPath: /ldifworkingdir + resources: +{{ toYaml .Values.initResources | indent 10 }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: {{ .Chart.Name }}-init-tls + image: busybox + command: ['sh', '-c', 'cp /tls/* /certs'] + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: tls + mountPath: /tls + - name: certs + mountPath: /certs + resources: +{{ toYaml .Values.initResources | indent 10 }} + {{- if .Values.tls.CA.enabled }} + - name: {{ .Chart.Name }}-init-catls + image: busybox + command: ['sh', '-c', 'cp /catls/ca.crt /certs'] + volumeMounts: + - name: catls + mountPath: /catls + - name: certs + mountPath: /certs + resources: +{{ toYaml .Values.initResources | indent 10 }} + {{- end }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} +{{- if .Values.customLdifFiles }} + args: [--copy-service] +{{- end }} + ports: + - name: ldap-port + containerPort: 389 + - name: ssl-ldap-port + containerPort: 636 + envFrom: + - configMapRef: + name: {{ template "openldap.fullname" . }}-env + - secretRef: + name: {{ template "openldap.secretName" . }} + volumeMounts: + - name: data + mountPath: /var/lib/ldap + subPath: data + - name: data + mountPath: /etc/ldap/slapd.d + subPath: config-data + {{- if .Values.customLdifFiles }} + - name: ldifworkingdir + mountPath: /container/service/slapd/assets/config/bootstrap/ldif/custom + {{- end }} + {{- if .Values.tls.enabled }} + - name: certs + mountPath: /container/service/slapd/assets/certs + {{- end }} + env: + {{- if .Values.tls.enabled }} + - name: LDAP_TLS_CRT_FILENAME + value: tls.crt + - name: LDAP_TLS_KEY_FILENAME + value: tls.key + {{- if .Values.tls.CA.enabled }} + - name: LDAP_TLS_CA_CRT_FILENAME + value: ca.crt + {{- end }} + {{- end }} + livenessProbe: + tcpSocket: + port: ldap-port + initialDelaySeconds: 20 + periodSeconds: 10 + failureThreshold: 10 + readinessProbe: + tcpSocket: + port: ldap-port + initialDelaySeconds: 20 + periodSeconds: 10 + failureThreshold: 10 + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + {{- if .Values.customLdifFiles }} + - name: customldif + configMap: + name: {{ template "openldap.fullname" . }}-customldif + - name: ldifworkingdir + emptyDir: {} + {{- end }} + {{- if .Values.tls.enabled }} + - name: tls + secret: + secretName: {{ .Values.tls.secret }} + {{- if .Values.tls.CA.enabled }} + - name: catls + secret: + secretName: {{ .Values.tls.CA.secret }} + {{- end }} + {{- end }} + - name: certs + emptyDir: + medium: Memory + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "openldap.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end -}} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/pvc.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/pvc.yaml new file mode 100644 index 000000000..96d6c8680 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/pvc.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "openldap.fullname" . }} + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- end }} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/secret.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/secret.yaml new file mode 100644 index 000000000..9c7953acc --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/secret.yaml @@ -0,0 +1,18 @@ +{{ if not .Values.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "openldap.fullname" . }} + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +type: Opaque +data: + LDAP_ADMIN_PASSWORD: {{ .Values.adminPassword | default (randAlphaNum 32) | b64enc | quote }} + LDAP_CONFIG_PASSWORD: {{ .Values.configPassword | default (randAlphaNum 32) | b64enc | quote }} +{{ end }} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/service.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/service.yaml new file mode 100644 index 000000000..e1bb2d397 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/service.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} + name: {{ template "openldap.fullname" . }} + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +spec: + {{- with .Values.service.clusterIP }} + clusterIP: {{ . | quote }} + {{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP | quote }} +{{- end }} +{{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} +{{- end }} + ports: + - name: ldap-port + protocol: TCP + port: {{ .Values.service.ldapPort }} + targetPort: ldap-port + - name: ssl-ldap-port + protocol: TCP + port: {{ .Values.service.sslLdapPort }} + targetPort: ssl-ldap-port + selector: + app: {{ template "openldap.name" . }} + release: {{ .Release.Name }} + type: {{ .Values.service.type }} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/tests/openldap-test-runner.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/tests/openldap-test-runner.yaml new file mode 100644 index 000000000..cfcaf2183 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/tests/openldap-test-runner.yaml @@ -0,0 +1,50 @@ +{{- if .Values.test.enabled -}} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ template "openldap.fullname" . }}-test-{{ randAlphaNum 5 | lower }}" + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook": test-success +spec: + initContainers: + - name: test-framework + image: {{ .Values.test.image.repository }}:{{ .Values.test.image.tag }} + command: + - "bash" + - "-c" + - | + set -ex + # copy bats to tools dir + cp -R /usr/local/libexec/ /tools/bats/ + volumeMounts: + - mountPath: /tools + name: tools + containers: + - name: {{ .Release.Name }}-test + image: {{ .Values.test.image.repository }}:{{ .Values.test.image.tag }} + envFrom: + - secretRef: + name: {{ template "openldap.secretName" . }} + command: ["/tools/bats/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + - mountPath: /tools + name: tools + volumes: + - name: tests + configMap: + name: {{ template "openldap.fullname" . }}-tests + - name: tools + emptyDir: {} + restartPolicy: Never +{{- end -}} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/templates/tests/openldap-tests.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/templates/tests/openldap-tests.yaml new file mode 100644 index 000000000..1cdeb80b7 --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/templates/tests/openldap-tests.yaml @@ -0,0 +1,22 @@ +{{- if .Values.test.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "openldap.fullname" . }}-tests + labels: + app: {{ template "openldap.name" . }} + chart: {{ template "openldap.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.extraLabels }} +{{ toYaml .Values.extraLabels | indent 4 }} +{{- end }} +data: + run.sh: |- + @test "Testing connecting to slapd server" { + # Ideally, this should be in the docker image, but there is not a generic image we can use + # with bats and ldap-utils installed. It is not worth for now to push an image for this. + apt-get update && apt-get install -y ldap-utils + ldapsearch -x -H ldap://{{ template "openldap.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.ldapPort }} -b "dc=example,dc=org" -D "cn=admin,dc=example,dc=org" -w $LDAP_ADMIN_PASSWORD + } +{{- end -}} diff --git a/docker/mongodb-enterprise-tests/vendor/openldap/values.yaml b/docker/mongodb-enterprise-tests/vendor/openldap/values.yaml new file mode 100644 index 000000000..7779a539f --- /dev/null +++ b/docker/mongodb-enterprise-tests/vendor/openldap/values.yaml @@ -0,0 +1,117 @@ +# Default values for openldap. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +# Define deployment strategy - IMPORTANT: use rollingUpdate: null when use Recreate strategy. +# It prevents from merging with existing map keys which are forbidden. +strategy: {} + # type: RollingUpdate + # rollingUpdate: + # maxSurge: 1 + # maxUnavailable: 0 + # + # or + # + # type: Recreate + # rollingUpdate: null +image: + # From repository https://github.com/osixia/docker-openldap + repository: osixia/openldap + + tag: 1.2.5 + pullPolicy: IfNotPresent + +# Spcifies an existing secret to be used for admin and config user passwords +existingSecret: "" + +# settings for enabling TLS +tls: + enabled: false + secret: "" # The name of a kubernetes.io/tls type secret to use for TLS + CA: + enabled: false + secret: "" # The name of a generic secret to use for custom CA certificate (ca.crt) +## Add additional labels to all resources +extraLabels: {} +## Add additional annotations to pods +podAnnotations: {} +service: + annotations: {} + + ldapPort: 389 + sslLdapPort: 636 # Only used if tls.enabled is true + ## List of IP addresses at which the service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + loadBalancerIP: "" + loadBalancerSourceRanges: [] + type: ClusterIP + +# Default configuration for openldap as environment variables. These get injected directly in the container. +# Use the env variables from https://github.com/osixia/docker-openldap#beginner-guide +env: + LDAP_ORGANISATION: "Example Inc." + LDAP_DOMAIN: "example.org" + LDAP_BACKEND: "hdb" + LDAP_TLS: "true" + LDAP_TLS_ENFORCE: "false" + LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" + +# Default Passwords to use, stored as a secret. If unset, passwords are auto-generated. +# You can override these at install time with +# helm install openldap --set openldap.adminPassword=,openldap.configPassword= +# adminPassword: admin +# configPassword: config + +# Custom openldap configuration files used to override default settings +# customLdifFiles: + # 01-default-users.ldif: |- + # Predefine users here + +## Persist data to a persistent volume +persistence: + enabled: false + ## database data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessMode: ReadWriteOnce + size: 8Gi + # existingClaim: "" + +resources: {} + # requests: + # cpu: "100m" + # memory: "256Mi" + # limits: + # cpu: "500m" + # memory: "512Mi" + +initResources: {} + # requests: + # cpu: "100m" + # memory: "128Mi" + # limits: + # cpu: "100m" + # memory: "128Mi" + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +## test container details +test: + enabled: false + image: + repository: dduportal/bats + tag: 0.4.0 diff --git a/docs/investigation/pod-is-killed-while-agent-restores.md b/docs/investigation/pod-is-killed-while-agent-restores.md new file mode 100644 index 000000000..3254bcee6 --- /dev/null +++ b/docs/investigation/pod-is-killed-while-agent-restores.md @@ -0,0 +1,208 @@ +# Repro Pod is Killed During a Restore Operation + +## Summary + +In the middle of an automated database restore, customers found that the Pod was +being killed after 1 hour. They had to manually change the liveness probe to +increase the tolerance to many hours. + +It has been confirmed by the automation team that the Agent, when starting a +restore operation, will stop the database and download the restore files, copy +new files to /data and start the mongodb database once again. **During this +period the mongod process is down**, this is, there's no `PID` associated with +it, making the `Livenessprobe` to flag the Pods, causing a Pod restart. + +## References + +* https://jira.mongodb.org/browse/CLOUDP-84518 +* https://jira.mongodb.org/browse/CLOUDP-80199 + +## Materials + +In order to reproduce this scenario we need: + +- An Ops Manager instance with Backup configured +- Some way of pushing a reasonable amount of data + to a Replica set + +In this document, we describe how to do both. + +# Ops Manager Configuration + +The best possible alternative is to run the test `e2e_om_ops_manager_backup` +locally over your own cluster. This cluster should have enough capacity to hold: + ++ 1 Ops Manager instance ++ 1 Backup daemon ++ 3 Backup Replica Sets (blockstore, oplog and S3) ++ 1 AppDB Replica Set ++ 1 Test Replica Set (the one we'll fill with data and try to restore) + +## Set up Ops Manager with an E2E Test + + +Just execute the test with the test with the usual: + +```shell +make e2e test=e2e_om_ops_manager_backup_manual light=true +``` + +You'll have to wait around 20 minutes for this to finish. + +## Get a Database Dump to push to our new Replica Set + +Get the dump from S3 with: + +```shell +aws s3 cp s3://cloudp-84518/atlas-sample-database.zip . +``` + +And unzip: + +```shell +unzip atlas-sample-database.zip +``` + +The dump was obtained from Atlas, after loading a [sample +database](https://docs.atlas.mongodb.com/sample-data/) to it. + +## Upload data to the Running Database + +This will get a bit hacky, we'll restore the same dump *mutiple times*, into +databases with different names. + +1. Start a new Pod and download MongoDB Tools into it: + +``` +kubectl run mongothings -n mongodb --image ubuntu -i --tty --rm bash +apt update && apt install curl -y +curl -L https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.3.1.tgz -o /tmp/tools.tgz +tar xfz /tmp/tools.tgz +``` + +2. In a different shell, copy dump into Pod (takes like 2 minutes) + +``` +kubectl cp dump mongodb/mongothings:/tmp +``` + +3. Now go back to the other Pod and fill up the database. Each 10 iterations will fill the disk with about 3.5G + +``` +# 10 iterations will be about ~3.5 G +iterations=10 +mrestore="/mongodb-database-tools-ubuntu2004-x86_64-100.3.1/bin/mongorestore" +host="rs-non-fixed-0.rs-non-fixed-svc.mongodb.svc.cluster.local" +cd /tmp/dump + +for i in $(seq 1 $iterations); do + mkdir /tmp/dump-$i + for dir in *; do cp -r $dir "/tmp/dump-$i/$dir-$i"; done + + "${mrestore}" --host "${host}" /tmp/dump-$i + + rm -rf /tmp/dump-$i +done + +``` + + +# Backup & Restore + +## Enable Continuous Backup + +The `rs-fixed` and `rs-non-fixed` will have *Continuous Backup* enabled by this +point. We'll push some data into the database to cause a future restore to make +the database to fail. + +10 copies of the Atlas sample database generate a restore of around 7GB, then it +takes more than 3 minutes to `MakeBackupDataAvailable`. Which means that the Pod +will be restarted because of the LivenessProbe. + +``` +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +rs-non-fixed-0 1/1 Running 0 14m +rs-non-fixed-1 1/1 Running 0 14m +rs-non-fixed-2 1/1 Running 0 14m +``` + +For this investigation, we use 3 Pods; to make this easier to reproduce, we'll +wait for 60 minutes before starting the tests. + +Downloading the Backup file takes a long time (it is several GBs in size), and +this helps with the investigation. + +After some time, the Pods that are younger than 60 minutes get to running (and +Alive) state, but not the one that's older than 60 minutes. In order to fix it +I will proceed with the solution in next paragraph. + +## Pods keep being restarted + +The Pods, because they can't download the full 7GB of restore in less than 3 +minutes will be restarted, causing the whole process to start from scratch: the +agent has to be downloaded again, get the automation config, and start +downloading the restore archive. + +## How does the LivenessProbe works in this case + +The `LivenessProbe` is configured as follows: + +```golang +func databaseLivenessProbe() probes.Modification { + return probes.Apply( + probes.WithExecCommand([]string{databaseLivenessProbeCommand}), + probes.WithInitialDelaySeconds(60), + probes.WithTimeoutSeconds(30), + probes.WithPeriodSeconds(30), + probes.WithSuccessThreshold(1), + probes.WithFailureThreshold(6), + ) +} +``` + +The `LivenessProbe` has a 60 minutes tolerance to the absence of `mongod`, +`mongos` and `agent` processes. If we wait until the Pods are older than 1 hour, +and because the `LivenessProbe` is configured to fail after 6 tries +(`FailureThreshold`) and to probe every 30 seconds (`PeriodSeconds`), then the +Pods will be flag as failed after ~180 seconds. + +When describing one of the Pods we get: + +``` +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning Unhealthy 2m54s (x171 over 3h28m) kubelet Readiness probe failed: + Warning Unhealthy 2m53s (x10 over 162m) kubelet Liveness probe failed: + +``` + +And the Pods will report multiple restarts: + +```shell +kubectl get pods +NAME READY STATUS RESTARTS AGE +rs-non-fixed-0 0/1 Running 2 6h1m +rs-non-fixed-1 0/1 Running 1 3h19m +rs-non0fixed-2 0/1 Running 1 79m +``` + +# Solution + +If the `livenessProbe` is modified, to consider the agent process to be running, +as a sufficient condition for the Pod to be considered alive. The restore +operation succeeds after a reasonable amount of time. + +In this scenario, 2 MongoDB objects are provided: + +* `rs-fixed`: Uses a *modified* version of the `probe.sh` which will accept the + Agent's PID as sufficient condition for a Pod to be alive. +* `rs-non-fixed`: Uses the *regular* version of `probe.sh`. + + +`rs-fixed` survives a restore of a 3G snapshot, but `rs-non-fixed` does not. + +Both are identical, but `rs-fixed` has been configure with a special +LivenessProbe with the proposed fix, while `rs-non-fixed` is using the +LivenessProbe part of the Operator 1.10. diff --git a/docs/sharded-clusters/scaling-sequence-diagram.md b/docs/sharded-clusters/scaling-sequence-diagram.md new file mode 100644 index 000000000..39a52bf1d --- /dev/null +++ b/docs/sharded-clusters/scaling-sequence-diagram.md @@ -0,0 +1,90 @@ +```mermaid +--- +config: + theme: forest + noteAlign: left +--- +sequenceDiagram + title Sharded Cluster scaling process + + participant Operator + participant k8s + participant OM + participant Agent + Note over OM: OM never initiates connections or pushes the data to the Operator or Agents.
It's always the operator and the agents which are connecting to it.
The arrows' directions indicate only whether we update or just get the data from it. + Agent --> OM: Each running agent sends periodic pings to OM + note left of Agent: The time since last ping is reflected in WaitForAgentsToRegister
(via GET /groups//agents/AUTOMATION)
Ping delay > 1min means the agent process is DOWN + opt + note left of Operator: upgradeAllIfNeeded
(bumps AC version +1) + Operator->>OM: /groups//automationConfig/updateAgentVersions + Agent->>OM: re-plan + Note over Operator: It's fire and forget.
We don't wait for the goal state here.
It's executed once per 24h or with each operator startup.
It's executed for each managed database, so each operator restart
means bumps and re-plans for each MongoDB out there. + end + opt + note left of Operator: prepareScaleDownShardedCluster
(set vote=0, priority=0 for scaled down processes)) + Operator->>Operator: calculate to-be-scaled-down processes:
Iterate over all clusters (even unhealthy ones)
and gather all replicaset processes to be scaled down this
reconciliation. It uses scalers+ReplicasThisReconciliation,
so all replicasets are guaranteed to be scaled down one
by one across all clusters. + + Note over Operator: Mark one process at a time for any given replica set for scaling down
with votes=0, priority=0
We might mark more than one process at a time
if the processes are from different replicasets. + Operator->>OM: updateAutomationConfig with all to-be-scaled-down
processes marked with votes=0, priority=0
PUT /groups//automationConfig + Operator->>OM: WaitForGoalState (only on up&healthy to-be-scaled-down processes)
GET /groups//automationStatus
+ note right of Operator: Problem: currently, we wait not on all processes
here, but only on the processes to be scaled down. We should
ensure all (healthy) processes from any given replicaset are
in goal state. Especially for the case when the scaled down
processes are unhealthy -> we won't wait at all for the changes
applied to the replicaset + note right of Operator: note that there might be more processes to scale down from
any given replica set, but we'll be marking maximum of one
process at a time (but it can be multiple at once but from
different replicasets) + end + + opt + note left of Operator: AutomaticRecovery + Operator->>Operator: check the time spent in not Running phase
(r.deploymentState.Status.Phase != mdbstatus.PhaseRunning)
do nothing if the time since status.LastTransitionTime less than than
MDB_AUTOMATIC_RECOVERY_BACKOFF_TIME_S (default=20 mins)
+ Note right of Operator: run updateOmDeploymentShardedCluster
(publish AC first) + Note right of Operator: run createKubernetesResources
(publish sts first) + end + + opt + note left of Operator: RunInGivenOrder + Note over Operator: iterate over all desired statefulset configuration
(all statefulsets over all healthy clusters for all mongos, configSrv and shards)

if there is even one statefulset satisfying
publishAutomationConfigFirst function,
then the automation config is first published.
Otherwise it's statefulset first. In most cases it's statefulset first. + Note over Operator: it's the automation config first (true) when it's one of the following is satisfied:
- disabling TLS
- we're clearing TLS CA certificate
- we're clearing OM's CA cert set in project's configmap
- we're changing auth mechanism from X509 to a different one
- we're scaling down
- we're running in static arch and we're changing MongoDB's version + Note over Operator: it's the statefulset first (false) when one of the following is satisfied:
- there is no statefulset yet, or there is error getting it
- any other case not satisfied (it's a default case) + end + + opt + note left of Operator: updateOmDeploymentShardedCluster
(publish automation config) + opt waitForAgentsToRegister + OM ->> Operator: OM: we get all agent statuses
we traverse all pages from
GET /groups//agents/AUTOMATION) + Note over Operator: here the logic is executed for each component sequentially:
- we calculate the list of expected agents registered from *healthy* member clusters
- we get all the registered agents, filter out those not belonging to the currently processing component (replicaset or mongos)
- we filter out every agent that is DOWN (last ping >1min ago)
- if we have the same list of agents as expected we're done waiting
- if we don't have it yet, we retry (not requeue!), we're retrying for 10 times * 9s sleep
- we retry only when sleep times out + Note over Operator: what is the meaning of this wait:

- in order to push any automation config change we must have all the expected agents running and healthy
- we check only agents' pings to OM, not the goal state
- the list of agents is created according to ReplicasThisReconciliation, so we will always wait here for
the pod to be scaled up before we publish automation config (note in most cases it's the sts published before AC) + end + k8s ->> Operator: get current automation config
GET groups//automationConfig + opt + Note left of Operator: publishDeployment (finalizing=false, so not removing shards yet) + Note over Operator: we create the desired process list and the configuration according to resource's spec and ReplicasThisReconciliation for each component
important to note: we always uses spec.ShardCount when creating the processes for shard replicasets,
so if we're removing shards the operator immediately stops pushing *any* changes to the shards that are going to be removed.

The shards are not going to be removed from AC immediately though,
because we're always merging replicasets and processes with the current AC
and keep other shards until we do the finalizing=true stage.

The replicasets (shards) that are going to be removed are marked for draining and
the agents will initiate rebalancing of the data (it's the 1st stage of shard removal).

The information whether there are shards to be removed is returned and used to execute publishDeployment again after WairForReadyState + + Operator->>k8s: update merged automation config
PUT groups//automationConfig + end + k8s ->> Operator: get current automation config
GET groups//automationConfig + opt WaitForReadyState + Note over Operator: we use all the processes from the current automation config (d.GetProcessNames())
That means we're not only waiting for the goal state of the current expected processes,
but also for the processes from the draining replicasets and also from the unhealthy processes as well
(those which are not reporting pings anymore)
+ Note over Operator: Problem: this is the source of a deadlock. We shouldn't wait for the processes that are down,
because it won't ever succeed. But Mongos deadlock is more than that - it's healthy,
but it's not progressing until all DOWN processes are removed from the project. + OM ->> Operator: GET /groups//automationStatus + end + opt + Note left of Operator: publishDeployment (finalizing=true, so actually removing shard replicasets) + Note over Operator: This publishDeployment is executed only if there were shards scheduled to be removed in the previous publishDeployment.
Executing this after WaitForReadyState ensures the shards were drained and their data was rebalanced. + end + end + opt + note left of Operator: createKubernetesResources (publish statefulsets) + note over Operator: The order in which we publish statefulsets might differ.
In most cases the order is as follows:
- config server
- shards
- mongos

BUT if we're running in static arch AND we're downgrading MongoDB's version it's reversed:
- mongos
- shards
- config server

Publication of sts is unconditional. We only wait for the sts status after it's published.

For each healthy member cluster we publish statefulset with the number of replicas equal to ReplicasThisReconciliation, so maintaining one-at-a-time scaling. + + loop for each component (mongos, cs, each shard) but healthy member clusters only + Operator->>k8s: update statefulset, replicas=ReplicasThisReconciliation + end + loop healthy member clusters + Operator->>k8s: Get statefulset status + Operator ->> k8s: if ANY of statuses is NOT READY we exit and requeue + Note over Operator: statefulset is ready if all the pods are Ready (readiness probes reports ready see note below)
and there are desired number replicas in place (look for StatefulSetState.IsReady()) + Note over Operator: important to note: while we don't wait here for the agent's goal state explicitly, it's important to
understand that we're waiting for the pods' readiness.

And pod's readiness probe uses agent's health status in determining whether we're ready.
The readiness probe will return ready (simplifying):
- when the agent in goal state
- when the agent is in a WaitStep for longer than 15s (typically waiting for other nodes to perform changes)
- there is no plan to do (e.g. after fresh deployment, no process in AC yet for this agent) + end + end + Operator->>k8s: if scaling is still not finished -> update status to Pending + Operator->>k8s: delete statefulsets of removed shards + Operator->>k8s: update status to Pending +``` diff --git a/generate_ssdlc_report.py b/generate_ssdlc_report.py new file mode 100755 index 000000000..eff136647 --- /dev/null +++ b/generate_ssdlc_report.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 + +""" +SSDLC report + +At the moment, the following functionality has been implemented: + - Downloading SBOMs + - Replacing Date + - Replacing Release Version + +The script should be called manually from the project root. It will put generated files +into "./ssdlc-report/MEKO-$release" directory. +""" + +import enum +import json +import os +import pathlib +import subprocess +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from datetime import datetime +from queue import Queue +from typing import Dict, List + +import boto3 + +from lib.base_logger import logger +from scripts.evergreen.release.agent_matrix import ( + LATEST_OPERATOR_VERSION, + get_supported_version_for_image_matrix_handling, +) + +NUMBER_OF_THREADS = 15 +S3_BUCKET = "kubernetes-operators-sboms" + + +class Subreport(enum.Enum): + AGENT = ("Containerized MongoDB Agent", "scripts/ssdlc/templates/SSDLC Containerized MongoDB Agent ${VERSION}.md") + OPERATOR = ( + "Enterprise Kubernetes Operator", + "scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise Kubernetes Operator ${VERSION}.md", + ) + OPS_MANAGER = ( + "Containerized OpsManager", + "scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise OpsManager ${VERSION}.md", + ) + TESTING = ("not-used", "scripts/ssdlc/templates/SSDLC MongoDB Enterprise Operator Testing Report ${VERSION}.md") + + def __new__(cls, *args, **kwds): + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + + def __init__(self, sbom_subpath: str, template_path: str): + self.sbom_subpath = sbom_subpath + self.template_path = template_path + + +@dataclass +class SupportedImage: + versions: List[str] + name: str + image_pull_spec: str + ssdlc_report_name: str + sbom_file_names: List[str] + platforms: List[str] + subreport: Subreport + + +def get_release() -> Dict: + with open("release.json") as release: + return json.load(release) + + +def get_supported_images(release: Dict) -> dict[str, SupportedImage]: + logger.debug(f"Getting list of supported images") + supported_images: Dict[str, SupportedImage] = dict() + for supported_image in release["supportedImages"]: + ssdlc_name = release["supportedImages"][supported_image]["ssdlc_name"] + if "versions" in release["supportedImages"][supported_image]: + for tag in release["supportedImages"][supported_image]["versions"]: + if supported_image not in supported_images: + subreport = Subreport.OPERATOR + if supported_image == "ops-manager": + subreport = Subreport.OPS_MANAGER + supported_images[supported_image] = SupportedImage( + list(), supported_image, "", ssdlc_name, list(), ["linux/amd64"], subreport + ) + supported_images[supported_image].versions.append(tag) + + supported_images = filter_out_unsupported_images(supported_images) + supported_images = convert_to_image_names(supported_images) + supported_images["mongodb-agent-ubi"] = SupportedImage( + get_supported_version_for_image_matrix_handling("mongodb-agent", LATEST_OPERATOR_VERSION), + "mongodb-agent-ubi", + "quay.io/mongodb/mongodb-agent-ubi", + release["supportedImages"]["mongodb-agent"]["ssdlc_name"], + list(), + # Once MEKO supports both architectures, this should be re-enabled. + # ["linux/amd64", "linux/arm64"], + ["linux/amd64"], + Subreport.AGENT, + ) + + supported_images["mongodb-enterprise-cli"] = SupportedImage( + [release["mongodbOperator"]], + "mongodb-enterprise-cli", + "mongodb-enterprise-cli", + "MongoDB Enterprise Kubernetes Operator CLI", + list(), + ["linux/amd64", "linux/arm64", "darwin/amd64", "darwin/arm64"], + Subreport.OPERATOR, + ) + supported_images = filter_out_non_current_versions(release, supported_images) + logger.debug(f"Supported images: {supported_images}") + return supported_images + + +def convert_to_image_names(supported_images: Dict[str, SupportedImage]): + for supported_image in supported_images: + supported_images[supported_image].image_pull_spec = f"quay.io/mongodb/mongodb-enterprise-{supported_image}-ubi" + supported_images[supported_image].name = f"mongodb-enterprise-{supported_image}-ubi" + return supported_images + + +# noinspection PyShadowingNames +def filter_out_unsupported_images(supported_images: Dict[str, SupportedImage]): + if "mongodb-agent" in supported_images: + del supported_images["mongodb-agent"] + if "appdb-database" in supported_images: + del supported_images["appdb-database"] + if "mongodb-enterprise-server" in supported_images: + del supported_images["mongodb-enterprise-server"] + if "mongodb-kubernetes-operator-version-upgrade-post-start-hook" in supported_images: + del supported_images["mongodb-kubernetes-operator-version-upgrade-post-start-hook"] + if "mongodb-kubernetes-readinessprobe" in supported_images: + del supported_images["mongodb-kubernetes-readinessprobe"] + if "mongodb-kubernetes-operator" in supported_images: + del supported_images["mongodb-kubernetes-operator"] + return supported_images + + +def filter_out_non_current_versions(release: Dict, supported_images: Dict[str, SupportedImage]): + for supported_image_key in supported_images: + supported_image = supported_images[supported_image_key] + if supported_image.subreport == Subreport.OPERATOR: + supported_image.versions = filter(lambda x: x == release["mongodbOperator"], supported_image.versions) + return supported_images + + +def _download_augmented_sbom(s3_path: str, directory: str, sbom_name: str): + logger.debug(f"Downloading Augmented SBOM for {s3_path} into {directory}/{sbom_name}") + + s3 = boto3.client("s3") + response = s3.list_objects(Bucket=S3_BUCKET, Prefix=s3_path, MaxKeys=1) + if "Contents" not in response: + raise Exception(f"Could not download Augmented SBOM for {s3_path}") + else: + augmented_sbom_key = response["Contents"][0]["Key"] + pathlib.Path(directory).mkdir(parents=True, exist_ok=True) + with open(f"{directory}/{sbom_name}", "wb") as data: + s3.download_fileobj(S3_BUCKET, augmented_sbom_key, data) + + +def _queue_exception_handling(tasks_queue, ignore_sbom_download_errors: bool): + exceptions_found = False + for task in tasks_queue.queue: + if task.exception() is not None: + exceptions_found = True + logger.fatal(f"The following exception has been found when downloading SBOMs: {task.exception()}") + if exceptions_found: + if not ignore_sbom_download_errors: + raise Exception(f"Exception(s) found when downloading SBOMs") + + +def download_augmented_sboms( + report_path: str, supported_images: Dict[str, SupportedImage], ignore_sbom_download_errors: bool +) -> Dict[str, SupportedImage]: + tasks_queue = Queue() + with ProcessPoolExecutor(max_workers=NUMBER_OF_THREADS) as executor: + for supported_image_key in supported_images: + supported_image = supported_images[supported_image_key] + for tag in supported_image.versions: + for platform in supported_image.platforms: + platform_sanitized = platform.replace("/", "-") + s3_path = f"sboms/release/augmented/{supported_image.image_pull_spec}/{tag}/{platform_sanitized}" + sbom_name = f"{supported_image.name}_{tag}_{platform_sanitized}-augmented.json" + tasks_queue.put( + executor.submit( + _download_augmented_sbom, + s3_path, + f"{report_path}/{supported_image.subreport.sbom_subpath}", + sbom_name, + ) + ) + supported_image.sbom_file_names.append(sbom_name) + _queue_exception_handling(tasks_queue, ignore_sbom_download_errors) + return supported_images + + +def prepare_sbom_markdown(supported_images: Dict[str, SupportedImage], subreport: Subreport): + lines = "" + for supported_image_key in supported_images: + supported_image = supported_images[supported_image_key] + if supported_image.subreport == subreport: + lines = f"{lines}\n\t- {supported_image.ssdlc_report_name}:" + for sbom_location in supported_image.sbom_file_names: + lines = f"{lines}\n\t\t- [{sbom_location}](./{supported_image.subreport.sbom_subpath}/{sbom_location})" + return lines + + +def get_git_user_name() -> str: + res = subprocess.run(["git", "config", "user.name"], stdout=subprocess.PIPE) + return res.stdout.strip().decode() + + +def generate_ssdlc_report(ignore_sbom_download_errors: bool = False): + """Generates the SSDLC report. + + :param ignore_sbom_download_errors: True if downloading SBOM errors should be ignored. + :return: N/A + """ + logger.info(f"Producing SSDLC report for more manual edits") + release = get_release() + + operator_version = release["mongodbOperator"] + supported_images = get_supported_images(release) + report_path = os.getcwd() + "/ssdlc-report/MEKO-" + operator_version + + try: + if not os.path.exists(path=report_path): + os.makedirs(report_path) + + logger.info(f"Downloading SBOMs") + downloaded_sboms = download_augmented_sboms(report_path, supported_images, ignore_sbom_download_errors) + + for subreport in Subreport: + logger.info(f"Generating subreport {subreport.template_path}") + with open(subreport.template_path, "r") as report_template: + content = report_template.read() + + content = content.replace("${SBOMS}", prepare_sbom_markdown(downloaded_sboms, subreport)) + content = content.replace("${VERSION}", operator_version) + content = content.replace("${DATE}", datetime.today().strftime("%Y-%m-%d")) + content = content.replace("${RELEASE_TYPE}", "Minor") + content = content.replace("${AUTHOR}", get_git_user_name()) + + report_file_name = subreport.template_path.replace("${VERSION}", operator_version) + report_file_name = os.path.basename(report_file_name) + report_file_name = f"{report_path}/{report_file_name}" + with open(report_file_name, "w") as file: + file.write(content) + logger.info(f"Done") + except: + logger.exception(f"Could not generate report") + + +if __name__ == "__main__": + generate_ssdlc_report() diff --git a/generate_ssdlc_report_test.py b/generate_ssdlc_report_test.py new file mode 100755 index 000000000..95df7ede1 --- /dev/null +++ b/generate_ssdlc_report_test.py @@ -0,0 +1,50 @@ +""" +This file tests the SSDLC generation report. This file assumes it's been called from the project root +""" + +import json +import os +from typing import Dict + +import generate_ssdlc_report + + +def get_release() -> Dict: + with open("release.json") as release: + return json.load(release) + + +def test_report_generation(): + # Given + release = get_release() + current_version = release["mongodbOperator"] + current_directory = os.getcwd() + + # When + + # We ignore Augmented SBOM download errors for this test as we quite often have a few days in transition state. + # For example, when we release a new Ops Manager or Agent image, we upload the corresponding SBOM Lite + # on d+1. Then on d+2 we have the Augmented SBOM available for download. This situation is perfectly normal + # but causes this test to fail. Therefore, we ignore these errors here. + generate_ssdlc_report.generate_ssdlc_report(True) + + # Then + assert os.path.exists(f"{current_directory}/ssdlc-report/MEKO-{current_version}") + assert os.path.exists(f"{current_directory}/ssdlc-report/MEKO-{current_version}/Containerized MongoDB Agent") + assert os.listdir(f"{current_directory}/ssdlc-report/MEKO-{current_version}/Containerized MongoDB Agent") != [] + assert os.path.exists(f"{current_directory}/ssdlc-report/MEKO-{current_version}/Containerized OpsManager") + assert os.listdir(f"{current_directory}/ssdlc-report/MEKO-{current_version}/Containerized OpsManager") != [] + assert os.path.exists(f"{current_directory}/ssdlc-report/MEKO-{current_version}/Enterprise Kubernetes Operator") + assert os.listdir(f"{current_directory}/ssdlc-report/MEKO-{current_version}/Enterprise Kubernetes Operator") != [] + assert os.path.exists( + f"{current_directory}/ssdlc-report/MEKO-{current_version}/SSDLC Containerized MongoDB Agent {current_version}.md" + ) + assert os.path.exists( + f"{current_directory}/ssdlc-report/MEKO-{current_version}/SSDLC Containerized MongoDB Enterprise Kubernetes Operator {current_version}.md" + ) + assert os.path.exists( + f"{current_directory}/ssdlc-report/MEKO-{current_version}/SSDLC Containerized MongoDB Enterprise OpsManager {current_version}.md" + ) + assert os.path.exists( + f"{current_directory}/ssdlc-report/MEKO-{current_version}/SSDLC MongoDB Enterprise Operator Testing Report {current_version}.md" + ) diff --git a/go.mod b/go.mod index 5894aee21..448d42018 100644 --- a/go.mod +++ b/go.mod @@ -1,91 +1,117 @@ -module github.com/mongodb/mongodb-kubernetes-operator +// try to always update patch versions with `go get -u=patch ./...` -go 1.24.0 +module github.com/10gen/ops-manager-kubernetes require ( github.com/blang/semver v3.5.1+incompatible - github.com/go-logr/logr v1.4.2 + github.com/ghodss/yaml v1.0.0 + github.com/go-logr/zapr v1.3.0 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-retryablehttp v0.7.7 + github.com/hashicorp/vault/api v1.16.0 github.com/imdario/mergo v0.3.15 - github.com/spf13/cast v1.7.1 + github.com/mongodb/mongodb-kubernetes-operator v0.12.1-0.20250311084916-0e7208998804 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.21.0 + github.com/r3labs/diff/v3 v3.0.1 + github.com/spf13/cast v1.6.0 github.com/stretchr/objx v0.5.2 github.com/stretchr/testify v1.10.0 github.com/xdg/stringprep v1.0.3 - go.mongodb.org/mongo-driver v1.16.0 + github.com/yudai/gojsondiff v1.0.0 + go.mongodb.org/atlas v0.37.0 go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.37.0 + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.30.10 k8s.io/apimachinery v0.30.10 k8s.io/client-go v0.30.10 + k8s.io/code-generator v0.30.10 + k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.18.7 - sigs.k8s.io/yaml v1.4.0 ) -require google.golang.org/protobuf v1.33.0 // indirect +// force pin, until we update the direct dependencies +require google.golang.org/protobuf v1.36.1 // indirect require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect - github.com/moby/spdystream v0.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect - github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect + github.com/yudai/pp v2.0.1+incompatible // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.35.0 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.36.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/term v0.31.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect - k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) + +// to replace Community Operator to a locally cloned project run: +// go mod edit -replace github.com/mongodb/mongodb-kubernetes-operator=../mongodb-kubernetes-operator +// to replace it to a specific commit run: +// go mod edit -replace github.com/mongodb/mongodb-kubernetes-operator=github.com/mongodb/mongodb-kubernetes-operator@master + +go 1.24.0 diff --git a/go.sum b/go.sum index 4b8969f32..389c148c2 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,36 @@ -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -30,36 +41,68 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/vault/api v1.16.0 h1:nbEYGJiAPGzT9U4oWgaaB0g+Rj8E59QuHKyA5LhwQN4= +github.com/hashicorp/vault/api v1.16.0/go.mod h1:KhuUhzOD8lDSk29AtzNjgAu2kxRA9jL9NAbkFlqvkBA= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -68,8 +111,9 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -77,43 +121,69 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= -github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/mongodb/mongodb-kubernetes-operator v0.12.1-0.20250311084916-0e7208998804 h1:cdYYkNp0ER04dU7TNCG8mwbqaG0Nf1da2UgyQ7hWKZk= +github.com/mongodb/mongodb-kubernetes-operator v0.12.1-0.20250311084916-0e7208998804/go.mod h1:QjQRbqQK+qMOIFTxtp+y0FdazEDZNsBPGpKqcOreYB0= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= +github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/r3labs/diff/v3 v3.0.1 h1:CBKqf3XmNRHXKmdU7mZP1w7TV0pDyVCis1AUHtA4Xtg= +github.com/r3labs/diff/v3 v3.0.1/go.mod h1:f1S9bourRbiM66NskseyUdo0fTmEE0qKrikYJX63dgo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -122,26 +192,31 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= -go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.mongodb.org/atlas v0.37.0 h1:zQnO1o5+bVP9IotpAYpres4UjMD2F4nwNEFTZhNL4ck= +go.mongodb.org/atlas v0.37.0/go.mod h1:DJYtM+vsEpPEMSkQzJnFHrT0sP7ev6cseZc/GGjJYG8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -151,78 +226,88 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -236,8 +321,12 @@ k8s.io/apimachinery v0.30.10 h1:UflKuJeSSArttm05wjYP0GwpTlvjnMbDKFn6F7rKkKU= k8s.io/apimachinery v0.30.10/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.30.10 h1:C0oWM82QMvosIl/IdJhWfTUb7rIxM52rNSutFBknAVY= k8s.io/client-go v0.30.10/go.mod h1:OfTvt0yuo8VpMViOsgvYQb+tMJQLNWVBqXWkzdFXSq4= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/code-generator v0.30.10 h1:1p47NC8/zijsgCuqI0F20ErxsWE3VLUGEEcxoiweMeo= +k8s.io/code-generator v0.30.10/go.mod h1:b5HvR9KGVjQOK1fbnZfP/FL4Qe3Zox5CfXJ5Wp7tqQo= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= diff --git a/helm_chart/Chart.yaml b/helm_chart/Chart.yaml new file mode 100644 index 000000000..47dd194eb --- /dev/null +++ b/helm_chart/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: enterprise-operator +description: MongoDB Kubernetes Enterprise Operator +version: 1.32.0 +kubeVersion: '>=1.16-0' +type: application +keywords: +- mongodb +- database +- nosql +icon: https://mongodb-images-new.s3.eu-west-1.amazonaws.com/leaf-green-dark.png +home: https://github.com/mongodb/mongodb-enterprise-kubernetes +maintainers: +- name: MongoDB + email: support@mongodb.com diff --git a/helm_chart/crds/mongodb.com_mongodb.yaml b/helm_chart/crds/mongodb.com_mongodb.yaml new file mode 100644 index 000000000..a5b0287c2 --- /dev/null +++ b/helm_chart/crds/mongodb.com_mongodb.yaml @@ -0,0 +1,2662 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodb.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDB + listKind: MongoDBList + plural: mongodb + shortNames: + - mdb + singular: mongodb + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment. + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server. + jsonPath: .status.version + name: Version + type: string + - description: The type of MongoDB deployment. One of 'ReplicaSet', 'ShardedCluster' + and 'Standalone'. + jsonPath: .spec.type + name: Type + type: string + - description: The time since the MongoDB resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for the + mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the mongodb + processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + backup: + description: |- + Backup contains configuration options for configuring + backup for this MongoDB resource + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + autoTerminateOnDeletion: + description: |- + AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + when the MongoDB CR is deleted + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + client: + description: KMIP Client configuration + properties: + clientCertificatePrefix: + description: |- + A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + The names are generated using the following pattern: + KMIP Client Certificate (TLS Secret): + --kmip-client + KMIP Client Certificate Password: + --kmip-client-password + The expected key inside is called "password". + type: string + type: object + required: + - client + type: object + type: object + mode: + enum: + - enabled + - disabled + - terminated + type: string + snapshotSchedule: + properties: + clusterCheckpointIntervalMin: + enum: + - 15 + - 30 + - 60 + type: integer + dailySnapshotRetentionDays: + description: Number of days to retain daily snapshots. Setting + 0 will disable this rule. + maximum: 365 + minimum: 0 + type: integer + fullIncrementalDayOfWeek: + description: Day of the week when Ops Manager takes a full + snapshot. This ensures a recent complete backup. Ops Manager + sets the default value to SUNDAY. + enum: + - SUNDAY + - MONDAY + - TUESDAY + - WEDNESDAY + - THURSDAY + - FRIDAY + - SATURDAY + type: string + monthlySnapshotRetentionMonths: + description: Number of months to retain weekly snapshots. + Setting 0 will disable this rule. + maximum: 36 + minimum: 0 + type: integer + pointInTimeWindowHours: + description: Number of hours in the past for which a point-in-time + snapshot can be created. + enum: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 15 + - 30 + - 60 + - 90 + - 120 + - 180 + - 360 + type: integer + referenceHourOfDay: + description: Hour of the day to schedule snapshots using a + 24-hour clock, in UTC. + maximum: 23 + minimum: 0 + type: integer + referenceMinuteOfHour: + description: Minute of the hour to schedule snapshots, in + UTC. + maximum: 59 + minimum: 0 + type: integer + snapshotIntervalHours: + description: Number of hours between snapshots. + enum: + - 6 + - 8 + - 12 + - 24 + type: integer + snapshotRetentionDays: + description: Number of days to keep recent snapshots. + maximum: 365 + minimum: 1 + type: integer + weeklySnapshotRetentionWeeks: + description: Number of weeks to retain weekly snapshots. Setting + 0 will disable this rule + maximum: 365 + minimum: 0 + type: integer + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + format: hostname + type: string + configServerCount: + type: integer + configSrv: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + configSrvPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + duplicateServiceObjects: + description: |- + In few service mesh options for ex: Istio, by default we would need to duplicate the + service objects created per pod in all the clusters to enable DNS resolution. Users can + however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + type: boolean + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing MongoDB + to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added to the + externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + logLevel: + enum: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities and + tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Amount of members for this MongoDB Replica Set + type: integer + mongodsPerShardCount: + type: integer + mongos: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + mongosCount: + type: integer + mongosPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + persistent: + type: boolean + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + prometheus: + description: Prometheus configurations. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth Password. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode that + the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with a + CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + service: + description: |- + DEPRECATED please use `spec.statefulSet.spec.serviceName` to provide a custom service name. + this is an optional service, it will get the name "-service" in case not provided + type: string + shard: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + shardCount: + type: integer + shardOverrides: + description: |- + ShardOverrides allow for overriding the configuration of a specific shard. + It replaces deprecated spec.shard.shardSpecificPodSpec field. When spec.shard.shardSpecificPodSpec is still defined then + spec.shard.shardSpecificPodSpec is applied first to the particular shard and then spec.shardOverrides is applied on top + of that (if defined for the same shard). + items: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation + for the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItemOverride is almost exact copy of ClusterSpecItem object. + The object is used in ClusterSpecList in ShardedClusterComponentOverrideSpec in shard overrides. + The difference lies in some fields being optional, e.g. Members to make it possible to NOT override fields and rely on + what was set in top level shard configuration. + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + type: array + memberConfig: + description: Process configuration override for this shard. + Used in SingleCluster only. The number of items specified + must be >= spec.mongodsPerShardCount or spec.shardOverride.members. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Number of member nodes in this shard. Used only + in SingleCluster. For MultiCluster the number of members is + specified in ShardOverride.ClusterSpecList. + type: integer + podSpec: + description: The following override fields work for SingleCluster + only. For MultiCluster - fields from specific clusters are + used. + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + shardNames: + items: + type: string + minItems: 1 + type: array + statefulSet: + description: Statefulset override for this particular shard. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - shardNames + type: object + type: array + shardPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + shardSpecificPodSpec: + description: |- + ShardSpecificPodSpec allows you to provide a Statefulset override per shard. + DEPRECATED please use spec.shard.shardOverrides instead + items: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array + statefulSet: + description: |- + StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + the global one + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of MongoDB resources + It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - credentials + - type + - version + type: object + x-kubernetes-preserve-unknown-fields: true + status: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + configServerCount: + type: integer + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + members: + type: integer + message: + type: string + mongodsPerShardCount: + type: integer + mongosCount: + type: integer + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + shardCount: + type: integer + sizeStatusInClusters: + description: MongodbShardedSizeStatusInClusters describes the number + and sizes of replica sets members deployed across member clusters + properties: + configServerMongodsInClusters: + additionalProperties: + type: integer + type: object + mongosCountInClusters: + additionalProperties: + type: integer + type: object + shardMongodsInClusters: + additionalProperties: + type: integer + type: object + shardOverridesInClusters: + additionalProperties: + additionalProperties: + type: integer + type: object + type: object + type: object + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm_chart/crds/mongodb.com_mongodbmulticluster.yaml b/helm_chart/crds/mongodb.com_mongodbmulticluster.yaml new file mode 100644 index 000000000..0e1d65480 --- /dev/null +++ b/helm_chart/crds/mongodb.com_mongodbmulticluster.yaml @@ -0,0 +1,1074 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodbmulticluster.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBMultiCluster + listKind: MongoDBMultiClusterList + plural: mongodbmulticluster + shortNames: + - mdbmc + singular: mongodbmulticluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment. + jsonPath: .status.phase + name: Phase + type: string + - description: The time since the MongoDBMultiCluster resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for the + mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the mongodb + processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + backup: + description: |- + Backup contains configuration options for configuring + backup for this MongoDB resource + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + autoTerminateOnDeletion: + description: |- + AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + when the MongoDB CR is deleted + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + client: + description: KMIP Client configuration + properties: + clientCertificatePrefix: + description: |- + A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + The names are generated using the following pattern: + KMIP Client Certificate (TLS Secret): + --kmip-client + KMIP Client Certificate Password: + --kmip-client-password + The expected key inside is called "password". + type: string + type: object + required: + - client + type: object + type: object + mode: + enum: + - enabled + - disabled + - terminated + type: string + snapshotSchedule: + properties: + clusterCheckpointIntervalMin: + enum: + - 15 + - 30 + - 60 + type: integer + dailySnapshotRetentionDays: + description: Number of days to retain daily snapshots. Setting + 0 will disable this rule. + maximum: 365 + minimum: 0 + type: integer + fullIncrementalDayOfWeek: + description: Day of the week when Ops Manager takes a full + snapshot. This ensures a recent complete backup. Ops Manager + sets the default value to SUNDAY. + enum: + - SUNDAY + - MONDAY + - TUESDAY + - WEDNESDAY + - THURSDAY + - FRIDAY + - SATURDAY + type: string + monthlySnapshotRetentionMonths: + description: Number of months to retain weekly snapshots. + Setting 0 will disable this rule. + maximum: 36 + minimum: 0 + type: integer + pointInTimeWindowHours: + description: Number of hours in the past for which a point-in-time + snapshot can be created. + enum: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 15 + - 30 + - 60 + - 90 + - 120 + - 180 + - 360 + type: integer + referenceHourOfDay: + description: Hour of the day to schedule snapshots using a + 24-hour clock, in UTC. + maximum: 23 + minimum: 0 + type: integer + referenceMinuteOfHour: + description: Minute of the hour to schedule snapshots, in + UTC. + maximum: 59 + minimum: 0 + type: integer + snapshotIntervalHours: + description: Number of hours between snapshots. + enum: + - 6 + - 8 + - 12 + - 24 + type: integer + snapshotRetentionDays: + description: Number of days to keep recent snapshots. + maximum: 365 + minimum: 1 + type: integer + weeklySnapshotRetentionWeeks: + description: Number of weeks to retain weekly snapshots. Setting + 0 will disable this rule + maximum: 365 + minimum: 0 + type: integer + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + format: hostname + type: string + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added + to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the name + "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + duplicateServiceObjects: + description: |- + In few service mesh options for ex: Istio, by default we would need to duplicate the + service objects created per pod in all the clusters to enable DNS resolution. Users can + however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + type: boolean + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing MongoDB + to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added to the + externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + logLevel: + enum: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + type: string + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + persistent: + type: boolean + prometheus: + description: Prometheus configurations. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth Password. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode that + the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with a + CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + statefulSet: + description: |- + StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + the global one + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of MongoDB resources + It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - credentials + - type + - version + type: object + x-kubernetes-preserve-unknown-fields: true + status: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + clusterStatusList: + description: ClusterStatusList holds a list of clusterStatuses corresponding + to each cluster + properties: + clusterStatuses: + items: + description: |- + ClusterStatusItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + lastTransition: + type: string + members: + type: integer + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent + resource which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + warnings: + items: + type: string + type: array + required: + - phase + type: object + type: array + type: object + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm_chart/crds/mongodb.com_mongodbusers.yaml b/helm_chart/crds/mongodb.com_mongodbusers.yaml new file mode 100644 index 000000000..a81f0d449 --- /dev/null +++ b/helm_chart/crds/mongodb.com_mongodbusers.yaml @@ -0,0 +1,179 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodbusers.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBUser + listKind: MongoDBUserList + plural: mongodbusers + shortNames: + - mdbu + singular: mongodbuser + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the MongoDB User. + jsonPath: .status.phase + name: Phase + type: string + - description: The time since the MongoDB User resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + connectionStringSecretName: + type: string + db: + type: string + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + passwordSecretKeyRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + roles: + items: + properties: + db: + type: string + name: + type: string + required: + - db + - name + type: object + type: array + username: + type: string + required: + - db + - username + type: object + status: + properties: + db: + type: string + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + project: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + roles: + items: + properties: + db: + type: string + name: + type: string + required: + - db + - name + type: object + type: array + username: + type: string + warnings: + items: + type: string + type: array + required: + - db + - phase + - project + - username + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm_chart/crds/mongodb.com_opsmanagers.yaml b/helm_chart/crds/mongodb.com_opsmanagers.yaml new file mode 100644 index 000000000..f758cd571 --- /dev/null +++ b/helm_chart/crds/mongodb.com_opsmanagers.yaml @@ -0,0 +1,1950 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: opsmanagers.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBOpsManager + listKind: MongoDBOpsManagerList + plural: opsmanagers + shortNames: + - om + singular: opsmanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The number of replicas of MongoDBOpsManager. + jsonPath: .spec.replicas + name: Replicas + type: integer + - description: The version of MongoDBOpsManager. + jsonPath: .spec.version + name: Version + type: string + - description: The current state of the MongoDBOpsManager. + jsonPath: .status.opsManager.phase + name: State (OpsManager) + type: string + - description: The current state of the MongoDBOpsManager Application Database. + jsonPath: .status.applicationDatabase.phase + name: State (AppDB) + type: string + - description: The current state of the MongoDBOpsManager Backup Daemon. + jsonPath: .status.backup.phase + name: State (Backup) + type: string + - description: The time since the MongoDBOpsManager resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Warnings. + jsonPath: .status.warnings + name: Warnings + type: string + name: v1 + schema: + openAPIV3Schema: + description: The MongoDBOpsManager resource allows you to deploy Ops Manager + within your Kubernetes cluster + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + adminCredentials: + description: |- + AdminSecret is the secret for the first admin user to create + has the fields: "Username", "Password", "FirstName", "LastName" + type: string + applicationDatabase: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig are additional configurations that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: specify configuration like startup flags and automation + config settings for the AutomationAgent and MonitoringAgent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + automationConfig: + description: |- + AutomationConfigOverride holds any fields that will be merged on top of the Automation Config + that the operator creates for the AppDB. Currently only the process.disabled and logRotate field is recognized. + properties: + processes: + items: + description: OverrideProcess contains fields that we can + override on the AutomationConfig processes. + properties: + disabled: + type: boolean + logRotate: + description: CrdLogRotate is the crd definition of LogRotate + including fields in strings while the agent supports + them as float64 + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + name: + type: string + required: + - disabled + - name + type: object + type: array + replicaSet: + properties: + settings: + description: |- + MapWrapper is a wrapper for a map to be used by other structs. + The CRD generator does not support map[string]interface{} + on the top level and hence we need to work around this with + a wrapping struct. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + type: string + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added + to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica Set + maximum: 50 + minimum: 3 + type: integer + monitoringAgent: + description: |- + Specify configuration like startup flags just for the MonitoringAgent. + These take precedence over + the flags set in AutomationAgent + properties: + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + required: + - startupOptions + type: object + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + passwordSecretKeyRef: + description: |- + PasswordSecretKeyRef contains a reference to the secret which contains the password + for the mongodb-ops-manager SCRAM-SHA user + properties: + key: + type: string + name: + type: string + required: + - name + type: object + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + prometheus: + description: Enables Prometheus integration on the AppDB. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth + Password. + properties: + key: + description: Key is the key in the secret storing this + password. Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this + password. Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode + that the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of + auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with + a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + service: + description: this is an optional service, it will get the name + "-svc" in case not provided + type: string + topology: + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - version + type: object + backup: + description: Backup + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + blockStores: + items: + description: |- + DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores + Optionally references the user if the Mongodb is configured with authentication + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + required: + - mongodbResourceRef + - name + type: object + type: array + enabled: + description: Enabled indicates if Backups will be enabled for + this Ops Manager. + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + server: + description: KMIP Server configuration + properties: + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used for KMIP authentication + type: string + url: + description: |- + KMIP Server url in the following format: hostname:port + Valid examples are: + 10.10.10.3:5696 + my-kmip-server.mycorp.com:5696 + kmip-svc.svc.cluster.local:5696 + pattern: '[^\:]+:[0-9]{0,5}' + type: string + required: + - ca + - url + type: object + required: + - server + type: object + type: object + externalServiceEnabled: + type: boolean + fileSystemStores: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + headDB: + description: HeadDB specifies configuration options for the HeadDB + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + jvmParameters: + items: + type: string + type: array + logging: + properties: + LogBackAccessRef: + description: LogBackAccessRef points at a ConfigMap/key with + the logback access configuration file to mount on the Pod + properties: + name: + type: string + type: object + LogBackRef: + description: LogBackRef points at a ConfigMap/key with the + logback configuration file to mount on the Pod + properties: + name: + type: string + type: object + type: object + members: + description: Members indicate the number of backup daemon pods + to create. + minimum: 1 + type: integer + opLogStores: + description: OplogStoreConfigs describes the list of oplog store + configs used for backup + items: + description: |- + DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores + Optionally references the user if the Mongodb is configured with authentication + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + required: + - mongodbResourceRef + - name + type: object + type: array + queryableBackupSecretRef: + description: |- + QueryableBackupSecretRef references the secret which contains the pem file which is used + for queryable backup. This will be mounted into the Ops Manager pod. + properties: + name: + type: string + required: + - name + type: object + s3OpLogStores: + description: S3OplogStoreConfigs describes the list of s3 oplog + store configs used for backup. + items: + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + customCertificate: + description: |- + Set this to "true" to use the appDBCa as a CA to access S3. + Deprecated: This has been replaced by CustomCertificateSecretRefs, + In the future all custom certificates, which includes the appDBCa + for s3Config should be configured in CustomCertificateSecretRefs instead. + type: boolean + customCertificateSecretRefs: + description: |- + CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + that apply to the associated S3 bucket. + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + irsaEnabled: + description: |- + This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + type: boolean + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + pathStyleAccessEnabled: + type: boolean + s3BucketEndpoint: + type: string + s3BucketName: + type: string + s3RegionOverride: + type: string + s3SecretRef: + description: |- + S3SecretRef is the secret that contains the AWS credentials used to access S3 + It is optional because the credentials can be provided via AWS IRSA + properties: + name: + type: string + required: + - name + type: object + required: + - name + - pathStyleAccessEnabled + - s3BucketEndpoint + - s3BucketName + type: object + type: array + s3Stores: + items: + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + customCertificate: + description: |- + Set this to "true" to use the appDBCa as a CA to access S3. + Deprecated: This has been replaced by CustomCertificateSecretRefs, + In the future all custom certificates, which includes the appDBCa + for s3Config should be configured in CustomCertificateSecretRefs instead. + type: boolean + customCertificateSecretRefs: + description: |- + CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + that apply to the associated S3 bucket. + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + irsaEnabled: + description: |- + This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + type: boolean + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + pathStyleAccessEnabled: + type: boolean + s3BucketEndpoint: + type: string + s3BucketName: + type: string + s3RegionOverride: + type: string + s3SecretRef: + description: |- + S3SecretRef is the secret that contains the AWS credentials used to access S3 + It is optional because the credentials can be provided via AWS IRSA + properties: + name: + type: string + required: + - name + type: object + required: + - name + - pathStyleAccessEnabled + - s3BucketEndpoint + - s3BucketName + type: object + type: array + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - enabled + type: object + clusterDomain: + format: hostname + type: string + clusterName: + description: |- + Deprecated: This has been replaced by the ClusterDomain which should be + used instead + format: hostname + type: string + clusterSpecList: + items: + description: ClusterSpecOMItem defines members cluster details for + Ops Manager multi-cluster deployment. + properties: + backup: + description: |- + Backup contains settings to override from top-level `spec.backup` for this member cluster. + If the value is not set here, then the value is taken from `spec.backup`. + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + headDB: + description: HeadDB specifies configuration options for + the HeadDB + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + jvmParameters: + items: + type: string + type: array + members: + description: Members indicate the number of backup daemon + pods to create. + minimum: 0 + type: integer + statefulSet: + description: StatefulSetConfiguration specified optional + overrides for backup datemon statefulset. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + clusterDomain: + description: Cluster domain to override the default *.svc.cluster.local + if the default cluster domain has been changed on a cluster + level. + format: hostname + type: string + clusterName: + description: |- + ClusterName is name of the cluster where the Ops Manager Statefulset will be scheduled. + The operator is using ClusterName to find API credentials in `mongodb-enterprise-operator-member-list` config map to use for this member cluster. + If the credentials are not found, then the member cluster is considered unreachable and ignored in the reconcile process. + type: string + configuration: + additionalProperties: + type: string + description: |- + The configuration properties passed to Ops Manager and Backup Daemon in this cluster. + If specified (not empty) then this field overrides `spec.configuration` field entirely. + If not specified, then `spec.configuration` field is used for the Ops Manager and Backup Daemon instances in this cluster. + type: object + externalConnectivity: + description: |- + MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + accessing Ops Manager instances in this member cluster from outside the Kubernetes cluster. + If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be + directly passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this + Service when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to + this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, + this applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + jvmParameters: + description: |- + JVM parameters to pass to Ops Manager and Backup Daemon instances in this member cluster. + If specified (not empty) then this field overrides `spec.jvmParameters` field entirely. + If not specified, then `spec.jvmParameters` field is used for the Ops Manager and Backup Daemon instances in this cluster. + items: + type: string + type: array + members: + description: Number of Ops Manager instances in this member + cluster. + type: integer + statefulSet: + description: |- + Configure custom StatefulSet configuration to override in Ops Manager's statefulset in this member cluster. + If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + configuration: + additionalProperties: + type: string + description: The configuration properties passed to Ops Manager/Backup + Daemon + type: object + externalConnectivity: + description: |- + MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + accessing this Ops Manager resource from outside the Kubernetes cluster. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be directly + passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this Service + when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, this + applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + internalConnectivity: + description: |- + InternalConnectivity if set allows for overriding the settings of the default service + used for internal connectivity to the OpsManager servers. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be directly + passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this Service + when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, this + applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + jvmParameters: + description: Custom JVM parameters passed to the Ops Manager JVM + items: + type: string + type: array + logging: + properties: + LogBackAccessRef: + description: LogBackAccessRef points at a ConfigMap/key with the + logback access configuration file to mount on the Pod + properties: + name: + type: string + type: object + LogBackRef: + description: LogBackRef points at a ConfigMap/key with the logback + configuration file to mount on the Pod + properties: + name: + type: string + type: object + type: object + opsManagerURL: + description: |- + OpsManagerURL specified the URL with which the operator and AppDB monitoring agent should access Ops Manager instance (or instances). + When not set, the operator is using FQDN of Ops Manager's headless service `{name}-svc.{namespace}.svc.cluster.local` to connect to the instance. If that URL cannot be used, then URL in this field should be provided for the operator to connect to Ops Manager instances. + type: string + replicas: + minimum: 1 + type: integer + security: + description: Configure HTTPS. + properties: + certsSecretPrefix: + type: string + tls: + properties: + ca: + type: string + secretRef: + properties: + name: + type: string + required: + - name + type: object + type: object + type: object + statefulSet: + description: Configure custom StatefulSet configuration + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of Ops Manager deployment. + It defaults (and if not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + version: + type: string + required: + - applicationDatabase + - version + type: object + status: + properties: + applicationDatabase: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + clusterStatusList: + items: + properties: + clusterName: + type: string + members: + type: integer + type: object + type: array + configServerCount: + type: integer + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + members: + type: integer + message: + type: string + mongodsPerShardCount: + type: integer + mongosCount: + type: integer + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + shardCount: + type: integer + sizeStatusInClusters: + description: MongodbShardedSizeStatusInClusters describes the + number and sizes of replica sets members deployed across member + clusters + properties: + configServerMongodsInClusters: + additionalProperties: + type: integer + type: object + mongosCountInClusters: + additionalProperties: + type: integer + type: object + shardMongodsInClusters: + additionalProperties: + type: integer + type: object + shardOverridesInClusters: + additionalProperties: + additionalProperties: + type: integer + type: object + type: object + type: object + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + backup: + properties: + clusterStatusList: + items: + properties: + clusterName: + type: string + replicas: + type: integer + type: object + type: array + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + type: object + opsManager: + properties: + clusterStatusList: + items: + properties: + clusterName: + type: string + replicas: + type: integer + type: object + type: array + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + replicas: + type: integer + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + url: + type: string + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm_chart/templates/_helpers.tpl b/helm_chart/templates/_helpers.tpl new file mode 100644 index 000000000..816436a12 --- /dev/null +++ b/helm_chart/templates/_helpers.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* + Define the namespace in which the operator deployment will be running. + This can be used to override the installation of the operator in the same namespace as the helm release +*/}} +{{- define "mongodb-enterprise-operator.namespace" -}} +{{- if .Values.operator.namespace -}} +{{- .Values.operator.namespace -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} diff --git a/helm_chart/templates/database-roles.yaml b/helm_chart/templates/database-roles.yaml new file mode 100644 index 000000000..230ea158d --- /dev/null +++ b/helm_chart/templates/database-roles.yaml @@ -0,0 +1,86 @@ +{{ if .Values.operator.createResourcesServiceAccountsAndRoles }} + +{{- $watchNamespace := include "mongodb-enterprise-operator.namespace" . | list }} +{{- if .Values.operator.watchNamespace }} +{{- $watchNamespace = regexSplit "," .Values.operator.watchNamespace -1 }} +{{- end }} + + +{{- range $idx, $namespace := $watchNamespace }} + +{{- $namespaceBlock := printf "namespace: %s" $namespace }} +{{- if eq $namespace "*" }} +{{- $namespaceBlock = include "mongodb-enterprise-operator.namespace" $ | printf "namespace: %s" }} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-appdb + {{ $namespaceBlock }} +{{- if $.Values.registry.imagePullSecrets}} +imagePullSecrets: + - name: {{ $.Values.registry.imagePullSecrets }} +{{- end }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-database-pods + {{ $namespaceBlock }} +{{- if $.Values.registry.imagePullSecrets}} +imagePullSecrets: + - name: {{ $.Values.registry.imagePullSecrets }} +{{- end }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-ops-manager + {{ $namespaceBlock }} +{{- if $.Values.registry.imagePullSecrets}} +imagePullSecrets: + - name: {{ $.Values.registry.imagePullSecrets }} +{{- end }} + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + {{ $namespaceBlock }} +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + {{ $namespaceBlock }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-appdb +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-appdb + {{ $namespaceBlock }} + +{{- end }} +{{- end }}{{/* if .Values.operator.createResourcesServiceAccountsAndRoles */}} diff --git a/helm_chart/templates/operator-configmap.yaml b/helm_chart/templates/operator-configmap.yaml new file mode 100644 index 000000000..adc820238 --- /dev/null +++ b/helm_chart/templates/operator-configmap.yaml @@ -0,0 +1,17 @@ +{{ $ns := include "mongodb-enterprise-operator.namespace" . -}} +{{- if not (lookup "v1" "ConfigMap" $ns "mongodb-enterprise-operator-member-list") }} +{{- if .Values.multiCluster.clusters }} +--- +apiVersion: v1 +kind: ConfigMap +data: + {{- range .Values.multiCluster.clusters }} + {{ . }}: "" + {{- end }} +metadata: + namespace: {{$ns}} + name: mongodb-enterprise-operator-member-list + labels: + multi-cluster: "true" +{{- end }} +{{- end }} diff --git a/helm_chart/templates/operator-roles.yaml b/helm_chart/templates/operator-roles.yaml new file mode 100644 index 000000000..49b18ea78 --- /dev/null +++ b/helm_chart/templates/operator-roles.yaml @@ -0,0 +1,231 @@ +{{ if .Values.operator.createOperatorServiceAccount }} +{{- $watchNamespace := include "mongodb-enterprise-operator.namespace" . | list }} +{{- if .Values.operator.watchNamespace }} +{{- $watchNamespace = regexSplit "," .Values.operator.watchNamespace -1 }} +{{- $watchNamespace = concat $watchNamespace (include "mongodb-enterprise-operator.namespace" . | list) | uniq }} +{{- end }} + +{{- $roleScope := "Role" -}} +{{- if or (gt (len $watchNamespace) 1) (eq (first $watchNamespace) "*") }} +{{- $roleScope = "ClusterRole" }} +{{- end }} +--- +kind: {{ $roleScope }} +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.operator.name }} +{{- if eq $roleScope "Role" }} + namespace: {{ include "mongodb-enterprise-operator.namespace" . }} +{{- end }} +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + {{- if .Values.subresourceEnabled }} + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + {{- end }} +{{- if eq $roleScope "ClusterRole" }} + - apiGroups: + - '' + resources: + - namespaces + verbs: + - list + - watch +{{- end}} +{{ if .Values.operator.enablePVCResize }} + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +{{- end}} +{{- range $idx, $namespace := $watchNamespace }} + +{{- $namespaceBlock := "" }} +{{- if not (eq $namespace "*") }} +{{- $namespaceBlock = printf "namespace: %s" $namespace }} +{{- end }} + +--- +{{- if eq $namespace "*" }} +kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $.Values.operator.name }} + {{ $namespaceBlock }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: {{ $roleScope }} + name: {{ $.Values.operator.name }} +subjects: + - kind: ServiceAccount + name: {{ $.Values.operator.name }} + namespace: {{ include "mongodb-enterprise-operator.namespace" $ }} +{{- end }} + +{{- end }} +--- + +{{/* This cluster role and binding is necessary to allow the operator to automatically register ValidatingWebhookConfiguration. */}} +{{- if and .Values.operator.webhook.registerConfiguration .Values.operator.webhook.installClusterRole }} +{{- if not (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "mongodb-enterprise-operator-mongodb-webhook") }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +{{- end }} +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.operator.name }}-{{ include "mongodb-enterprise-operator.namespace" . }}-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: {{ .Values.operator.name }} + namespace: {{ include "mongodb-enterprise-operator.namespace" . }} + +{{- end }} + +{{- $clusterRoleName := printf "%s-cluster-telemetry" .Values.operator.name }} +{{- $telemetry := default dict .Values.operator.telemetry }} + +{{/* We can't use default here, as 0, false and nil as determined as unset and thus set the default value */}} +{{- if ne $telemetry.enabled false }} + {{- if ne $telemetry.installClusterRole false}} + +--- +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $clusterRoleName }} +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +{{- end}} +--- +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.operator.name }}-{{ include "mongodb-enterprise-operator.namespace" . }}-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $clusterRoleName }} +subjects: + - kind: ServiceAccount + name: {{ .Values.operator.name }} + namespace: {{ include "mongodb-enterprise-operator.namespace" . }} +{{- end }} diff --git a/helm_chart/templates/operator-sa.yaml b/helm_chart/templates/operator-sa.yaml new file mode 100644 index 000000000..fd3547a48 --- /dev/null +++ b/helm_chart/templates/operator-sa.yaml @@ -0,0 +1,23 @@ +{{ if .Values.operator.createOperatorServiceAccount }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.operator.name }} + namespace: {{ include "mongodb-enterprise-operator.namespace" . }} +{{- if .Values.registry.imagePullSecrets}} +imagePullSecrets: + - name: {{ .Values.registry.imagePullSecrets }} +{{- end }} + +{{- $watchNamespace := include "mongodb-enterprise-operator.namespace" . | list }} +{{- if .Values.operator.watchNamespace }} +{{- $watchNamespace = regexSplit "," .Values.operator.watchNamespace -1 }} +{{- $watchNamespace = concat $watchNamespace (include "mongodb-enterprise-operator.namespace" . | list) | uniq }} +{{- end }} + +{{- $roleScope := "Role" -}} +{{- if or (gt (len $watchNamespace) 1) (eq (first $watchNamespace) "*") }} +{{- $roleScope = "ClusterRole" }} +{{- end }} +{{- end }} {{/* if .Values.operator.createOperatorServiceAccount */}} diff --git a/helm_chart/templates/operator.yaml b/helm_chart/templates/operator.yaml new file mode 100644 index 000000000..3730518cf --- /dev/null +++ b/helm_chart/templates/operator.yaml @@ -0,0 +1,267 @@ +{{ $ns := include "mongodb-enterprise-operator.namespace" . -}} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.operator.name }} + namespace: {{$ns}} +spec: + replicas: {{ min 1 .Values.operator.replicas }} + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: {{ .Values.operator.name }} + app.kubernetes.io/instance: {{ .Values.operator.name }} + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: {{ .Values.operator.name }} + app.kubernetes.io/instance: {{ .Values.operator.name }} +{{- if .Values.operator.vaultSecretBackend }} + {{- if .Values.operator.vaultSecretBackend.enabled }} + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: "mongodbenterprise" + {{- if .Values.operator.vaultSecretBackend.tlsSecretRef }} + vault.hashicorp.com/tls-secret: {{ .Values.operator.vaultSecretBackend.tlsSecretRef }} + vault.hashicorp.com/ca-cert: /vault/tls/ca.crt + {{- end }} + {{- end }} +{{- end }} + spec: + serviceAccountName: {{ .Values.operator.name }} +{{- if not .Values.managedSecurityContext }} + securityContext: + runAsNonRoot: true + runAsUser: 2000 +{{- end }} +{{- if .Values.registry.imagePullSecrets}} + imagePullSecrets: + - name: {{ .Values.registry.imagePullSecrets }} +{{- end }} + containers: + - name: {{ .Values.operator.name }} + image: "{{ .Values.registry.operator }}/{{ .Values.operator.operator_image_name }}:{{ .Values.operator.version }}{{ .Values.operator.build }}" + imagePullPolicy: {{ .Values.registry.pullPolicy }} + {{- if .Values.operator.watchedResources }} + args: + {{- range .Values.operator.watchedResources }} + - -watch-resource={{ . }} + {{- end }} + {{- if .Values.multiCluster.clusters }} + - -watch-resource=mongodbmulticluster + {{- end }} + {{- range .Values.operator.additionalArguments }} + - {{ . }} + {{- end }} + command: + - /usr/local/bin/mongodb-enterprise-operator + {{- end }} + {{- if .Values.multiCluster.clusters }} + volumeMounts: + - mountPath: /etc/config/kubeconfig + name: kube-config-volume + {{- end }} + resources: + limits: + cpu: {{ .Values.operator.resources.limits.cpu }} + memory: {{ .Values.operator.resources.limits.memory }} + requests: + cpu: {{ .Values.operator.resources.requests.cpu }} + memory: {{ .Values.operator.resources.requests.memory }} + env: + - name: OPERATOR_ENV + value: {{ .Values.operator.env }} + - name: MDB_DEFAULT_ARCHITECTURE + value: {{ .Values.operator.mdbDefaultArchitecture }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.operator.vaultSecretBackend }} + {{- if .Values.operator.vaultSecretBackend.enabled }} + - name: SECRET_BACKEND + value: VAULT_BACKEND + {{- end }} + {{- end }} + - name: WATCH_NAMESPACE + {{- if .Values.operator.watchNamespace }} + value: "{{ .Values.operator.watchNamespace }}" + {{- else }} + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + {{- if eq .Values.managedSecurityContext true }} + - name: MANAGED_SECURITY_CONTEXT + value: 'true' + {{- end }} + {{- $telemetry := default dict .Values.operator.telemetry }} + {{- if eq $telemetry.enabled false }} + - name: MDB_OPERATOR_TELEMETRY_ENABLED + value: "false" + {{- end }} + {{- if eq $telemetry.collection.clusters.enabled false }} + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_CLUSTERS_ENABLED + value: "false" + {{- end }} + {{- if eq $telemetry.collection.deployments.enabled false }} + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_DEPLOYMENTS_ENABLED + value: "false" + {{- end }} + {{- if eq $telemetry.collection.operators.enabled false }} + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_OPERATORS_ENABLED + value: "false" + {{- end }} + {{- if $telemetry.collection.frequency}} + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "{{ $telemetry.collection.frequency }}" + {{- end }} + {{- if eq $telemetry.send.enabled false }} + - name: MDB_OPERATOR_TELEMETRY_SEND_ENABLED + value: "false" + {{- end }} + {{- if $telemetry.send.frequency}} + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "{{ .Values.operator.telemetry.send.frequency }}" + {{- end }} + {{- if $telemetry.send.baseUrl}} + - name: MDB_OPERATOR_TELEMETRY_SEND_BASEURL + value: "{{ $telemetry.send.baseUrl }}" + {{- end }} + {{- if .Values.multiCluster.clusterClientTimeout }} + - name: CLUSTER_CLIENT_TIMEOUT + value: "{{ .Values.multiCluster.clusterClientTimeout }}" + {{- end }} + {{- $mongodbEnterpriseDatabaseImageEnv := "MONGODB_ENTERPRISE_DATABASE_IMAGE" -}} + {{- $initDatabaseImageRepositoryEnv := "INIT_DATABASE_IMAGE_REPOSITORY" -}} + {{- $opsManagerImageRepositoryEnv := "OPS_MANAGER_IMAGE_REPOSITORY" -}} + {{- $agentImageRepository := "MDB_AGENT_IMAGE_REPOSITORY" -}} + {{- $initOpsManagerImageRepositoryEnv := "INIT_OPS_MANAGER_IMAGE_REPOSITORY" -}} + {{- $initAppDbImageRepositoryEnv := "INIT_APPDB_IMAGE_REPOSITORY" -}} + {{- $agentImageEnv := "AGENT_IMAGE" -}} + {{- $mongodbImageEnv := "MONGODB_IMAGE" -}} + {{- $initDatabaseVersion := print .Values.initDatabase.version (.Values.build | default "") -}} + {{- $databaseVersion := print .Values.database.version (.Values.build | default "") -}} + {{- $initOpsManagerVersion := print .Values.initOpsManager.version (.Values.build | default "") -}} + {{- $initAppDbVersion := print .Values.initAppDb.version (.Values.build | default "") -}} + {{- $agentVersion := .Values.agent.version }} + - name: IMAGE_PULL_POLICY + value: {{ .Values.registry.pullPolicy }} + # Database + - name: {{ $mongodbEnterpriseDatabaseImageEnv }} + value: {{ .Values.registry.database }}/{{ .Values.database.name }} + - name: {{ $initDatabaseImageRepositoryEnv }} + value: {{ .Values.registry.initDatabase }}/{{ .Values.initDatabase.name }} + - name: INIT_DATABASE_VERSION + value: {{ $initDatabaseVersion }} + - name: DATABASE_VERSION + value: {{ $databaseVersion }} + # Ops Manager + - name: {{ $opsManagerImageRepositoryEnv }} + value: {{ .Values.registry.opsManager }}/{{ .Values.opsManager.name }} + - name: {{ $initOpsManagerImageRepositoryEnv }} + value: {{ .Values.registry.initOpsManager }}/{{ .Values.initOpsManager.name }} + - name: INIT_OPS_MANAGER_VERSION + value: {{ $initOpsManagerVersion }} + # AppDB + - name: {{ $initAppDbImageRepositoryEnv }} + value: {{ .Values.registry.initAppDb }}/{{ .Values.initAppDb.name }} + - name: INIT_APPDB_VERSION + value: {{ $initAppDbVersion }} + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: {{ .Values.registry.pullPolicy }} + - name: {{ $agentImageEnv }} + value: "{{ $.Values.registry.agent }}/{{ $.Values.agent.name }}:{{ .Values.agent.version }}" + - name: {{ $agentImageRepository }} + value: "{{ $.Values.registry.agent }}/{{ $.Values.agent.name }}" + - name: {{ $mongodbImageEnv }} + value: {{ .Values.mongodb.name }} + - name: MONGODB_REPO_URL + value: {{ .Values.mongodb.repo }} + - name: MDB_IMAGE_TYPE + {{- if eq .Values.operator.mdbDefaultArchitecture "static" }} + value: "ubi9" + {{- else }} + value: {{ .Values.mongodb.imageType }} + {{- end }} + {{- if eq .Values.mongodb.appdbAssumeOldFormat true }} + - name: MDB_APPDB_ASSUME_OLD_FORMAT + value: 'true' + {{- end }} + {{- if eq .Values.multiCluster.performFailOver true }} + - name: PERFORM_FAILOVER + value: 'true' + {{- end }} + {{- if .Values.registry.imagePullSecrets }} + - name: IMAGE_PULL_SECRETS + value: {{ .Values.registry.imagePullSecrets }} + {{- end }} + {{- if not .Values.operator.webhook.registerConfiguration }} + - name: MDB_WEBHOOK_REGISTER_CONFIGURATION + value: "false" + {{- end }} + {{- if .Values.operator.maxConcurrentReconciles }} + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "{{ .Values.operator.maxConcurrentReconciles }}" + {{- end }} + {{- if .Values.relatedImages }} + - name: RELATED_IMAGE_{{ $mongodbEnterpriseDatabaseImageEnv }}_{{ $databaseVersion | replace "." "_" | replace "-" "_" }} + value: "{{ .Values.registry.database }}/{{ .Values.database.name }}:{{ $databaseVersion }}" + - name: RELATED_IMAGE_{{ $initDatabaseImageRepositoryEnv }}_{{ $initDatabaseVersion | replace "." "_" | replace "-" "_" }} + value: "{{ .Values.registry.initDatabase }}/{{ .Values.initDatabase.name }}:{{ $initDatabaseVersion }}" + - name: RELATED_IMAGE_{{ $initOpsManagerImageRepositoryEnv }}_{{ $initOpsManagerVersion | replace "." "_" | replace "-" "_" }} + value: "{{ .Values.registry.initOpsManager }}/{{ .Values.initOpsManager.name }}:{{ $initOpsManagerVersion }}" + - name: RELATED_IMAGE_{{ $initAppDbImageRepositoryEnv }}_{{ $initAppDbVersion | replace "." "_" | replace "-" "_" }} + value: "{{ .Values.registry.initAppDb }}/{{ .Values.initAppDb.name }}:{{ $initAppDbVersion }}" + {{- range $version := .Values.relatedImages.agent }} + - name: RELATED_IMAGE_{{ $agentImageEnv }}_{{ $version | replace "." "_" | replace "-" "_" }} + value: "{{ $.Values.registry.agent }}/{{ $.Values.agent.name }}:{{ $version }}" + {{- end }} + {{- range $version := .Values.relatedImages.opsManager }} + - name: RELATED_IMAGE_{{ $opsManagerImageRepositoryEnv }}_{{ $version | replace "." "_" | replace "-" "_" }} + value: "{{ $.Values.registry.opsManager }}/{{ $.Values.opsManager.name }}:{{ $version }}" + {{- end }} + # since the official server images end with a different suffix we can re-use the same $mongodbImageEnv + {{- range $version := .Values.relatedImages.mongodb }} + - name: RELATED_IMAGE_{{ $mongodbImageEnv }}_{{ $version | replace "." "_" | replace "-" "_" }} + value: "{{ $.Values.mongodb.repo }}/{{ $.Values.mongodb.name }}:{{ $version }}" + {{- end }} + # mongodbLegacyAppDb will be deleted in 1.23 release + {{- range $version := .Values.relatedImages.mongodbLegacyAppDb }} + - name: RELATED_IMAGE_{{ $mongodbImageEnv }}_{{ $version | replace "." "_" | replace "-" "_" }} + value: "{{ $.Values.mongodbLegacyAppDb.repo }}/{{ $.Values.mongodbLegacyAppDb.name }}:{{ $version }}" + {{- end }} + {{- end }} + {{- if .Values.customEnvVars }} + {{- range split "&" .Values.customEnvVars }} + - name: {{ (split "=" .)._0 }} + value: '{{ (split "=" .)._1 }}' + {{- end }} + {{- end }} +{{- if .Values.multiCluster.clusters }} + volumes: + - name: kube-config-volume + secret: + defaultMode: 420 + secretName: {{ .Values.multiCluster.kubeConfigSecretName }} +{{- end }} + +{{- with .Values.operator }} + {{- with .nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} + + diff --git a/helm_chart/templates/secret-config.yaml b/helm_chart/templates/secret-config.yaml new file mode 100644 index 000000000..fa6433832 --- /dev/null +++ b/helm_chart/templates/secret-config.yaml @@ -0,0 +1,23 @@ +{{- if .Values.operator.vaultSecretBackend }} + {{- if .Values.operator.vaultSecretBackend.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: secret-configuration + namespace: {{ include "mongodb-enterprise-operator.namespace" . }} +data: + {{- if .Values.operator.vaultSecretBackend.tlsSecretRef }} + VAULT_SERVER_ADDRESS: https://vault.vault.svc.cluster.local:8200 + {{ else }} + VAULT_SERVER_ADDRESS: http://vault.vault.svc.cluster.local:8200 + {{ end }} + OPERATOR_SECRET_BASE_PATH: mongodbenterprise/operator + DATABASE_SECRET_BASE_PATH: mongodbenterprise/database + OPS_MANAGER_SECRET_BASE_PATH: mongodbenterprise/opsmanager + APPDB_SECRET_BASE_PATH: mongodbenterprise/appdb + {{- if .Values.operator.vaultSecretBackend.tlsSecretRef }} + TLS_SECRET_REF: vault-tls + {{ end }} +{{ end }} +{{ end }} diff --git a/helm_chart/values-multi-cluster.yaml b/helm_chart/values-multi-cluster.yaml new file mode 100644 index 000000000..c7798ddc7 --- /dev/null +++ b/helm_chart/values-multi-cluster.yaml @@ -0,0 +1,15 @@ +operator: + # Name that will be assigned to most internal Kubernetes objects like Deployment, ServiceAccount, Role etc. + name: mongodb-enterprise-operator-multi-cluster + +multiCluster: + # Specify if we want to deploy the operator in multi-cluster mode + clusters: + [ + "MDB_CLUSTER_1_FULL_NAME", + "MDB_CLUSTER_2_FULL_NAME", + "MDB_CLUSTER_3_FULL_NAME", + ] + kubeConfigSecretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + performFailOver: true + clusterClientTimeout: 10 diff --git a/helm_chart/values-openshift.yaml b/helm_chart/values-openshift.yaml new file mode 100644 index 000000000..08546e50d --- /dev/null +++ b/helm_chart/values-openshift.yaml @@ -0,0 +1,141 @@ +# Name of the Namespace to use +namespace: mongodb + +# Set this to true if your cluster is managing SecurityContext for you. +# If running OpenShift (Cloud, Minishift, etc.), set this to true. +managedSecurityContext: true + +operator: + webhook: + # registerConfiguration setting (default: true) controls if the operator should automatically register ValidatingWebhookConfiguration and if required for it cluster-wide roles should be installed. + # + # Setting false: + # - Adds env var MDB_WEBHOOK_REGISTER_CONFIGURATION=false to the operator deployment. + # - ClusterRole and ClusterRoleBinding required to manage ValidatingWebhookConfigurations will not be installed + # - The operator will not create ValidatingWebhookConfigurations upon startup. + # - The operator will not create the service for the webhook. If the `operator-webhook` service was created before, it will be deleted. + # - The operator will still expose the webhook's endpoint on port on MDB_WEBHOOK_PORT (if not specified, the operator uses a default 1993) in case the ValidatingWebhookConfigurations is configured externally (e.g. in OLM/OpenShift) or by the administrator manually. + # + # Setting true: + # - It's the default setting, behaviour of the operator w.r.t. webhook configuration is the same as before. + # - operator-webhook service will be created by the operator + # - ClusterRole and ClusterRoleBinding required to manage ValidatingWebhookConfigurations will be installed. + # - ValidatingWebhookConfigurations will be managed by the operator (requires cluster permissions) + registerConfiguration: true + +# Versions listed here are used to populate RELATED_IMAGE_ env variables in the operator deployment. +# Environment variables prefixed with RELATED_IMAGE_ are used by operator-sdk to generate relatedImages section +# with sha256 digests pinning for the certified operator bundle with disconnected environment feature enabled. +# https://docs.openshift.com/container-platform/4.14/operators/operator_sdk/osdk-generating-csvs.html#olm-enabling-operator-for-restricted-network_osdk-generating-csvs + version: 1.32.0 +relatedImages: + opsManager: + - 6.0.25 + - 6.0.26 + - 6.0.27 + - 7.0.13 + - 7.0.14 + - 7.0.15 + - 8.0.4 + - 8.0.5 + - 8.0.6 + mongodb: + - 4.4.0-ubi8 + - 4.4.1-ubi8 + - 4.4.2-ubi8 + - 4.4.3-ubi8 + - 4.4.4-ubi8 + - 4.4.5-ubi8 + - 4.4.6-ubi8 + - 4.4.7-ubi8 + - 4.4.8-ubi8 + - 4.4.9-ubi8 + - 4.4.10-ubi8 + - 4.4.11-ubi8 + - 4.4.12-ubi8 + - 4.4.13-ubi8 + - 4.4.14-ubi8 + - 4.4.15-ubi8 + - 4.4.16-ubi8 + - 4.4.17-ubi8 + - 4.4.18-ubi8 + - 4.4.19-ubi8 + - 4.4.20-ubi8 + - 4.4.21-ubi8 + - 5.0.0-ubi8 + - 5.0.1-ubi8 + - 5.0.2-ubi8 + - 5.0.3-ubi8 + - 5.0.4-ubi8 + - 5.0.5-ubi8 + - 5.0.6-ubi8 + - 5.0.7-ubi8 + - 5.0.8-ubi8 + - 5.0.9-ubi8 + - 5.0.10-ubi8 + - 5.0.11-ubi8 + - 5.0.12-ubi8 + - 5.0.13-ubi8 + - 5.0.14-ubi8 + - 5.0.15-ubi8 + - 5.0.16-ubi8 + - 5.0.17-ubi8 + - 5.0.18-ubi8 + - 6.0.0-ubi8 + - 6.0.1-ubi8 + - 6.0.2-ubi8 + - 6.0.3-ubi8 + - 6.0.4-ubi8 + - 6.0.5-ubi8 + - 8.0.0-ubi8 + - 8.0.0-ubi9 + agent: + - 107.0.13.8702-1 + - 107.0.13.8702-1_1.30.0 + - 107.0.13.8702-1_1.31.0 + - 107.0.13.8702-1_1.32.0 + - 107.0.15.8741-1 + - 107.0.15.8741-1_1.30.0 + - 107.0.15.8741-1_1.31.0 + - 107.0.15.8741-1_1.32.0 + - 108.0.2.8729-1 + - 108.0.4.8770-1 + - 108.0.4.8770-1_1.30.0 + - 108.0.4.8770-1_1.31.0 + - 108.0.4.8770-1_1.32.0 + - 108.0.6.8796-1 + - 108.0.6.8796-1_1.30.0 + - 108.0.6.8796-1_1.31.0 + - 108.0.6.8796-1_1.32.0 + - 12.0.33.7866-1 + - 12.0.33.7866-1_1.30.0 + - 12.0.33.7866-1_1.31.0 + - 12.0.33.7866-1_1.32.0 + - 12.0.34.7888-1 + - 12.0.34.7888-1_1.30.0 + - 12.0.34.7888-1_1.31.0 + - 12.0.34.7888-1_1.32.0 + - 12.0.35.7911-1 + - 12.0.35.7911-1_1.30.0 + - 12.0.35.7911-1_1.31.0 + - 12.0.35.7911-1_1.32.0 + - 13.32.0.9397-1 + - 13.32.0.9397-1_1.30.0 + - 13.32.0.9397-1_1.31.0 + - 13.32.0.9397-1_1.32.0 + mongodbLegacyAppDb: + - 4.2.11-ent + - 4.2.2-ent + - 4.2.24-ent + - 4.2.6-ent + - 4.2.8-ent + - 4.4.0-ent + - 4.4.11-ent + - 4.4.4-ent + - 4.4.21-ent + - 5.0.1-ent + - 5.0.5-ent + - 5.0.6-ent + - 5.0.7-ent + - 5.0.14-ent + - 5.0.18-ent diff --git a/helm_chart/values.yaml b/helm_chart/values.yaml new file mode 100644 index 000000000..35adff966 --- /dev/null +++ b/helm_chart/values.yaml @@ -0,0 +1,173 @@ +# Set this to true if your cluster is managing SecurityContext for you. +# If running OpenShift (Cloud, Minishift, etc.), set this to true. +managedSecurityContext: false + +operator: + # Execution environment for the operator, dev or prod. Use dev for more verbose logging + env: prod + + # Default architecture for the operator. + # Values are "static" and "non-static: + mdbDefaultArchitecture: non-static + + # Name that will be assigned to most internal Kubernetes objects like Deployment, ServiceAccount, Role etc. + name: mongodb-enterprise-operator + + # Name of the operator image + operator_image_name: mongodb-enterprise-operator-ubi + + # Name of the deployment of the operator pod + deployment_name: mongodb-enterprise-operator + + # Version of mongodb-enterprise-operator + version: 1.32.0 + + # The Custom Resources that will be watched by the Operator. Needs to be changed if only some of the CRDs are installed + watchedResources: + - mongodb + - opsmanagers + - mongodbusers + + nodeSelector: {} + + tolerations: [] + + affinity: {} + + # operator cpu requests and limits + resources: + requests: + cpu: 500m + memory: 200Mi + limits: + cpu: 1100m + memory: 1Gi + + # Control how many reconciles can be performed in parallel. + # It sets MaxConcurrentReconciles https://pkg.go.dev/github.com/kubernetes-sigs/controller-runtime/pkg/controller#Options). + # Increasing the number of concurrent reconciles will decrease the time needed to reconcile all watched resources. + # But it might result in increased load on Ops Manager API, K8S API server and will require allocating more cpu and memory for the operator deployment. + # + # This setting works independently for all watched CRD types, so setting doesn't mean the operator will use only 4 workers in total, but + # each CRD type (MongoDB, MongoDBMultiCluster, MongoDBOpsManager, MongoDBUser) will be reconciled with 4 workers, making it + # 4*4=20 workers in total. Memory usage depends on the actual number of resources reconciles in parallel and is not allocated upfront. + maxConcurrentReconciles: 1 + + # Create operator service account and roles + # if false, then templates/operator-roles.yaml is excluded + createOperatorServiceAccount: true + + # Set to false to NOT create service accounts and roles for the resources managed by the operator + # It might be necessary to disable it to avoid conflicts when + # kubectl mongodb plugin is used to configure multi-cluster resources + createResourcesServiceAccountsAndRoles: true + + # Set to false to not create the RBAC for enabling access to the PVC for resizing for the operator + enablePVCResize: true + + vaultSecretBackend: + # set to true if you want the operator to store secrets in Vault + enabled: false + tlsSecretRef: '' + + # 0 or 1 is supported only + replicas: 1 + # additional arguments to pass on the operator's binary arguments, e.g. operator.additionalArguments={--v=9} to dump debug k8s networking to logs + additionalArguments: [] + + webhook: + # Controls whether the helm chart will install cluster role allowing to create ValidatingWebhookConfiguration. Default: true. + # Without the permissions, the operator will log errors when trying to configure admission webhooks, but will work correctly nonetheless. + installClusterRole: true + + # registerConfiguration setting (default: true) controls if the operator should automatically register ValidatingWebhookConfiguration and if required for it cluster-wide roles should be installed. + # DO NOT disable this setting if installing via helm. This setting is used for OLM installations. + # + # Setting false: + # - This setting is intended to be used ONLY when the operator is installed via OLM. Do not use it otherwise as the operator won't start due to missing webhook server certificates, which OLM provides automatically. + # - Adds env var MDB_WEBHOOK_REGISTER_CONFIGURATION=false to the operator deployment. + # - ClusterRole and ClusterRoleBinding required to manage ValidatingWebhookConfigurations will not be installed + # - The operator will not create ValidatingWebhookConfigurations upon startup. + # - The operator will not create the service for the webhook. If the `operator-webhook` service was created before, it will be deleted. + # - The operator will still expose the webhook's endpoint on port on MDB_WEBHOOK_PORT (if not specified, the operator uses a default 1993) in case the ValidatingWebhookConfigurations is configured externally (e.g. in OLM/OpenShift) or by the administrator manually. + # + # Setting true: + # - It's the default setting, behaviour of the operator w.r.t. webhook configuration is the same as before. + # - operator-webhook service will be created by the operator. + # - ClusterRole and ClusterRoleBinding required to manage ValidatingWebhookConfigurations will be installed. + # - ValidatingWebhookConfigurations will be managed by the operator. + registerConfiguration: true + + telemetry: + collection: + clusters: {} + deployments: {} + operators: {} + # Valid time units are "m", "h". Anything less than one minute defaults to 1h + frequency: 1h + # Enables sending the collected telemetry to MongoDB + send: + # 168 hours is one week + # Valid time units are "h". Anything less than one hours defaults to 168h + frequency: 168h + +## Database +database: + name: mongodb-enterprise-database-ubi + version: 1.32.0 + +initDatabase: + name: mongodb-enterprise-init-database-ubi + version: 1.32.0 + +## Ops Manager +opsManager: + name: mongodb-enterprise-ops-manager-ubi + +initOpsManager: + name: mongodb-enterprise-init-ops-manager-ubi + version: 1.32.0 + +## Application Database +initAppDb: + name: mongodb-enterprise-init-appdb-ubi + version: 1.32.0 + +agent: + name: mongodb-agent-ubi + version: 108.0.2.8729-1 + +mongodbLegacyAppDb: + name: mongodb-enterprise-appdb-database-ubi + repo: quay.io/mongodb + +# This is used by AppDB and by static containers to determine the image that the operator uses for databases. +mongodb: + name: mongodb-enterprise-server + repo: quay.io/mongodb + appdbAssumeOldFormat: false + imageType: ubi8 + +## Registry +registry: + imagePullSecrets: + pullPolicy: Always + # Specify if images are pulled from private registry + operator: quay.io/mongodb + database: quay.io/mongodb + initDatabase: quay.io/mongodb + initOpsManager: quay.io/mongodb + opsManager: quay.io/mongodb + initAppDb: quay.io/mongodb + appDb: quay.io/mongodb + agent: quay.io/mongodb + +multiCluster: + # Specify if we want to deploy the operator in multi-cluster mode + clusters: [] + kubeConfigSecretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + performFailOver: true + clusterClientTimeout: 10 +# Set this to false to disable subresource utilization +# It might be required on some versions of Openshift +subresourceEnabled: true diff --git a/inventories/agent.yaml b/inventories/agent.yaml new file mode 100644 index 000000000..3e7fa7d00 --- /dev/null +++ b/inventories/agent.yaml @@ -0,0 +1,60 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-agent-ubi + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-agent + +images: +- name: mongodb-agent + vars: + context: . + template_context: docker/mongodb-agent + platform: linux/amd64 + + stages: + - name: mongodb-agent-build-context + task_type: docker_build + dockerfile: docker/mongodb-agent/Dockerfile.builder + buildargs: + mongodb_tools_url_ubi: $(inputs.params.mongodb_tools_url_ubi) + mongodb_agent_url_ubi: $(inputs.params.mongodb_agent_url_ubi) + init_database_image: $(inputs.params.init_database_image) + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.version)-context + + - name: mongodb-agent-build-context-release + task_type: docker_build + tags: ["release"] + dockerfile: docker/mongodb-agent/Dockerfile.builder + buildargs: + mongodb_tools_url_ubi: $(inputs.params.mongodb_tools_url_ubi) + mongodb_agent_url_ubi: $(inputs.params.mongodb_agent_url_ubi) + init_database_image: $(inputs.params.init_database_image) + output: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context + + - name: mongodb-agent-build-ubi + task_type: docker_build + buildargs: + imagebase: $(inputs.params.registry)/mongodb-agent-ubi:$(inputs.params.version)-context + version: $(inputs.params.version) + dockerfile: docker/mongodb-agent/Dockerfile + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.version) + + - name: master-latest + task_type: tag_image + tags: [ "master" ] + source: + registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.version) + destination: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.agent_version)_latest + + - name: mongodb-agent-template-ubi + task_type: dockerfile_template + tags: ["release"] + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile diff --git a/inventories/agent_non_matrix.yaml b/inventories/agent_non_matrix.yaml new file mode 100644 index 000000000..21df88d92 --- /dev/null +++ b/inventories/agent_non_matrix.yaml @@ -0,0 +1,64 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-agent-ubi + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-agent + +images: + - name: mongodb-agent + vars: + context: . + template_context: docker/mongodb-agent-non-matrix + + platform: linux/$(inputs.params.architecture) + stages: + - name: mongodb-agent-context + task_type: docker_build + dockerfile: docker/mongodb-agent-non-matrix/Dockerfile.builder + tags: [ "ubi" ] + buildargs: + agent_version: $(inputs.params.version) + tools_version: $(inputs.params.tools_version) + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.version)-context-$(inputs.params.architecture) + + - name: mongodb-agent-build-context-release + task_type: docker_build + tags: ["release"] + dockerfile: docker/mongodb-agent-non-matrix/Dockerfile.builder + buildargs: + agent_version: $(inputs.params.version) + tools_version: $(inputs.params.tools_version) + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) + output: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context-$(inputs.params.architecture) + + - name: mongodb-agent-build + task_type: docker_build + tags: [ "ubi" ] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-agent-ubi:$(inputs.params.version)-context-$(inputs.params.architecture) + version: $(inputs.params.version) + dockerfile: docker/mongodb-agent-non-matrix/Dockerfile + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: $(inputs.params.version)-$(inputs.params.architecture) + - registry: $(inputs.params.registry)/mongodb-agent-ubi + tag: latest-$(inputs.params.architecture) + + - name: mongodb-agent-template-ubi + task_type: dockerfile_template + tags: ["release"] + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile \ No newline at end of file diff --git a/inventories/daily.yaml b/inventories/daily.yaml new file mode 100644 index 000000000..dd4551476 --- /dev/null +++ b/inventories/daily.yaml @@ -0,0 +1,41 @@ +vars: + # these variables are configured from the outside, in pipeline.py::image_config + quay_registry: quay.io/mongodb/ + s3_bucket_http: https://enterprise-operator-dockerfiles.s3.amazonaws.com/dockerfiles/ + ecr_registry_ubi: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/ + # ubi suffix is "-ubi" by default, but it's empty for mongodb-kubernetes-operator, readiness and versionhook images + ubi_suffix: "-ubi" + base_suffix: "" + architecture_suffix: "" + platform: "amd64" + +images: + - name: image-daily-build + vars: + context: . + platform: linux/$(inputs.params.platform) + stages: + - name: build-ubi + task_type: docker_build + tags: ["ubi"] + inputs: + - build_id + dockerfile: $(inputs.params.s3_bucket_http)/$(inputs.params.release_version)/ubi/Dockerfile + buildargs: + imagebase: $(inputs.params.quay_registry)$(inputs.params.base_suffix):$(inputs.params.release_version)-context$(inputs.params.architecture_suffix) + # This is required for correctly labeling the agent image and is not used + # in the other images. + version: $(inputs.params.release_version) + output: + - registry: $(inputs.params.quay_registry)$(inputs.params.ubi_suffix) + tag: $(inputs.params.release_version)$(inputs.params.architecture_suffix) + - registry: $(inputs.params.quay_registry)$(inputs.params.ubi_suffix) + tag: $(inputs.params.release_version)-b$(inputs.params.build_id)$(inputs.params.architecture_suffix) + # Below two coordinates are on pair with the e2e_om_ops_manager_upgrade test but + # doesn't seem to reflect the way we push things to Quay. + # The proper fix should be addressed in https://jira.mongodb.org/browse/CLOUDP-133709 + - registry: $(inputs.params.ecr_registry_ubi)$(inputs.params.ubi_suffix) + tag: $(inputs.params.release_version)$(inputs.params.architecture_suffix) + - registry: $(inputs.params.ecr_registry_ubi)$(inputs.params.ubi_suffix) + tag: $(inputs.params.release_version)-b$(inputs.params.build_id)$(inputs.params.architecture_suffix) + diff --git a/inventories/database.yaml b/inventories/database.yaml new file mode 100644 index 000000000..7f837d12e --- /dev/null +++ b/inventories/database.yaml @@ -0,0 +1,65 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-enterprise-database + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-enterprise-database + +images: +- name: database + vars: + context: docker/mongodb-enterprise-database + platform: linux/amd64 + + stages: + - name: database-build-context + task_type: docker_build + dockerfile: Dockerfile.builder + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-database-context + tag: $(inputs.params.version_id) + + - name: init-appdb-template-ubi + task_type: dockerfile_template + distro: ubi + tags: ["ubi"] + inputs: + - version + output: + - dockerfile: $(functions.tempfile) + + - name: database-build-ubi + task_type: docker_build + dockerfile: $(stages['init-appdb-template-ubi'].outputs[0].dockerfile) + tags: ["ubi"] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-enterprise-database-context:$(inputs.params.version_id) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-database-ubi + tag: $(inputs.params.version_id) + + - name: master-latest + task_type: tag_image + tags: ["master"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-database-ubi + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.registry)/mongodb-enterprise-database-ubi + tag: latest + + - name: database-release-context + task_type: tag_image + tags: ["release"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-database-context + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context + + - name: database-template-ubi + task_type: dockerfile_template + distro: ubi + tags: ["release"] + inputs: + - version + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile diff --git a/inventories/init_appdb.yaml b/inventories/init_appdb.yaml new file mode 100644 index 000000000..0287d8210 --- /dev/null +++ b/inventories/init_appdb.yaml @@ -0,0 +1,69 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-enterprise-init-appdb + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-enterprise-init-appdb + +images: +- name: init-appdb + vars: + context: . + template_context: docker/mongodb-enterprise-init-database + platform: linux/amd64 + + stages: + - name: init-appdb-build-context + task_type: docker_build + dockerfile: docker/mongodb-enterprise-init-database/Dockerfile.builder + buildargs: + mongodb_tools_url_ubi: $(inputs.params.mongodb_tools_url_ubi) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-appdb-context + tag: $(inputs.params.version_id) + + - name: init-appdb-template-ubi + task_type: dockerfile_template + template_file_extension: ubi_minimal + tags: ["ubi"] + inputs: + - is_appdb + output: + - dockerfile: $(functions.tempfile) + + - name: init-appdb-build-ubi + task_type: docker_build + tags: ["ubi"] + buildargs: + version: $(inputs.params.version) + imagebase: $(inputs.params.registry)/mongodb-enterprise-init-appdb-context:$(inputs.params.version_id) + dockerfile: $(stages['init-appdb-template-ubi'].outputs[0].dockerfile) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-appdb-ubi + tag: $(inputs.params.version_id) + + - name: master-latest + task_type: tag_image + tags: [ "master" ] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-init-appdb-ubi + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-appdb-ubi + tag: latest + + - name: init-appdb-release-context + task_type: tag_image + tags: ["release"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-init-appdb-context + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context + + - name: init-appdb-template-ubi + task_type: dockerfile_template + template_file_extension: ubi_minimal + tags: ["release"] + inputs: + - is_appdb + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile diff --git a/inventories/init_database.yaml b/inventories/init_database.yaml new file mode 100644 index 000000000..a2f9f2734 --- /dev/null +++ b/inventories/init_database.yaml @@ -0,0 +1,75 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-enterprise-init-database + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-enterprise-init-database + +images: +- name: init-database + vars: + context: . + template_context: docker/mongodb-enterprise-init-database + platform: linux/amd64 + + stages: + - name: init-database-build-context + task_type: docker_build + dockerfile: docker/mongodb-enterprise-init-database/Dockerfile.builder + buildargs: + mongodb_tools_url_ubi: $(inputs.params.mongodb_tools_url_ubi) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-database-context + tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-enterprise-init-database-context + tag: $(inputs.params.version) + + - name: init-database-template-ubi + task_type: dockerfile_template + template_file_extension: ubi_minimal + tags: ["ubi"] + inputs: + - is_appdb + output: + - dockerfile: $(functions.tempfile) + + - name: init-database-build-ubi + task_type: docker_build + tags: ["ubi"] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-enterprise-init-database-context:$(inputs.params.version_id) + version: $(inputs.params.version) + dockerfile: $(stages['init-database-template-ubi'].outputs[0].dockerfile) + inputs: + - is_appdb + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-database-ubi + tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/mongodb-enterprise-init-database-ubi + tag: $(inputs.params.version) + + - name: master-latest + task_type: tag_image + tags: ["master"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-init-database-ubi + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-database-ubi + tag: latest + + - name: init-database-release-context + task_type: tag_image + tags: ["release"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-init-database-context + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context + + - name: init-database-template-ubi + task_type: dockerfile_template + template_file_extension: ubi_minimal + tags: ["release"] + inputs: + - is_appdb + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile diff --git a/inventories/init_om.yaml b/inventories/init_om.yaml new file mode 100644 index 000000000..3df465e22 --- /dev/null +++ b/inventories/init_om.yaml @@ -0,0 +1,65 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-enterprise-init-ops-manager + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-enterprise-init-ops-manager + +images: +- name: init-ops-manager + vars: + context: docker/mongodb-enterprise-init-ops-manager + platform: linux/amd64 + + stages: + - name: init-ops-manager-build-context + task_type: docker_build + dockerfile: Dockerfile.builder + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-ops-manager-context + tag: $(inputs.params.version_id) + + - name: init-ops-manager-template-ubi + task_type: dockerfile_template + template_file_extension: ubi_minimal + tags: ["ubi"] + inputs: + - version + output: + - dockerfile: $(functions.tempfile) + + - name: init-ops-manager-build-ubi + task_type: docker_build + dockerfile: $(stages['init-ops-manager-template-ubi'].outputs[0].dockerfile) + tags: ["ubi"] + buildargs: + imagebase: $(inputs.params.registry)/mongodb-enterprise-init-ops-manager-context:$(inputs.params.version_id) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-ops-manager-ubi + tag: $(inputs.params.version_id) + + - name: master-latest + task_type: tag_image + tags: ["master"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-init-ops-manager-ubi + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.registry)/mongodb-enterprise-init-ops-manager-ubi + tag: latest + + - name: init-ops-manager-release-context + task_type: tag_image + tags: ["release"] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-init-ops-manager-context + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context + + - name: init-ops-manager-template-ubi + task_type: dockerfile_template + template_file_extension: ubi_minimal + tags: ["release"] + inputs: + - version + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile diff --git a/inventories/om.yaml b/inventories/om.yaml new file mode 100644 index 000000000..e4daf3103 --- /dev/null +++ b/inventories/om.yaml @@ -0,0 +1,62 @@ +vars: + quay_registry: quay.io/mongodb/mongodb-enterprise-ops-manager + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-enterprise-ops-manager + om_registry: 268558157000.dkr.ecr.us-east-1.amazonaws.com + +images: +- name: ops-manager + vars: + context: . + template_context: docker/mongodb-enterprise-ops-manager + platform: linux/amd64 + + stages: + - name: ops-manager-context + task_type: docker_build + dockerfile: docker/mongodb-enterprise-ops-manager/Dockerfile.builder + output: + - registry: $(inputs.params.registry)/ops-manager-context + tag: $(inputs.params.version_id) + + - name: ops-manager-template-ubi + task_type: dockerfile_template + template_file_extension: ubi + tags: ["ubi"] + inputs: + - om_download_url + - version + buildargs: + imagebase: $(inputs.params.registry)/ops-manager-context:$(inputs.params.version_id) + output: + - dockerfile: $(functions.tempfile) + + - name: ops-manager-build + task_type: docker_build + dockerfile: $(stages['ops-manager-template-ubi'].outputs[0].dockerfile) + tags: ["ubi"] + buildargs: + imagebase: $(inputs.params.registry)/ops-manager-context:$(inputs.params.version_id) + output: + - registry: $(inputs.params.om_registry)/dev/mongodb-enterprise-ops-manager-ubi + tag: $(inputs.params.version) + + ## Release tasks + - name: ops-manager-template + task_type: dockerfile_template + template_file_extension: ubi + tags: ["ubi", "release"] + inputs: + - om_download_url + - version + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile + + - name: ops-manager-context-release + task_type: tag_image + tags: ["release"] + source: + registry: $(inputs.params.registry)/ops-manager-context + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context diff --git a/inventories/test.yaml b/inventories/test.yaml new file mode 100644 index 000000000..bb62df1ca --- /dev/null +++ b/inventories/test.yaml @@ -0,0 +1,17 @@ +images: +- name: test + vars: + context: docker/mongodb-enterprise-tests + platform: linux/amd64 + + stages: + - name: build + task_type: docker_build + dockerfile: Dockerfile + buildargs: + PYTHON_VERSION: $(inputs.params.python_version) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-tests + tag: latest + - registry: $(inputs.params.registry)/mongodb-enterprise-tests + tag: $(inputs.params.version_id) diff --git a/inventory.yaml b/inventory.yaml index e2a37214c..e803dc1ea 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -1,315 +1,94 @@ vars: - registry: - # Default value but overwritten in pipeline.py - architecture: amd64 + registry: + quay_registry: quay.io/mongodb/mongodb-enterprise-operator + s3_bucket: s3://enterprise-operator-dockerfiles/dockerfiles/mongodb-enterprise-operator images: - - - name: agent - vars: - context: . - template_context: scripts/dev/templates/agent - +- name: operator + vars: + context: . + template_context: docker/mongodb-enterprise-operator + platform: linux/amd64 + inputs: + - version + - log_automation_config_diff + + stages: + + - name: operator-context-dockerfile + task_type: docker_build + dockerfile: docker/mongodb-enterprise-operator/Dockerfile.builder + buildargs: + release_version: $(inputs.params.version) + log_automation_config_diff: $(inputs.params.log_automation_config_diff) + use_race: "false" + output: + - registry: $(inputs.params.registry)/operator-context + tag: $(inputs.params.version_id) + + - name: operator-race-context-dockerfile + task_type: docker_build + dockerfile: docker/mongodb-enterprise-operator/Dockerfile.builder + buildargs: + release_version: $(inputs.params.version) + log_automation_config_diff: $(inputs.params.log_automation_config_diff) + use_race: "true" + output: + - registry: $(inputs.params.registry)/operator-context + tag: $(inputs.params.version_id)-race + + - name: operator-template-ubi + task_type: dockerfile_template + distro: ubi inputs: - - release_version - - tools_version - - image - - image_dev - - platform: linux/$(inputs.params.architecture) - stages: - - name: mongodb-agent-context - task_type: docker_build - dockerfile: scripts/dev/templates/agent/Dockerfile.builder - tags: [ "ubi" ] - buildargs: - agent_version: $(inputs.params.release_version) - tools_version: $(inputs.params.tools_version) - agent_distro: $(inputs.params.agent_distro) - tools_distro: $(inputs.params.tools_distro) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - - - name: agent-template-ubi - task_type: dockerfile_template - distro: ubi - tags: [ "ubi" ] - - output: - - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - - - name: mongodb-agent-build - task_type: docker_build - tags: [ "ubi" ] - - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) - agent_version: $(inputs.params.release_version) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-$(inputs.params.architecture) - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-$(inputs.params.architecture) - - - name: agent-template-ubi-s3 - task_type: dockerfile_template - tags: [ "ubi", "release" ] - distro: ubi - - inputs: - - release_version - - output: - - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/$(inputs.params.release_version)/ubi/Dockerfile - - - name: agent-context-ubi-release - task_type: docker_build - dockerfile: scripts/dev/templates/agent/Dockerfile.builder - tags: [ "ubi", "release" ] - buildargs: - agent_version: $(inputs.params.release_version) - tools_version: $(inputs.params.tools_version) - agent_distro: $(inputs.params.agent_distro) - tools_distro: $(inputs.params.tools_distro) - - labels: - quay.expires-after: Never - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - - - name: mongodb-agent-release - task_type: docker_build - tags: [ "ubi", "release" ] - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) - agent_version: $(inputs.params.release_version) - - labels: - quay.expires-after: Never - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-$(inputs.params.architecture) - - - name: readiness-probe - vars: - context: . - template_context: scripts/dev/templates/readiness - + - version + - debug + output: + - dockerfile: $(functions.tempfile) + + - name: operator-ubi-build + task_type: docker_build + dockerfile: $(stages['operator-template-ubi'].outputs[0].dockerfile) + buildargs: + imagebase: $(inputs.params.registry)/operator-context:$(inputs.params.version_id) + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-operator-ubi + tag: $(inputs.params.version_id) + + - name: operator-ubi-race-build + task_type: docker_build + dockerfile: $(stages['operator-template-ubi'].outputs[0].dockerfile) + buildargs: + imagebase: $(inputs.params.registry)/operator-context:$(inputs.params.version_id)-race + output: + - registry: $(inputs.params.registry)/mongodb-enterprise-operator-ubi + tag: $(inputs.params.version_id)-race + + - name: master-latest + task_type: tag_image + tags: [ "master" ] + source: + registry: $(inputs.params.registry)/mongodb-enterprise-operator-ubi + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.registry)/mongodb-enterprise-operator-ubi + tag: latest + + - name: operator-context-release + task_type: tag_image + tags: ["release"] + source: + registry: $(inputs.params.registry)/operator-context + tag: $(inputs.params.version_id) + destination: + - registry: $(inputs.params.quay_registry) + tag: $(inputs.params.version)-context + + - name: operator-template-ubi + task_type: dockerfile_template + tags: ["release"] + distro: ubi inputs: - - image - - image_dev - - platform: linux/$(inputs.params.architecture) - stages: - - name: readiness-init-context-build - task_type: docker_build - dockerfile: scripts/dev/templates/readiness/Dockerfile.builder - tags: [ "readiness-probe", "ubi" ] - labels: - quay.expires-after: 48h - - buildargs: - builder_image: $(inputs.params.builder_image) - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - - - name: readiness-template-ubi - task_type: dockerfile_template - tags: [ "ubi" ] - template_file_extension: readiness - - inputs: - - base_image - - output: - - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) - - - name: readiness-init-build - task_type: docker_build - tags: [ "readiness-probe", "ubi" ] - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) - - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-$(inputs.params.architecture) - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-$(inputs.params.architecture) - - - name: readiness-init-context-release - task_type: docker_build - dockerfile: scripts/dev/templates/readiness/Dockerfile.builder - tags: [ "readiness-probe", "release" , "ubi" ] - - labels: - quay.expires-after: Never - - buildargs: - builder_image: $(inputs.params.builder_image) - - inputs: - - release_version - - builder_image - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - - - name: readiness-template-release - task_type: dockerfile_template - tags: [ "readiness-probe", "release", "ubi" ] - template_file_extension: readiness - inputs: - - base_image - - release_version - - output: - - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.release_version) - - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-readinessprobe/$(inputs.params.release_version)/ubi/Dockerfile - - - name: readiness-init-build-release - task_type: docker_build - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.release_version) - tags: [ "readiness-probe", "release" , "ubi" ] - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) - - labels: - quay.expires-after: Never - - inputs: - - base_image - - release_version - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-$(inputs.params.architecture) - - - name: version-upgrade-hook - vars: - context: . - template_context: scripts/dev/templates/versionhook - - inputs: - - image - - image_dev - - platform: linux/$(inputs.params.architecture) - stages: - - name: version-upgrade-hook-context-build - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder - tags: [ "post-start-hook", "ubi" ] - - buildargs: - builder_image: $(inputs.params.builder_image) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) - - - name: version-post-start-hook-template-ubi - task_type: dockerfile_template - tags: [ "ubi" ] - template_file_extension: versionhook - - inputs: - - base_image - - output: - - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.version_id) - - - name: version-upgrade-hook-build - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.version_id) - tags: [ "post-start-hook", "ubi" ] - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) - - labels: - quay.expires-after: 48h - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: $(inputs.params.version_id)-$(inputs.params.architecture) - - registry: $(inputs.params.registry)/$(inputs.params.image_dev) - tag: latest-$(inputs.params.architecture) - - - name: version-upgrade-hook-context-release - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder - tags: [ "release", "post-start-hook", "ubi", ] - - labels: - quay.expires-after: Never - - buildargs: - builder_image: $(inputs.params.builder_image) - - inputs: - - release_version - - builder_image - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) - - - name: versionhook-template-release - task_type: dockerfile_template - tags: [ "post-start-hook", "release", "ubi" ] - template_file_extension: versionhook - inputs: - - base_image - - release_version - - output: - - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.release_version) - - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-operator-version-upgrade-post-start-hook/$(inputs.params.release_version)/ubi/Dockerfile - - - name: version-upgrade-hook-build-release - task_type: docker_build - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.release_version) - tags: [ "release", "post-start-hook", "ubi" ] - - buildargs: - imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) - - labels: - quay.expires-after: Never - - inputs: - - base_image - - release_version - - output: - - registry: $(inputs.params.registry)/$(inputs.params.image) - tag: $(inputs.params.release_version)-$(inputs.params.architecture) \ No newline at end of file + - version + output: + - dockerfile: $(inputs.params.s3_bucket)/$(inputs.params.version)/ubi/Dockerfile diff --git a/lib/base_logger.py b/lib/base_logger.py new file mode 100644 index 000000000..20fd33627 --- /dev/null +++ b/lib/base_logger.py @@ -0,0 +1,32 @@ +import logging +import os +import sys + +LOGLEVEL = os.environ.get("LOGLEVEL", "DEBUG").upper() + +# Create handlers to output Debug and Info logs to stdout, and above to stderr +# They are attached to each logger below +stdout_handler = logging.StreamHandler(sys.stdout) +stdout_handler.setLevel(logging.DEBUG) +stdout_handler.addFilter(lambda record: record.levelno <= logging.INFO) +stderr_handler = logging.StreamHandler(sys.stderr) +stderr_handler.setLevel(logging.WARNING) + +# Format the logs +formatter = logging.Formatter("%(levelname)-8s %(asctime)s [%(module)s] %(message)s") +stdout_handler.setFormatter(formatter) +stderr_handler.setFormatter(formatter) + +# Pipeline logger +logger = logging.getLogger("pipeline") +logger.setLevel(LOGLEVEL) +logger.propagate = False +logger.addHandler(stdout_handler) +logger.addHandler(stderr_handler) + +# Sonar logger +sonar_logger = logging.getLogger("sonar") +sonar_logger.setLevel(LOGLEVEL) +sonar_logger.propagate = False +sonar_logger.addHandler(stdout_handler) +sonar_logger.addHandler(stderr_handler) diff --git a/lib/sonar/.gitignore b/lib/sonar/.gitignore new file mode 100644 index 000000000..340963e85 --- /dev/null +++ b/lib/sonar/.gitignore @@ -0,0 +1,12 @@ +.DS_Store +.idea +*.log +tmp/ + +*.py[cod] +*.egg +build +htmlcov + +sonar.egg-info/* +sonar.iml diff --git a/lib/sonar/.pylintrc b/lib/sonar/.pylintrc new file mode 100644 index 000000000..ca7e33165 --- /dev/null +++ b/lib/sonar/.pylintrc @@ -0,0 +1,3 @@ +[MESSAGES CONTROL] + +disable=missing-docstring,empty-docstring,invalid-name diff --git a/lib/sonar/CODE_OF_CONDUCT.md b/lib/sonar/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..5fb8baa30 --- /dev/null +++ b/lib/sonar/CODE_OF_CONDUCT.md @@ -0,0 +1,6 @@ +# Code of Conduct + +This project has adopted the [MongoDB Code of +Conduct](https://www.mongodb.com/community-code-of-conduct). If you see any +violations of the above or have any other concerns or questions please contact +us using the following email alias: community-conduct@mongodb.com. diff --git a/lib/sonar/CONTRIBUTING.md b/lib/sonar/CONTRIBUTING.md new file mode 100644 index 000000000..4477cfa07 --- /dev/null +++ b/lib/sonar/CONTRIBUTING.md @@ -0,0 +1,18 @@ +# Contributing + +## Workflow + +MongoDB welcomes community contributions! If you’re interested in making a +contribution to Sonar, please follow the steps below before you start writing +any code: + +1. Sign the [contributor's agreement](http://www.mongodb.com/contributor). This + will allow us to review and accept contributions. +1. Fork the repository on GitHub. +1. Create a branch with a name that briefly describes your feature. +1. Implement your feature or bug fix. +1. Add new cases to `./test` that verify your bug fix or make sure no one + unintentionally breaks your feature in the future and run them with `python + -m pytest`. +1. Add comments around your new code that explain what's happening. +1. Commit and push your changes to your branch then submit a pull request. diff --git a/APACHE2 b/lib/sonar/LICENSE similarity index 100% rename from APACHE2 rename to lib/sonar/LICENSE diff --git a/lib/sonar/README.md b/lib/sonar/README.md new file mode 100644 index 000000000..3966f3d62 --- /dev/null +++ b/lib/sonar/README.md @@ -0,0 +1,96 @@ +# Sonar 🐳 + +Work with multiple Docker images easily. + +**Sonar is currently Work in Progress!** + +## What is Sonar + +Sonar is a tool that allows you to easily produce, template, build and publish +Dockerfiles and Docker images. It uses a declarative, multi-stage approach to +build Docker images. + +## Quick start + +Sonar can be used as a Python module or as a standalone program. Sonar will look +for an `inventory.yaml` file in your local directory that should contain a +collection of images to build and stages for each one of those images. A +different inventory file can be specified using `--inventory `. + +Sonar comes with an inventory file to be able to build itself, and to run its +unit tests. This [simple.yaml](inventories/simple.yaml) is: + +``` yaml +vars: + # start a local registry with: + # docker run -d -p 5000:5000 --restart=always --name registry registry:2 + registry: localhost:5000 + +images: +- name: sonar-test-runner + + vars: + context: . + + # First stage builds a Docker image. The resulting image will be + # pushed to the registry in the `output` section. + stages: + - name: build-sonar-tester-image + task_type: docker_build + + dockerfile: docker/Dockerfile + + output: + - registry: $(inputs.params.registry)/sonar-tester-image + tag: $(inputs.params.version_id) + + # Second stage pushes the previously built image into a new + # registry. + - name: tag-image + task_type: tag_image + + source: + registry: $(stages['build-sonar-tester-image'].output[0].registry) + tag: $(stages['build-sonar-tester-image'].output[0].tag) + + destination: + - registry: $(inputs.params.registry)/sonar-tester-image + tag: latest +``` + +To execute this inventory file, you can do: + +``` +$ python sonar.py --image sonar-test-runner --inventory inventories/simple.yaml + +[build-sonar-tester-image/docker_build] stage-started build-sonar-tester-image: 1/2 +[build-sonar-tester-image/docker_build] docker-image-push: localhost:5000/sonar-tester-image:8945563b-248e-4c03-bb0a-6cc15cff1a6e +[tag-image/tag_image] stage-started tag-image: 2/2 +[tag-image/tag_image] docker-image-push: localhost:5000/sonar-tester-image:latest +``` + +At the end of this phase, you'll have a Docker image tagged as +`localhost:5000/sonar-tester-image:latest` that you will be able to run with: + +``` +$ docker run localhost:5000/sonar-tester-image:latest +============================= test session starts ============================== +platform linux -- Python 3.9.4, pytest-6.2.4, py-1.10.0, pluggy-0.13.1 +rootdir: /src +collected 38 items + +test/test_build.py ... [ 7%] +test/test_context.py ......x..... [ 39%] +test/test_sign_image.py .. [ 44%] +test/test_sonar.py ........ [ 65%] +test/test_tag_image.py . [ 68%] +test/test_tags.py ........... [ 97%] +test/test_template.py . [100%] + +======================== 37 passed, 1 xfailed in 0.52s ========================= +``` + + +## Legal + +Sonar is released under the terms of the [Apache2 license](./LICENSE). diff --git a/lib/sonar/__init__.py b/lib/sonar/__init__.py new file mode 100644 index 000000000..a6d43718d --- /dev/null +++ b/lib/sonar/__init__.py @@ -0,0 +1,2 @@ +DCT_ENV_VARIABLE = "DOCKER_CONTENT_TRUST" +DCT_PASSPHRASE = "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE" diff --git a/lib/sonar/builders/__init__.py b/lib/sonar/builders/__init__.py new file mode 100644 index 000000000..bb000503c --- /dev/null +++ b/lib/sonar/builders/__init__.py @@ -0,0 +1,24 @@ +class SonarBuildError(Exception): + """Wrapper over docker.errors.BuildError""" + + pass + + +class SonarAPIError(Exception): + """Wrapper over docker.errors.APIError""" + + pass + + +def buildarg_from_dict(args): + if args is None: + return "" + + return " ".join(["--build-arg {}={}".format(k, v) for k, v in args.items()]) + + +def labels_from_dict(args): + if args is None: + return "" + + return " ".join(["--label {}={}".format(k, v) for k, v in args.items()]) diff --git a/lib/sonar/builders/docker.py b/lib/sonar/builders/docker.py new file mode 100644 index 000000000..c2acc3295 --- /dev/null +++ b/lib/sonar/builders/docker.py @@ -0,0 +1,204 @@ +import random +import subprocess +from typing import Dict, Optional + +import docker.errors +from opentelemetry import trace + +import docker +from lib.base_logger import logger + +from . import SonarAPIError + +TRACER = trace.get_tracer("evergreen-agent") + + +def docker_client() -> docker.DockerClient: + return docker.client.from_env(timeout=60 * 60 * 24) + + +@TRACER.start_as_current_span("docker_build") +def docker_build( + path: str, + dockerfile: str, + buildargs: Optional[Dict[str, str]] = None, + labels: Optional[Dict[str, str]] = None, + platform: Optional[str] = None, +): + """Builds a docker image.""" + + image_name = "sonar-docker-build-{}".format(random.randint(1, 10000)) + + logger.info("path: {}".format(path)) + logger.info("dockerfile: {}".format(dockerfile)) + logger.info("tag: {}".format(image_name)) + logger.info("buildargs: {}".format(buildargs)) + logger.info("labels: {}".format(labels)) + + try: + # docker build from docker-py has bugs resulting in errors or invalid platform when building with specified --platform=linux/amd64 on M1 + docker_build_cli( + path=path, + dockerfile=dockerfile, + tag=image_name, + buildargs=buildargs, + labels=labels, + platform=platform, + ) + + client = docker_client() + return client.images.get(image_name) + except docker.errors.APIError as e: + raise SonarAPIError from e + + +def _get_build_log(e: docker.errors.BuildError) -> str: + build_logs = "\n" + for item in e.build_log: + if "stream" not in item: + continue + item_str = item["stream"] + build_logs += item_str + return build_logs + + +def docker_build_cli( + path: str, + dockerfile: str, + tag: str, + buildargs: Optional[Dict[str, str]], + labels=Optional[Dict[str, str]], + platform=Optional[str], +): + dockerfile_path = dockerfile + # if dockerfile is relative it has to be set as relative to context (path) + if not dockerfile_path.startswith("/"): + dockerfile_path = f"{path}/{dockerfile_path}" + + args = get_docker_build_cli_args( + path=path, dockerfile=dockerfile_path, tag=tag, buildargs=buildargs, labels=labels, platform=platform + ) + + args_str = " ".join(args) + logger.info(f"executing cli docker build: {args_str}") + + cp = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if cp.returncode != 0: + raise SonarAPIError(cp.stderr) + + +def get_docker_build_cli_args( + path: str, + dockerfile: str, + tag: str, + buildargs: Optional[Dict[str, str]], + labels=Optional[Dict[str, str]], + platform=Optional[str], +): + args = ["docker", "buildx", "build", "--load", "--progress", "plain", path, "-f", dockerfile, "-t", tag] + if buildargs is not None: + for k, v in buildargs.items(): + args.append("--build-arg") + args.append(f"{k}={v}") + + if labels is not None: + for k, v in labels.items(): + args.append("--label") + args.append(f"{k}={v}") + + if platform is not None: + args.append("--platform") + args.append(platform) + + return args + + +def docker_pull( + image: str, + tag: str, +): + client = docker_client() + + try: + return client.images.pull(image, tag=tag) + except docker.errors.APIError as e: + raise SonarAPIError from e + + +def docker_tag( + image: docker.models.images.Image, + registry: str, + tag: str, +): + try: + return image.tag(registry, tag) + except docker.errors.APIError as e: + raise SonarAPIError from e + + +@TRACER.start_as_current_span("image_exists") +def image_exists(repository, tag): + """Check if a Docker image with the specified tag exists in the repository using efficient HEAD requests.""" + logger.info(f"checking image {tag}, exists in remote repository: {repository}") + + return check_registry_image_exists(repository, tag) + + +def check_registry_image_exists(repository, tag): + """Check if image exists in generic registries using HTTP HEAD requests.""" + import requests + + try: + # Determine registry URL format + parts = repository.split("/") + registry_domain = parts[0] + repository_path = "/".join(parts[1:]) + + # Construct URL for manifest check + url = f"https://{registry_domain}/v2/{repository_path}/manifests/{tag}" + headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"} + + # Make HEAD request instead of full manifest retrieval + response = requests.head(url, headers=headers, timeout=3) + return response.status_code == 200 + except Exception as e: + logger.warning(f"Error checking registry for {repository}:{tag}: {e}") + return False + + +@TRACER.start_as_current_span("docker_push") +def docker_push(registry: str, tag: str): + def inner_docker_push(should_raise=False): + + # We can't use docker-py here + # as it doesn't support DOCKER_CONTENT_TRUST + # env variable, which could be needed + cp = subprocess.run( + ["docker", "push", f"{registry}:{tag}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if cp.returncode != 0: + if should_raise: + raise SonarAPIError(cp.stderr) + + return False + + return True + + # We don't want to rebuild context images if they already exist. + # Context images should be out and immutable. + # This is especially important for base image changes like ubi8 to ubi9, we don't want to replace our existing + # agent-ubi8 with agent-ubi9 images and break for older operators. + # Instead of doing the hack here, we should instead either: + # - make sonar aware of context images + # - move the logic out of sonar to pipeline.py to all the places where we build context images + if "-context" in tag and image_exists(registry, tag): + logger.info(f"Image: {tag} in registry: {registry} already exists skipping pushing it") + else: + logger.info("Image does not exist remotely or is not a context image, pushing it!") + retries = 3 + while retries >= 0: + if inner_docker_push(retries == 0): + break + retries -= 1 diff --git a/lib/sonar/inventories/inventory-template.yaml b/lib/sonar/inventories/inventory-template.yaml new file mode 100644 index 000000000..372880709 --- /dev/null +++ b/lib/sonar/inventories/inventory-template.yaml @@ -0,0 +1,49 @@ +## This is a more complex inventory file. It has a few features on it: +## +## 1. A dockerfile can be a Jinja2 template, like `docker/Dockerfile.template` +## 2. This template dockerfile gets rendered into a concrete Dockerfile in a +## temp file on disk, using the $(functions.tempfile) function. +## 3. The name of this tempfile is passed further and used by a subsequent +## stage using the `$(stages['stage-name'].outputs[0].dockerfile)` +## +## To run this inventory you have to: +## +## ./sonar.py --image sonar-test-runner --inventory inventories/inventory-template.yaml +## + +vars: + # start a local registry with: + # docker run -d -p 5000:5000 --restart=always --name registry registry:2 + registry: localhost:5000 + +images: +- name: sonar-test-runner + + vars: + template_context: docker + context: . + + # First stage builds a Docker image. The resulting image will be + # pushed to the registry in the `output` section. + stages: + + - name: template-sonar + task_type: dockerfile_template + template_file_extension: 3.10rc # Template will be `Dockerfile.3.10rc` + + output: + # We will use $(functions.tempfile) to use a temporary file. The name of the + # temporary file will have to be accessed using + # `$(stages['stage-name']).outputs` afterwards. + - dockerfile: $(functions.tempfile) + + - name: build-sonar-tester-image + task_type: docker_build + + dockerfile: $(stages['template-sonar'].outputs[0].dockerfile) + + output: + - registry: $(inputs.params.registry)/sonar-template-test + tag: $(inputs.params.version_id) + - registry: $(inputs.params.registry)/sonar-template-test + tag: latest diff --git a/lib/sonar/inventories/simple.yaml b/lib/sonar/inventories/simple.yaml new file mode 100644 index 000000000..7d69c5a15 --- /dev/null +++ b/lib/sonar/inventories/simple.yaml @@ -0,0 +1,35 @@ +vars: + # start a local registry with: + # docker run -d -p 5000:5000 --restart=always --name registry registry:2 + registry: localhost:5000 + +images: +- name: sonar-test-runner + + vars: + context: . + + # First stage builds a Docker image. The resulting image will be + # pushed to the registry in the `output` section. + stages: + - name: build-sonar-tester-image + task_type: docker_build + + dockerfile: docker/Dockerfile + + output: + - registry: $(inputs.params.registry)/sonar-tester-image + tag: $(inputs.params.version_id) + + # Second stage pushes the previously built image into a new + # registry. + - name: tag-image + task_type: tag_image + + source: + registry: $(stages['build-sonar-tester-image'].outputs[0].registry) + tag: $(stages['build-sonar-tester-image'].outputs[0].tag) + + destination: + - registry: $(inputs.params.registry)/sonar-tester-image + tag: latest diff --git a/lib/sonar/sonar.py b/lib/sonar/sonar.py new file mode 100644 index 000000000..c97b2f0da --- /dev/null +++ b/lib/sonar/sonar.py @@ -0,0 +1,770 @@ +""" +sonar/sonar.py + +Implements Sonar's main functionality. +""" + +import json +import os +import re +import subprocess +import tempfile +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.request import urlretrieve + +import boto3 +import click +import yaml + +from lib.base_logger import logger + +from . import DCT_ENV_VARIABLE, DCT_PASSPHRASE +from .builders.docker import ( + SonarAPIError, + docker_build, + docker_pull, + docker_push, + docker_tag, +) +from .template import render + + +# pylint: disable=R0902 +@dataclass +class Context: + """ + Sonar's Execution Context. + + Holds information required for a run, including execution parameters, + inventory dictionary, tags (included and excluded). + """ + + inventory: Dict[str, str] + image: Dict[str, Dict] + + # Store parameters passed as arguments + parameters: Dict[str, str] + + skip_tags: List[str] = None + include_tags: List[str] = None + + stage: Dict = None + + # If continue_on_errors is set to true, errors will + # be captured and logged, but will not raise, and will + # not stop future tasks to be executed. + continue_on_errors: bool = True + + # If errors happened during the execution an exception + # will be raised. This can help on situations were some + # errors were captured (continue_on_errors == True) but + # we still want to fail the overall task. + fail_on_errors: bool = False + + # List of captured errors to report. + captured_errors: List[Exception] = field(default_factory=list) + + # Defines if running in pipeline mode, this is, the output + # is supposed to be consumable by the system calling sonar. + pipeline: bool = False + output: dict = field(default_factory=dict) + + # Generates a version_id to use if one is not present + stored_version_id: str = str(uuid.uuid4()) + + # stage_outputs is a dictionary of dictionaries. First dict + # has a key corresponding to the name of the stage, the dict + # you get from it is key/value (str, Any) with the values + # stored by given stage. + stage_outputs: Dict[str, List[Dict[str, Any]]] = field(default_factory=dict) + + # pylint: disable=C0103 + def I(self, string): + """ + I interpolates variables in string. + """ + return interpolate_vars(self, string, stage=self.stage) + + @property + def image_name(self): + """Returns current image name""" + return self.image["name"] + + @property + def version_id(self): + """Returns the version_id for this run. + + In evergreen context, it corresponds to Evergreen's run version_id, locally + a uuid is used as a way of having independent builds. + """ + return os.environ.get("version_id", self.stored_version_id) + + +def append_output_in_context(ctx: Context, stage_name: str, values: Dict) -> None: + """Stores a value as the output of the stage, so it can be consumed by future stages.""" + if stage_name not in ctx.stage_outputs.keys(): + # adds a new empty dictionary to this stage + # if there isn't one yet. + ctx.stage_outputs[stage_name] = list() + + ctx.stage_outputs[stage_name].append(values) + + +def find_inventory(inventory: Optional[str] = None): + """ + Finds the inventory file, and return it as a yaml object. + """ + if inventory is None: + inventory = "inventory.yaml" + + # pylint: disable=C0103 + with open(inventory, "r") as f: + return yaml.safe_load(f) + + +def find_image(image_name: str, inventory: str): + """ + Looks for an image of the given name in the inventory. + """ + for image in find_inventory(inventory)["images"]: + if image["name"] == image_name: + return image + + raise ValueError("Image {} not found".format(image_name)) + + +def find_variable_replacement(ctx: Context, variable: str, stage=None) -> str: + """ + Returns the variable *value* for this varable. + """ + if variable == "version_id": + return ctx.version_id + + replacement = None + # Find variable value on top level file + if "vars" in ctx.inventory: + if variable in ctx.inventory["vars"]: + replacement = ctx.inventory["vars"][variable] + + # Find top-level defined variables overrides, + # these might not be defined anywhere in the inventory file. + # maybe they should? + if variable in ctx.parameters: + replacement = ctx.parameters[variable] + + # Find variable value on image + if "vars" in ctx.image: + if variable in ctx.image["vars"]: + replacement = ctx.image["vars"][variable] + + # Find variables in stage + if stage is not None and "vars" in stage: + if variable in stage["vars"]: + replacement = stage["vars"][variable] + + # Find variable values on cli parameters + if "inputs" in ctx.image: + if variable in ctx.image["inputs"]: + # If in inputs then we get it form the parameters + replacement = ctx.parameters[variable] + + return replacement + + +def find_variable_replacements(ctx: Context, variables: List[str], stage=None) -> Dict[str, str]: + """ + Finds replacements for a list of variables. + """ + replacements = {} + for variable in variables: + value = find_variable_replacement(ctx, variable, stage) + if value is None: + raise ValueError("No value for variable {}".format(variable)) + + replacements[variable] = value + + return replacements + + +def execute_interpolatable_function(name: str) -> str: + if name == "tempfile": + tmp = tempfile.mkstemp() + # mkstemp returns a tuple, with the second element of it being + # the absolute path to the file. + return tmp[1] + + raise ValueError("Only supported function is 'tempfile'") + + +def find_variables_to_interpolate_from_stage(string: str) -> List[Any]: + """Finds a $(stage['stage-name'].outputs[])""" + var_finder_re = r"\$\(stages\[\'(?P[\w-]+)\'\]\.outputs\[(?P\d+)\]\.(?P\w+)" + + return re.findall(var_finder_re, string, re.UNICODE) + + +def find_variables_to_interpolate(string) -> List[str]: + """ + Returns a list of variables in the string that need to be interpolated. + """ + var_finder_re = r"\$\(inputs\.params\.(?P\w+)\)" + return re.findall(var_finder_re, string, re.UNICODE) + + +def find_functions_to_interpolate(string: str) -> List[Any]: + """Find functions to be interpolated.""" + var_finder_re = r"\$\(functions\.(?P\w+)\)" + + return re.findall(var_finder_re, string, re.UNICODE) + + +def interpolate_vars(ctx: Context, string: str, stage=None) -> str: + """ + For each variable to interpolate in string, finds its *value* and + replace it in the final string. + """ + variables = find_variables_to_interpolate(string) + replacements = find_variable_replacements(ctx, variables, stage) + + for variable in variables: + string = string.replace("$(inputs.params.{})".format(variable), replacements[variable]) + + variables = find_variables_to_interpolate_from_stage(string) + for stage, index, key in variables: + value = ctx.stage_outputs[stage][int(index)][key] + string = string.replace("$(stages['{}'].outputs[{}].{})".format(stage, index, key), value) + + functions = find_functions_to_interpolate(string) + for name in functions: + value = execute_interpolatable_function(name) + string = string.replace("$(functions.{})".format(name), value) + + return string + + +def build_add_statement(ctx, block) -> str: + """ + DEPRECATED: do not use + """ + stmt = "ADD " + if "from" in block: + stmt += "--from={} ".format(block["from"]) + + src = ctx.I(block["src"]) + dst = ctx.I(block["dst"]) + stmt += "{} {}\n".format(src, dst) + + return stmt + + +def find_docker_context(ctx: Context): + """ + Finds a docker context in multiple places in the inventory, image or stage. + """ + if ctx.stage is not None: + if "vars" in ctx.stage and "context" in ctx.stage["vars"]: + return ctx.stage["vars"]["context"] + + if "dockercontext" in ctx.stage: + return ctx.stage["dockercontext"] + + if "vars" in ctx.image and "context" in ctx.image["vars"]: + return ctx.image["vars"]["context"] + + raise ValueError("No context defined for image or stage") + + +def should_skip_stage(stage: Dict[str, str], skip_tags: List[str]) -> bool: + """ + Checks if this stage should be skipped. + """ + stage_tags = stage.get("tags", []) + if len(stage_tags) == 0: + return False + + return not set(stage_tags).isdisjoint(skip_tags) + + +def should_include_stage(stage: Dict[str, str], include_tags: List[str]) -> bool: + """ + Checks if this stage should be included in the run. If tags is empty, then + all stages should be run, included this one. + """ + stage_tags = stage.get("tags", []) + if len(include_tags) == 0: + # We don't have "include_tags" so all tasks should run + return True + + return not set(stage_tags).isdisjoint(include_tags) + + +def task_dockerfile_create(ctx: Context): + """Writes a simple Dockerfile from SCRATCH and ADD statements. This + is intended to build a 'context' Dockerfile, this is, a Dockerfile that's + not runnable but contains data. + + DEPRECATED: Use dockerfile_template or docker_build instead. + """ + output_dockerfile = ctx.I(ctx.stage["output"][0]["dockerfile"]) + fro = ctx.stage.get("from", "scratch") + + # pylint: disable=C0103 + with open("{}".format(output_dockerfile), "w") as fd: + fd.write("FROM {}\n".format(fro)) + for f in ctx.stage["static_files"]: + fd.write(build_add_statement(ctx, f)) + + echo(ctx, "dockerfile-save-location", output_dockerfile) + + +def get_secret(secret_name: str, region: str) -> str: + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region) + + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + + return get_secret_value_response.get("SecretString", "") + + +def get_private_key_id(registry: str, signer_name: str) -> str: + cp = subprocess.run( + ["docker", "trust", "inspect", registry], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + if cp.returncode != 0: + return SonarAPIError(cp.stderr) + + json_data = json.loads(cp.stdout) + assert len(json_data) != 0 + for signer in json_data[0]["Signers"]: + if signer_name == signer["Name"]: + assert len(signer["Keys"]) != 0 + return signer["Keys"][0]["ID"] + ".key" + + +def task_tag_image(ctx: Context): + """ + Pulls an image from source and pushes into destination. + """ + registry = ctx.I(ctx.stage["source"]["registry"]) + tag = ctx.I(ctx.stage["source"]["tag"]) + + image = docker_pull(registry, tag) + + for output in ctx.stage["destination"]: + registry = ctx.I(output["registry"]) + tag = ctx.I(output["tag"]) + echo( + ctx, + "docker-image-push", + "{}:{}".format(registry, tag), + ) + + docker_tag(image, registry, tag) + create_ecr_repository(registry) + try: + docker_push(registry, tag) + except SonarAPIError as e: + ctx.captured_errors.append(e) + if ctx.continue_on_errors: + echo(ctx, "docker-image-push/error", e) + else: + raise + + append_output_in_context(ctx, ctx.stage["name"], {"registry": registry, "tag": tag}) + + +def get_rendering_params(ctx: Context) -> Dict[str, str]: + """ + Finds rendering parameters for a template, based on the `inputs` section + of the stage. + """ + params = {} + for param in ctx.stage.get("inputs", {}): + params[param] = find_variable_replacement(ctx, param, ctx.stage) + + return params + + +def run_dockerfile_template(ctx: Context, dockerfile_context: str, distro: str) -> str: + """ + Renders a template and returns a file name pointing at the render. + """ + path = dockerfile_context + params = get_rendering_params(ctx) + + logger.debug("rendering params are:") + logger.debug(params) + + rendered = render(path, distro, params) + tmp = tempfile.NamedTemporaryFile(delete=False) + tmp.write(rendered.encode("utf-8")) + + return tmp.name + + +def interpolate_dict(ctx: Context, args: Dict[str, str]) -> Dict[str, str]: + """ + Returns a copy of the provided dictionary with their variables interpolated with values. + """ + copied_args = {} + # pylint: disable=C0103 + for k, v in args.items(): + copied_args[k] = ctx.I(v) + + return copied_args + + +def is_valid_ecr_repo(repo_name: str) -> bool: + """Returns true if repo_name is a ECR repository, it expectes + a domain part (*.amazonaws.com) and a repository part (/images/container-x/...).""" + rex = re.compile(r"^[0-9]{10,}\.dkr\.ecr\.[a-z]{2}\-[a-z]+\-[0-9]+\.amazonaws\.com/.+") + return rex.match(repo_name) is not None + + +def create_ecr_repository(tag: str): + """ + Creates ecr repository if it doesn't exist + """ + if not is_valid_ecr_repo(tag): + logger.info("Not an ECR repository: %s", tag) + return + + try: + no_tag = tag.partition(":")[0] + region = no_tag.split(".")[3] + repository_name = no_tag.partition("/")[2] + except IndexError: + logger.debug("Could not parse repository: %s", tag) + return + + logger.debug("Creating repository in %s with name %s", region, repository_name) + + client = boto3.client("ecr", region_name=region) + + try: + client.create_repository( + repositoryName=repository_name, + imageTagMutability="MUTABLE", + imageScanningConfiguration={"scanOnPush": False}, + ) + except client.exceptions.RepositoryAlreadyExistsException: + logger.debug("Repository already exists") + + +def echo(ctx: Context, entry_name: str, message: str, foreground: str = "white"): + """ + Echoes a message. + """ + + err = ctx.pipeline + section = ctx.output + + if ctx.pipeline: + image_name = ctx.image["name"] + if image_name not in ctx.output: + ctx.output[image_name] = {} + section = ctx.output[image_name] + + if ctx.stage is not None: + stage_name = ctx.stage["name"] + if stage_name not in ctx.output[image_name]: + ctx.output[image_name][stage_name] = {} + section = ctx.output[image_name][stage_name] + + section[entry_name] = message + + stage_title = "" + if ctx.stage: + stage_type = ctx.stage["task_type"] + stage_name = ctx.stage["name"] + stage_title = "[{}/{}] ".format(stage_name, stage_type) + + # If --pipeline, these messages go to stderr + + click.secho("{}{}: {}".format(stage_title, entry_name, message), fg=foreground, err=err) + + +def find_dockerfile(dockerfile: str): + """Returns a Dockerfile file location that can be local or remote. If remote it + will be downloaded into a temporary location first.""" + + if dockerfile.startswith("https://"): + tmpfile = tempfile.NamedTemporaryFile(delete=False) + urlretrieve(dockerfile, tmpfile.name) + + return tmpfile.name + + return dockerfile + + +def is_signing_enabled(output: Dict) -> bool: + return all( + key in output + for key in ( + "signer_name", + "key_secret_name", + "passphrase_secret_name", + "region", + ) + ) + + +def setup_signing_environment(ctx: Context, output: Dict) -> str: + os.environ[DCT_ENV_VARIABLE] = "1" + os.environ[DCT_PASSPHRASE] = get_secret(ctx.I(output["passphrase_secret_name"]), ctx.I(output["region"])) + # Asks docker trust inspect for the name the private key for the specified signer + # has to have + signing_key_name = get_private_key_id(ctx.I(output["registry"]), ctx.I(output["signer_name"])) + + # And writes the private key stored in the secret to the appropriate path + private_key = get_secret(ctx.I(output["key_secret_name"]), ctx.I(output["region"])) + docker_trust_path = f"{Path.home()}/.docker/trust/private" + Path(docker_trust_path).mkdir(parents=True, exist_ok=True) + with open(f"{docker_trust_path}/{signing_key_name}", "w+") as f: + f.write(private_key) + + return signing_key_name + + +def task_docker_build(ctx: Context): + """ + Builds a container image. + """ + docker_context = find_docker_context(ctx) + + platform = ctx.image.get("platform") + if platform: + platform = ctx.I(platform) + + dockerfile = find_dockerfile(ctx.I(ctx.stage["dockerfile"])) + + buildargs = interpolate_dict(ctx, ctx.stage.get("buildargs", {})) + + labels = interpolate_dict(ctx, ctx.stage.get("labels", {})) + + image = docker_build(docker_context, dockerfile, buildargs=buildargs, labels=labels, platform=platform) + + for output in ctx.stage["output"]: + registry = ctx.I(output["registry"]) + tag = ctx.I(output["tag"]) + sign = is_signing_enabled(output) + signing_key_name = "" + if sign: + signing_key_name = setup_signing_environment(ctx, output) + + echo(ctx, "docker-image-push", "{}:{}".format(registry, tag)) + docker_tag(image, registry, tag) + + create_ecr_repository(registry) + try: + docker_push(registry, tag) + except SonarAPIError as e: + ctx.captured_errors.append(e) + if ctx.continue_on_errors: + echo(ctx, "docker-image-push/error", e) + else: + raise + + append_output_in_context( + ctx, + ctx.stage["name"], + { + "registry": registry, + "tag": tag, + }, + ) + + if sign: + clear_signing_environment(signing_key_name) + + +def split_s3_location(s3loc: str) -> Tuple[str, str]: + if not s3loc.startswith("s3://"): + raise ValueError("{} is not a S3 URL".format(s3loc)) + + bucket, _, location = s3loc.partition("s3://")[2].partition("/") + + return bucket, location + + +def save_dockerfile(dockerfile: str, destination: str): + if destination.startswith("s3://"): + client = boto3.client("s3") + bucket, location = split_s3_location(destination) + client.upload_file(dockerfile, bucket, location, ExtraArgs={"ACL": "public-read"}) + else: + copyfile(dockerfile, destination) + + +def task_dockerfile_template(ctx: Context): + """ + Templates a dockerfile. + """ + docker_context = find_docker_context(ctx) + template_context = docker_context + + try: + template_context = ctx.image["vars"]["template_context"] + except KeyError: + pass + + template_file_extension = ctx.stage.get("template_file_extension") + if template_file_extension is None: + # Use distro as compatibility with pre 0.11 + template_file_extension = ctx.stage.get("distro") + + dockerfile = run_dockerfile_template(ctx, template_context, template_file_extension) + + for output in ctx.stage["output"]: + if "dockerfile" in output: + output_dockerfile = ctx.I(output["dockerfile"]) + save_dockerfile(dockerfile, output_dockerfile) + + echo(ctx, "dockerfile-save-location", output_dockerfile) + + append_output_in_context(ctx, ctx.stage["name"], {"dockerfile": output_dockerfile}) + + +def find_skip_tags(params: Optional[Dict[str, str]] = None) -> List[str]: + """Returns a list of tags passed in params that should be excluded from the build.""" + if params is None: + params = {} + + tags = params.get("skip_tags", []) + + if isinstance(tags, str): + tags = [t.strip() for t in tags.split(",") if t != ""] + + return tags + + +def find_include_tags(params: Optional[Dict[str, str]] = None) -> List[str]: + """Returns a list of tags passed in params that should be included in the build.""" + if params is None: + params = {} + + tags = params.get("include_tags", []) + + if isinstance(tags, str): + tags = [t.strip() for t in tags.split(",") if t != ""] + + return tags + + +def clear_signing_environment(key_to_remove: str): + # Note that this is not strictly needed + os.unsetenv(DCT_ENV_VARIABLE) + os.unsetenv(DCT_PASSPHRASE) + os.remove(f"{Path.home()}/.docker/trust/private/{key_to_remove}") + + +# pylint: disable=R0913, disable=R1710 +def process_image( + image_name: str, + skip_tags: Union[str, List[str]], + include_tags: Union[str, List[str]], + build_args: Optional[Dict[str, str]] = None, + inventory: Optional[str] = None, + build_options: Optional[Dict[str, str]] = None, +): + """ + Runs the Sonar process over an image, for an inventory and a set of configurations. + """ + if build_args is None: + build_args = {} + + ctx = build_context(image_name, skip_tags, include_tags, build_args, inventory, build_options) + + echo(ctx, "image_build_start", image_name, foreground="yellow") + + for idx, stage in enumerate(ctx.image.get("stages", [])): + ctx.stage = stage + name = ctx.stage["name"] + task = stage["task_type"] + if should_skip_stage(stage, ctx.skip_tags): + echo(ctx, "skipping-stage", name, foreground="green") + continue + + if not should_include_stage(stage, ctx.include_tags): + echo(ctx, "skipping-stage", name, foreground="green") + continue + + stages_len = len(ctx.image["stages"]) + + echo(ctx, f"stage-started {name} - task-started {task}", f"{idx + 1}/{stages_len}") + + if task == "dockerfile_create": + task_dockerfile_create(ctx) + elif task == "dockerfile_template": + task_dockerfile_template(ctx) + elif task == "docker_build": + task_docker_build(ctx) + elif task == "tag_image": + task_tag_image(ctx) + else: + raise NotImplementedError("task_type {} not supported".format(task)) + + if len(ctx.captured_errors) > 0 and ctx.fail_on_errors: + echo(ctx, "docker-image-push/captured-errors", ctx.captured_errors) + raise SonarAPIError(ctx.captured_errors[0]) + + if ctx.pipeline: + return ctx.output + + +def make_list_of_str(value: Union[None, str, List[str]]) -> List[str]: + """ + Returns a list of strings from multiple different types. + """ + if value is None: + return [] + + if isinstance(value, str): + if len(value) == 0: + return [] + + return [e.strip() for e in value.split(",") if e != ""] + + return value + + +def build_context( + image_name: str, + skip_tags: Union[str, List[str]], + include_tags: Union[str, List[str]], + build_args: Optional[Dict[str, str]] = None, + inventory: Optional[str] = None, + build_options: Optional[Dict[str, str]] = None, +) -> Context: + """A Context includes the whole inventory, the image to build, the current stage, + and the `I` interpolation function.""" + image = find_image(image_name, inventory) + + if build_args is None: + build_args = dict() + build_args = build_args.copy() + logger.debug("Should skip tags %s", skip_tags) + + if build_options is None: + build_options = {} + + context = Context( + inventory=find_inventory(inventory), + image=image, + parameters=build_args, + skip_tags=make_list_of_str(skip_tags), + include_tags=make_list_of_str(include_tags), + ) + + for k, v in build_options.items(): + if hasattr(context, k): + setattr(context, k, v) + + return context diff --git a/lib/sonar/template.py b/lib/sonar/template.py new file mode 100644 index 000000000..98d8e1a11 --- /dev/null +++ b/lib/sonar/template.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from typing import Dict + +import jinja2 + + +def render(path: str, template_name: str, parameters: Dict[str, str]) -> str: + """Returns a rendered Dockerfile. + + path indicates where in the filesystem the Dockerfiles are. + template_name references a Dockerfile. to render. + """ + env = jinja2.Environment(loader=jinja2.FileSystemLoader(path), undefined=jinja2.StrictUndefined) + + template = "Dockerfile" + if template_name is not None: + template = "Dockerfile.{}".format(template_name) + + return env.get_template(template).render(parameters) diff --git a/lib/sonar/test/__init__.py b/lib/sonar/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/sonar/test/test_build.py b/lib/sonar/test/test_build.py new file mode 100644 index 000000000..99d98b709 --- /dev/null +++ b/lib/sonar/test/test_build.py @@ -0,0 +1,130 @@ +from types import SimpleNamespace as sn +from unittest.mock import call, mock_open, patch + +from ..builders.docker import get_docker_build_cli_args +from ..sonar import find_dockerfile, process_image + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.urlretrieve") +@patch("sonar.sonar.create_ecr_repository") +def test_dockerfile_from_url( + patched_docker_build, + patched_docker_tag, + patched_docker_push, + patched_urlretrive, + patched_create_ecr_repository, +): + with open("lib/sonar/test/yaml_scenario6.yaml") as fd: + with patch("builtins.open", mock_open(read_data=fd.read())) as _mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=[], + include_tags=["test_dockerfile_from_url"], + build_args={}, + ) + + patched_urlretrive.assert_called_once() + patched_docker_build.assert_called_once() + patched_docker_tag.assert_called_once() + patched_docker_push.assert_called_once() + patched_create_ecr_repository.assert_called_once() + + +@patch("sonar.sonar.tempfile.NamedTemporaryFile", return_value=sn(name="random-filename")) +@patch("sonar.sonar.urlretrieve") +def test_find_dockerfile_fetches_file_from_url(patched_urlretrieve, patched_tempfile): + # If passed a dockerfile which starts with https:// + # make sure urlretrieve and NamedTemporaryFile is called + dockerfile = find_dockerfile("https://something") + + patched_urlretrieve.assert_called_once() + patched_tempfile.assert_called_once_with(delete=False) + assert dockerfile == "random-filename" + + patched_urlretrieve.reset_mock() + + # If dockerfile is a localfile, urlretrieve should not be called. + dockerfile = find_dockerfile("/localfile/somewhere") + patched_urlretrieve.assert_not_called() + assert dockerfile == "/localfile/somewhere" + + +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +def test_labels_are_passed_to_docker_build(_docker_build, _docker_tag): + _ = process_image( + image_name="image1", + skip_tags=[], + include_tags=[], + build_args={}, + build_options={}, + inventory="lib/sonar/test/yaml_scenario9.yaml", + ) + + # the labels have been specified in the test scenario and should be passed to docker_build. + calls = [ + call( + ".", + "Dockerfile", + buildargs={}, + labels={"label-0": "value-0", "label-1": "value-1", "label-2": "value-2"}, + platform=None, + ) + ] + + _docker_build.assert_has_calls(calls) + _docker_build.assert_called_once() + + +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +def test_platform_is_passed_to_docker_build(_docker_build, _docker_tag): + # platform in image is set + _ = process_image( + image_name="image1", + skip_tags=[], + include_tags=[], + build_args={}, + build_options={}, + inventory="lib/sonar/test/yaml_scenario11.yaml", + ) + + # platform is not specified + _ = process_image( + image_name="image2", + skip_tags=[], + include_tags=[], + build_args={}, + build_options={}, + inventory="lib/sonar/test/yaml_scenario11.yaml", + ) + + calls = [ + call(".", "Dockerfile", buildargs={}, labels={"label-0": "value-0"}, platform="linux/amd64"), + call(".", "Dockerfile", buildargs={}, labels={"label-1": "value-1"}, platform=None), + ] + + _docker_build.assert_has_calls(calls) + _docker_build.assert_called() + + +def test_get_docker_build_cli_args(): + assert "docker buildx build --load --progress plain . -f dockerfile -t image:latest" == " ".join( + get_docker_build_cli_args(".", "dockerfile", "image:latest", None, None, None) + ) + assert ( + "docker buildx build --load --progress plain . -f dockerfile -t image:latest --build-arg a=1 --build-arg long_arg=long_value --label l1=v1 --label l2=v2 --platform linux/amd64" + == " ".join( + get_docker_build_cli_args( + ".", + "dockerfile", + "image:latest", + {"a": "1", "long_arg": "long_value"}, + {"l1": "v1", "l2": "v2"}, + "linux/amd64", + ) + ) + ) diff --git a/lib/sonar/test/test_context.py b/lib/sonar/test/test_context.py new file mode 100644 index 000000000..ee985ecba --- /dev/null +++ b/lib/sonar/test/test_context.py @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- + +from unittest.mock import mock_open, patch + +import pytest + +from ..sonar import ( + Context, + append_output_in_context, + build_context, + find_skip_tags, + should_skip_stage, +) + + +# yaml_scenario0 +@pytest.fixture() +def ys0(): + return open("lib/sonar/test/yaml_scenario0.yaml").read() + + +@pytest.fixture() +def cs0(ys0): + with patch("builtins.open", mock_open(read_data=ys0)) as mock_file: + ctx = build_context( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={}, + ) + ctx.stage = ctx.image["stages"][0] + + return ctx + + +# yaml_scenario1 +@pytest.fixture() +def ys1(): + return open("lib/sonar/test/yaml_scenario1.yaml").read() + + +@pytest.fixture() +def cs1(ys1): + with patch("builtins.open", mock_open(read_data=ys1)) as mock_file: + ctx = build_context( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={}, + ) + ctx.stage = ctx.image["stages"][0] + + return ctx + + +# yaml_scenario2 +@pytest.fixture() +def ys2(): + return open("lib/sonar/test/yaml_scenario2.yaml").read() + + +@pytest.fixture() +def cs2(ys2): + with patch("builtins.open", mock_open(read_data=ys2)) as mock_file: + ctx = build_context( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={ + "image_input0": "🐳", + "image_input1": "🎄", + "non_defined_in_inventory": "yes", + }, + ) + ctx.stage = ctx.image["stages"][0] + + return ctx + + +def test_skip_tags(): + params = { + "some": "thing", + "skip_tags": "ubi,rhel", + } + + tags = find_skip_tags(params) + assert len(tags) == 2 + assert tags[0] == "ubi" + assert tags[1] == "rhel" + assert "skip_tags" in params + + tags = find_skip_tags() + assert tags == [] + + params = { + "some": "thing", + "skip_tags": "ubi", + } + + tags = find_skip_tags(params) + assert len(tags) == 1 + assert tags[0] == "ubi" + assert "skip_tags" in params + assert "some" in params + + +def test_should_skip_tags(): + stage = { + "name": "something", + "tags": ["tag0", "tag1"], + } + + assert should_skip_stage(stage, ["tag0"]) + assert should_skip_stage(stage, ["tag1"]) + assert not should_skip_stage(stage, ["another-tag"]) + + stage = { + "name": "something", + } + assert not should_skip_stage(stage, ["tag0"]) + + stage = { + "name": "something", + "tags": ["ubi"], + } + + assert not should_skip_stage(stage, ["ubuntu"]) + + +def test_build_context(cs0): + ctx = cs0 + assert ctx.image_name == "image0" + assert ctx.skip_tags == None + assert ctx.parameters == {} + + +def test_build_context(ys0): + with patch("builtins.open", mock_open(read_data=ys0)) as mock_file: + with pytest.raises(ValueError, match="Image image1 not found"): + build_context(image_name="image1", skip_tags=[], include_tags=[]) + + +def test_variable_interpolation0(cs1): + ctx = cs1 + + assert ctx.I("$(inputs.params.registry)/something") == "somereg/something" + with pytest.raises(KeyError): + ctx.I("$(inputs.params.input0)") + + +def test_variable_interpolation1(cs2): + ctx = cs2 + + # Inventory variables + assert ctx.I("$(inputs.params.inventory_var0)") == "inventory_var_value0" + assert ctx.I("$(inputs.params.inventory_var1)") == "inventory_var_value1" + with pytest.raises(ValueError): + ctx.I("$(inputs.params.inventory_var_non_existing)") + + # Parameters passed to function + assert ctx.I("$(inputs.params.image_input0)") == "🐳" + assert ctx.I("$(inputs.params.image_input1)") == "🎄" + with pytest.raises(ValueError): + ctx.I("$(inputs.params.image_input_non_existing)") + + # Image variables + assert ctx.I("$(inputs.params.image_var0)") == "image_var_value0" + assert ctx.I("$(inputs.params.image_var1)") == "image_var_value1" + with pytest.raises(ValueError): + ctx.I("$(inputs.params.image_var_non_existing)") + + # Stage variables + assert ctx.I("$(inputs.params.stage_var0)") == "stage_value0" + assert ctx.I("$(inputs.params.stage_var1)") == "stage_value1" + with pytest.raises(ValueError): + assert ctx.I("$(inputs.params.stage_var_non_existing)") == "stage_value2" + + # Parameters passed but not defined in inventory + assert ctx.I("$(inputs.params.non_defined_in_inventory)") == "yes" + + with pytest.raises(ValueError): + assert ctx.I("$(inputs.params.defined_nowhere)") + + +def test_variable_interpolation_stage_parameters(ys1): + with patch("builtins.open", mock_open(read_data=ys1)) as mock_file: + ctx = build_context( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={"input0": "value0", "input1": "value1"}, + ) + + ctx.stage = ctx.image["stages"][0] + + assert ctx.I("$(inputs.params.input0)") == "value0" + assert ctx.I("$(inputs.params.input1)") == "value1" + assert ctx.I("$(inputs.params.input0)/$(inputs.params.input1)") == "value0/value1" + assert ctx.I("$(inputs.params.input1)/$(inputs.params.input0)") == "value1/value0" + assert ( + ctx.I("some text $(inputs.params.input1)/$(inputs.params.input0) more text") + == "some text value1/value0 more text" + ) + + assert ctx.I("$(inputs.params.input0) 🐳") == "value0 🐳" + with pytest.raises(ValueError): + ctx.I("$(inputs.params.non_existing)") + + +@pytest.mark.xfail +def test_variable_interpolation_stage_parameters_funny(ys1): + """This test won't work and I'm not sure why: + 1. Maybe parsing the yaml file won't get the same unicode code? + 2. Regex won't capture it""" + with patch("builtins.open", mock_open(read_data=ys1)) as mock_file: + ctx = build_context( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={"🐳": "whale", "🎄": "tree"}, + ) + ctx.stage = ctx.image["stages"][0] + + assert ctx.I("$(inputs.params.🐳)") == "whale" + assert ctx.I("$(inputs.params.🎄)") == "tree" + + +@patch("sonar.sonar.find_image", return_value={}) +def test_build_context_skip_tags_from_str(_patched_find_image): + ctx = build_context( + inventory="lib/sonar/inventories/simple.yaml", + image_name="image-name", + skip_tags="skip0,skip1", + include_tags="included0, included1", + build_args={}, + ) + + assert ctx.skip_tags == ["skip0", "skip1"] + assert ctx.include_tags == ["included0", "included1"] + + +@patch("sonar.sonar.find_image", return_value={}) +def test_build_context_skip_tags_from_empty_str(_patched_find_image): + ctx = build_context( + inventory="lib/sonar/inventories/simple.yaml", + image_name="image-name", + skip_tags="", + include_tags="", + build_args={}, + ) + + assert ctx.skip_tags == [] + assert ctx.include_tags == [] + + +@patch("sonar.sonar.find_inventory", return_value={"images": {"name": "image-name"}}) +@patch("sonar.sonar.find_image", return_value={"name": "image-name"}) +def test_build_context_uses_any_inventory(patched_find_image, patched_find_inventory): + build_context( + image_name="image-name", + skip_tags="", + include_tags="", + build_args={}, + inventory="other-inventory.yaml", + ) + + patched_find_image.assert_called_once_with("image-name", "other-inventory.yaml") + patched_find_inventory.assert_called_once_with("other-inventory.yaml") + + +def test_use_specific_inventory(): + context = build_context( + image_name="image0", + skip_tags="", + include_tags="", + build_args={"input0": "my-value"}, + inventory="lib/sonar/test/yaml_scenario0.yaml", + ) + + assert context.image["name"] == "image0" + assert context.stage is None + + assert context.skip_tags == [] + assert context.include_tags == [] + + assert context.I("$(inputs.params.input0)") == "my-value" + + +def test_can_provide_generic_configuration(): + context = build_context( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={}, + inventory="lib/sonar/test/yaml_scenario0.yaml", + build_options={"invalid_options": False, "continue_on_errors": False}, + ) + + assert context.continue_on_errors is False + assert not hasattr(context, "invalid_options") + + +def test_can_store_in_context(): + ctx = Context( + inventory={}, + image="some-image", + parameters=[], + ) + + append_output_in_context(ctx, "stage0", {"key0": "value0", "key1": "value1", "key2": "value2"}) + + append_output_in_context(ctx, "stage0", {"key0": "value0", "key1": "value1", "key2": "value2", "key3": "value3"}) + + assert len(ctx.stage_outputs["stage0"]) == 2 + assert len(ctx.stage_outputs["stage0"][0]) == 3 + assert len(ctx.stage_outputs["stage0"][1]) == 4 + + assert ctx.stage_outputs["stage0"][0] == {"key0": "value0", "key1": "value1", "key2": "value2"} + + assert ctx.stage_outputs["stage0"][1] == {"key0": "value0", "key1": "value1", "key2": "value2", "key3": "value3"} + + append_output_in_context(ctx, "stage1", {"key0": "value0", "key1": "value1", "key2": "value2", "key3": "value3"}) + + assert len(ctx.stage_outputs) == 2 + assert len(ctx.stage_outputs["stage1"][0]) == 4 + + assert ctx.stage_outputs["stage1"][0] == {"key0": "value0", "key1": "value1", "key2": "value2", "key3": "value3"} + + assert ctx.I("$(stages['stage0'].outputs[0].key0)") == "value0" + assert ctx.I("$(stages['stage0'].outputs[1].key3)") == "value3" + + +def test_stages_output_and_variables(cs2: Context): + ctx = cs2 + append_output_in_context(ctx, "stage0", {"key0": "value0", "key1": "value1", "key2": "value2"}) + + append_output_in_context(ctx, "stage0", {"key0": "value0", "key1": "value1", "key2": "value2", "key3": "value3"}) + + assert ( + ctx.I("$(inputs.params.inventory_var0) -- $(stages['stage0'].outputs[0].key2)") + == "inventory_var_value0 -- value2" + ) + + assert ctx.I("$(inputs.params.image_input0) -- $(stages['stage0'].outputs[1].key3)") == "🐳 -- value3" diff --git a/lib/sonar/test/test_docker.py b/lib/sonar/test/test_docker.py new file mode 100644 index 000000000..e36840df0 --- /dev/null +++ b/lib/sonar/test/test_docker.py @@ -0,0 +1,45 @@ +from types import SimpleNamespace +from unittest.mock import Mock, call + +import pytest +from pytest_mock import MockerFixture + +from ..builders import SonarAPIError +from ..builders.docker import docker_push + + +def test_docker_push_is_retried(mocker: MockerFixture): + a = SimpleNamespace(returncode=1, stderr="some-error") + sp = mocker.patch("sonar.builders.docker.subprocess") + sp.PIPE = "|PIPE|" + sp.run.return_value = a + + with pytest.raises(SonarAPIError, match="some-error"): + docker_push("reg", "tag") + + # docker push is called 4 times, the last time it is called, it raises an exception + sp.run.assert_has_calls( + [ + call(["docker", "push", "reg:tag"], stdout="|PIPE|", stderr="|PIPE|"), + call(["docker", "push", "reg:tag"], stdout="|PIPE|", stderr="|PIPE|"), + call(["docker", "push", "reg:tag"], stdout="|PIPE|", stderr="|PIPE|"), + call(["docker", "push", "reg:tag"], stdout="|PIPE|", stderr="|PIPE|"), + ] + ) + + +def test_docker_push_is_retried_and_works(mocker: MockerFixture): + + ok = SimpleNamespace(returncode=0) + sp = mocker.patch("sonar.builders.docker.subprocess") + sp.PIPE = "|PIPE|" + sp.run = Mock() + sp.run.return_value = ok + + docker_push("reg", "tag") + + sp.run.assert_called_once_with( + ["docker", "push", "reg:tag"], + stdout="|PIPE|", + stderr="|PIPE|", + ) diff --git a/lib/sonar/test/test_sign_image.py b/lib/sonar/test/test_sign_image.py new file mode 100644 index 000000000..5edd284e0 --- /dev/null +++ b/lib/sonar/test/test_sign_image.py @@ -0,0 +1,155 @@ +import os +import os.path +from pathlib import Path +from unittest.mock import call, mock_open, patch + +import pytest +from sonar import DCT_ENV_VARIABLE, DCT_PASSPHRASE + +from ..sonar import is_signing_enabled, process_image + + +@pytest.fixture() +def ys7(): + return open("lib/sonar/test/yaml_scenario7.yaml").read() + + +@pytest.fixture() +def ys8(): + return open("lib/sonar/test/yaml_scenario8.yaml").read() + + +@patch("sonar.sonar.get_secret", return_value="SECRET") +@patch("sonar.sonar.get_private_key_id", return_value="abc.key") +@patch("sonar.sonar.clear_signing_environment") +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.urlretrieve") +@patch("sonar.sonar.create_ecr_repository") +def test_sign_image( + patched_create_ecr_repository, + patched_urlretrive, + patched_docker_build, + patched_docker_tag, + patched_docker_push, + patched_clear_signing_environment, + patched_get_private_key_id, + patched_get_secret, + ys7, +): + with patch("builtins.open", mock_open(read_data=ys7)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={}, + ) + + patched_clear_signing_environment.assert_called_once_with("abc.key") + assert os.environ.get(DCT_ENV_VARIABLE, "0") == "1" + assert os.environ.get(DCT_PASSPHRASE, "0") == "SECRET" + + secret_calls = [ + call("test/kube/passphrase", "us-east-1"), + call("test/kube/secret", "us-east-1"), + ] + + patched_get_secret.assert_has_calls(secret_calls) + patched_get_private_key_id.assert_called_once_with("foo", "evergreen_ci") + + +def test_is_signing_enabled(): + test_cases = [ + { + "input": { + "signer_name": "foo", + "key_secret_name": "key_name", + "passphrase_secret_name": "pass_name", + "region": "us-east-1", + }, + "result": True, + }, + { + "input": { + "key_secret_name": "key_name", + "passphrase_secret_name": "pass_name", + "region": "us-east-1", + }, + "result": False, + }, + { + "input": { + "signer_name": "foo", + "passphrase_secret_name": "pass_name", + "region": "us-east-1", + }, + "result": False, + }, + { + "input": { + "signer_name": "foo", + "key_secret_name": "key_name", + "region": "us-east-1", + }, + "result": False, + }, + { + "input": { + "signer_name": "foo", + "key_secret_name": "key_name", + "passphrase_secret_name": "pass_name", + }, + "result": False, + }, + ] + + for case in test_cases: + assert is_signing_enabled(case["input"]) == case["result"] + + +@patch("sonar.sonar.get_secret", return_value="SECRET") +@patch("sonar.sonar.get_private_key_id", return_value="abc.key") +@patch("sonar.sonar.clear_signing_environment") +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.urlretrieve") +@patch("sonar.sonar.create_ecr_repository") +def test_sign_image( + patched_create_ecr_repository, + patched_urlretrive, + patched_docker_build, + patched_docker_tag, + patched_docker_push, + patched_clear_signing_environment, + patched_get_private_key_id, + patched_get_secret, + ys8, +): + with patch("builtins.open", mock_open(read_data=ys8)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={}, + ) + + clear_calls = [call("abc.key"), call("abc.key"), call("abc.key")] + patched_clear_signing_environment.assert_has_calls(clear_calls) + assert os.environ.get(DCT_ENV_VARIABLE, "0") == "1" + assert os.environ.get(DCT_PASSPHRASE, "0") == "SECRET" + + secret_calls = [ + call("test/kube/passphrase", "us-east-1"), + call("test/kube/secret", "us-east-1"), + ] + + patched_get_secret.assert_has_calls(secret_calls) + + private_key_calls = [ + call("foo", "evergreen_ci"), + call("foo2", "evergreen_ci"), + call("foo3", "evergreen_ci_foo"), + ] + patched_get_private_key_id.assert_has_calls(private_key_calls) diff --git a/lib/sonar/test/test_sonar.py b/lib/sonar/test/test_sonar.py new file mode 100644 index 000000000..9341db5ce --- /dev/null +++ b/lib/sonar/test/test_sonar.py @@ -0,0 +1,167 @@ +import logging +from unittest.mock import Mock, call, patch + +import pytest + +from ..sonar import ( + SonarAPIError, + create_ecr_repository, + is_valid_ecr_repo, + process_image, +) + + +@patch("sonar.sonar.find_inventory", return_value={"images": {"name": "image-name"}}) +@patch("sonar.sonar.find_image", return_value={"name": "image-name"}) +def test_specific_inventory(patched_find_image, patched_find_inventory): + process_image( + image_name="image-name", + skip_tags=[], + include_tags=[], + build_args={}, + inventory="other-inventory.yaml", + ) + + patched_find_image.assert_called_once_with("image-name", "other-inventory.yaml") + patched_find_inventory.assert_called_once_with("other-inventory.yaml") + + +def test_repo_is_not_ecr(): + repos = ( + "quay.io/some-org/some-repo", + "scan.connect.redhat.com/ospid-10001000100-1000/some-repo", + "docker.io/some-more", + "1.dkr.ecr.us-east-1.amazonaws.com", # needs bigger account number + "1.dkr.ecr.us-east.amazonaws.com", # zone is not defined + ) + for repo in repos: + assert is_valid_ecr_repo(repo) is False + + +def test_repo_is_ecr(): + repos = ( + "123456789012.dkr.ecr.eu-west-1.amazonaws.com/some-other-repo", + "123456789012.dkr.ecr.us-east-1.amazonaws.com/something-else", + ) + for repo in repos: + assert is_valid_ecr_repo(repo) + + +@patch("sonar.sonar.boto3.client") +def test_create_ecr_repository_creates_repo_when_ecr_repo(patched_client: Mock): + returned_client = Mock() + patched_client.return_value = returned_client + + # repository with no tag + create_ecr_repository( + "123456789012.dkr.ecr.eu-west-1.amazonaws.com/some-other-repo", + ) + patched_client.assert_called_once() + returned_client.create_repository.assert_called_once_with( + repositoryName="some-other-repo", + imageTagMutability="MUTABLE", + imageScanningConfiguration={"scanOnPush": False}, + ) + patched_client.reset_mock() + + # repository with a tag + create_ecr_repository( + "123456789012.dkr.ecr.eu-west-1.amazonaws.com/some-other-repo:some-tag", + ) + patched_client.assert_called_once() + returned_client.create_repository.assert_called_once_with( + repositoryName="some-other-repo", + imageTagMutability="MUTABLE", + imageScanningConfiguration={"scanOnPush": False}, + ) + + +@patch("sonar.sonar.boto3.client") +def test_create_ecr_repository_doesnt_create_repo_when_not_ecr_repo( + patched_client: Mock, +): + returned_client = Mock() + patched_client.return_value = returned_client + + create_ecr_repository( + "my-private-repo.com/something", + ) + patched_client.assert_not_called() + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +def test_continue_on_errors(_docker_build, _docker_tag, mocked_docker_push): + """We'll mock a function that fails on first iteration but succeeds the seconds one.""" + mocked_docker_push.return_value = None + mocked_docker_push.side_effect = ["All ok!", SonarAPIError("fake-error"), "All ok!"] + + pipeline = process_image( + image_name="image1", + skip_tags=[], + include_tags=["test_continue_on_errors"], + build_args={}, + build_options={"pipeline": True, "continue_on_errors": True}, + inventory="lib/sonar/test/yaml_scenario6.yaml", + ) + + # Assert docker_push was called three times, even if one of them failed + assert mocked_docker_push.call_count == 3 + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +def test_do_not_continue_on_errors(_docker_build, _docker_tag, mocked_docker_push): + mocked_docker_push.return_value = None + mocked_docker_push.side_effect = [ + SonarAPIError("fake-error-should-not-continue"), + "All ok!", + ] + + with pytest.raises(SonarAPIError): + pipeline = process_image( + image_name="image1", + skip_tags=[], + include_tags=["test_continue_on_errors"], + build_args={}, + build_options={ + "pipeline": True, + "continue_on_errors": False, + }, + inventory="lib/sonar/test/yaml_scenario6.yaml", + ) + + # docker_push raised first time, only one call expected + assert mocked_docker_push.call_count == 1 + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +def test_fail_on_captured_errors(_docker_build, _docker_tag, mocked_docker_push): + mocked_docker_push.return_value = None + mocked_docker_push.side_effect = [ + "All ok!", + SonarAPIError("fake-error-should-not-continue"), + "All ok!", + ] + + with pytest.raises(SonarAPIError): + pipeline = process_image( + image_name="image1", + skip_tags=[], + include_tags=["test_continue_on_errors"], + build_args={}, + build_options={ + "pipeline": True, + "continue_on_errors": True, + "fail_on_errors": True, + }, + inventory="lib/sonar/test/yaml_scenario6.yaml", + ) + + # docker_push raised second time time, but allowed to continue, + # anyway, process_image still raised at the end! + assert mocked_docker_push.call_count == 3 diff --git a/lib/sonar/test/test_tag_image.py b/lib/sonar/test/test_tag_image.py new file mode 100644 index 000000000..e86226d10 --- /dev/null +++ b/lib/sonar/test/test_tag_image.py @@ -0,0 +1,50 @@ +from unittest.mock import call, mock_open, patch + +import pytest + +from ..sonar import process_image + + +@pytest.fixture() +def ys4(): + return open("lib/sonar/test/yaml_scenario4.yaml").read() + + +@patch("sonar.sonar.create_ecr_repository") +@patch("sonar.sonar.docker_pull", return_value="123") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_push") +def test_tag_image( + patched_docker_push, + patched_docker_tag, + patched_docker_pull, + patched_create_ecr_repository, + ys4, +): + with patch("builtins.open", mock_open(read_data=ys4)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=[], + include_tags=[], + build_args={}, + ) + + patched_docker_pull.assert_called_once_with("source-registry-0-test_value0", "source-tag-0-test_value1") + + tag_calls = [ + call("123", "dest-registry-0-test_value0", "dest-tag-0-test_value0-test_value1"), + call("123", "dest-registry-1-test_value0", "dest-tag-1-test_value0-test_value1"), + ] + patched_docker_tag.assert_has_calls(tag_calls) + + push_calls = [ + call("dest-registry-0-test_value0", "dest-tag-0-test_value0-test_value1"), + call("dest-registry-1-test_value0", "dest-tag-1-test_value0-test_value1"), + ] + patched_docker_push.assert_has_calls(push_calls) + + create_ecr_calls = [ + call("dest-registry-0-test_value0"), + call("dest-registry-1-test_value0"), + ] + patched_create_ecr_repository.assert_has_calls(create_ecr_calls) diff --git a/lib/sonar/test/test_tags.py b/lib/sonar/test/test_tags.py new file mode 100644 index 000000000..74a259f9e --- /dev/null +++ b/lib/sonar/test/test_tags.py @@ -0,0 +1,177 @@ +from unittest.mock import mock_open, patch + +import pytest + +from ..sonar import ( + find_include_tags, + find_skip_tags, + process_image, + should_include_stage, + should_skip_stage, +) + + +@pytest.fixture() +def ys3(): + return open("lib/sonar/test/yaml_scenario3.yaml").read() + + +def test_include_tags_empty_params(): + assert find_include_tags(None) == [] + assert find_include_tags({}) == [] + assert find_include_tags({"nop": 1}) == [] + + +def test_include_tags_is_list(): + assert find_include_tags({"include_tags": ["1", "2"]}) == ["1", "2"] + assert find_include_tags({"nop": 1, "include_tags": ["1", "2"]}) == ["1", "2"] + + +def test_include_tags_is_str(): + assert find_include_tags({"include_tags": ""}) == [] + assert find_include_tags({"include_tags": "1,2"}) == ["1", "2"] + assert find_include_tags({"include_tags": "hi"}) == ["hi"] + assert find_include_tags({"include_tags": "hi,"}) == ["hi"] + + +def test_skip_tags0(): + assert find_skip_tags({"skip_tags": ""}) == [] + assert find_skip_tags(None) == [] + assert find_skip_tags({}) == [] + assert find_skip_tags({"nop": 1}) == [] + assert find_skip_tags({"nop": 1, "skip_tags": []}) == [] + + assert find_skip_tags({"nop": 1, "skip_tags": ["1"]}) == ["1"] + assert find_skip_tags({"nop": 1, "skip_tags": ["1", "2"]}) == ["1", "2"] + + assert find_skip_tags({"nop": 1, "skip_tags": "1"}) == ["1"] + assert find_skip_tags({"nop": 1, "skip_tags": "1,2"}) == ["1", "2"] + assert find_skip_tags({"nop": 1, "skip_tags": "1, 2"}) == ["1", "2"] + assert find_skip_tags({"nop": 1, "skip_tags": "1, 2,"}) == ["1", "2"] + + +def test_should_include_stage(): + assert should_include_stage({"tags": ["a", "b"]}, []) + assert should_include_stage({"tags": ["a", "b"]}, ["a"]) + assert should_include_stage({"tags": ["a", "b"]}, ["b"]) + assert should_include_stage({"tags": ["a", "b"]}, ["a", "b"]) + assert should_include_stage({"tags": ["a", "b"]}, ["b", "a"]) + + assert should_include_stage({"tags": ["a", "b"]}, ["a", "c"]) + assert should_include_stage({"tags": ["a", "b"]}, ["b", "c"]) + + assert not should_include_stage({"tags": ["a", "b"]}, ["c"]) + assert not should_include_stage({"tags": ["b"]}, ["c"]) + assert not should_include_stage({"tags": []}, ["c"]) + + +def test_should_skip_stage(): + assert should_skip_stage({"tags": ["a", "b"]}, ["a"]) + assert should_skip_stage({"tags": ["a", "b"]}, ["a", "b"]) + assert should_skip_stage({"tags": ["a", "b"]}, ["a", "b", "c"]) + + assert not should_skip_stage({"tags": []}, []) + assert not should_skip_stage({"tags": []}, ["a"]) + assert not should_skip_stage({"tags": []}, ["a", "b"]) + assert not should_skip_stage({"tags": ["a"]}, ["b"]) + assert not should_skip_stage({"tags": ["a", "b"]}, []) + assert not should_skip_stage({"tags": ["a", "b"]}, ["c"]) + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.create_ecr_repository") +def test_include_tags_tag0( + _create_ecr_repository, + _docker_build, + _docker_tag, + _docker_push, + ys3, +): + """Only includes the stage with the corresponding tag.""" + + with patch("builtins.open", mock_open(read_data=ys3)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=[], + include_tags=["tag0"], + build_args={}, + build_options={"pipeline": True}, + ) + + assert "skipping-stage" not in pipeline["image0"]["stage0"] + assert pipeline["image0"]["stage1"] == {"skipping-stage": "stage1"} + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.create_ecr_repository") +def test_include_tags_tag0_tag1(_create_ecr_repository, _docker_build, _docker_tag, _docker_push, ys3): + """Only includes the stage with the corresponding tag.""" + with patch("builtins.open", mock_open(read_data=ys3)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=[], + include_tags=["tag0", "tag1"], + build_args={}, + build_options={"pipeline": True}, + ) + + assert "skipping-stage" not in pipeline["image0"]["stage0"] + assert "skipping-stage" not in pipeline["image0"]["stage1"] + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.create_ecr_repository") +def test_skip_tags1(_create_ecr_repository, _docker_build, _docker_tag, _docker_push, ys3): + """Only includes the stage with the corresponding tag.""" + with patch("builtins.open", mock_open(read_data=ys3)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=["tag0"], + include_tags=[], + build_args={}, + build_options={"pipeline": True}, + ) + + assert pipeline["image0"]["stage0"] == {"skipping-stage": "stage0"} + assert "skipping-stage" not in pipeline["image0"]["stage1"] + + +def test_skip_tags2(ys3): + """Only includes the stage with the corresponding tag.""" + with patch("builtins.open", mock_open(read_data=ys3)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=["tag0", "tag1"], + include_tags=[], + build_args={}, + build_options={"pipeline": True}, + ) + + assert pipeline["image0"]["stage0"] == {"skipping-stage": "stage0"} + assert pipeline["image0"]["stage1"] == {"skipping-stage": "stage1"} + + +@patch("sonar.sonar.docker_push") +@patch("sonar.sonar.docker_tag") +@patch("sonar.sonar.docker_build") +@patch("sonar.sonar.create_ecr_repository") +def test_skip_include_tags(_create_ecr_repository, _docker_build, _docker_tag, _docker_push, ys3): + """Only includes the stage with the corresponding tag.""" + + with patch("builtins.open", mock_open(read_data=ys3)) as mock_file: + pipeline = process_image( + image_name="image0", + skip_tags=["tag0"], + include_tags=["tag1"], + build_args={}, + build_options={"pipeline": True}, + ) + + assert pipeline["image0"]["stage0"] == {"skipping-stage": "stage0"} + assert "skipping-stage" not in pipeline["image0"]["stage1"] diff --git a/lib/sonar/test/test_template.py b/lib/sonar/test/test_template.py new file mode 100644 index 000000000..410f5ee26 --- /dev/null +++ b/lib/sonar/test/test_template.py @@ -0,0 +1,16 @@ +from unittest.mock import Mock, patch + +from ..sonar import process_image + + +@patch("sonar.sonar.render", return_value="") +def test_key_error_is_not_raised_on_empty_inputs(patched_render: Mock): + process_image( + image_name="image1", + skip_tags=[], + include_tags=[], + build_args={}, + build_options={}, + inventory="lib/sonar/test/yaml_scenario10.yaml", + ) + patched_render.assert_called() diff --git a/lib/sonar/test/yaml_scenario0.yaml b/lib/sonar/test/yaml_scenario0.yaml new file mode 100644 index 000000000..820f826e0 --- /dev/null +++ b/lib/sonar/test/yaml_scenario0.yaml @@ -0,0 +1,4 @@ +images: + - name: image0 + inputs: + - input0 diff --git a/lib/sonar/test/yaml_scenario1.yaml b/lib/sonar/test/yaml_scenario1.yaml new file mode 100644 index 000000000..442e5c38c --- /dev/null +++ b/lib/sonar/test/yaml_scenario1.yaml @@ -0,0 +1,19 @@ +vars: + registry: somereg + +images: + - name: image0 + vars: + context: . + + inputs: + - input0 + + stages: + - name: stage0 + task_type: docker_build + + dockerfile: Dockerfile + output: + - registry: $(inputs.params.registry)/something + tag: something diff --git a/lib/sonar/test/yaml_scenario10.yaml b/lib/sonar/test/yaml_scenario10.yaml new file mode 100644 index 000000000..8b239f4b3 --- /dev/null +++ b/lib/sonar/test/yaml_scenario10.yaml @@ -0,0 +1,17 @@ +vars: + registry: somereg + +images: + - name: image1 + vars: + context: . + + stages: + - name: stage0 + task_type: dockerfile_template + + + dockerfile: Dockerfile + output: + - registry: $(inputs.params.version_id)/something + tag: something diff --git a/lib/sonar/test/yaml_scenario11.yaml b/lib/sonar/test/yaml_scenario11.yaml new file mode 100644 index 000000000..7d02b7ff3 --- /dev/null +++ b/lib/sonar/test/yaml_scenario11.yaml @@ -0,0 +1,43 @@ +vars: + registry: somereg + +images: + - name: image1 + vars: + context: . + + inputs: + - input0 + + platform: linux/amd64 + + stages: + - name: stage0 + task_type: docker_build + + labels: + label-0: value-0 + + dockerfile: Dockerfile + output: + - registry: $(inputs.params.registry)/something + tag: something + + - name: image2 + vars: + context: . + + inputs: + - input0 + + stages: + - name: stage0 + task_type: docker_build + + labels: + label-1: value-1 + + dockerfile: Dockerfile + output: + - registry: $(inputs.params.registry)/something + tag: something diff --git a/lib/sonar/test/yaml_scenario2.yaml b/lib/sonar/test/yaml_scenario2.yaml new file mode 100644 index 000000000..8f58914cd --- /dev/null +++ b/lib/sonar/test/yaml_scenario2.yaml @@ -0,0 +1,26 @@ +vars: + inventory_var0: inventory_var_value0 + inventory_var1: inventory_var_value1 + +images: + - name: image0 + vars: + image_var0: image_var_value0 + image_var1: image_var_value1 + + inputs: + - image_input0 + - image_input1 + + stages: + - name: stage0 + task_type: docker_build + + vars: + stage_var0: stage_value0 + stage_var1: stage_value1 + + dockerfile: Dockerfile + output: + - registry: $(inputs.params.registry)/something + tag: something diff --git a/lib/sonar/test/yaml_scenario3.yaml b/lib/sonar/test/yaml_scenario3.yaml new file mode 100644 index 000000000..39d5f3af2 --- /dev/null +++ b/lib/sonar/test/yaml_scenario3.yaml @@ -0,0 +1,44 @@ +vars: + inventory_var0: inventory_var_value0 + inventory_var1: inventory_var_value1 + +images: + - name: image0 + vars: + image_var0: image_var_value0 + image_var1: image_var_value1 + + context: some-context + + inputs: + - image_input0 + - image_input1 + + stages: + - name: stage0 + task_type: docker_build + + vars: + stage_var0: stage_value0 + stage_var1: stage_value1 + + tags: ["tag0"] + + dockerfile: Dockerfile + output: + - registry: some-registry + tag: something + + - name: stage1 + task_type: docker_build + + vars: + stage_var0: stage_value0 + stage_var1: stage_value1 + + tags: ["tag1"] + + dockerfile: Dockerfile + output: + - registry: some-registry + tag: something diff --git a/lib/sonar/test/yaml_scenario4.yaml b/lib/sonar/test/yaml_scenario4.yaml new file mode 100644 index 000000000..e6984f184 --- /dev/null +++ b/lib/sonar/test/yaml_scenario4.yaml @@ -0,0 +1,23 @@ +vars: + test_var0: test_value0 + test_var1: test_value1 + +images: + - name: image0 + vars: + context: some-context + + stages: + - name: stage0 + task_type: tag_image + + source: + registry: source-registry-0-$(inputs.params.test_var0) + tag: source-tag-0-$(inputs.params.test_var1) + + destination: + - registry: dest-registry-0-$(inputs.params.test_var0) + tag: dest-tag-0-$(inputs.params.test_var0)-$(inputs.params.test_var1) + + - registry: dest-registry-1-$(inputs.params.test_var0) + tag: dest-tag-1-$(inputs.params.test_var0)-$(inputs.params.test_var1) diff --git a/lib/sonar/test/yaml_scenario6.yaml b/lib/sonar/test/yaml_scenario6.yaml new file mode 100644 index 000000000..888bf2d8d --- /dev/null +++ b/lib/sonar/test/yaml_scenario6.yaml @@ -0,0 +1,46 @@ +images: + - name: image0 + vars: + context: some-context + + stages: + - name: stage0 + tags: ["test_dockerfile_from_url"] + task_type: docker_build + + dockerfile: https://somedomain/dockerfile + output: + - registry: some-registry + tag: something + + - name: image1 + vars: + context: some-context + + stages: + - name: stage0 + task_type: docker_build + tags: ["test_continue_on_errors"] + + dockerfile: somedockerfile + output: + - registry: some-registry + tag: something + + - name: stage1 + task_type: docker_build + tags: ["test_continue_on_errors"] + + dockerfile: somedockerfile + output: + - registry: some-registry + tag: something + + - name: stage2 + task_type: docker_build + tags: ["test_continue_on_errors"] + + dockerfile: somedockerfile + output: + - registry: some-registry + tag: something diff --git a/lib/sonar/test/yaml_scenario7.yaml b/lib/sonar/test/yaml_scenario7.yaml new file mode 100644 index 000000000..3a36d68ba --- /dev/null +++ b/lib/sonar/test/yaml_scenario7.yaml @@ -0,0 +1,15 @@ +images: +- name: image0 + vars: + context: some-context + stages: + - name: stage-build0 + task_type: docker_build + dockerfile: https://somedomain/dockerfile + output: + - registry: foo + tag: bar + signer_name: evergreen_ci + key_secret_name: test/kube/secret + passphrase_secret_name: test/kube/passphrase + region: us-east-1 diff --git a/lib/sonar/test/yaml_scenario8.yaml b/lib/sonar/test/yaml_scenario8.yaml new file mode 100644 index 000000000..d2656122c --- /dev/null +++ b/lib/sonar/test/yaml_scenario8.yaml @@ -0,0 +1,27 @@ +images: +- name: image0 + vars: + context: some-context + stages: + - name: stage-build0 + task_type: docker_build + dockerfile: https://somedomain/dockerfile + + signing: &signing + signer_name: evergreen_ci + key_secret_name: test/kube/secret + passphrase_secret_name: test/kube/passphrase + region: us-east-1 + + + output: + - registry: foo + tag: bar + <<: *signing + - registry: foo2 + tag: bar2 + <<: *signing + - registry: foo3 + tag: bar3 + <<: *signing + signer_name: evergreen_ci_foo diff --git a/lib/sonar/test/yaml_scenario9.yaml b/lib/sonar/test/yaml_scenario9.yaml new file mode 100644 index 000000000..1439d52a6 --- /dev/null +++ b/lib/sonar/test/yaml_scenario9.yaml @@ -0,0 +1,24 @@ +vars: + registry: somereg + +images: + - name: image1 + vars: + context: . + + inputs: + - input0 + + stages: + - name: stage0 + task_type: docker_build + + labels: + label-0: value-0 + label-1: value-1 + label-2: value-2 + + dockerfile: Dockerfile + output: + - registry: $(inputs.params.registry)/something + tag: something diff --git a/licenses.csv b/licenses.csv index 931e7a9e4..51affb346 100644 --- a/licenses.csv +++ b/licenses.csv @@ -1,201 +1,86 @@ -github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT -github.com/blang/semver,https://github.com/blang/semver/blob/v3.5.1/LICENSE,MIT -github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.1.2/LICENSE.txt,MIT -github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC -github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.9.0/LICENSE,MIT -github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v4.12.0/LICENSE,BSD-3-Clause -github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause -github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE,BSD-3-Clause -github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.4.1/LICENSE,Apache-2.0 -github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 -github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.0/LICENSE,Apache-2.0 -github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.19.14/LICENSE,Apache-2.0 -github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause -github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 -github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.2/LICENSE,BSD-3-Clause -github.com/golang/snappy,https://github.com/golang/snappy/blob/v0.0.3/LICENSE,BSD-3-Clause -github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE,Apache-2.0 -github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.5.9/LICENSE,BSD-3-Clause -github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.1.0/LICENSE,Apache-2.0 -github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.0/LICENSE,BSD-3-Clause -github.com/hashicorp/errwrap,https://github.com/hashicorp/errwrap/blob/v1.0.0/LICENSE,MPL-2.0 -github.com/hashicorp/go-multierror,https://github.com/hashicorp/go-multierror/blob/v1.1.1/LICENSE,MPL-2.0 -github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.15/LICENSE,BSD-3-Clause -github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT -github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT -github.com/klauspost/compress,https://github.com/klauspost/compress/blob/v1.13.6/LICENSE,Apache-2.0 -github.com/klauspost/compress/internal/snapref,https://github.com/klauspost/compress/blob/v1.13.6/internal/snapref/LICENSE,BSD-3-Clause -github.com/klauspost/compress/zstd/internal/xxhash,https://github.com/klauspost/compress/blob/v1.13.6/zstd/internal/xxhash/LICENSE.txt,MIT -github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.6/LICENSE,MIT -github.com/matttproud/golang_protobuf_extensions/pbutil,https://github.com/matttproud/golang_protobuf_extensions/blob/v1.0.2/LICENSE,Apache-2.0 -github.com/moby/spdystream,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 -github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 -github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 -github.com/mongodb/mongodb-kubernetes-operator/api/v1,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/api/v1,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/cmd/manager,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness/testdata,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/cmd/versionhook,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers/construct,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers/construct,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers/validation,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers/watch,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/controllers/watch,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/agent,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/agent,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/authtypes,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/mocks,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/x509,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/x509,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/helm,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/lifecycle,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/pod,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/headless,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/headless,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/health,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/health,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/pod,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/pod,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/secret,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/apierrors,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/apierrors,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/constants,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/contains,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/result,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/state,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/state,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/status,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/status,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/versions,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/pkg/util/versions,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/feature_compatibility_version,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/prometheus,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_arbiter,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_authentication,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_change_version,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_connection_string_options,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_cross_namespace_deploy,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_custom_annotations_test_test,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_custom_persistent_volume,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_custom_role,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade_4_5,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade_5_6,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade_6_7,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mongod_config,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mongod_port_change_with_arbiters,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mongod_readiness,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mount_connection_string,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_multiple,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_operator_upgrade,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_recovery,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_scale,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_scale_down,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_recreate_mdbc,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_rotate,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_rotate_delete_sts,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_upgrade,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_x509,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/statefulset_arbitrary_config,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/statefulset_arbitrary_config_update,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/statefulset_delete,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/tlstests,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester,Unknown,Unknown -github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait,Unknown,Unknown -github.com/montanaflynn/stats,https://github.com/montanaflynn/stats/blob/1bf9dbcd8cbe/LICENSE,MIT -github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause -github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause -github.com/pmezard/go-difflib/difflib,https://github.com/pmezard/go-difflib/blob/v1.0.0/LICENSE,BSD-3-Clause -github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.14.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 -github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 -github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause -github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 -github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.6.0/LICENSE,MIT -github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause -github.com/stretchr/objx,https://github.com/stretchr/objx/blob/v0.5.1/LICENSE,MIT -github.com/stretchr/testify,https://github.com/stretchr/testify/blob/v1.8.4/LICENSE,MIT -github.com/xdg-go/pbkdf2,https://github.com/xdg-go/pbkdf2/blob/v1.0.0/LICENSE,Apache-2.0 -github.com/xdg-go/scram,https://github.com/xdg-go/scram/blob/v1.1.2/LICENSE,Apache-2.0 -github.com/xdg-go/stringprep,https://github.com/xdg-go/stringprep/blob/v1.0.4/LICENSE,Apache-2.0 -github.com/xdg/stringprep,https://github.com/xdg/stringprep/blob/v1.0.3/LICENSE,Apache-2.0 -github.com/youmark/pkcs8,https://github.com/youmark/pkcs8/blob/1be2e3e5546d/LICENSE,MIT -go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.13.1/LICENSE,Apache-2.0 -go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT -go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.17.0:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.17.0:LICENSE,BSD-3-Clause -golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/ee480838:LICENSE,BSD-3-Clause -golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause -golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause -golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause -gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 -google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.28.1/LICENSE,BSD-3-Clause -gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause -gopkg.in/natefinch/lumberjack.v2,https://github.com/natefinch/lumberjack/blob/v2.2.1/LICENSE,MIT -gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 -gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT -k8s.io/api,https://github.com/kubernetes/api/blob/v0.26.10/LICENSE,Apache-2.0 -k8s.io/apiextensions-apiserver/pkg,https://github.com/kubernetes/apiextensions-apiserver/blob/v0.26.10/LICENSE,Apache-2.0 -k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.26.10/LICENSE,Apache-2.0 -k8s.io/apimachinery/third_party/forked/golang,https://github.com/kubernetes/apimachinery/blob/v0.26.10/third_party/forked/golang/LICENSE,BSD-3-Clause -k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.26.10/LICENSE,Apache-2.0 -k8s.io/component-base/config,https://github.com/kubernetes/component-base/blob/v0.26.10/LICENSE,Apache-2.0 -k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.80.1/LICENSE,Apache-2.0 -k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/172d655c2280/LICENSE,Apache-2.0 -k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json,https://github.com/kubernetes/kube-openapi/blob/172d655c2280/pkg/internal/third_party/go-json-experiment/json/LICENSE,BSD-3-Clause -k8s.io/kube-openapi/pkg/validation/spec,https://github.com/kubernetes/kube-openapi/blob/172d655c2280/pkg/validation/spec/LICENSE,Apache-2.0 -k8s.io/utils,https://github.com/kubernetes/utils/blob/99ec85e7a448/LICENSE,Apache-2.0 -k8s.io/utils/internal/third_party/forked/golang/net,https://github.com/kubernetes/utils/blob/99ec85e7a448/internal/third_party/forked/golang/LICENSE,BSD-3-Clause -sigs.k8s.io/controller-runtime,https://github.com/kubernetes-sigs/controller-runtime/blob/v0.14.7/LICENSE,Apache-2.0 -sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/f223a00ba0e2/LICENSE,Apache-2.0 -sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.3/LICENSE,Apache-2.0 -sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.4.0/LICENSE,Apache-2.0 -sigs.k8s.io/yaml/goyaml.v2,https://github.com/kubernetes-sigs/yaml/blob/v1.4.0/goyaml.v2/LICENSE,Apache-2.0 + +github.com/beorn7/perks/quantile,v1.0.1,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT +github.com/blang/semver,v3.5.1,https://github.com/blang/semver/blob/v3.5.1/LICENSE,MIT +github.com/cenkalti/backoff/v4,v4.3.0,https://github.com/cenkalti/backoff/blob/v4.3.0/LICENSE,MIT +github.com/cespare/xxhash/v2,v2.3.0,https://github.com/cespare/xxhash/blob/v2.3.0/LICENSE.txt,MIT +github.com/davecgh/go-spew/spew,v1.1.1,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC +github.com/emicklei/go-restful/v3,v3.11.0,https://github.com/emicklei/go-restful/blob/v3.11.0/LICENSE,MIT +github.com/evanphx/json-patch/v5,v5.9.0,https://github.com/evanphx/json-patch/blob/v5.9.0/v5/LICENSE,BSD-3-Clause +github.com/fsnotify/fsnotify,v1.7.0,https://github.com/fsnotify/fsnotify/blob/v1.7.0/LICENSE,BSD-3-Clause +github.com/ghodss/yaml,v1.0.0,https://github.com/ghodss/yaml/blob/v1.0.0/LICENSE,MIT +github.com/go-jose/go-jose/v4,v4.0.5,https://github.com/go-jose/go-jose/blob/v4.0.5/LICENSE,Apache-2.0 +github.com/go-jose/go-jose/v4/json,v4.0.5,https://github.com/go-jose/go-jose/blob/v4.0.5/json/LICENSE,BSD-3-Clause +github.com/go-logr/logr,v1.4.2,https://github.com/go-logr/logr/blob/v1.4.2/LICENSE,Apache-2.0 +github.com/go-logr/zapr,v1.3.0,https://github.com/go-logr/zapr/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,v0.19.6,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,v0.20.2,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/swag,v0.22.3,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/gogo/protobuf,v1.3.2,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause +github.com/golang/groupcache/lru,v0.0.0-20210331224755-41bb18bfe9da,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 +github.com/golang/protobuf,v1.5.4,https://github.com/golang/protobuf/blob/v1.5.4/LICENSE,BSD-3-Clause +github.com/google/gnostic-models,v0.6.8,https://github.com/google/gnostic-models/blob/v0.6.8/LICENSE,Apache-2.0 +github.com/google/go-cmp/cmp,v0.6.0,https://github.com/google/go-cmp/blob/v0.6.0/LICENSE,BSD-3-Clause +github.com/google/go-querystring/query,v1.1.0,https://github.com/google/go-querystring/blob/v1.1.0/LICENSE,BSD-3-Clause +github.com/google/gofuzz,v1.2.0,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 +github.com/google/uuid,v1.6.0,https://github.com/google/uuid/blob/v1.6.0/LICENSE,BSD-3-Clause +github.com/hashicorp/errwrap,v1.1.0,https://github.com/hashicorp/errwrap/blob/v1.1.0/LICENSE,MPL-2.0 +github.com/hashicorp/go-cleanhttp,v0.5.2,https://github.com/hashicorp/go-cleanhttp/blob/v0.5.2/LICENSE,MPL-2.0 +github.com/hashicorp/go-multierror,v1.1.1,https://github.com/hashicorp/go-multierror/blob/v1.1.1/LICENSE,MPL-2.0 +github.com/hashicorp/go-retryablehttp,v0.7.7,https://github.com/hashicorp/go-retryablehttp/blob/v0.7.7/LICENSE,MPL-2.0 +github.com/hashicorp/go-rootcerts,v1.0.2,https://github.com/hashicorp/go-rootcerts/blob/v1.0.2/LICENSE,MPL-2.0 +github.com/hashicorp/go-secure-stdlib/parseutil,v0.1.6,https://github.com/hashicorp/go-secure-stdlib/blob/parseutil/v0.1.6/parseutil/LICENSE,MPL-2.0 +github.com/hashicorp/go-secure-stdlib/strutil,v0.1.2,https://github.com/hashicorp/go-secure-stdlib/blob/strutil/v0.1.2/strutil/LICENSE,MPL-2.0 +github.com/hashicorp/go-sockaddr,v1.0.2,https://github.com/hashicorp/go-sockaddr/blob/v1.0.2/LICENSE,MPL-2.0 +github.com/hashicorp/hcl,v1.0.0,https://github.com/hashicorp/hcl/blob/v1.0.0/LICENSE,MPL-2.0 +github.com/hashicorp/vault/api,v1.16.0,https://github.com/hashicorp/vault/blob/api/v1.16.0/api/LICENSE,MPL-2.0 +github.com/imdario/mergo,v0.3.15,https://github.com/imdario/mergo/blob/v0.3.15/LICENSE,BSD-3-Clause +github.com/josharian/intern,v1.0.0,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT +github.com/json-iterator/go,v1.1.12,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT +github.com/klauspost/compress,v1.17.11,https://github.com/klauspost/compress/blob/v1.17.11/LICENSE,Apache-2.0 +github.com/klauspost/compress/internal/snapref,v1.17.11,https://github.com/klauspost/compress/blob/v1.17.11/internal/snapref/LICENSE,BSD-3-Clause +github.com/klauspost/compress/zstd/internal/xxhash,v1.17.11,https://github.com/klauspost/compress/blob/v1.17.11/zstd/internal/xxhash/LICENSE.txt,MIT +github.com/mailru/easyjson,v0.7.7,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT +github.com/mitchellh/mapstructure,v1.5.0,https://github.com/mitchellh/mapstructure/blob/v1.5.0/LICENSE,MIT +github.com/modern-go/concurrent,v0.0.0-20180306012644-bacd9c7ef1dd,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 +github.com/modern-go/reflect2,v1.0.2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 +github.com/munnerz/goautoneg,v0.0.0-20191010083416-a7dc8b61c822,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause +github.com/pkg/errors,v0.9.1,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause +github.com/pmezard/go-difflib/difflib,v1.0.0,https://github.com/pmezard/go-difflib/blob/v1.0.0/LICENSE,BSD-3-Clause +github.com/prometheus/client_golang/internal/github.com/golang/gddo/httputil,v1.21.0,https://github.com/prometheus/client_golang/blob/v1.21.0/internal/github.com/golang/gddo/LICENSE,BSD-3-Clause +github.com/prometheus/client_golang/prometheus,v1.21.0,https://github.com/prometheus/client_golang/blob/v1.21.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,v0.6.1,https://github.com/prometheus/client_model/blob/v0.6.1/LICENSE,Apache-2.0 +github.com/prometheus/common,v0.62.0,https://github.com/prometheus/common/blob/v0.62.0/LICENSE,Apache-2.0 +github.com/prometheus/procfs,v0.15.1,https://github.com/prometheus/procfs/blob/v0.15.1/LICENSE,Apache-2.0 +github.com/r3labs/diff/v3,v3.0.1,https://github.com/r3labs/diff/blob/v3.0.1/LICENSE,MPL-2.0 +github.com/ryanuber/go-glob,v1.0.0,https://github.com/ryanuber/go-glob/blob/v1.0.0/LICENSE,MIT +github.com/spf13/cast,v1.6.0,https://github.com/spf13/cast/blob/v1.6.0/LICENSE,MIT +github.com/spf13/pflag,v1.0.5,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause +github.com/stretchr/objx,v0.5.2,https://github.com/stretchr/objx/blob/v0.5.2/LICENSE,MIT +github.com/stretchr/testify/assert,v1.10.0,https://github.com/stretchr/testify/blob/v1.10.0/LICENSE,MIT +github.com/vmihailenco/msgpack/v5,v5.3.5,https://github.com/vmihailenco/msgpack/blob/v5.3.5/LICENSE,BSD-2-Clause +github.com/vmihailenco/tagparser/v2,v2.0.0,https://github.com/vmihailenco/tagparser/blob/v2.0.0/LICENSE,BSD-2-Clause +github.com/xdg/stringprep,v1.0.3,https://github.com/xdg/stringprep/blob/v1.0.3/LICENSE,Apache-2.0 +go.uber.org/multierr,v1.11.0,https://github.com/uber-go/multierr/blob/v1.11.0/LICENSE.txt,MIT +go.uber.org/zap,v1.27.0,https://github.com/uber-go/zap/blob/v1.27.0/LICENSE,MIT +gomodules.xyz/jsonpatch/v2,v2.4.0,https://github.com/gomodules/jsonpatch/blob/v2.4.0/v2/LICENSE,Apache-2.0 +google.golang.org/protobuf,v1.36.1,https://github.com/protocolbuffers/protobuf-go/blob/v1.36.1/LICENSE,BSD-3-Clause +gopkg.in/inf.v0,v0.9.1,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/natefinch/lumberjack.v2,v2.2.1,https://github.com/natefinch/lumberjack/blob/v2.2.1/LICENSE,MIT +gopkg.in/yaml.v2,v2.4.0,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 +gopkg.in/yaml.v3,v3.0.1,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT +k8s.io/api,v0.30.10,https://github.com/kubernetes/api/blob/v0.30.10/LICENSE,Apache-2.0 +k8s.io/apiextensions-apiserver/pkg/apis/apiextensions,v0.30.1,https://github.com/kubernetes/apiextensions-apiserver/blob/v0.30.1/LICENSE,Apache-2.0 +k8s.io/apimachinery/pkg,v0.30.10,https://github.com/kubernetes/apimachinery/blob/v0.30.10/LICENSE,Apache-2.0 +k8s.io/apimachinery/third_party/forked/golang,v0.30.10,https://github.com/kubernetes/apimachinery/blob/v0.30.10/third_party/forked/golang/LICENSE,BSD-3-Clause +k8s.io/client-go,v0.30.10,https://github.com/kubernetes/client-go/blob/v0.30.10/LICENSE,Apache-2.0 +k8s.io/klog/v2,v2.130.1,https://github.com/kubernetes/klog/blob/v2.130.1/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg,v0.0.0-20240228011516-70dd3763d340,https://github.com/kubernetes/kube-openapi/blob/70dd3763d340/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json,v0.0.0-20240228011516-70dd3763d340,https://github.com/kubernetes/kube-openapi/blob/70dd3763d340/pkg/internal/third_party/go-json-experiment/json/LICENSE,BSD-3-Clause +k8s.io/kube-openapi/pkg/validation/spec,v0.0.0-20240228011516-70dd3763d340,https://github.com/kubernetes/kube-openapi/blob/70dd3763d340/pkg/validation/spec/LICENSE,Apache-2.0 +k8s.io/utils,v0.0.0-20240502163921-fe8a2dddb1d0,https://github.com/kubernetes/utils/blob/fe8a2dddb1d0/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang/net,v0.0.0-20240502163921-fe8a2dddb1d0,https://github.com/kubernetes/utils/blob/fe8a2dddb1d0/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/controller-runtime,v0.18.7,https://github.com/kubernetes-sigs/controller-runtime/blob/v0.18.7/LICENSE,Apache-2.0 +sigs.k8s.io/json,v0.0.0-20221116044647-bc3834ca7abd,https://github.com/kubernetes-sigs/json/blob/bc3834ca7abd/LICENSE,Apache-2.0 +sigs.k8s.io/structured-merge-diff/v4,v4.4.1,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.4.1/LICENSE,Apache-2.0 +sigs.k8s.io/yaml,v1.4.0,https://github.com/kubernetes-sigs/yaml/blob/v1.4.0/LICENSE,Apache-2.0 +sigs.k8s.io/yaml/goyaml.v2,v1.4.0,https://github.com/kubernetes-sigs/yaml/blob/v1.4.0/goyaml.v2/LICENSE,Apache-2.0 diff --git a/main.go b/main.go new file mode 100644 index 000000000..ba1a1b524 --- /dev/null +++ b/main.go @@ -0,0 +1,483 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "runtime/debug" + "slices" + "strconv" + "strings" + "sync" + + "github.com/go-logr/zapr" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + + mcoConstruct "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + corev1 "k8s.io/api/core/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + localruntime "runtime" + ctrl "sigs.k8s.io/controller-runtime" + runtime_cluster "sigs.k8s.io/controller-runtime/pkg/cluster" + kubelog "sigs.k8s.io/controller-runtime/pkg/log" + metricsServer "sigs.k8s.io/controller-runtime/pkg/metrics/server" + crWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" + + apiv1 "github.com/10gen/ops-manager-kubernetes/api/v1" + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbmultiv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/construct" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/telemetry" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" + "github.com/10gen/ops-manager-kubernetes/pkg/webhook" +) + +const ( + mongoDBCRDPlural = "mongodb" + mongoDBUserCRDPlural = "mongodbusers" + mongoDBOpsManagerCRDPlural = "opsmanagers" + mongoDBMultiClusterCRDPlural = "mongodbmulticluster" +) + +var ( + log *zap.SugaredLogger + operatorEnvOnce sync.Once + + // List of allowed operator environments. The first element of this list is + // considered the default one. + operatorEnvironments = []string{util.OperatorEnvironmentDev.String(), util.OperatorEnvironmentLocal.String(), util.OperatorEnvironmentProd.String()} + + scheme = runtime.NewScheme() + + // Default CRDs to watch (if not specified on the command line) + crds crdsToWatch +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(apiv1.AddToScheme(scheme)) + utilruntime.Must(corev1.AddToScheme(scheme)) + + // +kubebuilder:scaffold:scheme + + flag.Var(&crds, "watch-resource", "A Watch Resource specifies if the Operator should watch the given resource") +} + +// crdsToWatch is a custom Value implementation which can be +// used to receive command line arguments +type crdsToWatch []string + +func (c *crdsToWatch) Set(value string) error { + *c = append(*c, strings.ToLower(value)) + return nil +} + +func (c *crdsToWatch) String() string { + return strings.Join(*c, ",") +} + +// Trigger patch 3 +func main() { + flag.Parse() + // If no CRDs are specified, we set default to non-multicluster CRDs + if len(crds) == 0 { + crds = crdsToWatch{mongoDBCRDPlural, mongoDBUserCRDPlural, mongoDBOpsManagerCRDPlural} + } + + ctx := context.Background() + operator.OmUpdateChannel = make(chan event.GenericEvent) + + klog.InitFlags(nil) + initializeEnvironment() + + imageUrls := images.LoadImageUrlsFromEnv() + forceEnterprise := env.ReadBoolOrDefault(architectures.MdbAssumeEnterpriseImage, false) + initDatabaseNonStaticImageVersion := env.ReadOrDefault(construct.InitDatabaseVersionEnv, "latest") + databaseNonStaticImageVersion := env.ReadOrDefault(construct.DatabaseVersionEnv, "latest") + initAppdbVersion := env.ReadOrDefault(construct.InitAppdbVersionEnv, "latest") + initOpsManagerImageVersion := env.ReadOrDefault(util.InitOpsManagerVersion, "latest") + // Namespace where the operator is installed + currentNamespace := env.ReadOrPanic(util.CurrentNamespace) + + // Get a config to talk to the apiserver + cfg := ctrl.GetConfigOrDie() + + managerOptions := ctrl.Options{ + Scheme: scheme, + } + + namespacesToWatch := operator.GetWatchedNamespace() + if len(namespacesToWatch) > 1 || namespacesToWatch[0] != "" { + namespacesForCacheBuilder := namespacesToWatch + if !stringutil.Contains(namespacesToWatch, currentNamespace) { + namespacesForCacheBuilder = append(namespacesForCacheBuilder, currentNamespace) + } + defaultNamespaces := make(map[string]cache.Config) + for _, namespace := range namespacesForCacheBuilder { + defaultNamespaces[namespace] = cache.Config{} + } + managerOptions.Cache = cache.Options{ + DefaultNamespaces: defaultNamespaces, + } + } + + if isInLocalMode() { + // managerOptions.MetricsBindAddress = "127.0.0.1:8180" + managerOptions.Metrics = metricsServer.Options{ + BindAddress: "127.0.0.1:8180", + } + managerOptions.HealthProbeBindAddress = "127.0.0.1:8181" + } + + webhookOptions := setupWebhook(ctx, cfg, log, slices.Contains(crds, mongoDBMultiClusterCRDPlural), currentNamespace) + managerOptions.WebhookServer = crWebhook.NewServer(webhookOptions) + + mgr, err := ctrl.NewManager(cfg, managerOptions) + if err != nil { + log.Fatal(err) + } + log.Info("Registering Components.") + + // Setup Scheme for all resources + if err := apiv1.AddToScheme(scheme); err != nil { + log.Fatal(err) + } + + // memberClusterObjectsMap is a map of clusterName -> clusterObject + memberClusterObjectsMap := make(map[string]runtime_cluster.Cluster) + + if slices.Contains(crds, mongoDBMultiClusterCRDPlural) { + memberClustersNames, err := getMemberClusters(ctx, cfg, currentNamespace) + if err != nil { + log.Fatal(err) + } + + log.Infof("Watching Member clusters: %s", memberClustersNames) + + if len(memberClustersNames) == 0 { + log.Warnf("The operator did not detect any member clusters") + } + + memberClusterClients, err := multicluster.CreateMemberClusterClients(memberClustersNames, multicluster.GetKubeConfigPath()) + if err != nil { + log.Fatal(err) + } + + // Add the cluster object to the manager corresponding to each member clusters. + for k, v := range memberClusterClients { + var cluster runtime_cluster.Cluster + + cluster, err := runtime_cluster.New(v, func(options *runtime_cluster.Options) { + if len(namespacesToWatch) > 1 || namespacesToWatch[0] != "" { + defaultNamespaces := make(map[string]cache.Config) + for _, namespace := range namespacesToWatch { + defaultNamespaces[namespace] = cache.Config{} + } + options.Cache = cache.Options{ + DefaultNamespaces: defaultNamespaces, + } + } + }) + if err != nil { + // don't panic here but rather log the error, for example, error might happen when one of the cluster is + // unreachable, we would still like the operator to continue reconciliation on the other clusters. + log.Errorf("Failed to initialize client for cluster: %s, err: %s", k, err) + continue + } + + log.Infof("Adding cluster %s to cluster map.", k) + memberClusterObjectsMap[k] = cluster + if err = mgr.Add(cluster); err != nil { + log.Fatal(err) + } + } + } + + // Setup all Controllers + if slices.Contains(crds, mongoDBCRDPlural) { + if err := setupMongoDBCRD(ctx, mgr, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, memberClusterObjectsMap); err != nil { + log.Fatal(err) + } + } + if slices.Contains(crds, mongoDBOpsManagerCRDPlural) { + if err := setupMongoDBOpsManagerCRD(ctx, mgr, memberClusterObjectsMap, imageUrls, initAppdbVersion, initOpsManagerImageVersion); err != nil { + log.Fatal(err) + } + } + if slices.Contains(crds, mongoDBUserCRDPlural) { + if err := setupMongoDBUserCRD(ctx, mgr, memberClusterObjectsMap); err != nil { + log.Fatal(err) + } + } + if slices.Contains(crds, mongoDBMultiClusterCRDPlural) { + if err := setupMongoDBMultiClusterCRD(ctx, mgr, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, memberClusterObjectsMap); err != nil { + log.Fatal(err) + } + } + + for _, r := range crds { + log.Infof("Registered CRD: %s", r) + } + + if telemetry.IsTelemetryActivated() { + log.Info("Running telemetry component!") + telemetryRunnable, err := telemetry.NewLeaderRunnable(mgr, memberClusterObjectsMap, currentNamespace, imageUrls[mcoConstruct.MongodbImageEnv], imageUrls[util.NonStaticDatabaseEnterpriseImage], getOperatorEnv()) + if err != nil { + log.Errorf("Unable to enable telemetry; err: %s", err) + } + if err := mgr.Add(telemetryRunnable); err != nil { + log.Errorf("Unable to enable telemetry; err: %s", err) + } + } else { + log.Info("Not running telemetry component!") + } + + log.Info("Starting the Cmd.") + + // Start the Manager + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + log.Fatal(err) + } +} + +func setupMongoDBCRD(ctx context.Context, mgr manager.Manager, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, memberClusterObjectsMap map[string]runtime_cluster.Cluster) error { + if err := operator.AddStandaloneController(ctx, mgr, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise); err != nil { + return err + } + if err := operator.AddReplicaSetController(ctx, mgr, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise); err != nil { + return err + } + if err := operator.AddShardedClusterController(ctx, mgr, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, memberClusterObjectsMap); err != nil { + return err + } + return ctrl.NewWebhookManagedBy(mgr).For(&mdbv1.MongoDB{}).Complete() +} + +func setupMongoDBOpsManagerCRD(ctx context.Context, mgr manager.Manager, memberClusterObjectsMap map[string]runtime_cluster.Cluster, imageUrls images.ImageUrls, initAppdbVersion, initOpsManagerImageVersion string) error { + if err := operator.AddOpsManagerController(ctx, mgr, memberClusterObjectsMap, imageUrls, initAppdbVersion, initOpsManagerImageVersion); err != nil { + return err + } + return ctrl.NewWebhookManagedBy(mgr).For(&omv1.MongoDBOpsManager{}).Complete() +} + +func setupMongoDBUserCRD(ctx context.Context, mgr manager.Manager, memberClusterObjectsMap map[string]runtime_cluster.Cluster) error { + return operator.AddMongoDBUserController(ctx, mgr, memberClusterObjectsMap) +} + +func setupMongoDBMultiClusterCRD(ctx context.Context, mgr manager.Manager, imageUrls images.ImageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion string, forceEnterprise bool, memberClusterObjectsMap map[string]runtime_cluster.Cluster) error { + if err := operator.AddMultiReplicaSetController(ctx, mgr, imageUrls, initDatabaseNonStaticImageVersion, databaseNonStaticImageVersion, forceEnterprise, memberClusterObjectsMap); err != nil { + return err + } + return ctrl.NewWebhookManagedBy(mgr).For(&mdbmultiv1.MongoDBMultiCluster{}).Complete() +} + +// getMemberClusters retrieves the member clusters from the configmap util.MemberListConfigMapName +func getMemberClusters(ctx context.Context, cfg *rest.Config, currentNamespace string) ([]string, error) { + c, err := client.New(cfg, client.Options{}) + if err != nil { + panic(err) + } + + m := corev1.ConfigMap{} + err = c.Get(ctx, types.NamespacedName{Name: util.MemberListConfigMapName, Namespace: currentNamespace}, &m) + if err != nil { + return nil, err + } + + var members []string + for member := range m.Data { + members = append(members, member) + } + + return members, nil +} + +func isInLocalMode() bool { + return operatorEnvironments[1] == env.ReadOrPanic(util.OmOperatorEnv) +} + +// setupWebhook sets up the validation webhook for MongoDB resources in order +// to give people early warning when their MongoDB resources are wrong. +func setupWebhook(ctx context.Context, cfg *rest.Config, log *zap.SugaredLogger, multiClusterMode bool, currentNamespace string) crWebhook.Options { + // set webhook port — 1993 is chosen as Ben's birthday + webhookPort := env.ReadIntOrDefault(util.MdbWebhookPortEnv, 1993) + + // this is the default directory on Linux but setting it explicitly helps + // with cross-platform compatibility, specifically local development on MacOS + certDir := "/tmp/k8s-webhook-server/serving-certs/" + var webhookHost string + if isInLocalMode() { + webhookHost = "127.0.0.1" + } + + // create a kubernetes client that the webhook server can use. We can't reuse + // the one from the manager as it is not initialised yet. + webhookClient, err := client.New(cfg, client.Options{}) + if err != nil { + panic(err) + } + + // webhookServiceLocation is the name and namespace of the webhook service + // that will be created. + webhookServiceLocation := types.NamespacedName{ + Name: "operator-webhook", + Namespace: currentNamespace, + } + + if err := webhook.Setup(ctx, webhookClient, webhookServiceLocation, certDir, webhookPort, multiClusterMode, log); err != nil { + log.Errorf("could not set up webhook: %v", err) + } + + return crWebhook.Options{ + Port: webhookPort, + Host: webhookHost, + CertDir: certDir, + } +} + +func initializeEnvironment() { + omOperatorEnv := getOperatorEnv() + + initEnvVariables() + + log.Infof("Operator environment: %s", omOperatorEnv) + + if omOperatorEnv == util.OperatorEnvironmentDev || omOperatorEnv == util.OperatorEnvironmentLocal { + log.Infof("Operator build info:\n%s", getBuildSettingsString()) + } + + log.Infof("Operator version: %s", util.OperatorVersion) + log.Infof("Go Version: %s", localruntime.Version()) + log.Infof("Go OS/Arch: %s/%s", localruntime.GOOS, localruntime.GOARCH) + + printableEnvPrefixes := []string{ + "BACKUP_WAIT_", + "POD_WAIT_", + "OPERATOR_ENV", + "WATCH_NAMESPACE", + "MANAGED_SECURITY_CONTEXT", + "IMAGE_PULL_SECRETS", + "MONGODB_ENTERPRISE_", + "OPS_MANAGER_", + "KUBERNETES_", + "AGENT_IMAGE", + "MONGODB_", + "INIT_", + "MDB_", + } + + // Only env variables with one of these prefixes will be printed + env.PrintWithPrefix(printableEnvPrefixes) +} + +func getOperatorEnv() util.OperatorEnvironment { + operatorFromEnv := os.Getenv(util.OmOperatorEnv) + operatorEnv := util.OperatorEnvironment(operatorFromEnv) + if !validateOperatorEnv(operatorEnv) { + operatorEnvOnce.Do(func() { + log.Infof("Configured environment %s, not recognized. Must be one of %v", operatorEnv, operatorEnvironments) + log.Infof("Using default environment, %s, instead", util.OperatorEnvironmentDev) + }) + operatorEnv = util.OperatorEnvironmentDev + } + return operatorEnv +} + +// quoteKey reports whether key is required to be quoted. Taken from: 1.22.0 mod.go +func quoteKey(key string) bool { + return len(key) == 0 || strings.ContainsAny(key, "= \t\r\n\"`") +} + +// quoteValue reports whether value is required to be quoted. Taken from: 1.22.0 mod.go +func quoteValue(value string) bool { + return strings.ContainsAny(value, " \t\r\n\"`") +} + +func getBuildSettingsString() string { + var buf strings.Builder + info, _ := debug.ReadBuildInfo() + for _, s := range info.Settings { + key := s.Key + if quoteKey(key) { + key = strconv.Quote(key) + } + value := s.Value + if quoteValue(value) { + value = strconv.Quote(value) + } + buf.WriteString(fmt.Sprintf("build\t%s=%s\n", key, value)) + } + return buf.String() +} + +// initEnvVariables is the central place in application to initialize default configuration for the application (using +// env variables). Having the central place to manage defaults increases manageability and transparency of the application +// Method initializes variables only in case they are not specified already. +func initEnvVariables() { + env.EnsureVar(util.BackupDisableWaitSecondsEnv, util.DefaultBackupDisableWaitSeconds) + env.EnsureVar(util.BackupDisableWaitRetriesEnv, util.DefaultBackupDisableWaitRetries) + env.EnsureVar(util.OpsManagerMonitorAppDB, strconv.FormatBool(util.OpsManagerMonitorAppDBDefault)) +} + +func validateOperatorEnv(env util.OperatorEnvironment) bool { + return slices.Contains(operatorEnvironments[:], env.String()) +} + +func init() { + InitGlobalLogger() +} + +func InitGlobalLogger() { + omOperatorEnv := getOperatorEnv() + + var logger *zap.Logger + var e error + + switch omOperatorEnv { + case util.OperatorEnvironmentProd: + logger, e = zap.NewProduction() + case util.OperatorEnvironmentDev, util.OperatorEnvironmentLocal: + // Overriding the default stacktrace behavior - have them only for errors but not for warnings + logger, e = zap.NewDevelopment(zap.AddStacktrace(zap.ErrorLevel)) + default: + // if for some reason we didn't set a logger, let's be safe and default to prod + fmt.Println("No OPERATOR_ENV set, defaulting setting logger to prod") + logger, e = zap.NewProduction() + } + + if e != nil { + fmt.Println("Failed to create logger, will use the default one") + fmt.Println(e) + // in the worst case logger might stay nil, replacing everything with a nil logger, + // we don't want that + logger = zap.S().Desugar() + } + + // Set the global logger used by our operator + zap.ReplaceGlobals(logger) + // Set the logger for controller-runtime based on the general level of the operator + kubelog.SetLogger(zapr.NewLogger(logger)) + // Set the logger used by telemetry package + telemetry.ConfigureLogger() + + // Set the logger used by main.go + log = zap.S() +} diff --git a/.action_templates/e2e-fork-template.yaml b/mongodb-community-operator/.action_templates/e2e-fork-template.yaml similarity index 100% rename from .action_templates/e2e-fork-template.yaml rename to mongodb-community-operator/.action_templates/e2e-fork-template.yaml diff --git a/.action_templates/e2e-pr-template.yaml b/mongodb-community-operator/.action_templates/e2e-pr-template.yaml similarity index 100% rename from .action_templates/e2e-pr-template.yaml rename to mongodb-community-operator/.action_templates/e2e-pr-template.yaml diff --git a/.action_templates/e2e-single-template.yaml b/mongodb-community-operator/.action_templates/e2e-single-template.yaml similarity index 100% rename from .action_templates/e2e-single-template.yaml rename to mongodb-community-operator/.action_templates/e2e-single-template.yaml diff --git a/.action_templates/events/on-pull-request-master.yaml b/mongodb-community-operator/.action_templates/events/on-pull-request-master.yaml similarity index 100% rename from .action_templates/events/on-pull-request-master.yaml rename to mongodb-community-operator/.action_templates/events/on-pull-request-master.yaml diff --git a/.action_templates/events/on-push-master.yaml b/mongodb-community-operator/.action_templates/events/on-push-master.yaml similarity index 100% rename from .action_templates/events/on-push-master.yaml rename to mongodb-community-operator/.action_templates/events/on-push-master.yaml diff --git a/.action_templates/events/pull-request-target.yaml b/mongodb-community-operator/.action_templates/events/pull-request-target.yaml similarity index 100% rename from .action_templates/events/pull-request-target.yaml rename to mongodb-community-operator/.action_templates/events/pull-request-target.yaml diff --git a/.action_templates/events/single-e2e-workflow-dispatch.yaml b/mongodb-community-operator/.action_templates/events/single-e2e-workflow-dispatch.yaml similarity index 100% rename from .action_templates/events/single-e2e-workflow-dispatch.yaml rename to mongodb-community-operator/.action_templates/events/single-e2e-workflow-dispatch.yaml diff --git a/.action_templates/events/workflow-dispatch.yaml b/mongodb-community-operator/.action_templates/events/workflow-dispatch.yaml similarity index 100% rename from .action_templates/events/workflow-dispatch.yaml rename to mongodb-community-operator/.action_templates/events/workflow-dispatch.yaml diff --git a/.action_templates/jobs/display-github-context.yaml b/mongodb-community-operator/.action_templates/jobs/display-github-context.yaml similarity index 100% rename from .action_templates/jobs/display-github-context.yaml rename to mongodb-community-operator/.action_templates/jobs/display-github-context.yaml diff --git a/.action_templates/jobs/setup.yaml b/mongodb-community-operator/.action_templates/jobs/setup.yaml similarity index 100% rename from .action_templates/jobs/setup.yaml rename to mongodb-community-operator/.action_templates/jobs/setup.yaml diff --git a/.action_templates/jobs/single-test.yaml b/mongodb-community-operator/.action_templates/jobs/single-test.yaml similarity index 100% rename from .action_templates/jobs/single-test.yaml rename to mongodb-community-operator/.action_templates/jobs/single-test.yaml diff --git a/.action_templates/jobs/tests.yaml b/mongodb-community-operator/.action_templates/jobs/tests.yaml similarity index 100% rename from .action_templates/jobs/tests.yaml rename to mongodb-community-operator/.action_templates/jobs/tests.yaml diff --git a/.action_templates/steps/build-and-push-development-images.yaml b/mongodb-community-operator/.action_templates/steps/build-and-push-development-images.yaml similarity index 100% rename from .action_templates/steps/build-and-push-development-images.yaml rename to mongodb-community-operator/.action_templates/steps/build-and-push-development-images.yaml diff --git a/.action_templates/steps/cancel-previous.yaml b/mongodb-community-operator/.action_templates/steps/cancel-previous.yaml similarity index 100% rename from .action_templates/steps/cancel-previous.yaml rename to mongodb-community-operator/.action_templates/steps/cancel-previous.yaml diff --git a/.action_templates/steps/checkout-fork.yaml b/mongodb-community-operator/.action_templates/steps/checkout-fork.yaml similarity index 100% rename from .action_templates/steps/checkout-fork.yaml rename to mongodb-community-operator/.action_templates/steps/checkout-fork.yaml diff --git a/.action_templates/steps/checkout.yaml b/mongodb-community-operator/.action_templates/steps/checkout.yaml similarity index 100% rename from .action_templates/steps/checkout.yaml rename to mongodb-community-operator/.action_templates/steps/checkout.yaml diff --git a/.action_templates/steps/dump-and-upload-diagnostics-always.yaml b/mongodb-community-operator/.action_templates/steps/dump-and-upload-diagnostics-always.yaml similarity index 100% rename from .action_templates/steps/dump-and-upload-diagnostics-always.yaml rename to mongodb-community-operator/.action_templates/steps/dump-and-upload-diagnostics-always.yaml diff --git a/.action_templates/steps/dump-and-upload-diagnostics.yaml b/mongodb-community-operator/.action_templates/steps/dump-and-upload-diagnostics.yaml similarity index 100% rename from .action_templates/steps/dump-and-upload-diagnostics.yaml rename to mongodb-community-operator/.action_templates/steps/dump-and-upload-diagnostics.yaml diff --git a/.action_templates/steps/quay-login.yaml b/mongodb-community-operator/.action_templates/steps/quay-login.yaml similarity index 100% rename from .action_templates/steps/quay-login.yaml rename to mongodb-community-operator/.action_templates/steps/quay-login.yaml diff --git a/.action_templates/steps/run-test-matrix.yaml b/mongodb-community-operator/.action_templates/steps/run-test-matrix.yaml similarity index 100% rename from .action_templates/steps/run-test-matrix.yaml rename to mongodb-community-operator/.action_templates/steps/run-test-matrix.yaml diff --git a/.action_templates/steps/run-test-single.yaml b/mongodb-community-operator/.action_templates/steps/run-test-single.yaml similarity index 100% rename from .action_templates/steps/run-test-single.yaml rename to mongodb-community-operator/.action_templates/steps/run-test-single.yaml diff --git a/.action_templates/steps/save-run-status.yaml b/mongodb-community-operator/.action_templates/steps/save-run-status.yaml similarity index 100% rename from .action_templates/steps/save-run-status.yaml rename to mongodb-community-operator/.action_templates/steps/save-run-status.yaml diff --git a/.action_templates/steps/set-run-status.yaml b/mongodb-community-operator/.action_templates/steps/set-run-status.yaml similarity index 100% rename from .action_templates/steps/set-run-status.yaml rename to mongodb-community-operator/.action_templates/steps/set-run-status.yaml diff --git a/.action_templates/steps/set-up-qemu.yaml b/mongodb-community-operator/.action_templates/steps/set-up-qemu.yaml similarity index 100% rename from .action_templates/steps/set-up-qemu.yaml rename to mongodb-community-operator/.action_templates/steps/set-up-qemu.yaml diff --git a/.action_templates/steps/setup-and-install-python.yaml b/mongodb-community-operator/.action_templates/steps/setup-and-install-python.yaml similarity index 100% rename from .action_templates/steps/setup-and-install-python.yaml rename to mongodb-community-operator/.action_templates/steps/setup-and-install-python.yaml diff --git a/.action_templates/steps/setup-kind-cluster.yaml b/mongodb-community-operator/.action_templates/steps/setup-kind-cluster.yaml similarity index 100% rename from .action_templates/steps/setup-kind-cluster.yaml rename to mongodb-community-operator/.action_templates/steps/setup-kind-cluster.yaml diff --git a/mongodb-community-operator/.dockerignore b/mongodb-community-operator/.dockerignore new file mode 100644 index 000000000..4d9a61d4f --- /dev/null +++ b/mongodb-community-operator/.dockerignore @@ -0,0 +1,13 @@ +.github +.idea +zz_* +vendor/ +scripts/ +.git/ +bin/ +testbin/ +.mypy_cache/ +main +__debug_bin +# allow agent LICENSE +!scripts/dev/templates/agent/LICENSE diff --git a/mongodb-community-operator/.github/CODEOWNERS b/mongodb-community-operator/.github/CODEOWNERS new file mode 100644 index 000000000..db61cf612 --- /dev/null +++ b/mongodb-community-operator/.github/CODEOWNERS @@ -0,0 +1 @@ +* @mircea-cosbuc @lsierant @nammn @Julien-Ben @MaciejKaras @lucian-tosa @fealebenpae @m1kola \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/mongodb-community-operator/.github/ISSUE_TEMPLATE/bug_report.md similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to mongodb-community-operator/.github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/mongodb-community-operator/.github/ISSUE_TEMPLATE/config.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/config.yml rename to mongodb-community-operator/.github/ISSUE_TEMPLATE/config.yml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/mongodb-community-operator/.github/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from .github/PULL_REQUEST_TEMPLATE.md rename to mongodb-community-operator/.github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/config_files/config_lint.yaml b/mongodb-community-operator/.github/config_files/config_lint.yaml similarity index 100% rename from .github/config_files/config_lint.yaml rename to mongodb-community-operator/.github/config_files/config_lint.yaml diff --git a/.github/config_files/config_lint_clusterwide.yaml b/mongodb-community-operator/.github/config_files/config_lint_clusterwide.yaml similarity index 100% rename from .github/config_files/config_lint_clusterwide.yaml rename to mongodb-community-operator/.github/config_files/config_lint_clusterwide.yaml diff --git a/.github/config_files/config_lint_openshift.yaml b/mongodb-community-operator/.github/config_files/config_lint_openshift.yaml similarity index 100% rename from .github/config_files/config_lint_openshift.yaml rename to mongodb-community-operator/.github/config_files/config_lint_openshift.yaml diff --git a/mongodb-community-operator/.github/dependabot.yml b/mongodb-community-operator/.github/dependabot.yml new file mode 100644 index 000000000..eb3084c66 --- /dev/null +++ b/mongodb-community-operator/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly + day: monday + ignore: + - dependency-name: k8s.io/api + - dependency-name: k8s.io/apimachinery + - dependency-name: k8s.io/client-go + - dependency-name: k8s.io/code-generator + - dependency-name: sigs.k8s.io/controller-runtime + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + day: monday diff --git a/.github/workflows/close-stale-issues.yml b/mongodb-community-operator/.github/workflows/close-stale-issues.yml similarity index 100% rename from .github/workflows/close-stale-issues.yml rename to mongodb-community-operator/.github/workflows/close-stale-issues.yml diff --git a/.github/workflows/code-health.yml b/mongodb-community-operator/.github/workflows/code-health.yml similarity index 100% rename from .github/workflows/code-health.yml rename to mongodb-community-operator/.github/workflows/code-health.yml diff --git a/.github/workflows/comment-release-pr.yml b/mongodb-community-operator/.github/workflows/comment-release-pr.yml similarity index 100% rename from .github/workflows/comment-release-pr.yml rename to mongodb-community-operator/.github/workflows/comment-release-pr.yml diff --git a/.github/workflows/e2e-dispatch.yml b/mongodb-community-operator/.github/workflows/e2e-dispatch.yml similarity index 100% rename from .github/workflows/e2e-dispatch.yml rename to mongodb-community-operator/.github/workflows/e2e-dispatch.yml diff --git a/.github/workflows/e2e-fork.yml b/mongodb-community-operator/.github/workflows/e2e-fork.yml similarity index 100% rename from .github/workflows/e2e-fork.yml rename to mongodb-community-operator/.github/workflows/e2e-fork.yml diff --git a/.github/workflows/e2e.yml b/mongodb-community-operator/.github/workflows/e2e.yml similarity index 100% rename from .github/workflows/e2e.yml rename to mongodb-community-operator/.github/workflows/e2e.yml diff --git a/.github/workflows/go.yml b/mongodb-community-operator/.github/workflows/go.yml similarity index 100% rename from .github/workflows/go.yml rename to mongodb-community-operator/.github/workflows/go.yml diff --git a/.github/workflows/kubelinter-check.yml b/mongodb-community-operator/.github/workflows/kubelinter-check.yml similarity index 100% rename from .github/workflows/kubelinter-check.yml rename to mongodb-community-operator/.github/workflows/kubelinter-check.yml diff --git a/.github/workflows/main.yaml b/mongodb-community-operator/.github/workflows/main.yaml similarity index 100% rename from .github/workflows/main.yaml rename to mongodb-community-operator/.github/workflows/main.yaml diff --git a/.github/workflows/release-images.yml b/mongodb-community-operator/.github/workflows/release-images.yml similarity index 100% rename from .github/workflows/release-images.yml rename to mongodb-community-operator/.github/workflows/release-images.yml diff --git a/.github/workflows/release-single-image.yml b/mongodb-community-operator/.github/workflows/release-single-image.yml similarity index 100% rename from .github/workflows/release-single-image.yml rename to mongodb-community-operator/.github/workflows/release-single-image.yml diff --git a/.github/workflows/remove-label.yml b/mongodb-community-operator/.github/workflows/remove-label.yml similarity index 100% rename from .github/workflows/remove-label.yml rename to mongodb-community-operator/.github/workflows/remove-label.yml diff --git a/mongodb-community-operator/.gitignore b/mongodb-community-operator/.gitignore new file mode 100644 index 000000000..0229263df --- /dev/null +++ b/mongodb-community-operator/.gitignore @@ -0,0 +1,103 @@ +# Temporary Build Files +build/_output +build/_test +# Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* +# Org-mode +.org-id-locations +*_archive +# flymake-mode +*_flymake.* +# eshell files +/eshell/history +/eshell/lastdir +# elpa packages +/elpa/ +# reftex files +*.rel +# AUCTeX auto folder +/auto/ +# cask packages +.cask/ +dist/ +# Flycheck +flycheck_*.el +# server auth directory +/server/ +# projectiles files +.projectile +projectile-bookmarks.eld +# directory configuration +.dir-locals.el +# saveplace +places +# url cache +url/cache/ +# cedet +ede-projects.el +# smex +smex-items +# company-statistics +company-statistics-cache.el +# anaconda-mode +anaconda-mode/ +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +# Test binary, build with 'go test -c' +*.test +# Output of the go coverage tool, specifically when used with LiteIDE +*.out +### Vim ### +# swap +.sw[a-p] +.*.sw[a-p] +# session +Session.vim +# temporary +.netrwhist +# auto-generated tag files +tags +### VisualStudioCode ### +.vscode/* +.history +# End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode +*mypy_cache +bin/ +venv/ +local-config.json +.idea +vendor +__pycache__ +Dockerfile +Dockerfile_python_formatting +logs/* +testbin/bin +# OSX Trash +.DS_Store + +# ignore files generated by sonar +Dockerfile.ubi-* +Dockerfile.ubuntu-* + +diagnostics + +!test/test-app/Dockerfile + +Pipfile +Pipfile.lock +.community-operator-dev +*.iml diff --git a/.gitmodules b/mongodb-community-operator/.gitmodules similarity index 60% rename from .gitmodules rename to mongodb-community-operator/.gitmodules index 80d9434c7..ba9320f66 100644 --- a/.gitmodules +++ b/mongodb-community-operator/.gitmodules @@ -1,3 +1,3 @@ [submodule "helm-charts"] - path = helm-charts + path = mongodb-community-operator/helm-charts url = git@github.com:mongodb/helm-charts.git diff --git a/mongodb-community-operator/.golangci.yml b/mongodb-community-operator/.golangci.yml new file mode 100644 index 000000000..795e08728 --- /dev/null +++ b/mongodb-community-operator/.golangci.yml @@ -0,0 +1,61 @@ +--- +######################### +######################### +## Golang Linter rules ## +######################### +######################### + +# configure golangci-lint +# see https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml +issues: + exclude-rules: + - path: _test\.go + linters: + - dupl + - gosec + - goconst + - golint + text: "underscore" + - path: ^pkg\/util\/envvar + linters: + - forbidigo + - path: ^cmd\/(readiness|versionhook|manager)\/main\.go$ + linters: + - forbidigo +linters: + enable: + - govet + - errcheck + - staticcheck + - unused + - gosimple + - ineffassign + - typecheck + - rowserrcheck + - gosec + - unconvert + - forbidigo +linters-settings: + gosec: + excludes: + - G115 + forbidigo: + forbid: + - p: os\.(Getenv|LookupEnv|Environ|ExpandEnv) + pkg: os + msg: "Reading environemnt variables here is prohibited. Please read environment variables in the main package." + - p: os\.(Clearenv|Unsetenv|Setenv) + msg: "Modifying environemnt variables is prohibited." + pkg: os + - p: envvar\.(Read.*?|MergeWithOverride|GetEnvOrDefault) + pkg: github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar + msg: "Using this envvar package here is prohibited. Please work with environment variables in the main package." + # Rules with the `pkg` depend on it + analyze-types: true + +run: + modules-download-mode: mod + # timeout for analysis, e.g. 30s, 5m, default is 1m + timeout: 5m + # default concurrency is a available CPU number + concurrency: 4 diff --git a/mongodb-community-operator/APACHE2 b/mongodb-community-operator/APACHE2 new file mode 100644 index 000000000..6c13e8ea0 --- /dev/null +++ b/mongodb-community-operator/APACHE2 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2021] [MongoDB Inc.] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/CODE_OF_CONDUCT.md b/mongodb-community-operator/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to mongodb-community-operator/CODE_OF_CONDUCT.md diff --git a/LICENSE.md b/mongodb-community-operator/LICENSE.md similarity index 100% rename from LICENSE.md rename to mongodb-community-operator/LICENSE.md diff --git a/mongodb-community-operator/Makefile b/mongodb-community-operator/Makefile new file mode 100644 index 000000000..6f1811c8f --- /dev/null +++ b/mongodb-community-operator/Makefile @@ -0,0 +1,242 @@ +SHELL := /bin/bash + +MONGODB_COMMUNITY_CONFIG ?= $(HOME)/.community-operator-dev/config.json + +# Image URL to use all building/pushing image targets +REPO_URL := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) +OPERATOR_IMAGE := $(shell jq -r .operator_image < $(MONGODB_COMMUNITY_CONFIG)) +NAMESPACE := $(shell jq -r .namespace < $(MONGODB_COMMUNITY_CONFIG)) +UPGRADE_HOOK_IMG := $(shell jq -r .version_upgrade_hook_image < $(MONGODB_COMMUNITY_CONFIG)) +READINESS_PROBE_IMG := $(shell jq -r .readiness_probe_image < $(MONGODB_COMMUNITY_CONFIG)) +REGISTRY := $(shell jq -r .repo_url < $(MONGODB_COMMUNITY_CONFIG)) +AGENT_IMAGE_NAME := $(shell jq -r .agent_image < $(MONGODB_COMMUNITY_CONFIG)) +HELM_CHART ?= ./helm-charts/charts/community-operator + +STRING_SET_VALUES := --set namespace=$(NAMESPACE),versionUpgradeHook.name=$(UPGRADE_HOOK_IMG),readinessProbe.name=$(READINESS_PROBE_IMG),registry.operator=$(REPO_URL),operator.operatorImageName=$(OPERATOR_IMAGE),operator.version=latest,registry.agent=$(REGISTRY),registry.versionUpgradeHook=$(REGISTRY),registry.readinessProbe=$(REGISTRY),registry.operator=$(REGISTRY),versionUpgradeHook.version=latest,readinessProbe.version=latest,agent.version=latest,agent.name=$(AGENT_IMAGE_NAME) +STRING_SET_VALUES_LOCAL := $(STRING_SET_VALUES) --set operator.replicas=0 + +DOCKERFILE ?= operator +# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) +CRD_OPTIONS ?= "crd:crdVersions=v1" +RELEASE_NAME_HELM ?= mongodb-kubernetes-operator +TEST_NAMESPACE ?= $(NAMESPACE) + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +BASE_GO_PACKAGE = github.com/mongodb/mongodb-kubernetes-operator +GO_LICENSES = go-licenses +DISALLOWED_LICENSES = restricted # found reciprocal MPL-2.0 + +all: manager + +##@ Development + +fmt: ## Run go fmt against code + go fmt ./... + +vet: ## Run go vet against code + go vet ./... + +generate: controller-gen ## Generate code + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +$(GO_LICENSES): + @if ! which $@ &> /dev/null; then \ + go install github.com/google/go-licenses@latest; \ + fi + +licenses.csv: go.mod $(GO_LICENSES) ## Track licenses in a CSV file + @echo "Tracking licenses into file $@" + @echo "========================================" + GOOS=linux GOARCH=amd64 $(GO_LICENSES) csv --include_tests $(BASE_GO_PACKAGE)/... > $@ + +# We only check that go.mod is NOT newer than licenses.csv because the CI +# tends to generate slightly different results, so content comparison wouldn't work +licenses-tracked: ## Checks license.csv is up to date + @if [ go.mod -nt licenses.csv ]; then \ + echo "License.csv is stale! Please run 'make licenses.csv' and commit"; exit 1; \ + else echo "License.csv OK (up to date)"; fi + +.PHONY: check-licenses-compliance +check-licenses-compliance: licenses.csv ## Check licenses are compliant with our restrictions + @echo "Checking licenses not to be: $(DISALLOWED_LICENSES)" + @echo "============================================" + GOOS=linux GOARCH=amd64 $(GO_LICENSES) check --include_tests $(BASE_GO_PACKAGE)/... \ + --disallowed_types $(DISALLOWED_LICENSES) + @echo "--------------------" + @echo "Licenses check: PASS" + +.PHONY: check-licenses +check-licenses: licenses-tracked check-licenses-compliance ## Check license tracking & compliance + +TEST ?= ./pkg/... ./api/... ./cmd/... ./controllers/... ./test/e2e/util/mongotester/... +test: generate fmt vet manifests ## Run unit tests + go test $(options) $(TEST) -coverprofile cover.out + +manager: generate fmt vet ## Build operator binary + go build -o bin/manager ./cmd/manager/main.go + +run: install ## Run the operator against the configured Kubernetes cluster in ~/.kube/config + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ + go run ./cmd/manager/main.go + +debug: install install-rbac ## Run the operator in debug mode with dlv + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ + dlv debug ./cmd/manager/main.go + +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +controller-gen: ## Download controller-gen locally if necessary + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0) + +# Try to use already installed helm from PATH +ifeq (ok,$(shell test -f "$$(which helm)" && echo ok)) + HELM=$(shell which helm) +else + HELM=/usr/local/bin/helm +endif + +helm: ## Download helm locally if necessary + $(call install-helm) + +install-prerequisites-macos: ## installs prerequisites for macos development + scripts/dev/install_prerequisites.sh + +##@ Installation/Uninstallation + +install: manifests helm install-crd ## Install CRDs into a cluster + +install-crd: + kubectl apply -f config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml + +install-chart: uninstall-crd + $(HELM) upgrade --install $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) $(HELM_CHART) --namespace $(NAMESPACE) --create-namespace + +install-chart-local-operator: uninstall-crd + $(HELM) upgrade --install $(STRING_SET_VALUES_LOCAL) $(RELEASE_NAME_HELM) $(HELM_CHART) --namespace $(NAMESPACE) --create-namespace + +prepare-local-dev: generate-env-file install-chart-local-operator install-rbac setup-sas + +# patches all sas to use the local-image-registry +setup-sas: + scripts/dev/setup_sa.sh + +install-chart-with-tls-enabled: + $(HELM) upgrade --install --set createResource=true $(STRING_SET_VALUES) $(RELEASE_NAME_HELM) $(HELM_CHART) --namespace $(NAMESPACE) --create-namespace + +install-rbac: + $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml $(HELM_CHART) | kubectl apply -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl apply -f - + +uninstall-crd: + kubectl delete crd --ignore-not-found mongodbcommunity.mongodbcommunity.mongodb.com + +uninstall-chart: + $(HELM) uninstall $(RELEASE_NAME_HELM) -n $(NAMESPACE) + +uninstall-rbac: + $(HELM) template $(STRING_SET_VALUES) -s templates/database_roles.yaml $(HELM_CHART) | kubectl delete -f - + $(HELM) template $(STRING_SET_VALUES) -s templates/operator_roles.yaml $(HELM_CHART) | kubectl delete -f - + +uninstall: manifests helm uninstall-chart uninstall-crd ## Uninstall CRDs from a cluster + +##@ Deployment + +deploy: manifests helm install-chart install-crd ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config + +undeploy: uninstall-chart uninstall-crd ## UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config + +manifests: controller-gen ## Generate manifests e.g. CRD, RBAC etc. + $(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./..." output:crd:artifacts:config=config/crd/bases + cp config/crd/bases/* $(HELM_CHART)/crds + +##@ E2E + +# Run e2e tests locally using go build while also setting up a proxy in the shell to allow +# the test to run as if it were inside the cluster. This enables mongodb connectivity while running locally. +# "MDB_LOCAL_OPERATOR=true" ensures the operator pod is not spun up while running the e2e test - since you're +# running it locally. +e2e-telepresence: cleanup-e2e install ## Run e2e tests locally using go build while also setting up a proxy e.g. make e2e-telepresence test=replica_set cleanup=true + export MDB_LOCAL_OPERATOR=true; \ + telepresence connect; \ + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ + go test -v -timeout=30m -failfast $(options) ./test/e2e/$(test) ; \ + telepresence quit + +e2e-k8s: cleanup-e2e install e2e-image ## Run e2e test by deploying test image in kubernetes, you can provide e2e.py flags e.g. make e2e-k8s test=replica_set e2eflags="--perform-cleanup". + python scripts/dev/e2e.py $(e2eflags) --test $(test) + +e2e: cleanup-e2e install ## Run e2e test locally. e.g. make e2e test=replica_set cleanup=true + eval $$(scripts/dev/get_e2e_env_vars.py $(cleanup)); \ + go test -v -short -timeout=30m -failfast $(options) ./test/e2e/$(test) + +e2e-gh: ## Trigger a Github Action of the given test + scripts/dev/run_e2e_gh.sh $(test) + +cleanup-e2e: ## Cleans up e2e test env + kubectl delete mdbc,all,secrets -l e2e-test=true -n ${TEST_NAMESPACE} || true + # Most of the tests use StatefulSets, which in turn use stable storage. In order to + # avoid interleaving tests with each other, we need to drop them all. + kubectl delete pvc --all -n $(NAMESPACE) || true + kubectl delete pv --all -n $(NAMESPACE) || true + +generate-env-file: ## generates a local-test.env for local testing + mkdir -p .community-operator-dev + { python scripts/dev/get_e2e_env_vars.py | tee >(cut -d' ' -f2 > .community-operator-dev/local-test.env) ;} > .community-operator-dev/local-test.export.env + . .community-operator-dev/local-test.export.env + +##@ Image + +operator-image: ## Build and push the operator image + python pipeline.py --image-name operator $(IMG_BUILD_ARGS) + +e2e-image: ## Build and push e2e test image + python pipeline.py --image-name e2e $(IMG_BUILD_ARGS) + +agent-image: ## Build and push agent image + python pipeline.py --image-name agent $(IMG_BUILD_ARGS) + +readiness-probe-image: ## Build and push readiness probe image + python pipeline.py --image-name readiness-probe $(IMG_BUILD_ARGS) + +version-upgrade-post-start-hook-image: ## Build and push version upgrade post start hook image + python pipeline.py --image-name version-upgrade-hook $(IMG_BUILD_ARGS) + +all-images: operator-image e2e-image agent-image readiness-probe-image version-upgrade-post-start-hook-image ## create all required images + +define install-helm +@[ -f $(HELM) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 ;\ +chmod 700 get_helm.sh ;\ +./get_helm.sh ;\ +rm -rf $(TMP_DIR) ;\ +} +endef + +# go-install-tool will 'go install' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-install-tool +@[ -f $(1) ] || { \ +set -e ;\ +TMP_DIR=$$(mktemp -d) ;\ +cd $$TMP_DIR ;\ +go mod init tmp ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ +rm -rf $$TMP_DIR ;\ +} +endef + +help: ## Show this help screen. + @echo 'Usage: make ... ' + @echo '' + @echo 'Available targets are:' + @echo '' + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/mongodb-community-operator/PROJECT b/mongodb-community-operator/PROJECT new file mode 100644 index 000000000..fcd3ceff3 --- /dev/null +++ b/mongodb-community-operator/PROJECT @@ -0,0 +1,25 @@ +domain: mongodb.com +layout: +- go.kubebuilder.io/v3 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: mko-v1 +repo: github.com/mongodb/mongodb-kubernetes-operator +resources: +- api: + crdVersion: v1 + namespaced: true + group: mongodbcommunity + kind: MongoDBCommunity + version: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: mongodb.com + group: mongodbcommunity + kind: SimpleMongoDBCommunity + path: github.com/mongodb/mongodb-kubernetes-operator/api/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/mongodb-community-operator/README.md b/mongodb-community-operator/README.md new file mode 100644 index 000000000..8af30d276 --- /dev/null +++ b/mongodb-community-operator/README.md @@ -0,0 +1,82 @@ +# MongoDB Community Kubernetes Operator # + + + +This is a [Kubernetes Operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) which deploys MongoDB Community into Kubernetes clusters. + +If you are a MongoDB Enterprise customer, or need Enterprise features such as Backup, you can use the [MongoDB Enterprise Operator for Kubernetes](https://github.com/mongodb/mongodb-enterprise-kubernetes). + +Here is a talk from MongoDB Live 2020 about the Community Operator: +* [Run it in Kubernetes! Community and Enterprise MongoDB in Containers](https://www.youtube.com/watch?v=2Xszdg-4T6A&t=1368s) + +> **Note** +> +> Hi, I'm Dan Mckean 👋 I'm the Product Manager for MongoDB's support of Kubernetes. +> +> The [Community Operator](https://github.com/mongodb/mongodb-kubernetes-operator) is something I inherited when I started, but it doesn't get as much attention from us as we'd like, and we're trying to understand how it's used in order to establish it's future. It will help us establish exactly what level of support we can offer, and what sort of timeframe we aim to provide support in 🙂 +> +>Here's a super short survey (it's much easier for us to review all the feedback that way!): [https://docs.google.com/forms/d/e/1FAIpQLSfwrwyxBSlUyJ6AmC-eYlgW_3JEdfA48SB2i5--_WpiynMW2w/viewform?usp=sf_link](https://docs.google.com/forms/d/e/1FAIpQLSfwrwyxBSlUyJ6AmC-eYlgW_3JEdfA48SB2i5--_WpiynMW2w/viewform?usp=sf_link) +> +> If you'd rather email me instead: [dan.mckean@mongodb.com](mailto:dan.mckean@mongodb.com?subject=MongoDB%20Community%20Operator%20feedback) + +## Table of Contents + +- [Documentation](#documentation) +- [Supported Features](#supported-features) + - [Planned Features](#planned-features) +- [Contribute](#contribute) +- [License](#license) + +## Documentation + +See the [documentation](docs) to learn how to: + +1. [Install or upgrade](docs/install-upgrade.md) the Operator. +1. [Deploy and configure](docs/deploy-configure.md) MongoDB resources. +1. [Configure Logging](docs/logging.md) of the MongoDB resource components. +1. [Create a database user](docs/users.md) with SCRAM authentication. +1. [Secure MongoDB resource connections](docs/secure.md) using TLS. + +*NOTE: [MongoDB Enterprise Kubernetes Operator](https://www.mongodb.com/docs/kubernetes-operator/master/) docs are for the enterprise operator use case and NOT for the community operator. In addition to the docs mentioned above, you can refer to this [blog post](https://www.mongodb.com/blog/post/run-secure-containerized-mongodb-deployments-using-the-mongo-db-community-kubernetes-oper) as well to learn more about community operator deployment* + +## Supported Features + +The MongoDB Community Kubernetes Operator supports the following features: + +- Create [replica sets](https://www.mongodb.com/docs/manual/replication/) +- Upgrade and downgrade MongoDB server version +- Scale replica sets up and down +- Read from and write to the replica set while scaling, upgrading, and downgrading. These operations are done in an "always up" manner. +- Report MongoDB server state via the [MongoDBCommunity resource](/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml) `status` field +- Use any of the available [Docker MongoDB images](https://hub.docker.com/_/mongo/) +- Connect to the replica set from inside the Kubernetes cluster (no external connectivity) +- Secure client-to-server and server-to-server connections with TLS +- Create users with [SCRAM](https://www.mongodb.com/docs/manual/core/security-scram/) authentication +- Create custom roles +- Enable a [metrics target that can be used with Prometheus](docs/prometheus/README.md) + +## Contribute + +Before you contribute to the MongoDB Community Kubernetes Operator, please read: + +- [MongoDB Community Kubernetes Operator Architecture](docs/architecture.md) +- [Contributing to MongoDB Community Kubernetes Operator](docs/contributing.md) + +Please file issues before filing PRs. For PRs to be accepted, contributors must sign our [CLA](https://www.mongodb.com/legal/contributor-agreement). + +Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). + +## Linting + +This project uses the following linters upon every Pull Request: + +* `gosec` is a tool that find security problems in the code +* `Black` is a tool that verifies if Python code is properly formatted +* `MyPy` is a Static Type Checker for Python +* `Kube-linter` is a tool that verified if all Kubernetes YAML manifests are formatted correctly +* `Go vet` A built-in Go static checker +* `Snyk` The vulnerability scanner + +## License + +Please see the [LICENSE](LICENSE.md) file. diff --git a/SECURITY.md b/mongodb-community-operator/SECURITY.md similarity index 100% rename from SECURITY.md rename to mongodb-community-operator/SECURITY.md diff --git a/mongodb-community-operator/api/v1/doc.go b/mongodb-community-operator/api/v1/doc.go new file mode 100644 index 000000000..a6a3905a8 --- /dev/null +++ b/mongodb-community-operator/api/v1/doc.go @@ -0,0 +1,4 @@ +package v1 + +// +k8s:deepcopy-gen=package +// +versionName=v1 diff --git a/api/v1/groupversion_info.go b/mongodb-community-operator/api/v1/groupversion_info.go similarity index 100% rename from api/v1/groupversion_info.go rename to mongodb-community-operator/api/v1/groupversion_info.go diff --git a/api/v1/mongodbcommunity_types.go b/mongodb-community-operator/api/v1/mongodbcommunity_types.go similarity index 100% rename from api/v1/mongodbcommunity_types.go rename to mongodb-community-operator/api/v1/mongodbcommunity_types.go diff --git a/api/v1/mongodbcommunity_types_test.go b/mongodb-community-operator/api/v1/mongodbcommunity_types_test.go similarity index 100% rename from api/v1/mongodbcommunity_types_test.go rename to mongodb-community-operator/api/v1/mongodbcommunity_types_test.go diff --git a/mongodb-community-operator/api/v1/zz_generated.deepcopy.go b/mongodb-community-operator/api/v1/zz_generated.deepcopy.go new file mode 100644 index 000000000..df22b4876 --- /dev/null +++ b/mongodb-community-operator/api/v1/zz_generated.deepcopy.go @@ -0,0 +1,584 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import ( + "github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentConfiguration) DeepCopyInto(out *AgentConfiguration) { + *out = *in + if in.LogRotate != nil { + in, out := &in.LogRotate, &out.LogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } + if in.AuditLogRotate != nil { + in, out := &in.AuditLogRotate, &out.AuditLogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } + if in.SystemLog != nil { + in, out := &in.SystemLog, &out.SystemLog + *out = new(automationconfig.SystemLog) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentConfiguration. +func (in *AgentConfiguration) DeepCopy() *AgentConfiguration { + if in == nil { + return nil + } + out := new(AgentConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Authentication) DeepCopyInto(out *Authentication) { + *out = *in + if in.Modes != nil { + in, out := &in.Modes, &out.Modes + *out = make([]AuthMode, len(*in)) + copy(*out, *in) + } + if in.AgentCertificateSecret != nil { + in, out := &in.AgentCertificateSecret, &out.AgentCertificateSecret + *out = new(corev1.LocalObjectReference) + **out = **in + } + if in.IgnoreUnknownUsers != nil { + in, out := &in.IgnoreUnknownUsers, &out.IgnoreUnknownUsers + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Authentication. +func (in *Authentication) DeepCopy() *Authentication { + if in == nil { + return nil + } + out := new(Authentication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthenticationRestriction) DeepCopyInto(out *AuthenticationRestriction) { + *out = *in + if in.ClientSource != nil { + in, out := &in.ClientSource, &out.ClientSource + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ServerAddress != nil { + in, out := &in.ServerAddress, &out.ServerAddress + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthenticationRestriction. +func (in *AuthenticationRestriction) DeepCopy() *AuthenticationRestriction { + if in == nil { + return nil + } + out := new(AuthenticationRestriction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutomationConfigOverride) DeepCopyInto(out *AutomationConfigOverride) { + *out = *in + if in.Processes != nil { + in, out := &in.Processes, &out.Processes + *out = make([]OverrideProcess, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.ReplicaSet.DeepCopyInto(&out.ReplicaSet) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutomationConfigOverride. +func (in *AutomationConfigOverride) DeepCopy() *AutomationConfigOverride { + if in == nil { + return nil + } + out := new(AutomationConfigOverride) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomRole) DeepCopyInto(out *CustomRole) { + *out = *in + if in.Privileges != nil { + in, out := &in.Privileges, &out.Privileges + *out = make([]Privilege, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]Role, len(*in)) + copy(*out, *in) + } + if in.AuthenticationRestrictions != nil { + in, out := &in.AuthenticationRestrictions, &out.AuthenticationRestrictions + *out = make([]AuthenticationRestriction, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomRole. +func (in *CustomRole) DeepCopy() *CustomRole { + if in == nil { + return nil + } + out := new(CustomRole) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MapWrapper) DeepCopyInto(out *MapWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBCommunity) DeepCopyInto(out *MongoDBCommunity) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunity. +func (in *MongoDBCommunity) DeepCopy() *MongoDBCommunity { + if in == nil { + return nil + } + out := new(MongoDBCommunity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBCommunity) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBCommunityList) DeepCopyInto(out *MongoDBCommunityList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MongoDBCommunity, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunityList. +func (in *MongoDBCommunityList) DeepCopy() *MongoDBCommunityList { + if in == nil { + return nil + } + out := new(MongoDBCommunityList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MongoDBCommunityList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBCommunitySpec) DeepCopyInto(out *MongoDBCommunitySpec) { + *out = *in + if in.ReplicaSetHorizons != nil { + in, out := &in.ReplicaSetHorizons, &out.ReplicaSetHorizons + *out = make(ReplicaSetHorizonConfiguration, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = make(automationconfig.ReplicaSetHorizons, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + } + in.Security.DeepCopyInto(&out.Security) + if in.Users != nil { + in, out := &in.Users, &out.Users + *out = make([]MongoDBUser, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.StatefulSetConfiguration.DeepCopyInto(&out.StatefulSetConfiguration) + in.AgentConfiguration.DeepCopyInto(&out.AgentConfiguration) + in.AdditionalMongodConfig.DeepCopyInto(&out.AdditionalMongodConfig) + if in.AutomationConfigOverride != nil { + in, out := &in.AutomationConfigOverride, &out.AutomationConfigOverride + *out = new(AutomationConfigOverride) + (*in).DeepCopyInto(*out) + } + if in.Prometheus != nil { + in, out := &in.Prometheus, &out.Prometheus + *out = new(Prometheus) + **out = **in + } + in.AdditionalConnectionStringConfig.DeepCopyInto(&out.AdditionalConnectionStringConfig) + if in.MemberConfig != nil { + in, out := &in.MemberConfig, &out.MemberConfig + *out = make([]automationconfig.MemberOptions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunitySpec. +func (in *MongoDBCommunitySpec) DeepCopy() *MongoDBCommunitySpec { + if in == nil { + return nil + } + out := new(MongoDBCommunitySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBCommunityStatus) DeepCopyInto(out *MongoDBCommunityStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBCommunityStatus. +func (in *MongoDBCommunityStatus) DeepCopy() *MongoDBCommunityStatus { + if in == nil { + return nil + } + out := new(MongoDBCommunityStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongoDBUser) DeepCopyInto(out *MongoDBUser) { + *out = *in + out.PasswordSecretRef = in.PasswordSecretRef + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]Role, len(*in)) + copy(*out, *in) + } + in.AdditionalConnectionStringConfig.DeepCopyInto(&out.AdditionalConnectionStringConfig) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongoDBUser. +func (in *MongoDBUser) DeepCopy() *MongoDBUser { + if in == nil { + return nil + } + out := new(MongoDBUser) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MongodConfiguration) DeepCopyInto(out *MongodConfiguration) { + *out = *in + in.MapWrapper.DeepCopyInto(&out.MapWrapper) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MongodConfiguration. +func (in *MongodConfiguration) DeepCopy() *MongodConfiguration { + if in == nil { + return nil + } + out := new(MongodConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OverrideProcess) DeepCopyInto(out *OverrideProcess) { + *out = *in + if in.LogRotate != nil { + in, out := &in.LogRotate, &out.LogRotate + *out = new(automationconfig.CrdLogRotate) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverrideProcess. +func (in *OverrideProcess) DeepCopy() *OverrideProcess { + if in == nil { + return nil + } + out := new(OverrideProcess) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OverrideReplicaSet) DeepCopyInto(out *OverrideReplicaSet) { + *out = *in + if in.Id != nil { + in, out := &in.Id, &out.Id + *out = new(string) + **out = **in + } + in.Settings.DeepCopyInto(&out.Settings) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OverrideReplicaSet. +func (in *OverrideReplicaSet) DeepCopy() *OverrideReplicaSet { + if in == nil { + return nil + } + out := new(OverrideReplicaSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Privilege) DeepCopyInto(out *Privilege) { + *out = *in + in.Resource.DeepCopyInto(&out.Resource) + if in.Actions != nil { + in, out := &in.Actions, &out.Actions + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Privilege. +func (in *Privilege) DeepCopy() *Privilege { + if in == nil { + return nil + } + out := new(Privilege) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Prometheus) DeepCopyInto(out *Prometheus) { + *out = *in + out.PasswordSecretRef = in.PasswordSecretRef + out.TLSSecretRef = in.TLSSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Prometheus. +func (in *Prometheus) DeepCopy() *Prometheus { + if in == nil { + return nil + } + out := new(Prometheus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ReplicaSetHorizonConfiguration) DeepCopyInto(out *ReplicaSetHorizonConfiguration) { + { + in := &in + *out = make(ReplicaSetHorizonConfiguration, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = make(automationconfig.ReplicaSetHorizons, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicaSetHorizonConfiguration. +func (in ReplicaSetHorizonConfiguration) DeepCopy() ReplicaSetHorizonConfiguration { + if in == nil { + return nil + } + out := new(ReplicaSetHorizonConfiguration) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Resource) DeepCopyInto(out *Resource) { + *out = *in + if in.DB != nil { + in, out := &in.DB, &out.DB + *out = new(string) + **out = **in + } + if in.Collection != nil { + in, out := &in.Collection, &out.Collection + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resource. +func (in *Resource) DeepCopy() *Resource { + if in == nil { + return nil + } + out := new(Resource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Role) DeepCopyInto(out *Role) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Role. +func (in *Role) DeepCopy() *Role { + if in == nil { + return nil + } + out := new(Role) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretKeyReference) DeepCopyInto(out *SecretKeyReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyReference. +func (in *SecretKeyReference) DeepCopy() *SecretKeyReference { + if in == nil { + return nil + } + out := new(SecretKeyReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Security) DeepCopyInto(out *Security) { + *out = *in + in.Authentication.DeepCopyInto(&out.Authentication) + in.TLS.DeepCopyInto(&out.TLS) + if in.Roles != nil { + in, out := &in.Roles, &out.Roles + *out = make([]CustomRole, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Security. +func (in *Security) DeepCopy() *Security { + if in == nil { + return nil + } + out := new(Security) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetConfiguration) DeepCopyInto(out *StatefulSetConfiguration) { + *out = *in + in.SpecWrapper.DeepCopyInto(&out.SpecWrapper) + in.MetadataWrapper.DeepCopyInto(&out.MetadataWrapper) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetConfiguration. +func (in *StatefulSetConfiguration) DeepCopy() *StatefulSetConfiguration { + if in == nil { + return nil + } + out := new(StatefulSetConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetMetadataWrapper) DeepCopyInto(out *StatefulSetMetadataWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatefulSetSpecWrapper) DeepCopyInto(out *StatefulSetSpecWrapper) { + clone := in.DeepCopy() + *out = *clone +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLS) DeepCopyInto(out *TLS) { + *out = *in + out.CertificateKeySecret = in.CertificateKeySecret + if in.CaCertificateSecret != nil { + in, out := &in.CaCertificateSecret, &out.CaCertificateSecret + *out = new(corev1.LocalObjectReference) + **out = **in + } + if in.CaConfigMap != nil { + in, out := &in.CaConfigMap, &out.CaConfigMap + *out = new(corev1.LocalObjectReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLS. +func (in *TLS) DeepCopy() *TLS { + if in == nil { + return nil + } + out := new(TLS) + in.DeepCopyInto(out) + return out +} diff --git a/build/bin/entrypoint b/mongodb-community-operator/build/bin/entrypoint similarity index 100% rename from build/bin/entrypoint rename to mongodb-community-operator/build/bin/entrypoint diff --git a/build/bin/user_setup b/mongodb-community-operator/build/bin/user_setup similarity index 100% rename from build/bin/user_setup rename to mongodb-community-operator/build/bin/user_setup diff --git a/cmd/manager/main.go b/mongodb-community-operator/cmd/manager/main.go similarity index 100% rename from cmd/manager/main.go rename to mongodb-community-operator/cmd/manager/main.go diff --git a/cmd/readiness/main.go b/mongodb-community-operator/cmd/readiness/main.go similarity index 100% rename from cmd/readiness/main.go rename to mongodb-community-operator/cmd/readiness/main.go diff --git a/cmd/readiness/readiness_test.go b/mongodb-community-operator/cmd/readiness/readiness_test.go similarity index 100% rename from cmd/readiness/readiness_test.go rename to mongodb-community-operator/cmd/readiness/readiness_test.go diff --git a/cmd/readiness/testdata/config-current-version.json b/mongodb-community-operator/cmd/readiness/testdata/config-current-version.json similarity index 100% rename from cmd/readiness/testdata/config-current-version.json rename to mongodb-community-operator/cmd/readiness/testdata/config-current-version.json diff --git a/cmd/readiness/testdata/config-new-version.json b/mongodb-community-operator/cmd/readiness/testdata/config-new-version.json similarity index 100% rename from cmd/readiness/testdata/config-new-version.json rename to mongodb-community-operator/cmd/readiness/testdata/config-new-version.json diff --git a/cmd/readiness/testdata/health-status-deadlocked-waiting-for-correct-automation-credentials.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-deadlocked-waiting-for-correct-automation-credentials.json similarity index 100% rename from cmd/readiness/testdata/health-status-deadlocked-waiting-for-correct-automation-credentials.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-deadlocked-waiting-for-correct-automation-credentials.json diff --git a/cmd/readiness/testdata/health-status-deadlocked-with-prev-config.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-deadlocked-with-prev-config.json similarity index 100% rename from cmd/readiness/testdata/health-status-deadlocked-with-prev-config.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-deadlocked-with-prev-config.json diff --git a/cmd/readiness/testdata/health-status-deadlocked.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-deadlocked.json similarity index 100% rename from cmd/readiness/testdata/health-status-deadlocked.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-deadlocked.json diff --git a/cmd/readiness/testdata/health-status-enterprise-upgrade-interrupted.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-enterprise-upgrade-interrupted.json similarity index 100% rename from cmd/readiness/testdata/health-status-enterprise-upgrade-interrupted.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-enterprise-upgrade-interrupted.json diff --git a/cmd/readiness/testdata/health-status-error-tls.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-error-tls.json similarity index 100% rename from cmd/readiness/testdata/health-status-error-tls.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-error-tls.json diff --git a/cmd/readiness/testdata/health-status-no-deadlock.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-no-deadlock.json similarity index 100% rename from cmd/readiness/testdata/health-status-no-deadlock.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-no-deadlock.json diff --git a/cmd/readiness/testdata/health-status-no-plans.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-no-plans.json similarity index 100% rename from cmd/readiness/testdata/health-status-no-plans.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-no-plans.json diff --git a/cmd/readiness/testdata/health-status-no-processes.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-no-processes.json similarity index 100% rename from cmd/readiness/testdata/health-status-no-processes.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-no-processes.json diff --git a/cmd/readiness/testdata/health-status-no-replication.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-no-replication.json similarity index 100% rename from cmd/readiness/testdata/health-status-no-replication.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-no-replication.json diff --git a/cmd/readiness/testdata/health-status-not-readable-state.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-not-readable-state.json similarity index 100% rename from cmd/readiness/testdata/health-status-not-readable-state.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-not-readable-state.json diff --git a/cmd/readiness/testdata/health-status-ok-no-replica-status.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-ok-no-replica-status.json similarity index 100% rename from cmd/readiness/testdata/health-status-ok-no-replica-status.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-ok-no-replica-status.json diff --git a/cmd/readiness/testdata/health-status-ok-with-WaitForCorrectBinaries.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-ok-with-WaitForCorrectBinaries.json similarity index 100% rename from cmd/readiness/testdata/health-status-ok-with-WaitForCorrectBinaries.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-ok-with-WaitForCorrectBinaries.json diff --git a/cmd/readiness/testdata/health-status-ok.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-ok.json similarity index 100% rename from cmd/readiness/testdata/health-status-ok.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-ok.json diff --git a/cmd/readiness/testdata/health-status-pending.json b/mongodb-community-operator/cmd/readiness/testdata/health-status-pending.json similarity index 100% rename from cmd/readiness/testdata/health-status-pending.json rename to mongodb-community-operator/cmd/readiness/testdata/health-status-pending.json diff --git a/cmd/readiness/testdata/k8sobjects.go b/mongodb-community-operator/cmd/readiness/testdata/k8sobjects.go similarity index 100% rename from cmd/readiness/testdata/k8sobjects.go rename to mongodb-community-operator/cmd/readiness/testdata/k8sobjects.go diff --git a/cmd/versionhook/main.go b/mongodb-community-operator/cmd/versionhook/main.go similarity index 100% rename from cmd/versionhook/main.go rename to mongodb-community-operator/cmd/versionhook/main.go diff --git a/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml b/mongodb-community-operator/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml similarity index 100% rename from config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml rename to mongodb-community-operator/config/crd/bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml diff --git a/mongodb-community-operator/config/crd/kustomization.yaml b/mongodb-community-operator/config/crd/kustomization.yaml new file mode 100644 index 000000000..25eecc05f --- /dev/null +++ b/mongodb-community-operator/config/crd/kustomization.yaml @@ -0,0 +1,10 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/mongodbcommunity.mongodb.com_mongodbcommunity.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/mongodb-community-operator/config/crd/kustomizeconfig.yaml b/mongodb-community-operator/config/crd/kustomizeconfig.yaml new file mode 100644 index 000000000..ec5c150a9 --- /dev/null +++ b/mongodb-community-operator/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/mongodb-community-operator/config/default/kustomization.yaml b/mongodb-community-operator/config/default/kustomization.yaml new file mode 100644 index 000000000..bd972fd91 --- /dev/null +++ b/mongodb-community-operator/config/default/kustomization.yaml @@ -0,0 +1,6 @@ +namePrefix: "" + +resources: + - ../crd + - ../rbac + - ../manager diff --git a/config/local_run/kustomization.yaml b/mongodb-community-operator/config/local_run/kustomization.yaml similarity index 100% rename from config/local_run/kustomization.yaml rename to mongodb-community-operator/config/local_run/kustomization.yaml diff --git a/mongodb-community-operator/config/manager/kustomization.yaml b/mongodb-community-operator/config/manager/kustomization.yaml new file mode 100644 index 000000000..cb74a8d0e --- /dev/null +++ b/mongodb-community-operator/config/manager/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: mongodb-kubernetes-operator + newName: quay.io/mongodb/mongodb-kubernetes-operator:0.5.0 diff --git a/mongodb-community-operator/config/manager/manager.yaml b/mongodb-community-operator/config/manager/manager.yaml new file mode 100644 index 000000000..9013a8451 --- /dev/null +++ b/mongodb-community-operator/config/manager/manager.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + email: support@mongodb.com + labels: + owner: mongodb + name: mongodb-kubernetes-operator +spec: + replicas: 1 + selector: + matchLabels: + name: mongodb-kubernetes-operator + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + name: mongodb-kubernetes-operator + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: name + operator: In + values: + - mongodb-kubernetes-operator + topologyKey: kubernetes.io/hostname + containers: + - command: + - /usr/local/bin/entrypoint + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: mongodb-kubernetes-operator + - name: AGENT_IMAGE + value: quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1 + - name: VERSION_UPGRADE_HOOK_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-operator-version-upgrade-post-start-hook:1.0.9 + - name: READINESS_PROBE_IMAGE + value: quay.io/mongodb/mongodb-kubernetes-readinessprobe:1.0.22 + - name: MONGODB_IMAGE + value: mongodb-community-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + image: quay.io/mongodb/mongodb-kubernetes-operator:0.12.0 + imagePullPolicy: Always + name: mongodb-kubernetes-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + securityContext: + readOnlyRootFilesystem: true + runAsUser: 2000 + allowPrivilegeEscalation: false + securityContext: + seccompProfile: + type: RuntimeDefault + serviceAccountName: mongodb-kubernetes-operator diff --git a/mongodb-community-operator/config/rbac/kustomization.yaml b/mongodb-community-operator/config/rbac/kustomization.yaml new file mode 100644 index 000000000..f1fe88a33 --- /dev/null +++ b/mongodb-community-operator/config/rbac/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- role.yaml +- role_binding.yaml +- service_account.yaml +- service_account_database.yaml +- role_binding_database.yaml +- role_database.yaml diff --git a/mongodb-community-operator/config/rbac/role.yaml b/mongodb-community-operator/config/rbac/role.yaml new file mode 100644 index 000000000..6a9c42070 --- /dev/null +++ b/mongodb-community-operator/config/rbac/role.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: mongodb-kubernetes-operator +rules: +- apiGroups: + - "" + resources: + - pods + - services + - configmaps + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mongodbcommunity.mongodb.com + resources: + - mongodbcommunity + - mongodbcommunity/status + - mongodbcommunity/spec + - mongodbcommunity/finalizers + verbs: + - get + - patch + - list + - update + - watch diff --git a/mongodb-community-operator/config/rbac/role_binding.yaml b/mongodb-community-operator/config/rbac/role_binding.yaml new file mode 100644 index 000000000..b444f2d67 --- /dev/null +++ b/mongodb-community-operator/config/rbac/role_binding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-kubernetes-operator +subjects: +- kind: ServiceAccount + name: mongodb-kubernetes-operator +roleRef: + kind: Role + name: mongodb-kubernetes-operator + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/role_binding_database.yaml b/mongodb-community-operator/config/rbac/role_binding_database.yaml similarity index 100% rename from config/rbac/role_binding_database.yaml rename to mongodb-community-operator/config/rbac/role_binding_database.yaml diff --git a/config/rbac/role_database.yaml b/mongodb-community-operator/config/rbac/role_database.yaml similarity index 100% rename from config/rbac/role_database.yaml rename to mongodb-community-operator/config/rbac/role_database.yaml diff --git a/config/rbac/service_account.yaml b/mongodb-community-operator/config/rbac/service_account.yaml similarity index 100% rename from config/rbac/service_account.yaml rename to mongodb-community-operator/config/rbac/service_account.yaml diff --git a/config/rbac/service_account_database.yaml b/mongodb-community-operator/config/rbac/service_account_database.yaml similarity index 100% rename from config/rbac/service_account_database.yaml rename to mongodb-community-operator/config/rbac/service_account_database.yaml diff --git a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml b/mongodb-community-operator/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml similarity index 100% rename from config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml rename to mongodb-community-operator/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_custom_volume_cr.yaml diff --git a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml b/mongodb-community-operator/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml similarity index 100% rename from config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml rename to mongodb-community-operator/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_hostpath.yaml diff --git a/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_metadata.yaml b/mongodb-community-operator/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_metadata.yaml similarity index 100% rename from config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_metadata.yaml rename to mongodb-community-operator/config/samples/arbitrary_statefulset_configuration/mongodb.com_v1_metadata.yaml diff --git a/config/samples/external_access/agent-certificate.yaml b/mongodb-community-operator/config/samples/external_access/agent-certificate.yaml similarity index 100% rename from config/samples/external_access/agent-certificate.yaml rename to mongodb-community-operator/config/samples/external_access/agent-certificate.yaml diff --git a/config/samples/external_access/cert-manager-certificate.yaml b/mongodb-community-operator/config/samples/external_access/cert-manager-certificate.yaml similarity index 100% rename from config/samples/external_access/cert-manager-certificate.yaml rename to mongodb-community-operator/config/samples/external_access/cert-manager-certificate.yaml diff --git a/config/samples/external_access/cert-manager-issuer.yaml b/mongodb-community-operator/config/samples/external_access/cert-manager-issuer.yaml similarity index 100% rename from config/samples/external_access/cert-manager-issuer.yaml rename to mongodb-community-operator/config/samples/external_access/cert-manager-issuer.yaml diff --git a/config/samples/external_access/cert-x509.yaml b/mongodb-community-operator/config/samples/external_access/cert-x509.yaml similarity index 100% rename from config/samples/external_access/cert-x509.yaml rename to mongodb-community-operator/config/samples/external_access/cert-x509.yaml diff --git a/config/samples/external_access/external_services.yaml b/mongodb-community-operator/config/samples/external_access/external_services.yaml similarity index 100% rename from config/samples/external_access/external_services.yaml rename to mongodb-community-operator/config/samples/external_access/external_services.yaml diff --git a/config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml b/mongodb-community-operator/config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml similarity index 100% rename from config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml rename to mongodb-community-operator/config/samples/external_access/mongodb.com_v1_mongodbcommunity_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_additional_connection_string_options.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_additional_connection_string_options.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_additional_connection_string_options.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_additional_connection_string_options.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_additional_mongod_config_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_connection_string_secret_namespace.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_connection_string_secret_namespace.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_connection_string_secret_namespace.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_connection_string_secret_namespace.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_cr.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_cr.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_cr.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_cr_podantiaffinity.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_custom_role.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_custom_role.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_custom_role.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_custom_role.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_disabled_process_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_ignore_unkown_users_cr.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_ignore_unkown_users_cr.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_ignore_unkown_users_cr.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_ignore_unkown_users_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_openshift_cr.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_openshift_cr.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_openshift_cr.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_openshift_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_override_ac_setting.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_override_ac_setting.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_override_ac_setting.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_override_ac_setting.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_prometheus.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_prometheus.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_prometheus.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_prometheus.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_readiness_probe_values.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_specify_pod_resources.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_specify_pod_resources.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_specify_pod_resources.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_specify_pod_resources.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_tls_cr.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_tls_cr.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_tls_cr.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_tls_cr.yaml diff --git a/config/samples/mongodb.com_v1_mongodbcommunity_x509.yaml b/mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_x509.yaml similarity index 100% rename from config/samples/mongodb.com_v1_mongodbcommunity_x509.yaml rename to mongodb-community-operator/config/samples/mongodb.com_v1_mongodbcommunity_x509.yaml diff --git a/controllers/construct/build_statefulset_test.go b/mongodb-community-operator/controllers/construct/build_statefulset_test.go similarity index 100% rename from controllers/construct/build_statefulset_test.go rename to mongodb-community-operator/controllers/construct/build_statefulset_test.go diff --git a/controllers/construct/mongodbstatefulset.go b/mongodb-community-operator/controllers/construct/mongodbstatefulset.go similarity index 100% rename from controllers/construct/mongodbstatefulset.go rename to mongodb-community-operator/controllers/construct/mongodbstatefulset.go diff --git a/controllers/construct/mongodbstatefulset_test.go b/mongodb-community-operator/controllers/construct/mongodbstatefulset_test.go similarity index 100% rename from controllers/construct/mongodbstatefulset_test.go rename to mongodb-community-operator/controllers/construct/mongodbstatefulset_test.go diff --git a/controllers/mongodb_cleanup.go b/mongodb-community-operator/controllers/mongodb_cleanup.go similarity index 100% rename from controllers/mongodb_cleanup.go rename to mongodb-community-operator/controllers/mongodb_cleanup.go diff --git a/controllers/mongodb_cleanup_test.go b/mongodb-community-operator/controllers/mongodb_cleanup_test.go similarity index 100% rename from controllers/mongodb_cleanup_test.go rename to mongodb-community-operator/controllers/mongodb_cleanup_test.go diff --git a/controllers/mongodb_status_options.go b/mongodb-community-operator/controllers/mongodb_status_options.go similarity index 100% rename from controllers/mongodb_status_options.go rename to mongodb-community-operator/controllers/mongodb_status_options.go diff --git a/controllers/mongodb_status_options_test.go b/mongodb-community-operator/controllers/mongodb_status_options_test.go similarity index 100% rename from controllers/mongodb_status_options_test.go rename to mongodb-community-operator/controllers/mongodb_status_options_test.go diff --git a/controllers/mongodb_tls.go b/mongodb-community-operator/controllers/mongodb_tls.go similarity index 100% rename from controllers/mongodb_tls.go rename to mongodb-community-operator/controllers/mongodb_tls.go diff --git a/controllers/mongodb_tls_test.go b/mongodb-community-operator/controllers/mongodb_tls_test.go similarity index 100% rename from controllers/mongodb_tls_test.go rename to mongodb-community-operator/controllers/mongodb_tls_test.go diff --git a/controllers/mongodb_users.go b/mongodb-community-operator/controllers/mongodb_users.go similarity index 100% rename from controllers/mongodb_users.go rename to mongodb-community-operator/controllers/mongodb_users.go diff --git a/controllers/predicates/predicates.go b/mongodb-community-operator/controllers/predicates/predicates.go similarity index 100% rename from controllers/predicates/predicates.go rename to mongodb-community-operator/controllers/predicates/predicates.go diff --git a/controllers/prometheus.go b/mongodb-community-operator/controllers/prometheus.go similarity index 100% rename from controllers/prometheus.go rename to mongodb-community-operator/controllers/prometheus.go diff --git a/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go similarity index 100% rename from controllers/replica_set_controller.go rename to mongodb-community-operator/controllers/replica_set_controller.go diff --git a/controllers/replicaset_controller_test.go b/mongodb-community-operator/controllers/replicaset_controller_test.go similarity index 100% rename from controllers/replicaset_controller_test.go rename to mongodb-community-operator/controllers/replicaset_controller_test.go diff --git a/controllers/testdata/change_data_volume.yaml b/mongodb-community-operator/controllers/testdata/change_data_volume.yaml similarity index 100% rename from controllers/testdata/change_data_volume.yaml rename to mongodb-community-operator/controllers/testdata/change_data_volume.yaml diff --git a/controllers/testdata/custom_storage_class.yaml b/mongodb-community-operator/controllers/testdata/custom_storage_class.yaml similarity index 100% rename from controllers/testdata/custom_storage_class.yaml rename to mongodb-community-operator/controllers/testdata/custom_storage_class.yaml diff --git a/controllers/testdata/openshift_mdb.yaml b/mongodb-community-operator/controllers/testdata/openshift_mdb.yaml similarity index 100% rename from controllers/testdata/openshift_mdb.yaml rename to mongodb-community-operator/controllers/testdata/openshift_mdb.yaml diff --git a/controllers/testdata/specify_data_dir.yaml b/mongodb-community-operator/controllers/testdata/specify_data_dir.yaml similarity index 100% rename from controllers/testdata/specify_data_dir.yaml rename to mongodb-community-operator/controllers/testdata/specify_data_dir.yaml diff --git a/controllers/testdata/specify_net_port.yaml b/mongodb-community-operator/controllers/testdata/specify_net_port.yaml similarity index 100% rename from controllers/testdata/specify_net_port.yaml rename to mongodb-community-operator/controllers/testdata/specify_net_port.yaml diff --git a/controllers/testdata/tolerations_example.yaml b/mongodb-community-operator/controllers/testdata/tolerations_example.yaml similarity index 100% rename from controllers/testdata/tolerations_example.yaml rename to mongodb-community-operator/controllers/testdata/tolerations_example.yaml diff --git a/controllers/testdata/volume_claim_templates_mdb.yaml b/mongodb-community-operator/controllers/testdata/volume_claim_templates_mdb.yaml similarity index 100% rename from controllers/testdata/volume_claim_templates_mdb.yaml rename to mongodb-community-operator/controllers/testdata/volume_claim_templates_mdb.yaml diff --git a/controllers/validation/validation.go b/mongodb-community-operator/controllers/validation/validation.go similarity index 100% rename from controllers/validation/validation.go rename to mongodb-community-operator/controllers/validation/validation.go diff --git a/controllers/watch/watch.go b/mongodb-community-operator/controllers/watch/watch.go similarity index 100% rename from controllers/watch/watch.go rename to mongodb-community-operator/controllers/watch/watch.go diff --git a/controllers/watch/watch_test.go b/mongodb-community-operator/controllers/watch/watch_test.go similarity index 100% rename from controllers/watch/watch_test.go rename to mongodb-community-operator/controllers/watch/watch_test.go diff --git a/deploy/clusterwide/cluster_role.yaml b/mongodb-community-operator/deploy/clusterwide/cluster_role.yaml similarity index 100% rename from deploy/clusterwide/cluster_role.yaml rename to mongodb-community-operator/deploy/clusterwide/cluster_role.yaml diff --git a/deploy/clusterwide/cluster_role_binding.yaml b/mongodb-community-operator/deploy/clusterwide/cluster_role_binding.yaml similarity index 100% rename from deploy/clusterwide/cluster_role_binding.yaml rename to mongodb-community-operator/deploy/clusterwide/cluster_role_binding.yaml diff --git a/deploy/clusterwide/role-for-binding.yaml b/mongodb-community-operator/deploy/clusterwide/role-for-binding.yaml similarity index 100% rename from deploy/clusterwide/role-for-binding.yaml rename to mongodb-community-operator/deploy/clusterwide/role-for-binding.yaml diff --git a/deploy/e2e/role.yaml b/mongodb-community-operator/deploy/e2e/role.yaml similarity index 100% rename from deploy/e2e/role.yaml rename to mongodb-community-operator/deploy/e2e/role.yaml diff --git a/deploy/e2e/role_binding.yaml b/mongodb-community-operator/deploy/e2e/role_binding.yaml similarity index 100% rename from deploy/e2e/role_binding.yaml rename to mongodb-community-operator/deploy/e2e/role_binding.yaml diff --git a/deploy/e2e/service_account.yaml b/mongodb-community-operator/deploy/e2e/service_account.yaml similarity index 100% rename from deploy/e2e/service_account.yaml rename to mongodb-community-operator/deploy/e2e/service_account.yaml diff --git a/deploy/openshift/operator_openshift.yaml b/mongodb-community-operator/deploy/openshift/operator_openshift.yaml similarity index 100% rename from deploy/openshift/operator_openshift.yaml rename to mongodb-community-operator/deploy/openshift/operator_openshift.yaml diff --git a/docs/README.md b/mongodb-community-operator/docs/README.md similarity index 100% rename from docs/README.md rename to mongodb-community-operator/docs/README.md diff --git a/docs/RELEASE_NOTES.md b/mongodb-community-operator/docs/RELEASE_NOTES.md similarity index 100% rename from docs/RELEASE_NOTES.md rename to mongodb-community-operator/docs/RELEASE_NOTES.md diff --git a/docs/architecture.md b/mongodb-community-operator/docs/architecture.md similarity index 100% rename from docs/architecture.md rename to mongodb-community-operator/docs/architecture.md diff --git a/docs/build_operator_locally.md b/mongodb-community-operator/docs/build_operator_locally.md similarity index 100% rename from docs/build_operator_locally.md rename to mongodb-community-operator/docs/build_operator_locally.md diff --git a/docs/contributing.md b/mongodb-community-operator/docs/contributing.md similarity index 100% rename from docs/contributing.md rename to mongodb-community-operator/docs/contributing.md diff --git a/docs/deploy-configure.md b/mongodb-community-operator/docs/deploy-configure.md similarity index 100% rename from docs/deploy-configure.md rename to mongodb-community-operator/docs/deploy-configure.md diff --git a/docs/external_access.md b/mongodb-community-operator/docs/external_access.md similarity index 100% rename from docs/external_access.md rename to mongodb-community-operator/docs/external_access.md diff --git a/docs/grafana/sample_dashboard.json b/mongodb-community-operator/docs/grafana/sample_dashboard.json similarity index 100% rename from docs/grafana/sample_dashboard.json rename to mongodb-community-operator/docs/grafana/sample_dashboard.json diff --git a/docs/how-to-release.md b/mongodb-community-operator/docs/how-to-release.md similarity index 100% rename from docs/how-to-release.md rename to mongodb-community-operator/docs/how-to-release.md diff --git a/docs/install-upgrade.md b/mongodb-community-operator/docs/install-upgrade.md similarity index 100% rename from docs/install-upgrade.md rename to mongodb-community-operator/docs/install-upgrade.md diff --git a/docs/logging.md b/mongodb-community-operator/docs/logging.md similarity index 100% rename from docs/logging.md rename to mongodb-community-operator/docs/logging.md diff --git a/docs/prometheus/README.md b/mongodb-community-operator/docs/prometheus/README.md similarity index 100% rename from docs/prometheus/README.md rename to mongodb-community-operator/docs/prometheus/README.md diff --git a/docs/prometheus/issuer-and-cert.yaml b/mongodb-community-operator/docs/prometheus/issuer-and-cert.yaml similarity index 100% rename from docs/prometheus/issuer-and-cert.yaml rename to mongodb-community-operator/docs/prometheus/issuer-and-cert.yaml diff --git a/docs/prometheus/mongodb-prometheus-sample.yaml b/mongodb-community-operator/docs/prometheus/mongodb-prometheus-sample.yaml similarity index 100% rename from docs/prometheus/mongodb-prometheus-sample.yaml rename to mongodb-community-operator/docs/prometheus/mongodb-prometheus-sample.yaml diff --git a/docs/release-notes-template.md b/mongodb-community-operator/docs/release-notes-template.md similarity index 100% rename from docs/release-notes-template.md rename to mongodb-community-operator/docs/release-notes-template.md diff --git a/docs/resize-pvc.md b/mongodb-community-operator/docs/resize-pvc.md similarity index 100% rename from docs/resize-pvc.md rename to mongodb-community-operator/docs/resize-pvc.md diff --git a/docs/run-operator-locally.md b/mongodb-community-operator/docs/run-operator-locally.md similarity index 100% rename from docs/run-operator-locally.md rename to mongodb-community-operator/docs/run-operator-locally.md diff --git a/docs/secure.md b/mongodb-community-operator/docs/secure.md similarity index 100% rename from docs/secure.md rename to mongodb-community-operator/docs/secure.md diff --git a/docs/users.md b/mongodb-community-operator/docs/users.md similarity index 100% rename from docs/users.md rename to mongodb-community-operator/docs/users.md diff --git a/docs/x509-auth.md b/mongodb-community-operator/docs/x509-auth.md similarity index 100% rename from docs/x509-auth.md rename to mongodb-community-operator/docs/x509-auth.md diff --git a/mongodb-community-operator/go.mod b/mongodb-community-operator/go.mod new file mode 100644 index 000000000..5894aee21 --- /dev/null +++ b/mongodb-community-operator/go.mod @@ -0,0 +1,91 @@ +module github.com/mongodb/mongodb-kubernetes-operator + +go 1.24.0 + +require ( + github.com/blang/semver v3.5.1+incompatible + github.com/go-logr/logr v1.4.2 + github.com/hashicorp/go-multierror v1.1.1 + github.com/imdario/mergo v0.3.15 + github.com/spf13/cast v1.7.1 + github.com/stretchr/objx v0.5.2 + github.com/stretchr/testify v1.10.0 + github.com/xdg/stringprep v1.0.3 + go.mongodb.org/mongo-driver v1.16.0 + go.uber.org/zap v1.27.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 + k8s.io/api v0.30.10 + k8s.io/apimachinery v0.30.10 + k8s.io/client-go v0.30.10 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 + sigs.k8s.io/controller-runtime v0.18.7 + sigs.k8s.io/yaml v1.4.0 +) + +require google.golang.org/protobuf v1.33.0 // indirect + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/net v0.36.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.23.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) diff --git a/mongodb-community-operator/go.sum b/mongodb-community-operator/go.sum new file mode 100644 index 000000000..4b8969f32 --- /dev/null +++ b/mongodb-community-operator/go.sum @@ -0,0 +1,252 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= +go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.30.10 h1:2YvzRF/BELgCvxbQqFKaan5hnj2+y7JOuqu2WpVk3gg= +k8s.io/api v0.30.10/go.mod h1:Hyz3ZuK7jVLJBUFvwzDSGwxHuDdsrGs5RzF16wfHIn4= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.10 h1:UflKuJeSSArttm05wjYP0GwpTlvjnMbDKFn6F7rKkKU= +k8s.io/apimachinery v0.30.10/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.10 h1:C0oWM82QMvosIl/IdJhWfTUb7rIxM52rNSutFBknAVY= +k8s.io/client-go v0.30.10/go.mod h1:OfTvt0yuo8VpMViOsgvYQb+tMJQLNWVBqXWkzdFXSq4= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.18.7 h1:WDnx8LTRY8Fn1j/7B+S/R9MeDjWNAzpDBoaSvMSrQME= +sigs.k8s.io/controller-runtime v0.18.7/go.mod h1:L9r3fUZhID7Q9eK9mseNskpaTg2n11f/tlb8odyzJ4Y= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/mongodb-community-operator/hack/boilerplate.go.txt b/mongodb-community-operator/hack/boilerplate.go.txt new file mode 100644 index 000000000..45dbbbbcf --- /dev/null +++ b/mongodb-community-operator/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/helm-charts b/mongodb-community-operator/helm-charts similarity index 100% rename from helm-charts rename to mongodb-community-operator/helm-charts diff --git a/inventories/e2e-inventory.yaml b/mongodb-community-operator/inventories/e2e-inventory.yaml similarity index 100% rename from inventories/e2e-inventory.yaml rename to mongodb-community-operator/inventories/e2e-inventory.yaml diff --git a/inventories/operator-inventory.yaml b/mongodb-community-operator/inventories/operator-inventory.yaml similarity index 100% rename from inventories/operator-inventory.yaml rename to mongodb-community-operator/inventories/operator-inventory.yaml diff --git a/mongodb-community-operator/inventory.yaml b/mongodb-community-operator/inventory.yaml new file mode 100644 index 000000000..e2a37214c --- /dev/null +++ b/mongodb-community-operator/inventory.yaml @@ -0,0 +1,315 @@ +vars: + registry: + # Default value but overwritten in pipeline.py + architecture: amd64 + +images: + + - name: agent + vars: + context: . + template_context: scripts/dev/templates/agent + + inputs: + - release_version + - tools_version + - image + - image_dev + + platform: linux/$(inputs.params.architecture) + stages: + - name: mongodb-agent-context + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.builder + tags: [ "ubi" ] + buildargs: + agent_version: $(inputs.params.release_version) + tools_version: $(inputs.params.tools_version) + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) + + - name: agent-template-ubi + task_type: dockerfile_template + distro: ubi + tags: [ "ubi" ] + + output: + - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + + - name: mongodb-agent-build + task_type: docker_build + tags: [ "ubi" ] + + dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + + buildargs: + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) + agent_version: $(inputs.params.release_version) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: $(inputs.params.version_id)-$(inputs.params.architecture) + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: latest-$(inputs.params.architecture) + + - name: agent-template-ubi-s3 + task_type: dockerfile_template + tags: [ "ubi", "release" ] + distro: ubi + + inputs: + - release_version + + output: + - dockerfile: $(inputs.params.s3_bucket)/mongodb-agent/$(inputs.params.release_version)/ubi/Dockerfile + + - name: agent-context-ubi-release + task_type: docker_build + dockerfile: scripts/dev/templates/agent/Dockerfile.builder + tags: [ "ubi", "release" ] + buildargs: + agent_version: $(inputs.params.release_version) + tools_version: $(inputs.params.tools_version) + agent_distro: $(inputs.params.agent_distro) + tools_distro: $(inputs.params.tools_distro) + + labels: + quay.expires-after: Never + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image) + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) + + - name: mongodb-agent-release + task_type: docker_build + tags: [ "ubi", "release" ] + dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id) + + buildargs: + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) + agent_version: $(inputs.params.release_version) + + labels: + quay.expires-after: Never + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image) + tag: $(inputs.params.release_version)-$(inputs.params.architecture) + + - name: readiness-probe + vars: + context: . + template_context: scripts/dev/templates/readiness + + inputs: + - image + - image_dev + + platform: linux/$(inputs.params.architecture) + stages: + - name: readiness-init-context-build + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.builder + tags: [ "readiness-probe", "ubi" ] + labels: + quay.expires-after: 48h + + buildargs: + builder_image: $(inputs.params.builder_image) + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) + + - name: readiness-template-ubi + task_type: dockerfile_template + tags: [ "ubi" ] + template_file_extension: readiness + + inputs: + - base_image + + output: + - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) + + - name: readiness-init-build + task_type: docker_build + tags: [ "readiness-probe", "ubi" ] + dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.version_id) + + buildargs: + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) + + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: $(inputs.params.version_id)-$(inputs.params.architecture) + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: latest-$(inputs.params.architecture) + + - name: readiness-init-context-release + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.builder + tags: [ "readiness-probe", "release" , "ubi" ] + + labels: + quay.expires-after: Never + + buildargs: + builder_image: $(inputs.params.builder_image) + + inputs: + - release_version + - builder_image + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image) + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) + + - name: readiness-template-release + task_type: dockerfile_template + tags: [ "readiness-probe", "release", "ubi" ] + template_file_extension: readiness + inputs: + - base_image + - release_version + + output: + - dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.release_version) + - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-readinessprobe/$(inputs.params.release_version)/ubi/Dockerfile + + - name: readiness-init-build-release + task_type: docker_build + dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness-$(inputs.params.release_version) + tags: [ "readiness-probe", "release" , "ubi" ] + + buildargs: + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) + + labels: + quay.expires-after: Never + + inputs: + - base_image + - release_version + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image) + tag: $(inputs.params.release_version)-$(inputs.params.architecture) + + - name: version-upgrade-hook + vars: + context: . + template_context: scripts/dev/templates/versionhook + + inputs: + - image + - image_dev + + platform: linux/$(inputs.params.architecture) + stages: + - name: version-upgrade-hook-context-build + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder + tags: [ "post-start-hook", "ubi" ] + + buildargs: + builder_image: $(inputs.params.builder_image) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: $(inputs.params.version_id)-context-$(inputs.params.architecture) + + - name: version-post-start-hook-template-ubi + task_type: dockerfile_template + tags: [ "ubi" ] + template_file_extension: versionhook + + inputs: + - base_image + + output: + - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.version_id) + + - name: version-upgrade-hook-build + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.version_id) + tags: [ "post-start-hook", "ubi" ] + + buildargs: + imagebase: $(inputs.params.registry)/$(inputs.params.image_dev):$(inputs.params.version_id)-context-$(inputs.params.architecture) + + labels: + quay.expires-after: 48h + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: $(inputs.params.version_id)-$(inputs.params.architecture) + - registry: $(inputs.params.registry)/$(inputs.params.image_dev) + tag: latest-$(inputs.params.architecture) + + - name: version-upgrade-hook-context-release + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder + tags: [ "release", "post-start-hook", "ubi", ] + + labels: + quay.expires-after: Never + + buildargs: + builder_image: $(inputs.params.builder_image) + + inputs: + - release_version + - builder_image + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image) + tag: $(inputs.params.release_version)-context-$(inputs.params.architecture) + + - name: versionhook-template-release + task_type: dockerfile_template + tags: [ "post-start-hook", "release", "ubi" ] + template_file_extension: versionhook + inputs: + - base_image + - release_version + + output: + - dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.release_version) + - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-operator-version-upgrade-post-start-hook/$(inputs.params.release_version)/ubi/Dockerfile + + - name: version-upgrade-hook-build-release + task_type: docker_build + dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook-$(inputs.params.release_version) + tags: [ "release", "post-start-hook", "ubi" ] + + buildargs: + imagebase: $(inputs.params.registry)/$(inputs.params.image):$(inputs.params.release_version)-context-$(inputs.params.architecture) + + labels: + quay.expires-after: Never + + inputs: + - base_image + - release_version + + output: + - registry: $(inputs.params.registry)/$(inputs.params.image) + tag: $(inputs.params.release_version)-$(inputs.params.architecture) \ No newline at end of file diff --git a/mongodb-community-operator/licenses.csv b/mongodb-community-operator/licenses.csv new file mode 100644 index 000000000..931e7a9e4 --- /dev/null +++ b/mongodb-community-operator/licenses.csv @@ -0,0 +1,201 @@ +github.com/beorn7/perks/quantile,https://github.com/beorn7/perks/blob/v1.0.1/LICENSE,MIT +github.com/blang/semver,https://github.com/blang/semver/blob/v3.5.1/LICENSE,MIT +github.com/cespare/xxhash/v2,https://github.com/cespare/xxhash/blob/v2.1.2/LICENSE.txt,MIT +github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC +github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.9.0/LICENSE,MIT +github.com/evanphx/json-patch,https://github.com/evanphx/json-patch/blob/v4.12.0/LICENSE,BSD-3-Clause +github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause +github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.6.0/LICENSE,BSD-3-Clause +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.4.1/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.20.0/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.19.14/LICENSE,Apache-2.0 +github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause +github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 +github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.2/LICENSE,BSD-3-Clause +github.com/golang/snappy,https://github.com/golang/snappy/blob/v0.0.3/LICENSE,BSD-3-Clause +github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE,Apache-2.0 +github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.5.9/LICENSE,BSD-3-Clause +github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.1.0/LICENSE,Apache-2.0 +github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.0/LICENSE,BSD-3-Clause +github.com/hashicorp/errwrap,https://github.com/hashicorp/errwrap/blob/v1.0.0/LICENSE,MPL-2.0 +github.com/hashicorp/go-multierror,https://github.com/hashicorp/go-multierror/blob/v1.1.1/LICENSE,MPL-2.0 +github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.15/LICENSE,BSD-3-Clause +github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT +github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT +github.com/klauspost/compress,https://github.com/klauspost/compress/blob/v1.13.6/LICENSE,Apache-2.0 +github.com/klauspost/compress/internal/snapref,https://github.com/klauspost/compress/blob/v1.13.6/internal/snapref/LICENSE,BSD-3-Clause +github.com/klauspost/compress/zstd/internal/xxhash,https://github.com/klauspost/compress/blob/v1.13.6/zstd/internal/xxhash/LICENSE.txt,MIT +github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.6/LICENSE,MIT +github.com/matttproud/golang_protobuf_extensions/pbutil,https://github.com/matttproud/golang_protobuf_extensions/blob/v1.0.2/LICENSE,Apache-2.0 +github.com/moby/spdystream,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 +github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 +github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 +github.com/mongodb/mongodb-kubernetes-operator/api/v1,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/api/v1,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/cmd/manager,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/cmd/readiness/testdata,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/cmd/versionhook,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers/construct,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers/construct,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers/predicates,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers/validation,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers/watch,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/controllers/watch,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/agent,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/agent,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/authtypes,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/mocks,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scram,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/x509,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/x509,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/helm,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/configmap,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/container,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/lifecycle,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/pod,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/resourcerequirements,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/config,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/headless,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/headless,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/health,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/health,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/pod,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/pod,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/readiness/secret,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/apierrors,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/apierrors,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/constants,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/contains,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/functions,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/generate,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/result,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/scale,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/state,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/state,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/status,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/status,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/versions,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/pkg/util/versions,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/feature_compatibility_version,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/prometheus,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_arbiter,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_authentication,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_change_version,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_connection_string_options,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_cross_namespace_deploy,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_custom_annotations_test_test,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_custom_persistent_volume,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_custom_role,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade_4_5,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade_5_6,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_enterprise_upgrade_6_7,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mongod_config,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mongod_port_change_with_arbiters,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mongod_readiness,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_mount_connection_string,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_multiple,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_operator_upgrade,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_recovery,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_scale,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_scale_down,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_recreate_mdbc,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_rotate,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_rotate_delete_sts,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_tls_upgrade,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/replica_set_x509,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/statefulset_arbitrary_config,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/statefulset_arbitrary_config_update,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/statefulset_delete,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/tlstests,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester,Unknown,Unknown +github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait,Unknown,Unknown +github.com/montanaflynn/stats,https://github.com/montanaflynn/stats/blob/1bf9dbcd8cbe/LICENSE,MIT +github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause +github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause +github.com/pmezard/go-difflib/difflib,https://github.com/pmezard/go-difflib/blob/v1.0.0/LICENSE,BSD-3-Clause +github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.14.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause +github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 +github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.6.0/LICENSE,MIT +github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause +github.com/stretchr/objx,https://github.com/stretchr/objx/blob/v0.5.1/LICENSE,MIT +github.com/stretchr/testify,https://github.com/stretchr/testify/blob/v1.8.4/LICENSE,MIT +github.com/xdg-go/pbkdf2,https://github.com/xdg-go/pbkdf2/blob/v1.0.0/LICENSE,Apache-2.0 +github.com/xdg-go/scram,https://github.com/xdg-go/scram/blob/v1.1.2/LICENSE,Apache-2.0 +github.com/xdg-go/stringprep,https://github.com/xdg-go/stringprep/blob/v1.0.4/LICENSE,Apache-2.0 +github.com/xdg/stringprep,https://github.com/xdg/stringprep/blob/v1.0.3/LICENSE,Apache-2.0 +github.com/youmark/pkcs8,https://github.com/youmark/pkcs8/blob/1be2e3e5546d/LICENSE,MIT +go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.13.1/LICENSE,Apache-2.0 +go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.10.0/LICENSE.txt,MIT +go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.26.0/LICENSE.txt,MIT +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.17.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.17.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/ee480838:LICENSE,BSD-3-Clause +golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.15.0:LICENSE,BSD-3-Clause +golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.14.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.28.1/LICENSE,BSD-3-Clause +gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/natefinch/lumberjack.v2,https://github.com/natefinch/lumberjack/blob/v2.2.1/LICENSE,MIT +gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 +gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT +k8s.io/api,https://github.com/kubernetes/api/blob/v0.26.10/LICENSE,Apache-2.0 +k8s.io/apiextensions-apiserver/pkg,https://github.com/kubernetes/apiextensions-apiserver/blob/v0.26.10/LICENSE,Apache-2.0 +k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.26.10/LICENSE,Apache-2.0 +k8s.io/apimachinery/third_party/forked/golang,https://github.com/kubernetes/apimachinery/blob/v0.26.10/third_party/forked/golang/LICENSE,BSD-3-Clause +k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.26.10/LICENSE,Apache-2.0 +k8s.io/component-base/config,https://github.com/kubernetes/component-base/blob/v0.26.10/LICENSE,Apache-2.0 +k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.80.1/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/172d655c2280/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json,https://github.com/kubernetes/kube-openapi/blob/172d655c2280/pkg/internal/third_party/go-json-experiment/json/LICENSE,BSD-3-Clause +k8s.io/kube-openapi/pkg/validation/spec,https://github.com/kubernetes/kube-openapi/blob/172d655c2280/pkg/validation/spec/LICENSE,Apache-2.0 +k8s.io/utils,https://github.com/kubernetes/utils/blob/99ec85e7a448/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang/net,https://github.com/kubernetes/utils/blob/99ec85e7a448/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/controller-runtime,https://github.com/kubernetes-sigs/controller-runtime/blob/v0.14.7/LICENSE,Apache-2.0 +sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/f223a00ba0e2/LICENSE,Apache-2.0 +sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.3/LICENSE,Apache-2.0 +sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.4.0/LICENSE,Apache-2.0 +sigs.k8s.io/yaml/goyaml.v2,https://github.com/kubernetes-sigs/yaml/blob/v1.4.0/goyaml.v2/LICENSE,Apache-2.0 diff --git a/mypy.ini b/mongodb-community-operator/mypy.ini similarity index 100% rename from mypy.ini rename to mongodb-community-operator/mypy.ini diff --git a/mongodb-community-operator/pipeline.py b/mongodb-community-operator/pipeline.py new file mode 100644 index 000000000..f288d1a5b --- /dev/null +++ b/mongodb-community-operator/pipeline.py @@ -0,0 +1,294 @@ +import argparse +import json +import subprocess +import sys +from typing import Dict, List, Set +from scripts.ci.base_logger import logger +from scripts.ci.images_signing import ( + sign_image, + verify_signature, + mongodb_artifactory_login, +) + +from scripts.dev.dev_config import load_config, DevConfig +from sonar.sonar import process_image + +# These image names must correspond to prefixes in release.json, developer configuration and inventories +VALID_IMAGE_NAMES = { + "agent", + "readiness-probe", + "version-upgrade-hook", + "operator", + "e2e", +} + +AGENT_DISTRO_KEY = "agent_distro" +TOOLS_DISTRO_KEY = "tools_distro" + +AGENT_DISTROS_PER_ARCH = { + "amd64": {AGENT_DISTRO_KEY: "rhel8_x86_64", TOOLS_DISTRO_KEY: "rhel88-x86_64"}, + "arm64": {AGENT_DISTRO_KEY: "amzn2_aarch64", TOOLS_DISTRO_KEY: "rhel88-aarch64"}, +} + + +def load_release() -> Dict: + with open("release.json") as f: + return json.load(f) + + +def build_image_args(config: DevConfig, image_name: str) -> Dict[str, str]: + release = load_release() + + # Naming in pipeline : readiness-probe, naming in dev config : readiness_probe_image + image_name_prefix = image_name.replace("-", "_") + + # Default config + arguments = { + "builder": "true", + # Defaults to "" if empty, e2e has no release version + "release_version": release.get(image_name, ""), + "tools_version": "", + "image": getattr(config, f"{image_name_prefix}_image"), + # Defaults to "" if empty, e2e has no dev image + "image_dev": getattr(config, f"{image_name_prefix}_image_dev", ""), + "registry": config.repo_url, + "s3_bucket": config.s3_bucket, + "builder_image": release["golang-builder-image"], + "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", + "inventory": "inventory.yaml", + "skip_tags": config.skip_tags, # Include skip_tags + "include_tags": config.include_tags, # Include include_tags + } + + # Handle special cases + if image_name == "operator": + arguments["inventory"] = "inventories/operator-inventory.yaml" + + if image_name == "e2e": + arguments.pop("builder", None) + arguments["base_image"] = release["golang-builder-image"] + arguments["inventory"] = "inventories/e2e-inventory.yaml" + + if image_name == "agent": + arguments["tools_version"] = release["agent-tools-version"] + + return arguments + + +def sign_and_verify(registry: str, tag: str) -> None: + sign_image(registry, tag) + verify_signature(registry, tag) + + +def build_and_push_image( + image_name: str, + config: DevConfig, + args: Dict[str, str], + architectures: Set[str], + release: bool, + sign: bool, + insecure: bool = False, +) -> None: + if sign: + mongodb_artifactory_login() + for arch in architectures: + image_tag = f"{image_name}" + args["architecture"] = arch + if image_name == "agent": + args[AGENT_DISTRO_KEY] = AGENT_DISTROS_PER_ARCH[arch][AGENT_DISTRO_KEY] + args[TOOLS_DISTRO_KEY] = AGENT_DISTROS_PER_ARCH[arch][TOOLS_DISTRO_KEY] + process_image( + image_tag, + build_args=args, + inventory=args["inventory"], + skip_tags=args["skip_tags"], + include_tags=args["include_tags"], + ) + if release: + registry = args["registry"] + "/" + args["image"] + context_tag = args["release_version"] + "-context-" + arch + release_tag = args["release_version"] + "-" + arch + if sign: + sign_and_verify(registry, context_tag) + sign_and_verify(registry, release_tag) + + if args["image_dev"]: + image_to_push = args["image_dev"] + elif image_name == "e2e": + # If no image dev (only e2e is concerned) we push the normal image + image_to_push = args["image"] + else: + raise Exception("Dev image must be specified") + + push_manifest(config, architectures, image_to_push, insecure) + + if config.gh_run_id: + push_manifest(config, architectures, image_to_push, insecure, config.gh_run_id) + + if release: + registry = args["registry"] + "/" + args["image"] + context_tag = args["release_version"] + "-context" + push_manifest( + config, architectures, args["image"], insecure, args["release_version"] + ) + push_manifest(config, architectures, args["image"], insecure, context_tag) + if sign: + sign_and_verify(registry, args["release_version"]) + sign_and_verify(registry, context_tag) + + +""" +Generates docker manifests by running the following commands: +1. Clear existing manifests +docker manifest rm config.repo_url/image:tag +2. Create the manifest +docker manifest create config.repo_url/image:tag --amend config.repo_url/image:tag-amd64 --amend config.repo_url/image:tag-arm64 +3. Push the manifest +docker manifest push config.repo_url/image:tag +""" + + +def push_manifest( + config: DevConfig, + architectures: Set[str], + image_name: str, + insecure: bool = False, + image_tag: str = "latest", +) -> None: + logger.info(f"Pushing manifest for {image_tag}") + final_manifest = "{0}/{1}:{2}".format(config.repo_url, image_name, image_tag) + remove_args = ["docker", "manifest", "rm", final_manifest] + logger.info("Removing existing manifest") + run_cli_command(remove_args, fail_on_error=False) + + create_args = [ + "docker", + "manifest", + "create", + final_manifest, + ] + + if insecure: + create_args.append("--insecure") + + for arch in architectures: + create_args.extend(["--amend", final_manifest + "-" + arch]) + + logger.info("Creating new manifest") + run_cli_command(create_args) + + push_args = ["docker", "manifest", "push", final_manifest] + logger.info("Pushing new manifest") + run_cli_command(push_args) + + +# Raises exceptions by default +def run_cli_command(args: List[str], fail_on_error: bool = True) -> None: + command = " ".join(args) + logger.debug(f"Running: {command}") + try: + cp = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + check=False, + ) + except Exception as e: + logger.error(f" Command raised the following exception: {e}") + if fail_on_error: + raise Exception + else: + logger.warning("Continuing...") + return + + if cp.returncode != 0: + error_msg = cp.stderr.decode().strip() + stdout = cp.stdout.decode().strip() + logger.error(f"Error running command") + logger.error(f"stdout:\n{stdout}") + logger.error(f"stderr:\n{error_msg}") + if fail_on_error: + raise Exception + else: + logger.warning("Continuing...") + return + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--image-name", type=str) + parser.add_argument("--release", action="store_true", default=False) + parser.add_argument( + "--arch", + choices=["amd64", "arm64"], + nargs="+", + help="for daily builds only, specify the list of architectures to build for images", + ) + parser.add_argument("--tag", type=str) + parser.add_argument("--sign", action="store_true", default=False) + parser.add_argument("--insecure", action="store_true", default=False) + return parser.parse_args() + + +""" +Takes arguments: +--image-name : The name of the image to build, must be one of VALID_IMAGE_NAMES +--release : We push the image to the registry only if this flag is set +--architecture : List of architectures to build for the image +--sign : Sign images with our private key if sign is set (only for release) + +Run with --help for more information +Example usage : `python pipeline.py --image-name agent --release --sign` + +Builds and push the docker image to the registry +Many parameters are defined in the dev configuration, default path is : ~/.community-operator-dev/config.json +""" + + +def main() -> int: + args = _parse_args() + + image_name = args.image_name + if image_name not in VALID_IMAGE_NAMES: + logger.error( + f"Invalid image name: {image_name}. Valid options are: {VALID_IMAGE_NAMES}" + ) + return 1 + + # Handle dev config + config: DevConfig = load_config() + config.gh_run_id = args.tag + + # Warn user if trying to release E2E tests + if args.release and image_name == "e2e": + logger.warning( + "Warning : releasing E2E test will fail because E2E image has no release version" + ) + + # Skipping release tasks by default + if not args.release: + config.ensure_skip_tag("release") + if args.sign: + logger.warning("--sign flag has no effect without --release") + + if args.arch: + arch_set = set(args.arch) + else: + # Default is multi-arch + arch_set = {"amd64", "arm64"} + logger.info(f"Building for architectures: {','.join(arch_set)}") + + if not args.sign: + logger.warning("--sign flag not provided, images won't be signed") + + image_args = build_image_args(config, image_name) + + build_and_push_image( + image_name, config, image_args, arch_set, args.release, args.sign, args.insecure + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pkg/agent/agent_readiness.go b/mongodb-community-operator/pkg/agent/agent_readiness.go similarity index 100% rename from pkg/agent/agent_readiness.go rename to mongodb-community-operator/pkg/agent/agent_readiness.go diff --git a/pkg/agent/agent_readiness_test.go b/mongodb-community-operator/pkg/agent/agent_readiness_test.go similarity index 100% rename from pkg/agent/agent_readiness_test.go rename to mongodb-community-operator/pkg/agent/agent_readiness_test.go diff --git a/pkg/agent/agentflags.go b/mongodb-community-operator/pkg/agent/agentflags.go similarity index 100% rename from pkg/agent/agentflags.go rename to mongodb-community-operator/pkg/agent/agentflags.go diff --git a/pkg/agent/agentflags_test.go b/mongodb-community-operator/pkg/agent/agentflags_test.go similarity index 100% rename from pkg/agent/agentflags_test.go rename to mongodb-community-operator/pkg/agent/agentflags_test.go diff --git a/pkg/agent/agenthealth.go b/mongodb-community-operator/pkg/agent/agenthealth.go similarity index 100% rename from pkg/agent/agenthealth.go rename to mongodb-community-operator/pkg/agent/agenthealth.go diff --git a/pkg/agent/replica_set_port_manager.go b/mongodb-community-operator/pkg/agent/replica_set_port_manager.go similarity index 100% rename from pkg/agent/replica_set_port_manager.go rename to mongodb-community-operator/pkg/agent/replica_set_port_manager.go diff --git a/pkg/agent/replica_set_port_manager_test.go b/mongodb-community-operator/pkg/agent/replica_set_port_manager_test.go similarity index 100% rename from pkg/agent/replica_set_port_manager_test.go rename to mongodb-community-operator/pkg/agent/replica_set_port_manager_test.go diff --git a/pkg/authentication/authentication.go b/mongodb-community-operator/pkg/authentication/authentication.go similarity index 100% rename from pkg/authentication/authentication.go rename to mongodb-community-operator/pkg/authentication/authentication.go diff --git a/pkg/authentication/authentication_test.go b/mongodb-community-operator/pkg/authentication/authentication_test.go similarity index 100% rename from pkg/authentication/authentication_test.go rename to mongodb-community-operator/pkg/authentication/authentication_test.go diff --git a/pkg/authentication/authtypes/authtypes.go b/mongodb-community-operator/pkg/authentication/authtypes/authtypes.go similarity index 100% rename from pkg/authentication/authtypes/authtypes.go rename to mongodb-community-operator/pkg/authentication/authtypes/authtypes.go diff --git a/pkg/authentication/mocks/mocks.go b/mongodb-community-operator/pkg/authentication/mocks/mocks.go similarity index 100% rename from pkg/authentication/mocks/mocks.go rename to mongodb-community-operator/pkg/authentication/mocks/mocks.go diff --git a/pkg/authentication/scram/scram.go b/mongodb-community-operator/pkg/authentication/scram/scram.go similarity index 100% rename from pkg/authentication/scram/scram.go rename to mongodb-community-operator/pkg/authentication/scram/scram.go diff --git a/pkg/authentication/scram/scram_enabler.go b/mongodb-community-operator/pkg/authentication/scram/scram_enabler.go similarity index 100% rename from pkg/authentication/scram/scram_enabler.go rename to mongodb-community-operator/pkg/authentication/scram/scram_enabler.go diff --git a/pkg/authentication/scram/scram_enabler_test.go b/mongodb-community-operator/pkg/authentication/scram/scram_enabler_test.go similarity index 100% rename from pkg/authentication/scram/scram_enabler_test.go rename to mongodb-community-operator/pkg/authentication/scram/scram_enabler_test.go diff --git a/pkg/authentication/scram/scram_test.go b/mongodb-community-operator/pkg/authentication/scram/scram_test.go similarity index 100% rename from pkg/authentication/scram/scram_test.go rename to mongodb-community-operator/pkg/authentication/scram/scram_test.go diff --git a/pkg/authentication/scramcredentials/scram_credentials.go b/mongodb-community-operator/pkg/authentication/scramcredentials/scram_credentials.go similarity index 100% rename from pkg/authentication/scramcredentials/scram_credentials.go rename to mongodb-community-operator/pkg/authentication/scramcredentials/scram_credentials.go diff --git a/pkg/authentication/scramcredentials/scram_credentials_test.go b/mongodb-community-operator/pkg/authentication/scramcredentials/scram_credentials_test.go similarity index 100% rename from pkg/authentication/scramcredentials/scram_credentials_test.go rename to mongodb-community-operator/pkg/authentication/scramcredentials/scram_credentials_test.go diff --git a/pkg/authentication/x509/x509.go b/mongodb-community-operator/pkg/authentication/x509/x509.go similarity index 100% rename from pkg/authentication/x509/x509.go rename to mongodb-community-operator/pkg/authentication/x509/x509.go diff --git a/pkg/authentication/x509/x509_enabler.go b/mongodb-community-operator/pkg/authentication/x509/x509_enabler.go similarity index 100% rename from pkg/authentication/x509/x509_enabler.go rename to mongodb-community-operator/pkg/authentication/x509/x509_enabler.go diff --git a/pkg/authentication/x509/x509_enabler_test.go b/mongodb-community-operator/pkg/authentication/x509/x509_enabler_test.go similarity index 100% rename from pkg/authentication/x509/x509_enabler_test.go rename to mongodb-community-operator/pkg/authentication/x509/x509_enabler_test.go diff --git a/pkg/authentication/x509/x509_test.go b/mongodb-community-operator/pkg/authentication/x509/x509_test.go similarity index 100% rename from pkg/authentication/x509/x509_test.go rename to mongodb-community-operator/pkg/authentication/x509/x509_test.go diff --git a/pkg/automationconfig/automation_config.go b/mongodb-community-operator/pkg/automationconfig/automation_config.go similarity index 100% rename from pkg/automationconfig/automation_config.go rename to mongodb-community-operator/pkg/automationconfig/automation_config.go diff --git a/pkg/automationconfig/automation_config_builder.go b/mongodb-community-operator/pkg/automationconfig/automation_config_builder.go similarity index 100% rename from pkg/automationconfig/automation_config_builder.go rename to mongodb-community-operator/pkg/automationconfig/automation_config_builder.go diff --git a/pkg/automationconfig/automation_config_secret.go b/mongodb-community-operator/pkg/automationconfig/automation_config_secret.go similarity index 100% rename from pkg/automationconfig/automation_config_secret.go rename to mongodb-community-operator/pkg/automationconfig/automation_config_secret.go diff --git a/pkg/automationconfig/automation_config_secret_test.go b/mongodb-community-operator/pkg/automationconfig/automation_config_secret_test.go similarity index 100% rename from pkg/automationconfig/automation_config_secret_test.go rename to mongodb-community-operator/pkg/automationconfig/automation_config_secret_test.go diff --git a/pkg/automationconfig/automation_config_test.go b/mongodb-community-operator/pkg/automationconfig/automation_config_test.go similarity index 100% rename from pkg/automationconfig/automation_config_test.go rename to mongodb-community-operator/pkg/automationconfig/automation_config_test.go diff --git a/pkg/automationconfig/zz_generated.deepcopy.go b/mongodb-community-operator/pkg/automationconfig/zz_generated.deepcopy.go similarity index 100% rename from pkg/automationconfig/zz_generated.deepcopy.go rename to mongodb-community-operator/pkg/automationconfig/zz_generated.deepcopy.go diff --git a/pkg/helm/helm.go b/mongodb-community-operator/pkg/helm/helm.go similarity index 100% rename from pkg/helm/helm.go rename to mongodb-community-operator/pkg/helm/helm.go diff --git a/pkg/kube/annotations/annotations.go b/mongodb-community-operator/pkg/kube/annotations/annotations.go similarity index 100% rename from pkg/kube/annotations/annotations.go rename to mongodb-community-operator/pkg/kube/annotations/annotations.go diff --git a/pkg/kube/client/client.go b/mongodb-community-operator/pkg/kube/client/client.go similarity index 100% rename from pkg/kube/client/client.go rename to mongodb-community-operator/pkg/kube/client/client.go diff --git a/pkg/kube/client/client_test.go b/mongodb-community-operator/pkg/kube/client/client_test.go similarity index 100% rename from pkg/kube/client/client_test.go rename to mongodb-community-operator/pkg/kube/client/client_test.go diff --git a/pkg/kube/client/mocked_client.go b/mongodb-community-operator/pkg/kube/client/mocked_client.go similarity index 100% rename from pkg/kube/client/mocked_client.go rename to mongodb-community-operator/pkg/kube/client/mocked_client.go diff --git a/pkg/kube/client/mocked_client_test.go b/mongodb-community-operator/pkg/kube/client/mocked_client_test.go similarity index 100% rename from pkg/kube/client/mocked_client_test.go rename to mongodb-community-operator/pkg/kube/client/mocked_client_test.go diff --git a/pkg/kube/client/mocked_manager.go b/mongodb-community-operator/pkg/kube/client/mocked_manager.go similarity index 100% rename from pkg/kube/client/mocked_manager.go rename to mongodb-community-operator/pkg/kube/client/mocked_manager.go diff --git a/pkg/kube/configmap/configmap.go b/mongodb-community-operator/pkg/kube/configmap/configmap.go similarity index 100% rename from pkg/kube/configmap/configmap.go rename to mongodb-community-operator/pkg/kube/configmap/configmap.go diff --git a/pkg/kube/configmap/configmap_builder.go b/mongodb-community-operator/pkg/kube/configmap/configmap_builder.go similarity index 100% rename from pkg/kube/configmap/configmap_builder.go rename to mongodb-community-operator/pkg/kube/configmap/configmap_builder.go diff --git a/pkg/kube/configmap/configmap_test.go b/mongodb-community-operator/pkg/kube/configmap/configmap_test.go similarity index 100% rename from pkg/kube/configmap/configmap_test.go rename to mongodb-community-operator/pkg/kube/configmap/configmap_test.go diff --git a/pkg/kube/container/container_test.go b/mongodb-community-operator/pkg/kube/container/container_test.go similarity index 100% rename from pkg/kube/container/container_test.go rename to mongodb-community-operator/pkg/kube/container/container_test.go diff --git a/pkg/kube/container/container_util.go b/mongodb-community-operator/pkg/kube/container/container_util.go similarity index 100% rename from pkg/kube/container/container_util.go rename to mongodb-community-operator/pkg/kube/container/container_util.go diff --git a/pkg/kube/container/containers.go b/mongodb-community-operator/pkg/kube/container/containers.go similarity index 100% rename from pkg/kube/container/containers.go rename to mongodb-community-operator/pkg/kube/container/containers.go diff --git a/pkg/kube/lifecycle/lifecyle.go b/mongodb-community-operator/pkg/kube/lifecycle/lifecyle.go similarity index 100% rename from pkg/kube/lifecycle/lifecyle.go rename to mongodb-community-operator/pkg/kube/lifecycle/lifecyle.go diff --git a/pkg/kube/persistentvolumeclaim/pvc.go b/mongodb-community-operator/pkg/kube/persistentvolumeclaim/pvc.go similarity index 100% rename from pkg/kube/persistentvolumeclaim/pvc.go rename to mongodb-community-operator/pkg/kube/persistentvolumeclaim/pvc.go diff --git a/pkg/kube/pod/pod.go b/mongodb-community-operator/pkg/kube/pod/pod.go similarity index 100% rename from pkg/kube/pod/pod.go rename to mongodb-community-operator/pkg/kube/pod/pod.go diff --git a/pkg/kube/podtemplatespec/podspec_template.go b/mongodb-community-operator/pkg/kube/podtemplatespec/podspec_template.go similarity index 100% rename from pkg/kube/podtemplatespec/podspec_template.go rename to mongodb-community-operator/pkg/kube/podtemplatespec/podspec_template.go diff --git a/pkg/kube/podtemplatespec/podspec_template_test.go b/mongodb-community-operator/pkg/kube/podtemplatespec/podspec_template_test.go similarity index 100% rename from pkg/kube/podtemplatespec/podspec_template_test.go rename to mongodb-community-operator/pkg/kube/podtemplatespec/podspec_template_test.go diff --git a/pkg/kube/probes/probes.go b/mongodb-community-operator/pkg/kube/probes/probes.go similarity index 100% rename from pkg/kube/probes/probes.go rename to mongodb-community-operator/pkg/kube/probes/probes.go diff --git a/pkg/kube/resourcerequirements/resource_requirements.go b/mongodb-community-operator/pkg/kube/resourcerequirements/resource_requirements.go similarity index 100% rename from pkg/kube/resourcerequirements/resource_requirements.go rename to mongodb-community-operator/pkg/kube/resourcerequirements/resource_requirements.go diff --git a/pkg/kube/resourcerequirements/resource_requirements_test.go b/mongodb-community-operator/pkg/kube/resourcerequirements/resource_requirements_test.go similarity index 100% rename from pkg/kube/resourcerequirements/resource_requirements_test.go rename to mongodb-community-operator/pkg/kube/resourcerequirements/resource_requirements_test.go diff --git a/pkg/kube/secret/secret.go b/mongodb-community-operator/pkg/kube/secret/secret.go similarity index 100% rename from pkg/kube/secret/secret.go rename to mongodb-community-operator/pkg/kube/secret/secret.go diff --git a/pkg/kube/secret/secret_builder.go b/mongodb-community-operator/pkg/kube/secret/secret_builder.go similarity index 100% rename from pkg/kube/secret/secret_builder.go rename to mongodb-community-operator/pkg/kube/secret/secret_builder.go diff --git a/pkg/kube/secret/secret_test.go b/mongodb-community-operator/pkg/kube/secret/secret_test.go similarity index 100% rename from pkg/kube/secret/secret_test.go rename to mongodb-community-operator/pkg/kube/secret/secret_test.go diff --git a/mongodb-community-operator/pkg/kube/service/service.go b/mongodb-community-operator/pkg/kube/service/service.go new file mode 100644 index 000000000..abb749acf --- /dev/null +++ b/mongodb-community-operator/pkg/kube/service/service.go @@ -0,0 +1,46 @@ +package service + +import ( + "context" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Getter interface { + GetService(ctx context.Context, objectKey client.ObjectKey) (corev1.Service, error) +} + +type Updater interface { + UpdateService(ctx context.Context, service corev1.Service) error +} + +type Creator interface { + CreateService(ctx context.Context, service corev1.Service) error +} + +type Deleter interface { + DeleteService(ctx context.Context, objectKey client.ObjectKey) error +} + +type GetDeleter interface { + Getter + Deleter +} + +type GetUpdater interface { + Getter + Updater +} + +type GetUpdateCreator interface { + Getter + Updater + Creator +} + +type GetUpdateCreateDeleter interface { + Getter + Updater + Creator + Deleter +} diff --git a/pkg/kube/service/service_builder.go b/mongodb-community-operator/pkg/kube/service/service_builder.go similarity index 100% rename from pkg/kube/service/service_builder.go rename to mongodb-community-operator/pkg/kube/service/service_builder.go diff --git a/pkg/kube/statefulset/merge_statefulset_test.go b/mongodb-community-operator/pkg/kube/statefulset/merge_statefulset_test.go similarity index 100% rename from pkg/kube/statefulset/merge_statefulset_test.go rename to mongodb-community-operator/pkg/kube/statefulset/merge_statefulset_test.go diff --git a/pkg/kube/statefulset/statefulset.go b/mongodb-community-operator/pkg/kube/statefulset/statefulset.go similarity index 100% rename from pkg/kube/statefulset/statefulset.go rename to mongodb-community-operator/pkg/kube/statefulset/statefulset.go diff --git a/pkg/kube/statefulset/statefulset_builder.go b/mongodb-community-operator/pkg/kube/statefulset/statefulset_builder.go similarity index 100% rename from pkg/kube/statefulset/statefulset_builder.go rename to mongodb-community-operator/pkg/kube/statefulset/statefulset_builder.go diff --git a/pkg/kube/statefulset/statefulset_test.go b/mongodb-community-operator/pkg/kube/statefulset/statefulset_test.go similarity index 100% rename from pkg/kube/statefulset/statefulset_test.go rename to mongodb-community-operator/pkg/kube/statefulset/statefulset_test.go diff --git a/pkg/readiness/config/config.go b/mongodb-community-operator/pkg/readiness/config/config.go similarity index 100% rename from pkg/readiness/config/config.go rename to mongodb-community-operator/pkg/readiness/config/config.go diff --git a/pkg/readiness/headless/headless.go b/mongodb-community-operator/pkg/readiness/headless/headless.go similarity index 100% rename from pkg/readiness/headless/headless.go rename to mongodb-community-operator/pkg/readiness/headless/headless.go diff --git a/pkg/readiness/headless/headless_test.go b/mongodb-community-operator/pkg/readiness/headless/headless_test.go similarity index 100% rename from pkg/readiness/headless/headless_test.go rename to mongodb-community-operator/pkg/readiness/headless/headless_test.go diff --git a/pkg/readiness/health/health.go b/mongodb-community-operator/pkg/readiness/health/health.go similarity index 100% rename from pkg/readiness/health/health.go rename to mongodb-community-operator/pkg/readiness/health/health.go diff --git a/pkg/readiness/health/health_test.go b/mongodb-community-operator/pkg/readiness/health/health_test.go similarity index 100% rename from pkg/readiness/health/health_test.go rename to mongodb-community-operator/pkg/readiness/health/health_test.go diff --git a/pkg/readiness/pod/podannotation.go b/mongodb-community-operator/pkg/readiness/pod/podannotation.go similarity index 100% rename from pkg/readiness/pod/podannotation.go rename to mongodb-community-operator/pkg/readiness/pod/podannotation.go diff --git a/pkg/readiness/pod/podannotation_test.go b/mongodb-community-operator/pkg/readiness/pod/podannotation_test.go similarity index 100% rename from pkg/readiness/pod/podannotation_test.go rename to mongodb-community-operator/pkg/readiness/pod/podannotation_test.go diff --git a/pkg/readiness/pod/podpatcher.go b/mongodb-community-operator/pkg/readiness/pod/podpatcher.go similarity index 100% rename from pkg/readiness/pod/podpatcher.go rename to mongodb-community-operator/pkg/readiness/pod/podpatcher.go diff --git a/pkg/readiness/secret/automationconfig.go b/mongodb-community-operator/pkg/readiness/secret/automationconfig.go similarity index 100% rename from pkg/readiness/secret/automationconfig.go rename to mongodb-community-operator/pkg/readiness/secret/automationconfig.go diff --git a/pkg/readiness/secret/secretreader.go b/mongodb-community-operator/pkg/readiness/secret/secretreader.go similarity index 100% rename from pkg/readiness/secret/secretreader.go rename to mongodb-community-operator/pkg/readiness/secret/secretreader.go diff --git a/pkg/util/apierrors/apierrors.go b/mongodb-community-operator/pkg/util/apierrors/apierrors.go similarity index 100% rename from pkg/util/apierrors/apierrors.go rename to mongodb-community-operator/pkg/util/apierrors/apierrors.go diff --git a/pkg/util/apierrors/apierrors_test.go b/mongodb-community-operator/pkg/util/apierrors/apierrors_test.go similarity index 100% rename from pkg/util/apierrors/apierrors_test.go rename to mongodb-community-operator/pkg/util/apierrors/apierrors_test.go diff --git a/pkg/util/constants/constants.go b/mongodb-community-operator/pkg/util/constants/constants.go similarity index 100% rename from pkg/util/constants/constants.go rename to mongodb-community-operator/pkg/util/constants/constants.go diff --git a/pkg/util/contains/contains.go b/mongodb-community-operator/pkg/util/contains/contains.go similarity index 100% rename from pkg/util/contains/contains.go rename to mongodb-community-operator/pkg/util/contains/contains.go diff --git a/pkg/util/envvar/envvars.go b/mongodb-community-operator/pkg/util/envvar/envvars.go similarity index 100% rename from pkg/util/envvar/envvars.go rename to mongodb-community-operator/pkg/util/envvar/envvars.go diff --git a/pkg/util/envvar/envvars_test.go b/mongodb-community-operator/pkg/util/envvar/envvars_test.go similarity index 100% rename from pkg/util/envvar/envvars_test.go rename to mongodb-community-operator/pkg/util/envvar/envvars_test.go diff --git a/pkg/util/functions/functions.go b/mongodb-community-operator/pkg/util/functions/functions.go similarity index 100% rename from pkg/util/functions/functions.go rename to mongodb-community-operator/pkg/util/functions/functions.go diff --git a/mongodb-community-operator/pkg/util/generate/generate.go b/mongodb-community-operator/pkg/util/generate/generate.go new file mode 100644 index 000000000..338e0d1b8 --- /dev/null +++ b/mongodb-community-operator/pkg/util/generate/generate.go @@ -0,0 +1,88 @@ +package generate + +import ( + "crypto/rand" + "crypto/sha1" // nolint + "crypto/sha256" + "encoding/base64" + "hash" + "unicode" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials" +) + +// final key must be between 6 and at most 1024 characters +func KeyFileContents() (string, error) { + return generateRandomString(500) +} + +// RandomValidDNS1123Label generates a random fixed-length string with characters in a certain range. +func RandomValidDNS1123Label(n int) (string, error) { + str, err := RandomFixedLengthStringOfSize(n) + if err != nil { + return "", err + } + + runes := []rune(str) + + // Make sure that any letters are lowercase and that if any non-alphanumeric characters appear they are set to '0'. + for i, r := range runes { + if unicode.IsLetter(r) { + runes[i] = unicode.ToLower(r) + } else if !unicode.IsNumber(r) { + runes[i] = rune('0') + } + } + + return string(runes), nil +} + +func RandomFixedLengthStringOfSize(n int) (string, error) { + b, err := generateRandomBytes(n) + return base64.URLEncoding.EncodeToString(b)[:n], err +} + +// Salts generates 2 different salts. The first is for the sha1 algorithm +// the second is for sha256 +func Salts() ([]byte, []byte, error) { + sha1Salt, err := salt(sha1.New) + if err != nil { + return nil, nil, err + } + + sha256Salt, err := salt(sha256.New) + if err != nil { + return nil, nil, err + } + return sha1Salt, sha256Salt, nil +} + +// salt will create a salt which can be used to compute Scram Sha credentials based on the given hashConstructor. +// sha1.New should be used for MONGODB-CR/SCRAM-SHA-1 and sha256.New should be used for SCRAM-SHA-256 +func salt(hashConstructor func() hash.Hash) ([]byte, error) { + saltSize := hashConstructor().Size() - scramcredentials.RFC5802MandatedSaltSize + salt, err := RandomFixedLengthStringOfSize(20) + + if err != nil { + return nil, err + } + shaBytes32 := sha256.Sum256([]byte(salt)) + + // the algorithms expect a salt of a specific size. + return shaBytes32[:saltSize], nil +} + +func generateRandomBytes(size int) ([]byte, error) { + b := make([]byte, size) + _, err := rand.Read(b) + if err != nil { + return nil, err + } + + return b, nil +} + +func generateRandomString(numBytes int) (string, error) { + b, err := generateRandomBytes(numBytes) + return base64.StdEncoding.EncodeToString(b), err +} diff --git a/pkg/util/merge/merge.go b/mongodb-community-operator/pkg/util/merge/merge.go similarity index 100% rename from pkg/util/merge/merge.go rename to mongodb-community-operator/pkg/util/merge/merge.go diff --git a/pkg/util/merge/merge_automationconfigs.go b/mongodb-community-operator/pkg/util/merge/merge_automationconfigs.go similarity index 100% rename from pkg/util/merge/merge_automationconfigs.go rename to mongodb-community-operator/pkg/util/merge/merge_automationconfigs.go diff --git a/pkg/util/merge/merge_automationconfigs_test.go b/mongodb-community-operator/pkg/util/merge/merge_automationconfigs_test.go similarity index 100% rename from pkg/util/merge/merge_automationconfigs_test.go rename to mongodb-community-operator/pkg/util/merge/merge_automationconfigs_test.go diff --git a/pkg/util/merge/merge_ephemeral_container.go b/mongodb-community-operator/pkg/util/merge/merge_ephemeral_container.go similarity index 100% rename from pkg/util/merge/merge_ephemeral_container.go rename to mongodb-community-operator/pkg/util/merge/merge_ephemeral_container.go diff --git a/pkg/util/merge/merge_podtemplate_spec.go b/mongodb-community-operator/pkg/util/merge/merge_podtemplate_spec.go similarity index 100% rename from pkg/util/merge/merge_podtemplate_spec.go rename to mongodb-community-operator/pkg/util/merge/merge_podtemplate_spec.go diff --git a/pkg/util/merge/merge_service_spec.go b/mongodb-community-operator/pkg/util/merge/merge_service_spec.go similarity index 100% rename from pkg/util/merge/merge_service_spec.go rename to mongodb-community-operator/pkg/util/merge/merge_service_spec.go diff --git a/pkg/util/merge/merge_statefulset.go b/mongodb-community-operator/pkg/util/merge/merge_statefulset.go similarity index 100% rename from pkg/util/merge/merge_statefulset.go rename to mongodb-community-operator/pkg/util/merge/merge_statefulset.go diff --git a/pkg/util/merge/merge_test.go b/mongodb-community-operator/pkg/util/merge/merge_test.go similarity index 100% rename from pkg/util/merge/merge_test.go rename to mongodb-community-operator/pkg/util/merge/merge_test.go diff --git a/pkg/util/result/reconciliationresults.go b/mongodb-community-operator/pkg/util/result/reconciliationresults.go similarity index 100% rename from pkg/util/result/reconciliationresults.go rename to mongodb-community-operator/pkg/util/result/reconciliationresults.go diff --git a/pkg/util/scale/scale.go b/mongodb-community-operator/pkg/util/scale/scale.go similarity index 100% rename from pkg/util/scale/scale.go rename to mongodb-community-operator/pkg/util/scale/scale.go diff --git a/pkg/util/state/statemachine.go b/mongodb-community-operator/pkg/util/state/statemachine.go similarity index 100% rename from pkg/util/state/statemachine.go rename to mongodb-community-operator/pkg/util/state/statemachine.go diff --git a/pkg/util/state/statemachine_test.go b/mongodb-community-operator/pkg/util/state/statemachine_test.go similarity index 100% rename from pkg/util/state/statemachine_test.go rename to mongodb-community-operator/pkg/util/state/statemachine_test.go diff --git a/pkg/util/status/status.go b/mongodb-community-operator/pkg/util/status/status.go similarity index 100% rename from pkg/util/status/status.go rename to mongodb-community-operator/pkg/util/status/status.go diff --git a/pkg/util/status/status_test.go b/mongodb-community-operator/pkg/util/status/status_test.go similarity index 100% rename from pkg/util/status/status_test.go rename to mongodb-community-operator/pkg/util/status/status_test.go diff --git a/pkg/util/versions/versions.go b/mongodb-community-operator/pkg/util/versions/versions.go similarity index 100% rename from pkg/util/versions/versions.go rename to mongodb-community-operator/pkg/util/versions/versions.go diff --git a/pkg/util/versions/versions_test.go b/mongodb-community-operator/pkg/util/versions/versions_test.go similarity index 100% rename from pkg/util/versions/versions_test.go rename to mongodb-community-operator/pkg/util/versions/versions_test.go diff --git a/mongodb-community-operator/release.json b/mongodb-community-operator/release.json new file mode 100644 index 000000000..88983b5b3 --- /dev/null +++ b/mongodb-community-operator/release.json @@ -0,0 +1,8 @@ +{ + "golang-builder-image": "golang:1.24", + "operator": "0.12.0", + "version-upgrade-hook": "1.0.9", + "readiness-probe": "1.0.22", + "agent": "108.0.2.8729-1", + "agent-tools-version": "100.10.0" +} diff --git a/mongodb-community-operator/requirements.txt b/mongodb-community-operator/requirements.txt new file mode 100644 index 000000000..e264e2e09 --- /dev/null +++ b/mongodb-community-operator/requirements.txt @@ -0,0 +1,20 @@ +git+https://github.com/mongodb/sonar@bc7bf7732851425421f3cfe2a19cf50b0460e633 +github-action-templates==0.0.4 +docker==7.1.0 +kubernetes==26.1.0 +jinja2==3.1.4 +MarkupSafe==2.0.1 +PyYAML==6.0.1 +black==24.3.0 +mypy==0.961 +tqdm==v4.66.3 +boto3==1.16.21 +pymongo==4.6.3 +dnspython==2.6.1 +requests==2.32.3 +ruamel.yaml==0.17.9 +semver==2.13.0 +rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools==78.0.1 # not directly required, pinned by Snyk to avoid a vulnerability +certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability +urllib3<2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/scripts/ci/base_logger.py b/mongodb-community-operator/scripts/ci/base_logger.py similarity index 100% rename from scripts/ci/base_logger.py rename to mongodb-community-operator/scripts/ci/base_logger.py diff --git a/scripts/ci/config.json b/mongodb-community-operator/scripts/ci/config.json similarity index 100% rename from scripts/ci/config.json rename to mongodb-community-operator/scripts/ci/config.json diff --git a/scripts/ci/create_kind_cluster.sh b/mongodb-community-operator/scripts/ci/create_kind_cluster.sh similarity index 100% rename from scripts/ci/create_kind_cluster.sh rename to mongodb-community-operator/scripts/ci/create_kind_cluster.sh diff --git a/scripts/ci/determine_required_releases.py b/mongodb-community-operator/scripts/ci/determine_required_releases.py similarity index 100% rename from scripts/ci/determine_required_releases.py rename to mongodb-community-operator/scripts/ci/determine_required_releases.py diff --git a/scripts/ci/dump_diagnostics.sh b/mongodb-community-operator/scripts/ci/dump_diagnostics.sh similarity index 100% rename from scripts/ci/dump_diagnostics.sh rename to mongodb-community-operator/scripts/ci/dump_diagnostics.sh diff --git a/scripts/ci/images_signing.py b/mongodb-community-operator/scripts/ci/images_signing.py similarity index 100% rename from scripts/ci/images_signing.py rename to mongodb-community-operator/scripts/ci/images_signing.py diff --git a/scripts/ci/update_release.py b/mongodb-community-operator/scripts/ci/update_release.py similarity index 100% rename from scripts/ci/update_release.py rename to mongodb-community-operator/scripts/ci/update_release.py diff --git a/mongodb-community-operator/scripts/dev/__init__.py b/mongodb-community-operator/scripts/dev/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/dev/dev_config.py b/mongodb-community-operator/scripts/dev/dev_config.py similarity index 100% rename from scripts/dev/dev_config.py rename to mongodb-community-operator/scripts/dev/dev_config.py diff --git a/scripts/dev/e2e.py b/mongodb-community-operator/scripts/dev/e2e.py similarity index 100% rename from scripts/dev/e2e.py rename to mongodb-community-operator/scripts/dev/e2e.py diff --git a/scripts/dev/edit_cluster_config.sh b/mongodb-community-operator/scripts/dev/edit_cluster_config.sh similarity index 100% rename from scripts/dev/edit_cluster_config.sh rename to mongodb-community-operator/scripts/dev/edit_cluster_config.sh diff --git a/scripts/dev/generate_github_actions.py b/mongodb-community-operator/scripts/dev/generate_github_actions.py similarity index 100% rename from scripts/dev/generate_github_actions.py rename to mongodb-community-operator/scripts/dev/generate_github_actions.py diff --git a/scripts/dev/get_e2e_env_vars.py b/mongodb-community-operator/scripts/dev/get_e2e_env_vars.py similarity index 100% rename from scripts/dev/get_e2e_env_vars.py rename to mongodb-community-operator/scripts/dev/get_e2e_env_vars.py diff --git a/scripts/dev/install_prerequisites.sh b/mongodb-community-operator/scripts/dev/install_prerequisites.sh similarity index 100% rename from scripts/dev/install_prerequisites.sh rename to mongodb-community-operator/scripts/dev/install_prerequisites.sh diff --git a/scripts/dev/k8s_conditions.py b/mongodb-community-operator/scripts/dev/k8s_conditions.py similarity index 100% rename from scripts/dev/k8s_conditions.py rename to mongodb-community-operator/scripts/dev/k8s_conditions.py diff --git a/scripts/dev/k8s_request_data.py b/mongodb-community-operator/scripts/dev/k8s_request_data.py similarity index 100% rename from scripts/dev/k8s_request_data.py rename to mongodb-community-operator/scripts/dev/k8s_request_data.py diff --git a/scripts/dev/run_e2e_gh.sh b/mongodb-community-operator/scripts/dev/run_e2e_gh.sh similarity index 100% rename from scripts/dev/run_e2e_gh.sh rename to mongodb-community-operator/scripts/dev/run_e2e_gh.sh diff --git a/mongodb-community-operator/scripts/dev/setup_kind_cluster.sh b/mongodb-community-operator/scripts/dev/setup_kind_cluster.sh new file mode 100755 index 000000000..3178f2878 --- /dev/null +++ b/mongodb-community-operator/scripts/dev/setup_kind_cluster.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +#### +# This file is copy-pasted from https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/scripts/dev/setup_kind_cluster.sh +# Do not edit !!! +#### + +function usage() { + echo "Deploy local registry and create kind cluster configured to use this registry. Local Docker registry is deployed at localhost:5000. + +Usage: + setup_kind_cluster.sh [-n ] [-r] + setup_kind_cluster.sh [-h] + setup_kind_cluster.sh [-n ] [-e] [-r] + +Options: + -n (optional) Set kind cluster name to . Creates kubeconfig in ~/.kube/. The default name is 'kind' if not set. + -e (optional) Export newly created kind cluster's credentials to ~/.kube/ and set current kubectl context. + -h (optional) Shows this screen. + -r (optional) Recreate cluster if needed + -p (optional) Network reserved for Pods, e.g. 10.244.0.0/16 + -s (optional) Network reserved for Services, e.g. 10.96.0.0/16 +" + exit 0 +} + +cluster_name=${CLUSTER_NAME:-"kind"} +export_kubeconfig=0 +recreate=0 +pod_network="10.244.0.0/16" +service_network="10.96.0.0/16" +while getopts ':p:s:n:her' opt; do + case $opt in + (n) cluster_name=$OPTARG;; + (e) export_kubeconfig=1;; + (r) recreate=1;; + (p) pod_network=$OPTARG;; + (s) service_network=$OPTARG;; + (h) usage;; + (*) usage;; + esac +done +shift "$((OPTIND-1))" + +kubeconfig_path="$HOME/.kube/${cluster_name}" + +# create the kind network early unless it already exists. +# it would normally be created automatically by kind but we +# need it earlier to get the IP address of our registry. +docker network create kind || true + +# adapted from https://kind.sigs.k8s.io/docs/user/local-registry/ +# create registry container unless it already exists +reg_name='kind-registry' +reg_port='5000' +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network kind --name "${reg_name}" registry:2 +fi + +if [ "${recreate}" != 0 ]; then + kind delete cluster --name "${cluster_name}" || true +fi + +# create a cluster with the local registry enabled in containerd +cat <" variable with the clustername you want in this spec. +# iii. Set the "" of the cluster previosuly setup. +# iv. Provive a non-overlapping CIDR for this new cluster +# v. kops create -f cluster.yaml +# verify by deploying a pod in cluster1 and make sure it's curl"able" from cluster2 +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: null + name: +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudProvider: aws + configBase: s3://kube-om-state-store/ + containerRuntime: docker + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master- + name: a + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master- + name: a + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: 1.20.10 + masterPublicName: api. + networkID: + # configure the CNI, default "kubenet" CNI which KOPS uses won't work + networking: + amazonvpc: {} + # select a non overlapping CIDR block, separate from the one previously deployed, else + # the API server will get confused seeing the same IP coming from two different clusters + nonMasqueradeCIDR: 172.20.64.0/19 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: + name: + type: Public + zone: + topology: + dns: + type: Public + masters: public + nodes: public +--- +# master instance group +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: null + labels: + kops.k8s.io/cluster: + name: master- +spec: + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20210315 + machineType: + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: master- + role: Master + rootVolumeSize: 16 + subnets: + - + +--- +# node instance group +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: null + labels: + kops.k8s.io/cluster: + name: nodes- +spec: + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20210315 + machineType: + maxSize: 4 + minSize: 4 + nodeLabels: + kops.k8s.io/instancegroup: nodes- + role: Node + rootVolumeSize: 40 + subnets: + - diff --git a/multi_cluster/create_security_groups.py b/multi_cluster/create_security_groups.py new file mode 100755 index 000000000..162c94eaf --- /dev/null +++ b/multi_cluster/create_security_groups.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import sys +from typing import List + +import boto3 + + +def get_instance_groups_for_cluster(cluster_name: str) -> List[str]: + """ + :param cluster_name: name of the cluster + :return: list of instance groups associated with that cluster + """ + return [f"nodes.{cluster_name}", f"masters.{cluster_name}"] + + +def get_other_instance_groups_for_cluster(cluster_name: str, all_clusters: List[str]) -> List[str]: + """ + :param cluster_name: the name of the cluster + :param all_clusters: a list of all clusters + :return: a list of instance group names that need to have rules added for + """ + other_instance_groups = [] + for cluster in all_clusters: + if cluster == cluster_name: + continue + other_instance_groups.extend(get_instance_groups_for_cluster(cluster)) + return other_instance_groups + + +def get_all_instance_groups(cluster_names: List[str]) -> List[str]: + """ + :param cluster_names: list of all cluster names. + :return: list of all instance group names. + """ + all_instance_groups = [] + for cluster in cluster_names: + all_instance_groups.extend(get_instance_groups_for_cluster(cluster)) + return all_instance_groups + + +def get_security_group_by_name(security_groups, name: str): + """ + :param security_groups: list of all security group objects. + :param name: name of the desired security group. + :return: + """ + return next(iter([sg for sg in security_groups if sg.group_name == name])) + + +def _add_all_traffic_security_group_rule(sg0, sg1, vpc_id): + """ + :param sg0: security group that the ingres will be added to. + :param sg1: security group that the inbound rule will be added to. + :param vpc_id: vpc id of both security groups. + """ + sg0.authorize_ingress( + IpPermissions=[ + { + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + "IpProtocol": "-1", + "Ipv6Ranges": [], + "PrefixListIds": [], + "UserIdGroupPairs": [{"VpcId": vpc_id, "GroupId": sg1.id, "UserId": "268558157000"}], + }, + ] + ) + + +def main( + region: str, + vpc_id: str, + cluster_names: List[str], +): + """ + This script creates inbound rules allowing all traffic between all instances groups + associated with all of the clusters provided. + + + :param region: the aws region. + :param vpc_id: the vpc id associated with the clusters. + :param cluster_names: the names of the clusters. + :return: None + """ + ec2 = boto3.resource("ec2", region_name=region) + + security_groups = ec2.security_groups.all() + for cluster in cluster_names: + cluster_instance_groups = get_instance_groups_for_cluster(cluster) + other_instance_group_names = get_other_instance_groups_for_cluster(cluster, cluster_names) + + for instance_group in cluster_instance_groups: + instance_group_sg = get_security_group_by_name(security_groups, instance_group) + for other_instance_group in other_instance_group_names: + other_instance_group_sg = get_security_group_by_name(security_groups, other_instance_group) + print(f"adding rule for {instance_group_sg.group_name} to {other_instance_group_sg.group_name}") + try: + _add_all_traffic_security_group_rule(instance_group_sg, other_instance_group_sg, vpc_id) + except Exception as e: + print(e) + + +if __name__ == "__main__": + if len(sys.argv) != 4: + raise ValueError("Usage: create_security_groups.py ") + + region = sys.argv[1] + vpc_id = sys.argv[2] + cluster_names = sys.argv[3].split(",") + + main(region, vpc_id, cluster_names) diff --git a/multi_cluster/setup_multi_cluster_environment.sh b/multi_cluster/setup_multi_cluster_environment.sh new file mode 100755 index 000000000..b23f2ec83 --- /dev/null +++ b/multi_cluster/setup_multi_cluster_environment.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +aws_region="eu-west-2" +region="${aws_region}a" +cluster_one_name="e2e.operator.mongokubernetes.com" +cluster_two_name="e2e.cluster1.mongokubernetes.com" +cluster_three_name="e2e.cluster2.mongokubernetes.com" +master_size="m5.large" +node_size="t2.medium" + +# forward slash needs to be escaped for sed +# these are all non overlapping ranges. +cluster_one_cidr="172.20.32.0\/19" +cluster_two_cidr="172.20.64.0\/19" +cluster_three_cidr="172.20.0.0\/19" + +if ! kops validate cluster "${cluster_one_name}"; then + echo "Kops cluster \"${cluster_one_name}\" doesn't exist" + sed -e "s//${cluster_one_name}/g" -e "s//${region}/g" -e "s//${cluster_one_cidr}/g" -e '//d' -e "s//${master_size}/g" -e "s//${node_size}/g" < cluster.yaml | kops create -f - + kops create secret --name ${cluster_one_name} sshpublickey admin -i ~/.ssh/id_rsa.pub + kops update cluster --name ${cluster_one_name} --yes + echo "Waiting until kops cluster gets ready..." + kops export kubecfg "${cluster_one_name}" --admin=87600h + kops validate cluster "${cluster_one_name}" --wait 20m +fi +kops export kubecfg "${cluster_one_name}" --admin=87600h + + +VPC_ID="$(aws ec2 describe-vpcs --region ${aws_region} --filters Name=tag:Name,Values=${cluster_one_name} | jq -r .Vpcs[].VpcId)" +echo "VPC ID is ${VPC_ID}" + +if ! kops validate cluster "${cluster_two_name}"; then + echo "Kops cluster \"${cluster_two_name}\" doesn't exist" + sed -e "s//${cluster_two_name}/g" -e "s//${region}/g" -e "s//${cluster_two_cidr}/g" -e "s//${VPC_ID}/g" -e "s//${master_size}/g" -e "s//${node_size}/g" < cluster.yaml | kops create -f - + kops create secret --name ${cluster_two_name} sshpublickey admin -i ~/.ssh/id_rsa.pub + kops update cluster --name ${cluster_two_name} --yes + echo "Waiting until kops cluster ${cluster_two_name} gets ready..." + + kops export kubecfg "${cluster_two_name}" --admin=87600h + kops validate cluster "${cluster_two_name}" --wait 20m +fi +kops export kubecfg "${cluster_two_name}" --admin=87600h + + +if ! kops validate cluster "${cluster_three_name}"; then + echo "Kops cluster \"${cluster_three_name}\" doesn't exist" + sed -e "s//${cluster_three_name}/g" -e "s//${region}/g" -e "s//${cluster_three_cidr}/g" -e "s//${VPC_ID}/g" -e "s//${master_size}/g" -e "s//${node_size}/g" < cluster.yaml | kops create -f - + kops create secret --name ${cluster_three_name} sshpublickey admin -i ~/.ssh/id_rsa.pub + kops update cluster --name ${cluster_three_name} --yes + echo "Waiting until kops cluster ${cluster_three_name} gets ready..." + + kops export kubecfg "${cluster_three_name}" --admin=87600h + kops validate cluster "${cluster_three_name}" --wait 20m +fi +kops export kubecfg "${cluster_three_name}" --admin=87600h + + +./create_security_groups.py "${aws_region}" "${VPC_ID}" "${cluster_one_name},${cluster_two_name},${cluster_three_name}" diff --git a/multi_cluster/tools/README.md b/multi_cluster/tools/README.md new file mode 100644 index 000000000..b9e09744d --- /dev/null +++ b/multi_cluster/tools/README.md @@ -0,0 +1,19 @@ +### Installing Istio in e2e clusters + +The script is intended to install Istio in the multi E2E clusters that we have currently deployed. + +Steps to run the script and verify it: + +* Install the istioctl binary: + `curl -sL https://istio.io/downloadIstioctl | ISTIO_VERSION=1.9.1 sh -` + `export PATH=$PATH:$HOME/.istioctl/bin` + +* Export cluster variables: + `export CTX_CLUSTER1=e2e.cluster1.mongokubernetes.com` + + `export CTX_CLUSTER2=e2e.cluster2.mongokubernetes.com ` + + +* Run the script : `sh ./install_istio.sh` + +* [Verify the Istio installation](https://istio.io/latest/docs/setup/install/multicluster/verify/) \ No newline at end of file diff --git a/multi_cluster/tools/download_istio.sh b/multi_cluster/tools/download_istio.sh new file mode 100755 index 000000000..0fb687682 --- /dev/null +++ b/multi_cluster/tools/download_istio.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +export VERSION=${VERSION:-1.16.1} +ISTIO_SCRIPT_CHECKSUM="254c6bd6aa5b8ac8c552561c84d8e9b3a101d9e613e2a8edd6db1f19c1871dbf" + +echo "Checking if we need to download Istio ${VERSION}" +if [ ! -d "istio-${VERSION}" ]; then + echo "Downloading Istio ${VERSION}" + curl -O https://raw.githubusercontent.com/istio/istio/d710dfc2f95adb9399e1656165fa5ac22f6e1a16/release/downloadIstioCandidate.sh + echo "${ISTIO_SCRIPT_CHECKSUM} downloadIstioCandidate.sh" | sha256sum --check + ISTIO_VERSION=${VERSION} sh downloadIstioCandidate.sh +else + echo "Istio ${VERSION} already downloaded... Skipping." +fi diff --git a/multi_cluster/tools/install_istio.sh b/multi_cluster/tools/install_istio.sh new file mode 100755 index 000000000..8e65b56fc --- /dev/null +++ b/multi_cluster/tools/install_istio.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +set -eux + +export CTX_CLUSTER1=${CTX_CLUSTER1:-e2e.cluster1.mongokubernetes.com} +export CTX_CLUSTER2=${CTX_CLUSTER2:-e2e.cluster2.mongokubernetes.com} +export CTX_CLUSTER3=${CTX_CLUSTER3:-e2e.cluster3.mongokubernetes.com} +export VERSION=${VERSION:-1.12.8} + +IS_KIND="false" +if [[ $CTX_CLUSTER1 = kind* ]]; then + IS_KIND="true" +fi + +source multi_cluster/tools/download_istio.sh + +# +cd istio-${VERSION} +## perform cleanup prior to install +bin/istioctl x uninstall --context="${CTX_CLUSTER1}" --purge --skip-confirmation & +bin/istioctl x uninstall --context="${CTX_CLUSTER2}" --purge --skip-confirmation & +bin/istioctl x uninstall --context="${CTX_CLUSTER3}" --purge --skip-confirmation & +wait + +rm -rf certs +mkdir -p certs +pushd certs + +# create root trust for the clusters +make -f ../tools/certs/Makefile.selfsigned.mk "root-ca" +# FIXME: I'm not sure why, but Istio's makefiles seem to fail on my Mac when generating those certs. +# The funny thing is that they are generated fine once I rerun the targets. +# This probably requires a bit more investigation or upgrading Istio to the latest version. +make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER1}-cacerts" || make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER1}-cacerts" +make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER2}-cacerts" || make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER2}-cacerts" +make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER3}-cacerts" || make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER3}-cacerts" + +# create cluster secret objects with the certs and keys +kubectl --context="${CTX_CLUSTER1}" delete ns istio-system || true +kubectl --context="${CTX_CLUSTER1}" create ns istio-system +kubectl --context="${CTX_CLUSTER1}" create secret generic cacerts -n istio-system \ + --from-file=${CTX_CLUSTER1}/ca-cert.pem \ + --from-file=${CTX_CLUSTER1}/ca-key.pem \ + --from-file=${CTX_CLUSTER1}/root-cert.pem \ + --from-file=${CTX_CLUSTER1}/cert-chain.pem + +kubectl --context="${CTX_CLUSTER2}" delete ns istio-system || true +kubectl --context="${CTX_CLUSTER2}" create ns istio-system +kubectl --context="${CTX_CLUSTER2}" create secret generic cacerts -n istio-system \ + --from-file=${CTX_CLUSTER2}/ca-cert.pem \ + --from-file=${CTX_CLUSTER2}/ca-key.pem \ + --from-file=${CTX_CLUSTER2}/root-cert.pem \ + --from-file=${CTX_CLUSTER2}/cert-chain.pem + +kubectl --context="${CTX_CLUSTER3}" delete ns istio-system || true +kubectl --context="${CTX_CLUSTER3}" create ns istio-system +kubectl --context="${CTX_CLUSTER3}" create secret generic cacerts -n istio-system \ + --from-file=${CTX_CLUSTER3}/ca-cert.pem \ + --from-file=${CTX_CLUSTER3}/ca-key.pem \ + --from-file=${CTX_CLUSTER3}/root-cert.pem \ + --from-file=${CTX_CLUSTER3}/cert-chain.pem +popd + +# install IstioOperator in clusters +cat <cluster1.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + tag: ${VERSION} + meshConfig: + defaultConfig: + terminationDrainDuration: 30s + proxyMetadata: + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + ISTIO_META_DNS_CAPTURE: "true" + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster1 + network: network1 +EOF + +bin/istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml -y & + +cat <cluster2.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + tag: ${VERSION} + meshConfig: + defaultConfig: + terminationDrainDuration: 30s + proxyMetadata: + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + ISTIO_META_DNS_CAPTURE: "true" + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster2 + network: network1 +EOF + +bin/istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml -y & + +cat <cluster3.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + tag: ${VERSION} + meshConfig: + defaultConfig: + terminationDrainDuration: 30s + proxyMetadata: + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + ISTIO_META_DNS_CAPTURE: "true" + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster3 + network: network1 +EOF + +bin/istioctl install --context="${CTX_CLUSTER3}" -f cluster3.yaml -y & + +wait + +CLUSTER_1_ADDITIONAL_OPTS="" +CLUSTER_2_ADDITIONAL_OPTS="" +CLUSTER_3_ADDITIONAL_OPTS="" +if [[ $IS_KIND == "true" ]]; then + CLUSTER_1_ADDITIONAL_OPTS="--server https://$(kubectl --context=${CTX_CLUSTER1} get node e2e-cluster-1-control-plane -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'):6443" + CLUSTER_2_ADDITIONAL_OPTS="--server https://$(kubectl --context=${CTX_CLUSTER2} get node e2e-cluster-2-control-plane -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'):6443" + CLUSTER_3_ADDITIONAL_OPTS="--server https://$(kubectl --context=${CTX_CLUSTER3} get node e2e-cluster-3-control-plane -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}'):6443" +fi + +# enable endpoint discovery +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER1}" \ + -n istio-system \ + --name=cluster1 ${CLUSTER_1_ADDITIONAL_OPTS} | + kubectl apply -f - --context="${CTX_CLUSTER2}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER1}" \ + -n istio-system \ + --name=cluster1 ${CLUSTER_1_ADDITIONAL_OPTS} | + kubectl apply -f - --context="${CTX_CLUSTER3}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + -n istio-system \ + --name=cluster2 ${CLUSTER_2_ADDITIONAL_OPTS} | + kubectl apply -f - --context="${CTX_CLUSTER1}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + -n istio-system \ + --name=cluster2 ${CLUSTER_2_ADDITIONAL_OPTS} | + kubectl apply -f - --context="${CTX_CLUSTER3}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER3}" \ + -n istio-system \ + --name=cluster3 ${CLUSTER_3_ADDITIONAL_OPTS} | + kubectl apply -f - --context="${CTX_CLUSTER1}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER3}" \ + -n istio-system \ + --name=cluster3 ${CLUSTER_3_ADDITIONAL_OPTS} | + kubectl apply -f - --context="${CTX_CLUSTER2}" +# disable namespace injection explicitly for istio-system namespace +kubectl --context="${CTX_CLUSTER1}" label namespace istio-system istio-injection=disabled +kubectl --context="${CTX_CLUSTER2}" label namespace istio-system istio-injection=disabled +kubectl --context="${CTX_CLUSTER3}" label namespace istio-system istio-injection=disabled + +# Skipping the cleanup for now. Otherwise we won't have the tools to diagnose issues +#cd .. +#rm -r istio-${VERSION} +#rm -f cluster1.yaml cluster2.yaml cluster3.yaml diff --git a/multi_cluster/tools/install_istio_central.sh b/multi_cluster/tools/install_istio_central.sh new file mode 100755 index 000000000..da6f84477 --- /dev/null +++ b/multi_cluster/tools/install_istio_central.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eux + +export VERSION=${VERSION:-1.14.2} + +export CTX_CLUSTER=${CTX_CLUSTER:-e2e.operator.mongokubernetes.com} + +source multi_cluster/tools/download_istio.sh +cd istio-${VERSION} + +bin/istioctl x uninstall --context="${CTX_CLUSTER}" --purge --skip-confirmation +bin/istioctl install --context="${CTX_CLUSTER}" --set profile=default --set meshConfig.outboundTrafficPolicy.mode=REGISTRY_ONLY --skip-confirmation diff --git a/pipeline.py b/pipeline.py old mode 100644 new mode 100755 index f288d1a5b..4b3a869c6 --- a/pipeline.py +++ b/pipeline.py @@ -1,140 +1,293 @@ +#!/usr/bin/env python3 + +"""This pipeline script knows about the details of our Docker images +and where to fetch and calculate parameters. It uses Sonar.py +to produce the final images.""" + import argparse +import copy import json +import os +import random +import shutil import subprocess import sys -from typing import Dict, List, Set -from scripts.ci.base_logger import logger -from scripts.ci.images_signing import ( +import tarfile +import time +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from queue import Queue +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union + +import requests +import semver +from opentelemetry import context +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as OTLPSpanGrpcExporter, +) +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import ( + SynchronousMultiSpanProcessor, + Tracer, + TracerProvider, +) +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags +from packaging.version import Version + +import docker +from lib.base_logger import logger +from lib.sonar.sonar import process_image +from scripts.evergreen.release.agent_matrix import ( + get_supported_operator_versions, + get_supported_version_for_image_matrix_handling, +) +from scripts.evergreen.release.images_signing import ( + mongodb_artifactory_login, sign_image, verify_signature, - mongodb_artifactory_login, ) +from scripts.evergreen.release.sbom import generate_sbom, generate_sbom_for_cli -from scripts.dev.dev_config import load_config, DevConfig -from sonar.sonar import process_image - -# These image names must correspond to prefixes in release.json, developer configuration and inventories -VALID_IMAGE_NAMES = { - "agent", - "readiness-probe", - "version-upgrade-hook", - "operator", - "e2e", -} - -AGENT_DISTRO_KEY = "agent_distro" -TOOLS_DISTRO_KEY = "tools_distro" - -AGENT_DISTROS_PER_ARCH = { - "amd64": {AGENT_DISTRO_KEY: "rhel8_x86_64", TOOLS_DISTRO_KEY: "rhel88-x86_64"}, - "arm64": {AGENT_DISTRO_KEY: "amzn2_aarch64", TOOLS_DISTRO_KEY: "rhel88-aarch64"}, -} - - -def load_release() -> Dict: - with open("release.json") as f: - return json.load(f) - - -def build_image_args(config: DevConfig, image_name: str) -> Dict[str, str]: - release = load_release() - - # Naming in pipeline : readiness-probe, naming in dev config : readiness_probe_image - image_name_prefix = image_name.replace("-", "_") - - # Default config - arguments = { - "builder": "true", - # Defaults to "" if empty, e2e has no release version - "release_version": release.get(image_name, ""), - "tools_version": "", - "image": getattr(config, f"{image_name_prefix}_image"), - # Defaults to "" if empty, e2e has no dev image - "image_dev": getattr(config, f"{image_name_prefix}_image_dev", ""), - "registry": config.repo_url, - "s3_bucket": config.s3_bucket, - "builder_image": release["golang-builder-image"], - "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest", - "inventory": "inventory.yaml", - "skip_tags": config.skip_tags, # Include skip_tags - "include_tags": config.include_tags, # Include include_tags - } +TRACER = trace.get_tracer("evergreen-agent") - # Handle special cases - if image_name == "operator": - arguments["inventory"] = "inventories/operator-inventory.yaml" - if image_name == "e2e": - arguments.pop("builder", None) - arguments["base_image"] = release["golang-builder-image"] - arguments["inventory"] = "inventories/e2e-inventory.yaml" +def _setup_tracing(): + trace_id = os.environ.get("otel_trace_id") + parent_id = os.environ.get("otel_parent_id") + endpoint = os.environ.get("otel_collector_endpoint") + if any(value is None for value in [trace_id, parent_id, endpoint]): + logger.info("tracing environment variables are missing, not configuring tracing") + return + logger.info(f"parent_id is {parent_id}") + logger.info(f"trace_id is {trace_id}") + logger.info(f"endpoint is {endpoint}") + span_context = SpanContext( + trace_id=int(trace_id, 16), + span_id=int(parent_id, 16), + is_remote=False, + # Magic number needed for our OTEL collector + trace_flags=TraceFlags(0x01), + ) + ctx = trace.set_span_in_context(NonRecordingSpan(span_context)) + context.attach(ctx) + sp = SynchronousMultiSpanProcessor() + span_processor = BatchSpanProcessor( + OTLPSpanGrpcExporter( + endpoint=endpoint, + ) + ) + sp.add_span_processor(span_processor) + resource = Resource(attributes={SERVICE_NAME: "evergreen-agent"}) + provider = TracerProvider(resource=resource, active_span_processor=sp) + trace.set_tracer_provider(provider) - if image_name == "agent": - arguments["tools_version"] = release["agent-tools-version"] - return arguments +DEFAULT_IMAGE_TYPE = "ubi" +DEFAULT_NAMESPACE = "default" +# QUAY_REGISTRY_URL sets the base registry for all release build stages. Context images and daily builds will push the +# final images to the registry specified here. +# This makes it easy to use ECR to test changes on the pipeline before pushing to Quay. +QUAY_REGISTRY_URL = os.environ.get("QUAY_REGISTRY", "quay.io/mongodb") -def sign_and_verify(registry: str, tag: str) -> None: - sign_image(registry, tag) - verify_signature(registry, tag) +@dataclass +class BuildConfiguration: + image_type: str + base_repository: str + namespace: str -def build_and_push_image( - image_name: str, - config: DevConfig, - args: Dict[str, str], - architectures: Set[str], - release: bool, - sign: bool, - insecure: bool = False, -) -> None: - if sign: - mongodb_artifactory_login() - for arch in architectures: - image_tag = f"{image_name}" - args["architecture"] = arch - if image_name == "agent": - args[AGENT_DISTRO_KEY] = AGENT_DISTROS_PER_ARCH[arch][AGENT_DISTRO_KEY] - args[TOOLS_DISTRO_KEY] = AGENT_DISTROS_PER_ARCH[arch][TOOLS_DISTRO_KEY] - process_image( - image_tag, - build_args=args, - inventory=args["inventory"], - skip_tags=args["skip_tags"], - include_tags=args["include_tags"], + include_tags: list[str] + skip_tags: list[str] + + builder: str = "docker" + parallel: bool = False + parallel_factor: int = 0 + architecture: Optional[List[str]] = None + sign: bool = False + all_agents: bool = False + + pipeline: bool = True + debug: bool = True + + def build_args(self, args: Optional[Dict[str, str]] = None) -> Dict[str, str]: + if args is None: + args = {} + args = args.copy() + + args["registry"] = self.base_repository + + return args + + def get_skip_tags(self) -> list[str]: + return make_list_of_str(self.skip_tags) + + def get_include_tags(self) -> list[str]: + return make_list_of_str(self.include_tags) + + def is_release_step_executed(self) -> bool: + if "release" in self.get_skip_tags(): + return False + if "release" in self.get_include_tags(): + return True + return len(self.get_include_tags()) == 0 + + +def make_list_of_str(value: Union[None, str, List[str]]) -> List[str]: + if value is None: + return [] + + if isinstance(value, str): + return [e.strip() for e in value.split(",")] + + return value + + +def get_tools_distro(tools_version: str) -> Dict[str, str]: + new_rhel_tool_version = "100.10.0" + default_distro = {"arm": "rhel90-aarch64", "amd": "rhel90-x86_64"} + if Version(tools_version) >= Version(new_rhel_tool_version): + return {"arm": "rhel93-aarch64", "amd": "rhel93-x86_64"} + return default_distro + + +def operator_build_configuration( + builder: str, + parallel: bool, + debug: bool, + architecture: Optional[List[str]] = None, + sign: bool = False, + all_agents: bool = False, + parallel_factor: int = 0, +) -> BuildConfiguration: + bc = BuildConfiguration( + image_type=os.environ.get("distro", DEFAULT_IMAGE_TYPE), + base_repository=os.environ["BASE_REPO_URL"], + namespace=os.environ.get("namespace", DEFAULT_NAMESPACE), + skip_tags=make_list_of_str(os.environ.get("skip_tags")), + include_tags=make_list_of_str(os.environ.get("include_tags")), + builder=builder, + parallel=parallel, + all_agents=all_agents or bool(os.environ.get("all_agents", False)), + debug=debug, + architecture=architecture, + sign=sign, + parallel_factor=parallel_factor, + ) + + logger.info(f"is_running_in_patch: {is_running_in_patch()}") + logger.info(f"is_running_in_evg_pipeline: {is_running_in_evg_pipeline()}") + if is_running_in_patch() or not is_running_in_evg_pipeline(): + logger.info( + f"Running build not in evg pipeline (is_running_in_evg_pipeline={is_running_in_evg_pipeline()}) " + f"or in pipeline but not from master (is_running_in_patch={is_running_in_patch()}). " + "Adding 'master' tag to skip to prevent publishing to the latest dev image." ) - if release: - registry = args["registry"] + "/" + args["image"] - context_tag = args["release_version"] + "-context-" + arch - release_tag = args["release_version"] + "-" + arch - if sign: - sign_and_verify(registry, context_tag) - sign_and_verify(registry, release_tag) - - if args["image_dev"]: - image_to_push = args["image_dev"] - elif image_name == "e2e": - # If no image dev (only e2e is concerned) we push the normal image - image_to_push = args["image"] + bc.skip_tags.append("master") + + return bc + + +def is_running_in_evg_pipeline(): + return os.getenv("RUNNING_IN_EVG", "") == "true" + + +class MissingEnvironmentVariable(Exception): + pass + + +def should_pin_at() -> Optional[Tuple[str, str]]: + """Gets the value of the pin_tag_at to tag the images with. + + Returns its value split on :. + """ + # We need to return something so `partition` does not raise + # AttributeError + is_patch = is_running_in_patch() + + try: + pinned = os.environ["pin_tag_at"] + except KeyError: + raise MissingEnvironmentVariable(f"pin_tag_at environment variable does not exist, but is required") + if is_patch: + if pinned == "00:00": + raise Exception("Pinning to midnight during a patch is not supported. Please pin to another date!") + + hour, _, minute = pinned.partition(":") + return hour, minute + + +def is_running_in_patch(): + is_patch = os.environ.get("is_patch") + return is_patch is not None and is_patch.lower() == "true" + + +def build_id() -> str: + """Returns the current UTC time in ISO8601 date format. + + If running in Evergreen and `created_at` expansion is defined, use the + datetime defined in that variable instead. + + It is possible to pin this time at midnight (00:00) for periodic builds. If + running a manual build, then the Evergreen `pin_tag_at` variable needs to be + set to the empty string, in which case, the image tag suffix will correspond + to the current timestamp. + + """ + + date = datetime.now(timezone.utc) + try: + created_at = os.environ["created_at"] + date = datetime.strptime(created_at, "%y_%m_%d_%H_%M_%S") + except KeyError: + pass + + hour, minute = should_pin_at() + if hour and minute: + logger.info(f"we are pinning to, hour: {hour}, minute: {minute}") + date = date.replace(hour=int(hour), minute=int(minute), second=0) else: - raise Exception("Dev image must be specified") + logger.warning(f"hour and minute cannot be extracted from provided pin_tag_at env, pinning to now") - push_manifest(config, architectures, image_to_push, insecure) + string_time = date.strftime("%Y%m%dT%H%M%SZ") - if config.gh_run_id: - push_manifest(config, architectures, image_to_push, insecure, config.gh_run_id) + return string_time - if release: - registry = args["registry"] + "/" + args["image"] - context_tag = args["release_version"] + "-context" - push_manifest( - config, architectures, args["image"], insecure, args["release_version"] - ) - push_manifest(config, architectures, args["image"], insecure, context_tag) - if sign: - sign_and_verify(registry, args["release_version"]) - sign_and_verify(registry, context_tag) + +def get_release() -> Dict: + with open("release.json") as release: + return json.load(release) + + +def get_git_release_tag() -> tuple[str, bool]: + release_env_var = os.getenv("triggered_by_git_tag") + + # that means we are in a release and only return the git_tag; otherwise we want to return the patch_id + # appended to ensure the image created is unique and does not interfere + if release_env_var is not None: + return release_env_var, True + + patch_id = os.environ.get("version_id", "latest") + return patch_id, False + + +def copy_into_container(client, src, dst): + """Copies a local file into a running container.""" + + os.chdir(os.path.dirname(src)) + srcname = os.path.basename(src) + with tarfile.open(src + ".tar", mode="w") as tar: + tar.add(srcname) + + name, dst = dst.split(":") + container = client.containers.get(name) + + with open(src + ".tar", "rb") as fd: + container.put_archive(os.path.dirname(dst), fd.read()) """ @@ -148,147 +301,1197 @@ def build_and_push_image( """ -def push_manifest( - config: DevConfig, - architectures: Set[str], - image_name: str, - insecure: bool = False, - image_tag: str = "latest", -) -> None: - logger.info(f"Pushing manifest for {image_tag}") - final_manifest = "{0}/{1}:{2}".format(config.repo_url, image_name, image_tag) - remove_args = ["docker", "manifest", "rm", final_manifest] - logger.info("Removing existing manifest") - run_cli_command(remove_args, fail_on_error=False) - - create_args = [ +# This method calls docker directly on the command line, this is different from the rest of the code which uses +# Sonar as an interface to docker. We decided to keep this asymmetry for now, as Sonar will be removed soon. + + +def create_and_push_manifest(image: str, tag: str) -> None: + final_manifest = image + ":" + tag + + args = [ "docker", "manifest", "create", final_manifest, + "--amend", + final_manifest + "-amd64", + "--amend", + final_manifest + "-arm64", ] + args_str = " ".join(args) + logger.debug(f"creating new manifest: {args_str}") + cp = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if insecure: - create_args.append("--insecure") - - for arch in architectures: - create_args.extend(["--amend", final_manifest + "-" + arch]) + if cp.returncode != 0: + raise Exception(cp.stderr) - logger.info("Creating new manifest") - run_cli_command(create_args) + args = ["docker", "manifest", "push", final_manifest] + args_str = " ".join(args) + logger.info(f"pushing new manifest: {args_str}") + cp = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - push_args = ["docker", "manifest", "push", final_manifest] - logger.info("Pushing new manifest") - run_cli_command(push_args) + if cp.returncode != 0: + raise Exception(cp.stderr) -# Raises exceptions by default -def run_cli_command(args: List[str], fail_on_error: bool = True) -> None: - command = " ".join(args) - logger.debug(f"Running: {command}") +def try_get_platform_data(client, image): + """Helper function to try and retrieve platform data.""" try: - cp = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True, - check=False, - ) + return client.images.get_registry_data(image) except Exception as e: - logger.error(f" Command raised the following exception: {e}") - if fail_on_error: - raise Exception - else: - logger.warning("Continuing...") - return + logger.error("Failed to get registry data for image: {0}. Error: {1}".format(image, str(e))) + return None - if cp.returncode != 0: - error_msg = cp.stderr.decode().strip() - stdout = cp.stdout.decode().strip() - logger.error(f"Error running command") - logger.error(f"stdout:\n{stdout}") - logger.error(f"stderr:\n{error_msg}") - if fail_on_error: - raise Exception + +""" +Checks if a docker image supports AMD and ARM platforms by inspecting the registry data. + +:param str image: The image name and tag +""" + + +def check_multi_arch(image: str, suffix: str) -> bool: + client = docker.from_env() + platforms = ["linux/amd64", "linux/arm64"] + + for img in [image, image + suffix]: + reg_data = try_get_platform_data(client, img) + if reg_data is not None and all(reg_data.has_platform(p) for p in platforms): + logger.info("Base image {} supports multi architecture, building for ARM64 and AMD64".format(img)) + return True + + logger.info("Base image {} is single-arch, building only for AMD64.".format(img)) + return False + + +@TRACER.start_as_current_span("sonar_build_image") +def sonar_build_image( + image_name: str, + build_configuration: BuildConfiguration, + args: Dict[str, str] = None, + inventory="inventory.yaml", + with_sbom: bool = True, +): + """Calls sonar to build `image_name` with arguments defined in `args`.""" + span = trace.get_current_span() + span.set_attribute("meko.image_name", image_name) + span.set_attribute("meko.inventory", inventory) + if args: + span.set_attribute("meko.build_args", str(args)) + + build_options = { + # Will continue building an image if it finds an error. See next comment. + "continue_on_errors": True, + # But will still fail after all the tasks have completed + "fail_on_errors": True, + "pipeline": build_configuration.pipeline, + } + + logger.info(f"Sonar config bc: {build_configuration}, args: {args}, for image: {image_name}") + + process_image( + image_name, + skip_tags=build_configuration.get_skip_tags(), + include_tags=build_configuration.get_include_tags(), + build_args=build_configuration.build_args(args), + inventory=inventory, + build_options=build_options, + ) + + if with_sbom: + produce_sbom(build_configuration, args) + + +@TRACER.start_as_current_span("produce_sbom") +def produce_sbom(build_configuration, args): + span = trace.get_current_span() + if not is_running_in_evg_pipeline(): + logger.info("Skipping SBOM Generation (enabled only for EVG)") + return + + try: + image_pull_spec = args["quay_registry"] + args.get("ubi_suffix", "") + except KeyError: + logger.error(f"Could not find image pull spec. Args: {args}, BuildConfiguration: {build_configuration}") + logger.error(f"Skipping SBOM generation") + return + + try: + image_tag = args["release_version"] + span.set_attribute("meko.release_version", image_tag) + except KeyError: + logger.error(f"Could not find image tag. Args: {args}, BuildConfiguration: {build_configuration}") + logger.error(f"Skipping SBOM generation") + return + + image_pull_spec = f"{image_pull_spec}:{image_tag}" + print(f"Producing SBOM for image: {image_pull_spec} args: {args}") + + if "platform" in args: + if args["platform"] == "arm64": + platform = "linux/arm64" + elif args["platform"] == "amd64": + platform = "linux/amd64" else: - logger.warning("Continuing...") - return + # TODO: return here? + logger.error(f"Unrecognized architectures in {args}. Skipping SBOM generation") + else: + platform = "linux/amd64" + generate_sbom(image_pull_spec, platform) -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--image-name", type=str) - parser.add_argument("--release", action="store_true", default=False) - parser.add_argument( - "--arch", - choices=["amd64", "arm64"], - nargs="+", - help="for daily builds only, specify the list of architectures to build for images", + +def build_tests_image(build_configuration: BuildConfiguration): + """ + Builds image used to run tests. + """ + image_name = "test" + + # helm directory needs to be copied over to the tests docker context. + helm_src = "helm_chart" + helm_dest = "docker/mongodb-enterprise-tests/helm_chart" + requirements_dest = "docker/mongodb-enterprise-tests/requirements.txt" + public_src = "public" + public_dest = "docker/mongodb-enterprise-tests/public" + + # Remove existing directories/files if they exist + shutil.rmtree(helm_dest, ignore_errors=True) + shutil.rmtree(public_dest, ignore_errors=True) + + # Copy directories and files (recursive copy) + shutil.copytree(helm_src, helm_dest) + shutil.copytree(public_src, public_dest) + shutil.copyfile("release.json", "docker/mongodb-enterprise-tests/release.json") + shutil.copyfile("requirements.txt", requirements_dest) + + python_version = os.getenv("PYTHON_VERSION", "") + if python_version == "": + raise Exception("Missing PYTHON_VERSION environment variable") + + buildargs = dict({"python_version": python_version}) + + sonar_build_image(image_name, build_configuration, buildargs, "inventories/test.yaml") + + +def build_operator_image(build_configuration: BuildConfiguration): + """Calculates arguments required to build the operator image, and starts the build process.""" + # In evergreen, we can pass test_suffix env to publish the operator to a quay + # repository with a given suffix. + test_suffix = os.environ.get("test_suffix", "") + log_automation_config_diff = os.environ.get("LOG_AUTOMATION_CONFIG_DIFF", "false") + version, _ = get_git_release_tag() + + args = { + "version": version, + "log_automation_config_diff": log_automation_config_diff, + "test_suffix": test_suffix, + "debug": build_configuration.debug, + } + + logger.info(f"Building Operator args: {args}") + + build_image_generic(build_configuration, "operator", "inventory.yaml", args) + + +def build_database_image(build_configuration: BuildConfiguration): + """ + Builds a new database image. + """ + release = get_release() + version = release["databaseImageVersion"] + args = {"version": version} + build_image_generic(build_configuration, "database", "inventories/database.yaml", args) + + +def build_CLI_SBOM(build_configuration: BuildConfiguration): + if not is_running_in_evg_pipeline(): + logger.info("Skipping SBOM Generation (enabled only for EVG)") + return + + if build_configuration.architecture is None or len(build_configuration.architecture) == 0: + architectures = ["linux/amd64", "linux/arm64", "darwin/arm64", "darwin/amd64"] + elif "arm64" in build_configuration.architecture: + architectures = ["linux/arm64", "darwin/arm64"] + elif "amd64" in build_configuration.architecture: + architectures = ["linux/amd64", "darwin/amd64"] + else: + logger.error(f"Unrecognized architectures {build_configuration.architecture}. Skipping SBOM generation") + return + + release = get_release() + version = release["mongodbOperator"] + + for architecture in architectures: + generate_sbom_for_cli(version, architecture) + + +def build_operator_image_patch(build_configuration: BuildConfiguration): + """This function builds the operator locally and pushed into an existing + Docker image. This is the fastest way I could image we can do this.""" + + client = docker.from_env() + # image that we know is where we build operator. + image_repo = ( + build_configuration.base_repository + "/" + build_configuration.image_type + "/mongodb-enterprise-operator" ) - parser.add_argument("--tag", type=str) - parser.add_argument("--sign", action="store_true", default=False) - parser.add_argument("--insecure", action="store_true", default=False) - return parser.parse_args() + image_tag = "latest" + repo_tag = image_repo + ":" + image_tag + logger.debug(f"Pulling image: {repo_tag}") + try: + image = client.images.get(repo_tag) + except docker.errors.ImageNotFound: + logger.debug("Operator image does not exist locally. Building it now") + build_operator_image(build_configuration) + return -""" -Takes arguments: ---image-name : The name of the image to build, must be one of VALID_IMAGE_NAMES ---release : We push the image to the registry only if this flag is set ---architecture : List of architectures to build for the image ---sign : Sign images with our private key if sign is set (only for release) + logger.debug("Done") + too_old = datetime.now() - timedelta(hours=3) + image_timestamp = datetime.fromtimestamp( + image.history()[0]["Created"] + ) # Layer 0 is the latest added layer to this Docker image. [-1] is the FROM layer. + + if image_timestamp < too_old: + logger.info("Current operator image is too old, will rebuild it completely first") + build_operator_image(build_configuration) + return + + container_name = "mongodb-enterprise-operator" + operator_binary_location = "/usr/local/bin/mongodb-enterprise-operator" + try: + client.containers.get(container_name).remove() + logger.debug(f"Removed {container_name}") + except docker.errors.NotFound: + pass + + container = client.containers.run(repo_tag, name=container_name, entrypoint="sh", detach=True) + + logger.debug("Building operator with debugging symbols") + subprocess.run(["make", "manager"], check=True, stdout=subprocess.PIPE) + logger.debug("Done building the operator") + + copy_into_container( + client, + os.getcwd() + "/docker/mongodb-enterprise-operator/content/mongodb-enterprise-operator", + container_name + ":" + operator_binary_location, + ) + + # Commit changes on disk as a tag + container.commit( + repository=image_repo, + tag=image_tag, + ) + # Stop this container so we can use it next time + container.stop() + container.remove() + + logger.info("Pushing operator to {}:{}".format(image_repo, image_tag)) + client.images.push( + repository=image_repo, + tag=image_tag, + ) + + +def get_supported_variants_for_image(image: str) -> List[str]: + return get_release()["supportedImages"][image]["variants"] + + +def image_config( + image_name: str, + name_prefix: str = "mongodb-enterprise-", + s3_bucket: str = "enterprise-operator-dockerfiles", + ubi_suffix: str = "-ubi", + base_suffix: str = "", +) -> Tuple[str, Dict[str, str]]: + """Generates configuration for an image suitable to be passed + to Sonar. + + It returns a dictionary with registries and S3 configuration.""" + args = { + "quay_registry": "{}/{}{}".format(QUAY_REGISTRY_URL, name_prefix, image_name), + "ecr_registry_ubi": "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/{}{}".format(name_prefix, image_name), + "s3_bucket_http": "https://{}.s3.amazonaws.com/dockerfiles/{}{}".format(s3_bucket, name_prefix, image_name), + "ubi_suffix": ubi_suffix, + "base_suffix": base_suffix, + } + + return image_name, args + + +def args_for_daily_image(image_name: str) -> Dict[str, str]: + """Returns configuration for an image to be able to be pushed with Sonar. + + This includes the quay_registry and ospid corresponding to RedHat's project id. + """ + image_configs = [ + image_config("appdb"), + image_config("database"), + image_config("init-appdb"), + image_config("agent"), + image_config("init-database"), + image_config("init-ops-manager"), + image_config("operator"), + image_config("ops-manager"), + image_config("mongodb-agent", name_prefix="", ubi_suffix="-ubi", base_suffix="-ubi"), + image_config( + image_name="mongodb-kubernetes-operator", + name_prefix="", + s3_bucket="enterprise-operator-dockerfiles", + # community ubi image does not have a suffix in its name + ubi_suffix="", + ), + image_config( + image_name="mongodb-kubernetes-readinessprobe", + ubi_suffix="", + name_prefix="", + s3_bucket="enterprise-operator-dockerfiles", + ), + image_config( + image_name="mongodb-kubernetes-operator-version-upgrade-post-start-hook", + ubi_suffix="", + name_prefix="", + s3_bucket="enterprise-operator-dockerfiles", + ), + ] + + images = {k: v for k, v in image_configs} + return images[image_name] + + +def is_version_in_range(version: str, min_version: str, max_version: str) -> bool: + """Check if the version is in the range""" + try: + parsed_version = semver.VersionInfo.parse(version) + if parsed_version.prerelease: + logger.info(f"Excluding {version} from range {min_version}-{max_version} because it's a pre-release") + return False + version_without_rc = semver.VersionInfo.finalize_version(parsed_version) + except ValueError: + version_without_rc = version + if min_version and max_version: + return version_without_rc.match(">=" + min_version) and version_without_rc.match("<" + max_version) + return True + + +def get_versions_to_rebuild(supported_versions, min_version, max_version): + # this means we only want to release one version, we cannot rely on the below range function + # since the agent does not follow semver for comparison + if (min_version and max_version) and (min_version == max_version): + return [min_version] + return filter(lambda x: is_version_in_range(x, min_version, max_version), supported_versions) + + +def get_versions_to_rebuild_per_operator_version(supported_versions, operator_version): + """ + This function returns all versions sliced by a specific operator version. + If the input is `onlyAgents` then it only returns agents without the operator suffix. + """ + versions_to_rebuild = [] + + for version in supported_versions: + if operator_version == "onlyAgents": + # 1_ works because we append the operator version via "_", all agents end with "1". + if "1_" not in version: + versions_to_rebuild.append(version) + else: + if operator_version in version: + versions_to_rebuild.append(version) + return versions_to_rebuild + + +class TracedThreadPoolExecutor(ThreadPoolExecutor): + """Implementation of :class:ThreadPoolExecutor that will pass context into sub tasks.""" + + def __init__(self, tracer: Tracer, *args, **kwargs): + self.tracer = tracer + super().__init__(*args, **kwargs) + + def with_otel_context(self, c: otel_context.Context, fn: Callable): + otel_context.attach(c) + return fn() + + def submit(self, fn, *args, **kwargs): + """Submit a new task to the thread pool.""" + + # get the current otel context + c = otel_context.get_current() + if c: + return super().submit( + lambda: self.with_otel_context(c, lambda: fn(*args, **kwargs)), + ) + else: + return super().submit(lambda: fn(*args, **kwargs)) -Run with --help for more information -Example usage : `python pipeline.py --image-name agent --release --sign` -Builds and push the docker image to the registry -Many parameters are defined in the dev configuration, default path is : ~/.community-operator-dev/config.json """ +Starts the daily build process for an image. This function works for all images we support, for community and +enterprise operator. The list of supported image_name is defined in get_builder_function_for_image_name. +Builds an image for each version listed in ./release.json +The registry used to pull base image and output the daily build is configured in the image_config function, it is passed +as an argument to the inventories/daily.yaml file. +If the context image supports both ARM and AMD architectures, both will be built. +""" -def main() -> int: - args = _parse_args() - image_name = args.image_name - if image_name not in VALID_IMAGE_NAMES: - logger.error( - f"Invalid image name: {image_name}. Valid options are: {VALID_IMAGE_NAMES}" - ) - return 1 +def build_image_daily( + image_name: str, # corresponds to the image_name in the release.json + min_version: str = None, + max_version: str = None, + operator_version: str = None, +): + """Builds a daily image.""" + + def get_architectures_set(build_configuration, args): + """Determine the set of architectures to build for""" + arch_set = set(build_configuration.architecture) if build_configuration.architecture else set() + if arch_set == {"arm64"}: + raise ValueError("Building for ARM64 only is not supported yet") + + # Automatic architecture detection is the default behavior if 'arch' argument isn't specified + if arch_set == set(): + if check_multi_arch( + image=args["quay_registry"] + args["ubi_suffix"] + ":" + args["release_version"], + suffix="-context", + ): + arch_set = {"amd64", "arm64"} + else: + # When nothing specified and single-arch, default to amd64 + arch_set = {"amd64"} + + return arch_set + + def create_and_push_manifests(args: dict): + """Create and push manifests for all registries.""" + registries = [args["ecr_registry_ubi"], args["quay_registry"]] + tags = [args["release_version"], args["release_version"] + "-b" + args["build_id"]] + for registry in registries: + for tag in tags: + create_and_push_manifest(registry + args["ubi_suffix"], tag) + + def sign_image_concurrently(executor, args, futures, arch=None): + v = args["release_version"] + logger.info(f"Enqueuing signing task for version: {v}") + future = executor.submit(sign_image_in_repositories, args, arch) + futures.append(future) + + @TRACER.start_as_current_span("inner") + def inner(build_configuration: BuildConfiguration): + supported_versions = get_supported_version_for_image_matrix_handling(image_name) + variants = get_supported_variants_for_image(image_name) - # Handle dev config - config: DevConfig = load_config() - config.gh_run_id = args.tag + args = args_for_daily_image(image_name) + args["build_id"] = build_id() + + completed_versions = set() + + filtered_versions = get_versions_to_rebuild(supported_versions, min_version, max_version) + if operator_version: + filtered_versions = get_versions_to_rebuild_per_operator_version(filtered_versions, operator_version) + + logger.info("Building Versions for {}: {}".format(image_name, filtered_versions)) + + with TracedThreadPoolExecutor(TRACER) as executor: + futures = [] + for version in filtered_versions: + build_configuration = copy.deepcopy(build_configuration) + if build_configuration.include_tags is None: + build_configuration.include_tags = [] + build_configuration.include_tags.extend(variants) + + logger.info("Rebuilding {} with variants {}".format(version, variants)) + args["release_version"] = version + + arch_set = get_architectures_set(build_configuration, args) + + if version not in completed_versions: + if arch_set == {"amd64", "arm64"}: + # We need to release the non context amd64 and arm64 images first before we can create the sbom + for arch in arch_set: + # Suffix to append to images name for multi-arch (see usage in daily.yaml inventory) + args["architecture_suffix"] = f"-{arch}" + args["platform"] = arch + sonar_build_image( + "image-daily-build", + build_configuration, + args, + inventory="inventories/daily.yaml", + with_sbom=False, + ) + if build_configuration.sign: + sign_image_concurrently(executor, copy.deepcopy(args), futures, arch) + create_and_push_manifests(args) + for arch in arch_set: + args["architecture_suffix"] = f"-{arch}" + args["platform"] = arch + logger.info(f"Enqueuing SBOM production task for image: {version}") + future = executor.submit(produce_sbom, build_configuration, copy.deepcopy(args)) + futures.append(future) + if build_configuration.sign: + sign_image_concurrently(executor, copy.deepcopy(args), futures) + else: + # No suffix for single arch images + args["architecture_suffix"] = "" + args["platform"] = "amd64" + sonar_build_image( + "image-daily-build", + build_configuration, + args, + inventory="inventories/daily.yaml", + ) + if build_configuration.sign: + sign_image_concurrently(executor, copy.deepcopy(args), futures) + completed_versions.add(version) + + # wait for all signings to be done + logger.info("Waiting for all tasks to complete...") + encountered_error = False + # all the futures contain concurrent sbom and signing tasks + for future in futures: + try: + future.result() + except Exception as e: + logger.error(f"Error in future: {e}") + encountered_error = True + + executor.shutdown(wait=True) + logger.info("All tasks completed.") + + # we execute them concurrently with retries, if one of them eventually fails, we fail the whole task + if encountered_error: + logger.info("Some tasks failed.") + exit(1) + + return inner + + +@TRACER.start_as_current_span("sign_image_in_repositories") +def sign_image_in_repositories(args: Dict[str, str], arch: str = None): + span = trace.get_current_span() + repository = args["quay_registry"] + args["ubi_suffix"] + tag = args["release_version"] + if arch: + tag = f"{tag}-{arch}" + + span.set_attribute("meko.tag", tag) + + sign_image(repository, tag) + verify_signature(repository, tag) + + +def find_om_in_releases(om_version: str, releases: Dict[str, str]) -> Optional[str]: + """ + There are a few alternatives out there that allow for json-path or xpath-type + traversal of Json objects in Python, I don't have time to look for one of + them now but I have to do at some point. + """ + for release in releases: + if release["version"] == om_version: + for platform in release["platform"]: + if platform["package_format"] == "deb" and platform["arch"] == "x86_64": + for package in platform["packages"]["links"]: + if package["name"] == "tar.gz": + return package["download_link"] + return None + + +def get_om_releases() -> Dict[str, str]: + """Returns a dictionary representation of the Json document holdin all the OM + releases. + """ + ops_manager_release_archive = ( + "https://info-mongodb-com.s3.amazonaws.com/com-download-center/ops_manager_release_archive.json" + ) - # Warn user if trying to release E2E tests - if args.release and image_name == "e2e": - logger.warning( - "Warning : releasing E2E test will fail because E2E image has no release version" + return requests.get(ops_manager_release_archive).json() + + +def find_om_url(om_version: str) -> str: + """Gets a download URL for a given version of OM.""" + releases = get_om_releases() + + current_release = find_om_in_releases(om_version, releases["currentReleases"]) + if current_release is None: + current_release = find_om_in_releases(om_version, releases["oldReleases"]) + + if current_release is None: + raise ValueError("Ops Manager version {} could not be found".format(om_version)) + + return current_release + + +def build_init_om_image(build_configuration: BuildConfiguration): + release = get_release() + init_om_version = release["initOpsManagerVersion"] + args = {"version": init_om_version} + build_image_generic(build_configuration, "init-ops-manager", "inventories/init_om.yaml", args) + + +def build_om_image(build_configuration: BuildConfiguration): + # Make this a parameter for the Evergreen build + # https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds + om_version = os.environ.get("om_version") + if om_version is None: + raise ValueError("`om_version` should be defined.") + + om_download_url = os.environ.get("om_download_url", "") + if om_download_url == "": + om_download_url = find_om_url(om_version) + + args = { + "version": om_version, + "om_download_url": om_download_url, + } + build_image_generic(build_configuration, "ops-manager", "inventories/om.yaml", args) + + +def build_image_generic( + config: BuildConfiguration, + image_name: str, + inventory_file: str, + extra_args: dict = None, + registry_address: str = None, + is_multi_arch: bool = False, + multi_arch_args_list: list = None, + is_run_in_parallel: bool = False, +): + if not multi_arch_args_list: + multi_arch_args_list = [extra_args or {}] + + version = multi_arch_args_list[0].get("version", "") # the version is the same in multi-arch for each item + registry = f"{QUAY_REGISTRY_URL}/mongodb-enterprise-{image_name}" if not registry_address else registry_address + + for args in multi_arch_args_list: # in case we are building multiple architectures + args["quay_registry"] = registry + sonar_build_image(image_name, config, args, inventory_file, False) + if is_multi_arch: + # we only push the manifests of the context images here, + # since daily rebuilds will push the manifests for the proper images later + create_and_push_manifest(registry_address, f"{version}-context") + if not config.is_release_step_executed(): + # Normally daily rebuild would create and push the manifests for the non-context images. + # But since we don't run daily rebuilds on ecr image builds, we can do that step instead here. + # We only need to push manifests for multi-arch images. + create_and_push_manifest(registry_address, version) + if config.sign and config.is_release_step_executed(): + sign_and_verify_context_image(registry, version) + if config.is_release_step_executed() and version and QUAY_REGISTRY_URL in registry: + logger.info( + f"finished building context images, releasing them now via daily builds process for" + f" image: {image_name} and version: {version}!" ) + # Sleep for a random time between 0 and 5 seconds to distribute daily builds better, + # as we do a lot of things there that require network connections like: + # - Kondukto uploads, downloads + # - image verification and signings + # - manifest creations + # - docker image pushes + # - etc. + if is_run_in_parallel: + time.sleep(random.uniform(0, 5)) + build_image_daily(image_name, version, version)(config) + + +def sign_and_verify_context_image(registry, version): + sign_image(registry, version + "-context") + verify_signature(registry, version + "-context") + + +def build_init_appdb(build_configuration: BuildConfiguration): + release = get_release() + version = release["initAppDbVersion"] + base_url = "https://fastdl.mongodb.org/tools/db/" + mongodb_tools_url_ubi = "{}{}".format(base_url, release["mongodbToolsBundle"]["ubi"]) + args = {"version": version, "is_appdb": True, "mongodb_tools_url_ubi": mongodb_tools_url_ubi} + build_image_generic(build_configuration, "init-appdb", "inventories/init_appdb.yaml", args) + + +def build_agent_in_sonar( + build_configuration: BuildConfiguration, + image_version, + init_database_image, + mongodb_tools_url_ubi, + mongodb_agent_url_ubi: str, + agent_version, +): + args = { + "version": image_version, + "mongodb_tools_url_ubi": mongodb_tools_url_ubi, + "mongodb_agent_url_ubi": mongodb_agent_url_ubi, + "init_database_image": init_database_image, + } + + agent_quay_registry = QUAY_REGISTRY_URL + f"/mongodb-agent-ubi" + args["quay_registry"] = agent_quay_registry + args["agent_version"] = agent_version + + build_image_generic( + config=build_configuration, + image_name="mongodb-agent", + inventory_file="inventories/agent.yaml", + extra_args=args, + registry_address=agent_quay_registry, + is_run_in_parallel=True, + ) + + +def build_multi_arch_agent_in_sonar( + build_configuration: BuildConfiguration, + image_version, + tools_version, +): + """ + Creates the multi-arch non-operator suffixed version of the agent. + This is a drop-in replacement for the agent + release from MCO. + This should only be called during releases. + Which will lead to a release of the multi-arch + images to quay and ecr. + """ + + logger.info(f"building multi-arch base image for: {image_version}") + is_release = build_configuration.is_release_step_executed() + args = { + "version": image_version, + "tools_version": tools_version, + } + + arch_arm = { + "agent_distro": "amzn2_aarch64", + "tools_distro": get_tools_distro(tools_version=tools_version)["arm"], + "architecture": "arm64", + } + arch_amd = { + "agent_distro": "rhel9_x86_64", + "tools_distro": get_tools_distro(tools_version=tools_version)["amd"], + "architecture": "amd64", + } + + new_rhel_tool_version = "100.10.0" + if Version(tools_version) >= Version(new_rhel_tool_version): + arch_arm["tools_distro"] = "rhel93-aarch64" + arch_amd["tools_distro"] = "rhel93-x86_64" + + ecr_registry = os.environ.get("REGISTRY", "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev") + ecr_agent_registry = ecr_registry + f"/mongodb-agent-ubi" + quay_agent_registry = QUAY_REGISTRY_URL + f"/mongodb-agent-ubi" + joined_args = list() + for arch in [arch_arm, arch_amd]: + joined_args.append(args | arch) + build_image_generic( + config=build_configuration, + image_name="mongodb-agent", + inventory_file="inventories/agent_non_matrix.yaml", + multi_arch_args_list=joined_args, + registry_address=quay_agent_registry if is_release else ecr_agent_registry, + is_multi_arch=True, + is_run_in_parallel=True, + ) + + +def build_agent_default_case(build_configuration: BuildConfiguration): + """ + Build the agent only for the latest operator for patches and operator releases. + + See more information in the function: build_agent_on_agent_bump + """ + release = get_release() - # Skipping release tasks by default - if not args.release: - config.ensure_skip_tag("release") - if args.sign: - logger.warning("--sign flag has no effect without --release") + operator_version, is_release = get_git_release_tag() - if args.arch: - arch_set = set(args.arch) + # We need to release [all agents x latest operator] on operator releases + if is_release: + agent_versions_to_build = gather_all_supported_agent_versions(release) + # We only need [latest agents (for each OM major version and for CM) x patch ID] for patches else: - # Default is multi-arch - arch_set = {"amd64", "arm64"} - logger.info(f"Building for architectures: {','.join(arch_set)}") + agent_versions_to_build = gather_latest_agent_versions(release) + + logger.info(f"Building Agent versions: {agent_versions_to_build} for Operator versions: {operator_version}") + + tasks_queue = Queue() + max_workers = 1 + if build_configuration.parallel: + max_workers = None + if build_configuration.parallel_factor > 0: + max_workers = build_configuration.parallel_factor + with ProcessPoolExecutor(max_workers=max_workers) as executor: + logger.info(f"running with factor of {max_workers}") + for agent_version in agent_versions_to_build: + # We don't need to keep create and push the same image on every build. + # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. + if build_configuration.is_release_step_executed() or build_configuration.all_agents: + tasks_queue.put( + executor.submit( + build_multi_arch_agent_in_sonar, + build_configuration, + agent_version[0], + agent_version[1], + ) + ) + _build_agent_operator( + agent_version, build_configuration, executor, operator_version, tasks_queue, is_release + ) + + queue_exception_handling(tasks_queue) + + +def build_agent_on_agent_bump(build_configuration: BuildConfiguration): + """ + Build the agent matrix (operator version x agent version), triggered by PCT. + + We have three cases where we need to build the agent: + - e2e test runs + - operator releases + - OM/CM bumps via PCT + + We don't require building a full matrix on e2e test runs and operator releases. + "Operator releases" and "e2e test runs" require only the latest operator x agents + + In OM/CM bumps, we release a new agent which we potentially require to release to older operators as well. + This function takes care of that. + """ + release = get_release() + is_release = build_configuration.is_release_step_executed() + + if build_configuration.all_agents: + # We need to release [all agents x latest operator] on operator releases to make e2e tests work + # This was changed previously in https://github.com/10gen/ops-manager-kubernetes/pull/3960 + agent_versions_to_build = gather_all_supported_agent_versions(release) + else: + # we only need to release the latest images, we don't need to re-push old images, as we don't clean them up anymore. + agent_versions_to_build = gather_latest_agent_versions(release) + + legacy_agent_versions_to_build = release["supportedImages"]["mongodb-agent"]["versions"] + + tasks_queue = Queue() + max_workers = 1 + if build_configuration.parallel: + max_workers = None + if build_configuration.parallel_factor > 0: + max_workers = build_configuration.parallel_factor + with ProcessPoolExecutor(max_workers=max_workers) as executor: + logger.info(f"running with factor of {max_workers}") + + # We need to regularly push legacy agents, otherwise ecr lifecycle policy will expire them. + # We only need to push them once in a while to ecr, so no quay required + if not is_release: + for legacy_agent in legacy_agent_versions_to_build: + tasks_queue.put( + executor.submit( + build_multi_arch_agent_in_sonar, + build_configuration, + legacy_agent, + # we assume that all legacy agents are build using that tools version + "100.9.4", + ) + ) + + for agent_version in agent_versions_to_build: + # We don't need to keep create and push the same image on every build. + # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. + if build_configuration.is_release_step_executed() or build_configuration.all_agents: + tasks_queue.put( + executor.submit( + build_multi_arch_agent_in_sonar, + build_configuration, + agent_version[0], + agent_version[1], + ) + ) + for operator_version in get_supported_operator_versions(): + logger.info(f"Building Agent versions: {agent_version} for Operator versions: {operator_version}") + _build_agent_operator( + agent_version, build_configuration, executor, operator_version, tasks_queue, is_release + ) + + queue_exception_handling(tasks_queue) + + +def queue_exception_handling(tasks_queue): + exceptions_found = False + for task in tasks_queue.queue: + if task.exception() is not None: + exceptions_found = True + logger.fatal(f"The following exception has been found when building: {task.exception()}") + if exceptions_found: + raise Exception( + f"Exception(s) found when processing Agent images. \nSee also previous logs for more info\nFailing the build" + ) + + +def _build_agent_operator( + agent_version: Tuple[str, str], + build_configuration: BuildConfiguration, + executor: ProcessPoolExecutor, + operator_version: str, + tasks_queue: Queue, + use_quay: bool = False, +): + agent_distro = "rhel9_x86_64" + tools_version = agent_version[1] + tools_distro = get_tools_distro(tools_version)["amd"] + image_version = f"{agent_version[0]}_{operator_version}" + mongodb_tools_url_ubi = ( + f"https://downloads.mongodb.org/tools/db/mongodb-database-tools-{tools_distro}-{tools_version}.tgz" + ) + mongodb_agent_url_ubi = f"https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-{agent_version[0]}.{agent_distro}.tar.gz" + # We use Quay if not in a patch + # We could rely on input params (quay_registry or registry), but it makes templating more complex in the inventory + non_quay_registry = os.environ.get("REGISTRY", "268558157000.dkr.ecr.us-east-1.amazonaws.com/dev") + base_init_database_repo = QUAY_REGISTRY_URL if use_quay else non_quay_registry + init_database_image = f"{base_init_database_repo}/mongodb-enterprise-init-database-ubi:{operator_version}" + + tasks_queue.put( + executor.submit( + build_agent_in_sonar, + build_configuration, + image_version, + init_database_image, + mongodb_tools_url_ubi, + mongodb_agent_url_ubi, + agent_version[0], + ) + ) + + +def gather_all_supported_agent_versions(release: Dict) -> List[Tuple[str, str]]: + # This is a list of a tuples - agent version and corresponding tools version + agent_versions_to_build = list() + agent_versions_to_build.append( + ( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], + ) + ) + for _, om in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].items(): + agent_versions_to_build.append((om["agent_version"], om["tools_version"])) + + # lets not build the same image multiple times + return sorted(list(set(agent_versions_to_build))) + + +def gather_latest_agent_versions(release: Dict) -> List[Tuple[str, str]]: + """ + This function is used when we release a new agent via OM bump. + That means we will need to release that agent with all supported operators. + Since we don't want to release all agents again, we only release the latest, which will contain the newly added one + :return: the latest agent for each major version + """ + agent_versions_to_build = list() + agent_versions_to_build.append( + ( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], + ) + ) + + latest_versions = {} + + for version in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].keys(): + parsed_version = semver.VersionInfo.parse(version) + major_version = parsed_version.major + if major_version in latest_versions: + latest_parsed_version = semver.VersionInfo.parse(str(latest_versions[major_version])) + latest_versions[major_version] = max(parsed_version, latest_parsed_version) + else: + latest_versions[major_version] = version + + for major_version, latest_version in latest_versions.items(): + agent_versions_to_build.append( + ( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ + "agent_version" + ], + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ + "tools_version" + ], + ) + ) + + # TODO: Remove this once we don't need to use OM 7.0.12 in the OM Multicluster DR tests + # https://jira.mongodb.org/browse/CLOUDP-297377 + agent_versions_to_build.append(("107.0.12.8669-1", "100.10.0")) + + return sorted(list(set(agent_versions_to_build))) + + +def get_builder_function_for_image_name() -> Dict[str, Callable]: + """Returns a dictionary of image names that can be built.""" + + image_builders = { + "cli": build_CLI_SBOM, + "test": build_tests_image, + "operator": build_operator_image, + "operator-quick": build_operator_image_patch, + "database": build_database_image, + "agent-pct": build_agent_on_agent_bump, + "agent": build_agent_default_case, + # + # Init images + "init-appdb": build_init_appdb, + "init-database": build_init_database, + "init-ops-manager": build_init_om_image, + # + # Daily builds + "operator-daily": build_image_daily("operator"), + "appdb-daily": build_image_daily("appdb"), + "database-daily": build_image_daily("database"), + "init-appdb-daily": build_image_daily("init-appdb"), + "init-database-daily": build_image_daily("init-database"), + "init-ops-manager-daily": build_image_daily("init-ops-manager"), + "ops-manager-6-daily": build_image_daily("ops-manager", min_version="6.0.0", max_version="7.0.0"), + "ops-manager-7-daily": build_image_daily("ops-manager", min_version="7.0.0", max_version="8.0.0"), + "ops-manager-8-daily": build_image_daily("ops-manager", min_version="8.0.0", max_version="9.0.0"), + # + # Ops Manager image + "ops-manager": build_om_image, + # This only builds the agents without the operator suffix + "mongodb-agent-daily": build_image_daily("mongodb-agent", operator_version="onlyAgents"), + # Community images + "mongodb-kubernetes-readinessprobe-daily": build_image_daily( + "mongodb-kubernetes-readinessprobe", + ), + "mongodb-kubernetes-operator-version-upgrade-post-start-hook-daily": build_image_daily( + "mongodb-kubernetes-operator-version-upgrade-post-start-hook", + ), + "mongodb-kubernetes-operator-daily": build_image_daily("mongodb-kubernetes-operator"), + } + + # since we only support the last 3 operator versions, we can build the following names which each matches to an + # operator version we support and rebuild: + # - mongodb-agent-daily-1 + # - mongodb-agent-daily-2 + # - mongodb-agent-daily-3 + # get_supported_operator_versions returns the last three supported operator versions in a sorted manner + i = 1 + for operator_version in get_supported_operator_versions(): + image_builders[f"mongodb-agent-{i}-daily"] = build_image_daily( + "mongodb-agent", operator_version=operator_version + ) + i += 1 + + return image_builders + + +# TODO: nam static: remove this once static containers becomes the default +def build_init_database(build_configuration: BuildConfiguration): + release = get_release() + version = release["initDatabaseVersion"] # comes from release.json + base_url = "https://fastdl.mongodb.org/tools/db/" + mongodb_tools_url_ubi = "{}{}".format(base_url, release["mongodbToolsBundle"]["ubi"]) + args = {"version": version, "mongodb_tools_url_ubi": mongodb_tools_url_ubi, "is_appdb": False} + build_image_generic(build_configuration, "init-database", "inventories/init_database.yaml", args) + + +def build_image(image_name: str, build_configuration: BuildConfiguration): + """Builds one of the supported images by its name.""" + get_builder_function_for_image_name()[image_name](build_configuration) + + +def build_all_images( + images: Iterable[str], + builder: str, + debug: bool = False, + parallel: bool = False, + architecture: Optional[List[str]] = None, + sign: bool = False, + all_agents: bool = False, + parallel_factor: int = 0, +): + """Builds all the images in the `images` list.""" + build_configuration = operator_build_configuration( + builder, parallel, debug, architecture, sign, all_agents, parallel_factor + ) + if sign: + mongodb_artifactory_login() + for image in images: + build_image(image, build_configuration) + + +def calculate_images_to_build( + images: List[str], include: Optional[List[str]], exclude: Optional[List[str]] +) -> Set[str]: + """ + Calculates which images to build based on the `images`, `include` and `exclude` sets. + + >>> calculate_images_to_build(["a", "b"], ["a"], ["b"]) + ... ["a"] + """ + + if not include and not exclude: + return set(images) + include = set(include or []) + exclude = set(exclude or []) + images = set(images or []) + + for image in include.union(exclude): + if image not in images: + raise ValueError("Image definition {} not found".format(image)) + + images_to_build = include.intersection(images) + if exclude: + images_to_build = images.difference(exclude) + return images_to_build + + +def main(): + _setup_tracing() + + parser = argparse.ArgumentParser() + parser.add_argument("--include", action="append") + parser.add_argument("--exclude", action="append") + parser.add_argument("--builder", default="docker", type=str) + parser.add_argument("--list-images", action="store_true") + parser.add_argument("--parallel", action="store_true", default=False) + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument( + "--arch", + choices=["amd64", "arm64"], + nargs="+", + help="for daily builds only, specify the list of architectures to build for images", + ) + parser.add_argument("--sign", action="store_true", default=False) + parser.add_argument( + "--parallel-factor", + type=int, + default=0, + help="the factor on how many agents are build in parallel. 0 means all CPUs will be used", + ) + parser.add_argument( + "--all-agents", + action="store_true", + default=False, + help="optional parameter to be able to push " + "all non operator suffixed agents, even if we are not in a release", + ) + args = parser.parse_args() + + if args.list_images: + print(get_builder_function_for_image_name().keys()) + sys.exit(0) + + if args.arch == ["arm64"]: + print("Building for arm64 only is not supported yet") + sys.exit(1) if not args.sign: logger.warning("--sign flag not provided, images won't be signed") - image_args = build_image_args(config, image_name) + images_to_build = calculate_images_to_build( + list(get_builder_function_for_image_name().keys()), args.include, args.exclude + ) - build_and_push_image( - image_name, config, image_args, arch_set, args.release, args.sign, args.insecure + build_all_images( + images_to_build, + args.builder, + debug=args.debug, + parallel=args.parallel, + architecture=args.arch, + sign=args.sign, + all_agents=args.all_agents, + parallel_factor=args.parallel_factor, ) - return 0 if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/pipeline_test.py b/pipeline_test.py new file mode 100644 index 000000000..ee6b37ad4 --- /dev/null +++ b/pipeline_test.py @@ -0,0 +1,227 @@ +import os +import subprocess +import unittest +from unittest.mock import patch + +import pytest + +from pipeline import ( + calculate_images_to_build, + gather_all_supported_agent_versions, + gather_latest_agent_versions, + get_versions_to_rebuild, + get_versions_to_rebuild_per_operator_version, + is_version_in_range, + operator_build_configuration, +) +from scripts.evergreen.release.images_signing import run_command_with_retries + +release_json = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "cloud_manager": "13.19.0.8937-1", + "cloud_manager_tools": "100.9.4", + "ops_manager": { + "6.0.0": {"agent_version": "12.0.30.7791-1", "tools_version": "100.9.4"}, + "6.0.21": {"agent_version": "12.0.29.7785-1", "tools_version": "100.9.4"}, + "6.0.22": {"agent_version": "12.0.30.7791-1", "tools_version": "100.9.4"}, + "7.0.0": {"agent_version": "107.0.1.8507-1", "tools_version": "100.9.4"}, + "7.0.1": {"agent_version": "107.0.1.8507-1", "tools_version": "100.9.4"}, + "7.0.2": {"agent_version": "107.0.2.8531-1", "tools_version": "100.9.4"}, + "7.0.3": {"agent_version": "107.0.3.8550-1", "tools_version": "100.9.4"}, + "6.0.23": {"agent_version": "12.0.31.7825-1", "tools_version": "100.9.4"}, + "7.0.4": {"agent_version": "107.0.4.8567-1", "tools_version": "100.9.4"}, + "7.0.6": {"agent_version": "107.0.6.8587-1", "tools_version": "100.9.4"}, + "7.0.7": {"agent_version": "107.0.7.8596-1", "tools_version": "100.9.4"}, + "7.0.10": {"agent_version": "107.0.10.8627-1", "tools_version": "100.9.5"}, + "7.0.11": {"agent_version": "107.0.11.8645-1", "tools_version": "100.10.0"}, + }, + } + } + } +} + + +def test_operator_build_configuration(): + with patch.dict(os.environ, {"distro": "a_distro", "BASE_REPO_URL": "somerepo/url", "namespace": "something"}): + config = operator_build_configuration("builder", True, False) + assert config.image_type == "a_distro" + assert config.base_repository == "somerepo/url" + assert config.namespace == "something" + + +def test_operator_build_configuration_defaults(): + with patch.dict( + os.environ, + { + "BASE_REPO_URL": "", + }, + ): + config = operator_build_configuration("builder", True, False) + assert config.image_type == "ubi" + assert config.base_repository == "" + assert config.namespace == "default" + + +@pytest.mark.parametrize( + "test_case", + [ + (["a", "b", "c"], ["a"], ["b"], {"a", "c"}), + (["a", "b", "c"], ["a", "b"], None, {"a", "b"}), + (["a", "b", "c"], None, ["a"], {"b", "c"}), + (["a", "b", "c"], [], [], {"a", "b", "c"}), + (["a", "b", "c"], ["d"], None, ValueError), + (["a", "b", "c"], None, ["d"], ValueError), + ([], ["a"], ["b"], ValueError), + (["a", "b", "c"], None, None, {"a", "b", "c"}), + ], +) +def test_calculate_images_to_build(test_case): + images, include, exclude, expected = test_case + if expected is ValueError: + with pytest.raises(ValueError): + calculate_images_to_build(images, include, exclude) + else: + assert calculate_images_to_build(images, include, exclude) == expected + + +@pytest.mark.parametrize( + "version,min_version,max_version,expected", + [ + # When one bound is empty or None, always return True + ("7.0.0", "8.0.0", "", True), + ("7.0.0", "8.0.0", None, True), + ("9.0.0", "", "8.0.0", True), + # Upper bound is excluded + ("8.1.1", "8.0.0", "8.1.1", False), + # Lower bound is included + ("8.0.0", "8.0.0", "8.1.1", True), + # Test some values + ("8.5.2", "8.5.1", "8.5.3", True), + ("8.5.2", "8.5.3", "8.4.2", False), + ], +) +def test_is_version_in_range(version, min_version, max_version, expected): + assert is_version_in_range(version, min_version, max_version) == expected + + +@pytest.mark.parametrize( + "description,case", + [ + ("No skip or include tags", {"skip_tags": [], "include_tags": [], "expected": True}), + ("Include 'release' only", {"skip_tags": [], "include_tags": ["release"], "expected": True}), + ("Skip 'release' only", {"skip_tags": ["release"], "include_tags": [], "expected": False}), + ("Include non-release, no skip", {"skip_tags": [], "include_tags": ["test", "deploy"], "expected": False}), + ("Skip non-release, no include", {"skip_tags": ["test", "deploy"], "include_tags": [], "expected": True}), + ("Include and skip 'release'", {"skip_tags": ["release"], "include_tags": ["release"], "expected": False}), + ( + "Skip non-release, include 'release'", + {"skip_tags": ["test", "deploy"], "include_tags": ["release"], "expected": True}, + ), + ], +) +def test_is_release_step_executed(description, case): + config = operator_build_configuration("builder", True, False) + config.skip_tags = case["skip_tags"] + config.include_tags = case["include_tags"] + result = config.is_release_step_executed() + assert result == case["expected"], f"Test failed: {description}. Expected {case['expected']}, got {result}." + + +def test_build_latest_agent_versions(): + latest_agents = gather_latest_agent_versions(release_json) + expected_agents = [ + ("107.0.11.8645-1", "100.10.0"), + # TODO: Remove this once we don't need to use OM 7.0.12 in the OM Multicluster DR tests + # https://jira.mongodb.org/browse/CLOUDP-297377 + ("107.0.12.8669-1", "100.10.0"), + ("12.0.31.7825-1", "100.9.4"), + ("13.19.0.8937-1", "100.9.4"), + ] + assert latest_agents == expected_agents + + +def test_get_versions_to_rebuild_same_version(): + supported_versions = gather_all_supported_agent_versions(release_json) + agents = get_versions_to_rebuild(supported_versions, "6.0.0_1.26.0", "6.0.0_1.26.0") + assert len(agents) == 1 + assert agents[0] == "6.0.0_1.26.0" + + +def test_get_versions_to_rebuild_multiple_versions(): + supported_versions = ["6.0.0", "6.0.1", "6.0.21", "6.11.0", "7.0.0"] + expected_agents = ["6.0.0", "6.0.1", "6.0.21"] + agents = get_versions_to_rebuild(supported_versions, "6.0.0", "6.10.0") + actual_agents = [] + for a in agents: + actual_agents.append(a) + assert actual_agents == expected_agents + + +def test_get_versions_to_rebuild_multiple_versions_per_operator(): + supported_versions = ["107.0.1.8507-1_1.27.0", "107.0.1.8507-1_1.28.0", "100.0.1.8507-1_1.28.0"] + expected_agents = ["107.0.1.8507-1_1.28.0", "100.0.1.8507-1_1.28.0"] + agents = get_versions_to_rebuild_per_operator_version(supported_versions, "1.28.0") + assert agents == expected_agents + + +def test_get_versions_to_rebuild_multiple_versions_per_operator_only_non_suffixed(): + supported_versions = ["107.0.1.8507-1_1.27.0", "107.0.10.8627-1-arm64", "100.0.10.8627-1"] + expected_agents = ["107.0.10.8627-1-arm64", "100.0.10.8627-1"] + agents = get_versions_to_rebuild_per_operator_version(supported_versions, "onlyAgents") + assert agents == expected_agents + + +command = ["echo", "Hello World"] + + +class TestRunCommandWithRetries(unittest.TestCase): + @patch("subprocess.run") + @patch("time.sleep", return_value=None) # to avoid actual sleeping during tests + def test_successful_command(self, mock_sleep, mock_run): + # Mock a successful command execution + mock_run.return_value = subprocess.CompletedProcess(command, 0, stdout="Hello World", stderr="") + + result = run_command_with_retries(command) + self.assertEqual(result.stdout, "Hello World") + mock_run.assert_called_once() + mock_sleep.assert_not_called() + + @patch("subprocess.run") + @patch("time.sleep", return_value=None) # to avoid actual sleeping during tests + def test_retryable_error(self, mock_sleep, mock_run): + # Mock a retryable error first, then a successful command execution + mock_run.side_effect = [ + subprocess.CalledProcessError(500, command, stderr="500 Internal Server Error"), + subprocess.CompletedProcess(command, 0, stdout="Hello World", stderr=""), + ] + + result = run_command_with_retries(command) + self.assertEqual(result.stdout, "Hello World") + self.assertEqual(mock_run.call_count, 2) + mock_sleep.assert_called_once() + + @patch("subprocess.run") + @patch("time.sleep", return_value=None) # to avoid actual sleeping during tests + def test_non_retryable_error(self, mock_sleep, mock_run): + # Mock a non-retryable error + mock_run.side_effect = subprocess.CalledProcessError(1, command, stderr="1 Some Error") + + with self.assertRaises(subprocess.CalledProcessError): + run_command_with_retries(command) + + self.assertEqual(mock_run.call_count, 1) + mock_sleep.assert_not_called() + + @patch("subprocess.run") + @patch("time.sleep", return_value=None) # to avoid actual sleeping during tests + def test_all_retries_fail(self, mock_sleep, mock_run): + # Mock all attempts to fail with a retryable error + mock_run.side_effect = subprocess.CalledProcessError(500, command, stderr="500 Internal Server Error") + + with self.assertRaises(subprocess.CalledProcessError): + run_command_with_retries(command, retries=3) + + self.assertEqual(mock_run.call_count, 3) + self.assertEqual(mock_sleep.call_count, 2) diff --git a/pkg/agentVersionManagement/get_agent_version.go b/pkg/agentVersionManagement/get_agent_version.go new file mode 100644 index 000000000..87469e429 --- /dev/null +++ b/pkg/agentVersionManagement/get_agent_version.go @@ -0,0 +1,250 @@ +package agentVersionManagement + +import ( + "encoding/json" + "os" + "strconv" + "strings" + "sync" + + "github.com/blang/semver" + "go.uber.org/zap" + "golang.org/x/xerrors" + + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/om" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +const ( + MappingFilePathEnv = "MDB_OM_VERSION_MAPPING_PATH" + mappingFileDefaultPath = "/usr/local/om_version_mapping.json" +) + +var om6StaticContainersSupport = semver.Version{ + Major: 6, + Minor: 0, + Patch: 21, +} + +var om7StaticContainersSupport = semver.Version{ + Major: 7, + Minor: 0, + Patch: 0, +} + +var initializationMutex sync.Mutex + +// AgentVersionManager handles the retrieval of agent versions. +// See https://docs.google.com/document/d/1rsj_Ng3IRGv74y1yMTiztfc0LEpBVizjGWFQTaYBz60/edit#heading=h.9frez6xnhit0 +type AgentVersionManager struct { + // This structure contains a map from semantic versions to semantic versions (string) + omToAgentVersionMapping map[omv1.OpsManagerVersion]omv1.AgentVersion + + // Major version to latest OM version + latestOMVersionsByMajor map[string]string + + // Agent version for Cloud Manager + agentVersionCM string +} + +var ( + versionManager *AgentVersionManager + lastUsedMappingPath string +) + +func newAgentVersionManager(omVersionToAgentVersion map[omv1.OpsManagerVersion]omv1.AgentVersion, cmVersion string) *AgentVersionManager { + omVersionsByMajor := make(map[string]string) + + if cmVersion == "" { + zap.S().Warnf("No version provided for Cloud Manager agent") + } + + for omVersion := range omVersionToAgentVersion { + majorOmVersion := getMajorVersion(string(omVersion)) + + if currentVersion, exists := omVersionsByMajor[majorOmVersion]; !exists || isLaterVersion(string(omVersion), currentVersion) { + omVersionsByMajor[majorOmVersion] = string(omVersion) + } + } + + return &AgentVersionManager{ + omToAgentVersionMapping: omVersionToAgentVersion, + latestOMVersionsByMajor: omVersionsByMajor, + agentVersionCM: cmVersion, + } +} + +// isLaterVersion compares two semantic versions and returns true if the first is later than the second +func isLaterVersion(version1, version2 string) bool { + splitVersion := func(version string) []int { + var parts []int + for _, part := range strings.Split(strings.Split(version, "-")[0], ".") { + if num, err := strconv.Atoi(part); err == nil { + parts = append(parts, num) + } + } + return parts + } + + v1Parts := splitVersion(version1) + v2Parts := splitVersion(version2) + + for i := 0; i < len(v1Parts) && i < len(v2Parts); i++ { + if v1Parts[i] != v2Parts[i] { + return v1Parts[i] > v2Parts[i] + } + } + return len(v1Parts) > len(v2Parts) +} + +func InitializeAgentVersionManager(mappingFilePath string) (*AgentVersionManager, error) { + m, cmVersion, err := readReleaseFile(mappingFilePath) + if err != nil { + return nil, err + } + return newAgentVersionManager(m, cmVersion), nil +} + +// GetAgentVersionManager returns the an instance of AgentVersionManager. +func GetAgentVersionManager() (*AgentVersionManager, error) { + initializationMutex.Lock() + defer initializationMutex.Unlock() + mappingFilePath := env.ReadOrDefault(MappingFilePathEnv, mappingFileDefaultPath) // nolint:forbidigo + if lastUsedMappingPath != mappingFilePath { + lastUsedMappingPath = mappingFilePath + var err error + + if versionManager, err = InitializeAgentVersionManager(mappingFilePath); err != nil { + return nil, err + } + } + + return versionManager, nil +} + +/* +The agent has the following mapping and support: +OM7 -> 107.0.x (meaning OM7 supports any agent version with major=107) +OM8 -> 108.0.x +OM6 would always use 12.0.x +CM -> Major.Minor can change in between updates and therefore this needs more special handling. +Unlike OM, there is no full guarantee that minor versions support each other for the same Major version. +*/ + +// GetAgentVersion returns the agent version to use with the Ops Manager +// readFromMapping is true in the case of AppDB, because they are started before OM, so we cannot rely on the endpoint +func (m *AgentVersionManager) GetAgentVersion(conn om.Connection, omVersion string, readFromMapping bool) (string, error) { + isCM := versionutil.OpsManagerVersion{VersionString: omVersion}.IsCloudManager() + if isCM { + return m.getAgentVersionForCloudManagerFromMapping() + } + + if readFromMapping { + return m.getClosestAgentVersionForOM(omVersion) + } + + supportsStatic, err := m.supportsStaticContainers(omVersion) + if err != nil { + return "", err + } + + if !supportsStatic { + return "", xerrors.Errorf("Ops Manager version %s does not support static containers, please use Ops Manager version of at least %s or %s", omVersion, om6StaticContainersSupport, om7StaticContainersSupport) + } + + version, err := m.getAgentVersionFromOpsManager(conn) + if err != nil { + return "", err + } + return addVersionSuffixIfAbsent(version), nil +} + +// supportsStaticContainers verifies whether the supplied omVersion supports static containers. +// Agent changes are not backported into older releases. +// Hence, we need to make sure that we run at least a specific agent version. +// The safest way for that is to make sure that the customer uses a specific omVersion, since that is tied to an +// agent version. +func (m *AgentVersionManager) supportsStaticContainers(omVersion string) (bool, error) { + omVersionConverted := versionutil.OpsManagerVersion{VersionString: omVersion} + givenVersion, err := omVersionConverted.Semver() + if err != nil { + return false, err + } + return givenVersion.GTE(om6StaticContainersSupport) || givenVersion.GTE(om7StaticContainersSupport), nil +} + +// ReleaseFile and following structs are used to unmarshall release.json fields +type ReleaseFile struct { + SupportedImages SupportedImages `json:"supportedImages"` +} + +type SupportedImages struct { + MongoDBAgent MongoDBAgent `json:"mongodb-agent"` +} + +type MongoDBAgent struct { + OpsManagerMapping omv1.OpsManagerVersionMapping `json:"opsManagerMapping"` +} + +// readReleaseFile reads the version mapping from the release.json file +func readReleaseFile(filePath string) (map[omv1.OpsManagerVersion]omv1.AgentVersion, string, error) { + var releaseFileContent ReleaseFile + + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, "", xerrors.Errorf("failed reading file %s: %w", filePath, err) + } + + if err = json.Unmarshal(fileBytes, &releaseFileContent); err != nil { + return nil, "", xerrors.Errorf("failed unmarshalling bytes from file %s: %w", filePath, err) + } + + mapping := releaseFileContent.SupportedImages.MongoDBAgent.OpsManagerMapping + + return mapping.OpsManager, mapping.CloudManager, nil +} + +func getMajorVersion(version string) string { + if version == "" { + return "" + } + return strings.Split(version, ".")[0] +} + +// getAgentVersionFromOpsManager retrieves the agent version by querying Ops Manager API. +func (m *AgentVersionManager) getAgentVersionFromOpsManager(conn om.Connection) (string, error) { + agentResponse, err := conn.ReadAgentVersion() + if err != nil { + return "", err + } + return agentResponse.AutomationVersion, nil +} + +func addVersionSuffixIfAbsent(version string) string { + if version == "" { + return "" + } + if strings.HasSuffix(version, "-1") { + return version + } + return version + "-1" +} + +func (m *AgentVersionManager) getAgentVersionForCloudManagerFromMapping() (string, error) { + if m.agentVersionCM != "" { + return m.agentVersionCM, nil + } + return "", xerrors.Errorf("No agent version found for Cloud Manager") +} + +func (m *AgentVersionManager) getClosestAgentVersionForOM(omVersion string) (string, error) { + if version, exists := m.omToAgentVersionMapping[omv1.OpsManagerVersion(omVersion)]; exists { + return version.AgentVersion, nil + } + majorOmVersion := getMajorVersion(omVersion) + latestAvailableOmVersion := m.latestOMVersionsByMajor[majorOmVersion] + latestAgentVersion := m.omToAgentVersionMapping[omv1.OpsManagerVersion(latestAvailableOmVersion)] // TODO: return smallest one for monitoring agent not automation agent + return addVersionSuffixIfAbsent(latestAgentVersion.AgentVersion), nil +} diff --git a/pkg/agentVersionManagement/get_agent_version_test.go b/pkg/agentVersionManagement/get_agent_version_test.go new file mode 100644 index 000000000..b452c5784 --- /dev/null +++ b/pkg/agentVersionManagement/get_agent_version_test.go @@ -0,0 +1,206 @@ +package agentVersionManagement + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/controllers/om" +) + +var jsonContents = ` +{ + "supportedImages": { + "mongodb-agent": { + "Description": "Agents corresponding to OpsManager 5.x and 6.x series", + "Description for specific versions": { + "11.0.5.6963-1": "An upgraded version for OM 5.0 we use for Operator-only deployments", + "12.0.28.7763-1": "OM 6 basic version", + "12.0.15.7646-1": "Community and Helm Charts version" + }, + "versions": [ + "12.0.4.7554-1", + "12.0.15.7646-1", + "12.0.21.7698-1", + "12.0.24.7719-1", + "12.0.25.7724-1", + "12.0.28.7763-1", + "12.0.29.7785-1", + "107.0.0.8465-1", + "107.0.1.8507-1" + ], + "opsManagerMapping": { + "cloud_manager": "13.10.0.8620-1", + "cloud_manager_tools": "100.9.4", + "ops_manager": { + "6.0.0": { + "agent_version": "12.0.30.7791-1", + "tools_version": "100.9.4" + }, + "7.0.0": { + "agent_version": "107.0.1.8507-1", + "tools_version": "100.9.4" + }, + "7.0.1": { + "agent_version": "107.0.1.8507-1", + "tools_version": "100.9.4" + }, + "7.0.2": { + "agent_version": "107.0.2.8531-1", + "tools_version": "100.9.4" + }, + "8.0.0-rc1": { + "agent_version": "108.0.0.8676-1", + "tools_version": "100.10.0" + } + } + }, + "legacyMonitoringOpsManagerMapping": { + "5.9": { + "agent_version": "12.0.4.7554-1" + }, + "6.0": { + "agent_version": "12.0.4.7554-1" + }, + "7.0": { + "agent_version": "107.0.0.8465-1" + } + }, + "variants": [ + "ubi" + ] + } + } +} +` + +func TestGetAgentVersionManager(t *testing.T) { + type args struct { + omConnection om.Connection + omVersion string + readFromMapping bool + } + + tempFilePath, closer := createTempMapping(t) + defer func() { + _ = closer() + }() + + getAgentVersionTests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "When OM Admin is not reachable reconcile again", + wantErr: true, + }, + { + name: "For om, return direct match if supported for static for version 7", + args: args{ + omConnection: om.NewEmptyMockedOmConnectionWithAgentVersion("11.0.5.6963-1", "11.0.0.11-1"), + omVersion: "7.0.0", + }, + want: "11.0.5.6963-1", + }, + { + name: "For om, return direct match if supported for static for version 6", + args: args{ + omConnection: om.NewEmptyMockedOmConnectionWithAgentVersion("11.0.5.6963-1", "11.0.0.11-1"), + omVersion: "6.0.21", + }, + want: "11.0.5.6963-1", + }, + { + name: "When we use CM, we query the 'cloud_manager' key in mapping", + args: args{ + omVersion: "v13.0.4.5666-1", + }, + want: "13.10.0.8620-1", + }, + { + name: "When read from mapping (appdb) and match, return direct match", + args: args{ + readFromMapping: true, + omVersion: "7.0.0", + }, + want: "107.0.1.8507-1", + }, + { + name: "Too old OM version", + args: args{ + omVersion: "6.0.10", + }, + wantErr: true, + }, + { + name: "Too old OM version 5", + args: args{ + omVersion: "5.0.10", + }, + wantErr: true, + }, + { + name: "Ops Manager RC interpreted correctly", + args: args{ + readFromMapping: true, + omVersion: "8.0.0-rc1", + }, + want: "108.0.0.8676-1", + }, + { + name: "Version not in mapping, does not matter since using connection", + args: args{ + omConnection: om.NewEmptyMockedOmConnectionWithAgentVersion("11.0.5.6963-1", "11.0.0.11-1"), + omVersion: "7.10.0", + }, + want: "11.0.5.6963-1", + }, + { + name: "When AppDB and no match, return latest for same major", + args: args{ + readFromMapping: true, + omVersion: "v13.11.4.5666-1", + }, + want: "13.10.0.8620-1", + }, + } + versionManager, err := InitializeAgentVersionManager(tempFilePath) + assert.NoError(t, err) + + for _, tt := range getAgentVersionTests { + t.Run(tt.name, func(t *testing.T) { + got, err := versionManager.GetAgentVersion(tt.args.omConnection, tt.args.omVersion, tt.args.readFromMapping) + if tt.wantErr { + assert.Error(t, err, "GetAgentVersion() should return an error") + } else { + assert.NoError(t, err, "GetAgentVersion() should not return an error") + } + assert.Equal(t, tt.want, got) + }) + } +} + +func createTempMapping(t *testing.T) (string, func() error) { + tempDirectory := t.TempDir() + tempFileName := "version_mapping.json" + tempFilePath := filepath.Join(tempDirectory, tempFileName) + + file, err := os.Create(tempFilePath) + if err != nil { + t.Errorf("Creating temp file failed: %s", err) + } + + _, err = file.Write([]byte(jsonContents)) + if err != nil { + t.Errorf("Writing JSON to file failed: %s", err) + } + _, err = InitializeAgentVersionManager(tempFilePath) + if err != nil { + t.Errorf("Initializing version manager failed: %s", err) + } + return tempFilePath, file.Close +} diff --git a/pkg/dependencymagnet/magnet.go b/pkg/dependencymagnet/magnet.go new file mode 100644 index 000000000..51c4ba93b --- /dev/null +++ b/pkg/dependencymagnet/magnet.go @@ -0,0 +1,7 @@ +package depenencymagnet + +import ( + // This is required to build both the Readiness Probe and Version Upgrade Hook. + // See docker/mongodb-enterprise-init-database/Dockerfile.builder + _ "gopkg.in/natefinch/lumberjack.v2" +) diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go new file mode 100644 index 000000000..189a5a983 --- /dev/null +++ b/pkg/dns/dns.go @@ -0,0 +1,130 @@ +package dns + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" +) + +func GetMultiPodName(stsName string, clusterNum, podNum int) string { + return fmt.Sprintf("%s-%d-%d", stsName, clusterNum, podNum) +} + +func GetMultiServiceName(stsName string, clusterNum, podNum int) string { + return fmt.Sprintf("%s-svc", GetMultiPodName(stsName, clusterNum, podNum)) +} + +func GetMultiHeadlessServiceName(stsName string, clusterNum int) string { + return fmt.Sprintf("%s-%d-svc", stsName, clusterNum) +} + +func GetServiceName(stsName string) string { + return fmt.Sprintf("%s-svc", stsName) +} + +func GetExternalServiceName(stsName string, podNum int) string { + return fmt.Sprintf("%s-%d-svc-external", stsName, podNum) +} + +func GetMultiExternalServiceName(stsName string, clusterNum, podNum int) string { + return fmt.Sprintf("%s-external", GetMultiServiceName(stsName, clusterNum, podNum)) +} + +func GetMultiServiceFQDN(stsName string, namespace string, clusterNum int, podNum int, clusterDomain string) string { + domain := "cluster.local" + if len(clusterDomain) > 0 { + domain = strings.TrimPrefix(clusterDomain, ".") + } + + return fmt.Sprintf("%s.%s.svc.%s", GetMultiServiceName(stsName, clusterNum, podNum), namespace, domain) +} + +func GetMultiServiceExternalDomain(stsName, externalDomain string, clusterNum, podNum int) string { + return fmt.Sprintf("%s.%s", GetMultiPodName(stsName, clusterNum, podNum), externalDomain) +} + +// GetMultiClusterProcessHostnames returns the agent hostnames, which they should be registered in OM in multi-cluster mode. +func GetMultiClusterProcessHostnames(stsName, namespace string, clusterNum, members int, clusterDomain string, externalDomain *string) []string { + hostnames, _ := GetMultiClusterProcessHostnamesAndPodNames(stsName, namespace, clusterNum, members, clusterDomain, externalDomain) + return hostnames +} + +func GetMultiClusterProcessHostnamesAndPodNames(stsName, namespace string, clusterNum, members int, clusterDomain string, externalDomain *string) ([]string, []string) { + hostnames := make([]string, 0) + podNames := make([]string, 0) + + for podNum := 0; podNum < members; podNum++ { + hostnames = append(hostnames, GetMultiClusterPodServiceFQDN(stsName, namespace, clusterNum, externalDomain, podNum, clusterDomain)) + podNames = append(podNames, GetMultiPodName(stsName, clusterNum, podNum)) + } + + return hostnames, podNames +} + +func GetMultiClusterPodServiceFQDN(stsName string, namespace string, clusterNum int, externalDomain *string, podNum int, clusterDomain string) string { + if externalDomain != nil { + return GetMultiServiceExternalDomain(stsName, *externalDomain, clusterNum, podNum) + } + return GetMultiServiceFQDN(stsName, namespace, clusterNum, podNum, clusterDomain) +} + +func GetServiceDomain(namespace string, clusterDomain string, externalDomain *string) string { + if externalDomain != nil { + return *externalDomain + } + if clusterDomain == "" { + clusterDomain = "cluster.local" + } + return fmt.Sprintf("%s.svc.%s", namespace, clusterDomain) +} + +// GetDnsForStatefulSet returns hostnames and names of pods in stateful set "set". This is a preferred way of getting hostnames +// it must be always used if it's possible to read the statefulset from Kubernetes +func GetDnsForStatefulSet(set appsv1.StatefulSet, clusterDomain string, externalDomain *string) ([]string, []string) { + return GetDnsForStatefulSetReplicasSpecified(set, clusterDomain, 0, externalDomain) +} + +// GetDnsForStatefulSetReplicasSpecified is similar to GetDnsForStatefulSet but expects the number of replicas to be specified +// (important for scale-down operations to support hostnames for old statefulset) +func GetDnsForStatefulSetReplicasSpecified(set appsv1.StatefulSet, clusterDomain string, replicas int, externalDomain *string) ([]string, []string) { + if replicas == 0 { + replicas = int(*set.Spec.Replicas) + } + return GetDNSNames(set.Name, set.Spec.ServiceName, set.Namespace, clusterDomain, replicas, externalDomain) +} + +// GetDNSNames returns hostnames and names of pods in stateful set, it's less preferable than "GetDnsForStatefulSet" and +// should be used only in situations when statefulset doesn't exist any more (the main example is when the mongodb custom +// resource is being deleted - then the dependant statefulsets cannot be read any more as they get into Terminated state) +func GetDNSNames(statefulSetName, service, namespace, clusterDomain string, replicas int, externalDomain *string) (hostnames, names []string) { + names = make([]string, replicas) + hostnames = make([]string, replicas) + + for i := 0; i < replicas; i++ { + names[i] = GetPodName(statefulSetName, i) + hostnames[i] = GetPodFQDN(names[i], service, namespace, clusterDomain, externalDomain) + } + return hostnames, names +} + +func GetPodFQDN(podName string, serviceName string, namespace string, clusterDomain string, externalDomain *string) string { + if externalDomain != nil && len(*externalDomain) > 0 { + return fmt.Sprintf("%s.%s", podName, *externalDomain) + } else { + return fmt.Sprintf("%s.%s", podName, GetServiceFQDN(serviceName, namespace, clusterDomain)) + } +} + +// GetServiceFQDN returns the FQDN for the service inside Kubernetes +func GetServiceFQDN(serviceName string, namespace string, clusterDomain string) string { + return fmt.Sprintf("%s.%s", serviceName, GetServiceDomain(namespace, clusterDomain, nil)) +} + +func GetPodName(stsName string, idx int) string { + return fmt.Sprintf("%s-%d", stsName, idx) +} + +func GetMultiStatefulSetName(replicaSetName string, clusterNum int) string { + return fmt.Sprintf("%s-%d", replicaSetName, clusterNum) +} diff --git a/pkg/dns/dns_test.go b/pkg/dns/dns_test.go new file mode 100644 index 000000000..b165271ba --- /dev/null +++ b/pkg/dns/dns_test.go @@ -0,0 +1,45 @@ +package dns + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func TestGetMultiClusterProcessHostnames(t *testing.T) { + assert.Equal(t, + []string{ + "om-db-0-0-svc.ns.svc.cluster.local", + "om-db-0-1-svc.ns.svc.cluster.local", + }, + GetMultiClusterProcessHostnames("om-db", "ns", 0, 2, "", nil), + ) + assert.Equal(t, + []string{ + "om-db-0-0-svc.ns.svc.cluster-2.local", + "om-db-0-1-svc.ns.svc.cluster-2.local", + }, + GetMultiClusterProcessHostnames("om-db", "ns", 0, 2, "cluster-2.local", nil), + ) + assert.Equal(t, + []string{ + "om-db-1-0-svc.ns.svc.cluster.local", + "om-db-1-1-svc.ns.svc.cluster.local", + "om-db-1-2-svc.ns.svc.cluster.local", + }, + GetMultiClusterProcessHostnames("om-db", "ns", 1, 3, "", nil), + ) + assert.Equal(t, + []string{}, + GetMultiClusterProcessHostnames("om-db", "ns", 1, 0, "", nil), + ) + assert.Equal(t, + []string{ + "om-db-0-0.some.domain", + "om-db-0-1.some.domain", + "om-db-0-2.some.domain", + }, + GetMultiClusterProcessHostnames("om-db", "ns", 0, 3, "", ptr.To("some.domain")), + ) +} diff --git a/pkg/fcv/fcv.go b/pkg/fcv/fcv.go new file mode 100644 index 000000000..a5146f75f --- /dev/null +++ b/pkg/fcv/fcv.go @@ -0,0 +1,46 @@ +package fcv + +import "github.com/10gen/ops-manager-kubernetes/pkg/util" + +func CalculateFeatureCompatibilityVersion(currentVersionFromCR string, fcvFromStatus string, fcvFromCR *string) string { + majorMinorVersionFromCR, setVersion, _ := util.MajorMinorVersion(currentVersionFromCR) + + // if fcvFromCR has been set by the customer + if fcvFromCR != nil { + convertedFcv := *fcvFromCR + + // If the fcvFromCR is set to util.AlwaysMatchVersionFCV, return the current version as is. + // It does not matter whether we upgraded or downgraded + if convertedFcv == util.AlwaysMatchVersionFCV { + return majorMinorVersionFromCR + } + + return convertedFcv + } + + // It is the first deployment; fcvFromStatus is empty since it's not been set yet. + // We can use the currentVersionFromCR as FCV + if fcvFromStatus == "" { + return majorMinorVersionFromCR + } + + lastAppliedMajorMinorVersion, setLastAppliedVersion, _ := util.MajorMinorVersion(fcvFromStatus + ".0") + // We don't support jumping 2 versions at once in fcvFromCR, in this case we need to use the higher fcvFromCR. + // We don't need to check the other way around, since mdb does not support downgrading by 2 versions. + if setVersion.Major-setLastAppliedVersion.Major >= 2 { + return majorMinorVersionFromCR + } + + // if no value is set, we want to use the lowest between new and old + comparisons, err := util.CompareVersions(currentVersionFromCR, fcvFromStatus+".0") + if err != nil { + return "" + } + + // return the smaller one from both + if comparisons == -1 { + return majorMinorVersionFromCR + } + + return lastAppliedMajorMinorVersion +} diff --git a/pkg/fcv/fcv_test.go b/pkg/fcv/fcv_test.go new file mode 100644 index 000000000..ac69f4d09 --- /dev/null +++ b/pkg/fcv/fcv_test.go @@ -0,0 +1,82 @@ +package fcv + +import ( + "reflect" + "testing" + + "k8s.io/utils/ptr" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +func TestCalculateFeatureCompatibilityVersion(t *testing.T) { + tests := []struct { + name string + newVersion string + lastAppliedFCVVersion string + currentFCV *string + expectedResult string + }{ + { + name: "FCV is set", + newVersion: "4.4.6", + lastAppliedFCVVersion: "4.2", + currentFCV: ptr.To("4.4"), + expectedResult: "4.4", + }, + { + name: "FCV is set and equal", + newVersion: "4.4.6", + lastAppliedFCVVersion: "4.4", + currentFCV: ptr.To("4.4"), + expectedResult: "4.4", + }, + { + name: "FCV is AlwaysMatchVersion, new version is smaller", + newVersion: "4.4.6", + lastAppliedFCVVersion: "5.0", + currentFCV: ptr.To(util.AlwaysMatchVersionFCV), + expectedResult: "4.4", + }, + { + name: "FCV is AlwaysMatchVersion, new version is higher", + newVersion: "5.0.8", + lastAppliedFCVVersion: "4.4", + currentFCV: ptr.To(util.AlwaysMatchVersionFCV), + expectedResult: "5.0", + }, + { + name: "FCV is nil, new version is higher", + newVersion: "5.4.6", + lastAppliedFCVVersion: "4.2", + expectedResult: "4.2", + }, + { + name: "FCV is nil, old version is higher", + newVersion: "4.2.8", + lastAppliedFCVVersion: "5.4", + expectedResult: "4.2", + }, + { + name: "FCV is nil, jumping 2 versions", + newVersion: "6.2.8", + lastAppliedFCVVersion: "4.4", + expectedResult: "6.2", + }, + { + name: "lastAppliedFCV is empty, first deployment", + newVersion: "6.2.8", + lastAppliedFCVVersion: "", + expectedResult: "6.2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := CalculateFeatureCompatibilityVersion(tt.newVersion, tt.lastAppliedFCVVersion, tt.currentFCV) + if !reflect.DeepEqual(result, tt.expectedResult) { + t.Errorf("expected %v, got %v", tt.expectedResult, result) + } + }) + } +} diff --git a/pkg/handler/enqueue_owner_multi.go b/pkg/handler/enqueue_owner_multi.go new file mode 100644 index 000000000..569d46554 --- /dev/null +++ b/pkg/handler/enqueue_owner_multi.go @@ -0,0 +1,55 @@ +package handler + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const MongoDBMultiResourceAnnotation = "MongoDBMultiResource" + +var _ handler.EventHandler = &EnqueueRequestForOwnerMultiCluster{} + +// EnqueueRequestForOwnerMultiCluster implements the EventHandler interface for multi-cluster callbacks. +// We cannot reuse the "EnqueueRequestForOwner" because it uses OwnerReference which doesn't work across clusters +type EnqueueRequestForOwnerMultiCluster struct{} + +func (e *EnqueueRequestForOwnerMultiCluster) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { + req := getOwnerMDBCRD(evt.Object.GetAnnotations(), evt.Object.GetNamespace()) + if req != (reconcile.Request{}) { + q.Add(req) + } +} + +func (e *EnqueueRequestForOwnerMultiCluster) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + reqs := []reconcile.Request{ + getOwnerMDBCRD(evt.ObjectOld.GetAnnotations(), evt.ObjectOld.GetNamespace()), + getOwnerMDBCRD(evt.ObjectNew.GetAnnotations(), evt.ObjectNew.GetNamespace()), + } + + for _, req := range reqs { + if req != (reconcile.Request{}) { + q.Add(req) + } + } +} + +func (e *EnqueueRequestForOwnerMultiCluster) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + req := getOwnerMDBCRD(evt.Object.GetAnnotations(), evt.Object.GetNamespace()) + q.Add(req) +} + +func (e *EnqueueRequestForOwnerMultiCluster) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { +} + +func getOwnerMDBCRD(annotations map[string]string, namespace string) reconcile.Request { + val, ok := annotations[MongoDBMultiResourceAnnotation] + if !ok { + return reconcile.Request{} + } + return reconcile.Request{NamespacedName: types.NamespacedName{Name: val, Namespace: namespace}} +} diff --git a/pkg/images/Imageurls.go b/pkg/images/Imageurls.go new file mode 100644 index 000000000..076282318 --- /dev/null +++ b/pkg/images/Imageurls.go @@ -0,0 +1,170 @@ +package images + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +// replaceImageTagOrDigestToTag returns the image with the tag or digest replaced to a given version +func replaceImageTagOrDigestToTag(image string, newVersion string) string { + // example: quay.io/mongodb/mongodb-agent@sha256:6a82abae27c1ba1133f3eefaad71ea318f8fa87cc57fe9355d6b5b817ff97f1a + if strings.Contains(image, "sha256:") { + imageSplit := strings.Split(image, "@") + imageSplit[len(imageSplit)-1] = newVersion + return strings.Join(imageSplit, ":") + } else { + // examples: + // - quay.io/mongodb/mongodb-agent:1234-567 + // - private-registry.local:3000/mongodb/mongodb-agent:1234-567 + // - mongodb + idx := strings.IndexRune(image, '/') + // If there is no domain separator in the image string or the segment before the slash does not contain + // a '.' or ':' and is not 'localhost' to indicate that the segment is a host, assume the image will be pulled from + // docker.io and the whole string represents the image. + if idx == -1 || (!strings.ContainsAny(image[:idx], ".:") && image[:idx] != "localhost") { + return fmt.Sprintf("%s:%s", image[:strings.LastIndex(image, ":")], newVersion) + } + + host := image[:idx] + imagePath := image[idx+1:] + + // If there is a ':' in the image path we can safely assume that it is a version separator. + if strings.Contains(imagePath, ":") { + imagePath = imagePath[:strings.LastIndex(imagePath, ":")] + } + return fmt.Sprintf("%s/%s:%s", host, imagePath, newVersion) + } +} + +// ImageUrls is a map of image names to their corresponding image URLs. +type ImageUrls map[string]string + +// LoadImageUrlsFromEnv reads all environment variables and selects +// the ones that are related to images for workloads. This includes env vars +// with RELATED_IMAGE_ prefix. +// RELATED_IMAGE_* env variables are set in Helm chart for OpenShift. +func LoadImageUrlsFromEnv() ImageUrls { + imageUrls := make(ImageUrls) + + for imageName, defaultValue := range map[string]string{ + // If you are considering adding a new variable here + // you at least one of the following should be true: + // - New env var has a RELATED_IMAGE_* counterpart + // - New env var contains an image URL or part of the URL + // and it will be used in container for MongoDB workfloads + construct.MongodbRepoUrlEnv: "", + construct.MongodbImageEnv: "", + util.InitOpsManagerImageUrl: "", + util.OpsManagerImageUrl: "", + util.InitDatabaseImageUrlEnv: "", + util.InitAppdbImageUrlEnv: "", + util.NonStaticDatabaseEnterpriseImage: "", + construct.AgentImageEnv: "", + architectures.MdbAgentImageRepo: architectures.MdbAgentImageRepoDefault, + } { + imageUrls[imageName] = env.ReadOrDefault(imageName, defaultValue) // nolint:forbidigo + } + + for _, env := range os.Environ() { // nolint:forbidigo + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 { + // Should never happen because os.Environ() returns key=value pairs + // but we are being defensive here. + continue + } + + if strings.HasPrefix(parts[0], "RELATED_IMAGE_") { + imageUrls[parts[0]] = parts[1] + } + } + return imageUrls +} + +// ContainerImage selects container image from imageUrls. +// It handles image digests when running in disconnected environment in OpenShift, where images +// are referenced by sha256 digest instead of tags. +// It works by convention by looking up RELATED_IMAGE_{imageName}_{versionUnderscored}. +// In case there is no RELATED_IMAGE_ defined it replaces digest or tag to version. +func ContainerImage(imageUrls ImageUrls, imageName string, version string) string { + versionUnderscored := strings.ReplaceAll(version, ".", "_") + versionUnderscored = strings.ReplaceAll(versionUnderscored, "-", "_") + relatedImageKey := fmt.Sprintf("RELATED_IMAGE_%s_%s", imageName, versionUnderscored) + + if relatedImage, ok := imageUrls[relatedImageKey]; ok { + return relatedImage + } + + imageURL := imageUrls[imageName] + + if strings.Contains(imageURL, ":") { + // here imageURL is not a host only but also with version or digest + // in that case we need to replace the version/digest. + // This is case with AGENT_IMAGE env variable which is provided as a full URL with version + // and not as a pair of host and version. + // In case AGENT_IMAGE is full URL with digest it will be replaced to given tag version, + // but most probably if it has digest, there is also RELATED_IMAGE defined, which will be picked up first. + return replaceImageTagOrDigestToTag(imageURL, version) + } + + return fmt.Sprintf("%s:%s", imageURL, version) +} + +func GetOfficialImage(imageUrls ImageUrls, version string, annotations map[string]string) string { + repoUrl := imageUrls[construct.MongodbRepoUrlEnv] + // TODO: rethink the logic of handling custom image types. We are currently only handling ubi9 and ubi8 and we never + // were really handling erroneus types, we just leave them be if specified (e.g. -ubuntu). + // envvar.GetEnvOrDefault(construct.MongoDBImageType, string(architectures.DefaultImageType)) + var imageType string + + if architectures.IsRunningStaticArchitecture(annotations) { + imageType = string(architectures.ImageTypeUBI9) + } else { + // For non-static architecture, we need to default to UBI8 to support customers running MongoDB versions < 6.0.4, + // which don't have UBI9 binaries. + imageType = string(architectures.ImageTypeUBI8) + } + + imageURL := imageUrls[construct.MongodbImageEnv] + + if strings.HasSuffix(repoUrl, "/") { + repoUrl = strings.TrimRight(repoUrl, "/") + } + + assumeOldFormat := envvar.ReadBool(util.MdbAppdbAssumeOldFormat) + if IsEnterpriseImage(imageURL) && !assumeOldFormat { + // 5.0.6-ent -> 5.0.6-ubi8 + if strings.HasSuffix(version, "-ent") { + version = fmt.Sprintf("%s%s", strings.TrimSuffix(version, "ent"), imageType) + } + // 5.0.6 -> 5.0.6-ubi8 + r := regexp.MustCompile("-.+$") + if !r.MatchString(version) { + version = version + "-" + imageType + } + if found, suffix := architectures.HasSupportedImageTypeSuffix(version); found { + version = fmt.Sprintf("%s%s", strings.TrimSuffix(version, suffix), imageType) + } + // if neither, let's not change it: 5.0.6-ubi8 -> 5.0.6-ubi8 + } + + mongoImageName := ContainerImage(imageUrls, construct.MongodbImageEnv, version) + + if strings.Contains(mongoImageName, "@sha256:") || strings.HasPrefix(mongoImageName, repoUrl) { + return mongoImageName + } + + return fmt.Sprintf("%s/%s", repoUrl, mongoImageName) +} + +func IsEnterpriseImage(mongodbImage string) bool { + return strings.Contains(mongodbImage, util.OfficialEnterpriseServerImageUrl) +} diff --git a/pkg/images/Imageurls_test.go b/pkg/images/Imageurls_test.go new file mode 100644 index 000000000..d3f8b11d6 --- /dev/null +++ b/pkg/images/Imageurls_test.go @@ -0,0 +1,231 @@ +package images + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mongodb/mongodb-kubernetes-operator/controllers/construct" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" +) + +func TestReplaceImageTagOrDigestToTag(t *testing.T) { + assert.Equal(t, "quay.io/mongodb/mongodb-agent:9876-54321", replaceImageTagOrDigestToTag("quay.io/mongodb/mongodb-agent:1234-567", "9876-54321")) + assert.Equal(t, "docker.io/mongodb/mongodb-enterprise-server:9876-54321", replaceImageTagOrDigestToTag("docker.io/mongodb/mongodb-enterprise-server:1234-567", "9876-54321")) + assert.Equal(t, "quay.io/mongodb/mongodb-agent:9876-54321", replaceImageTagOrDigestToTag("quay.io/mongodb/mongodb-agent@sha256:6a82abae27c1ba1133f3eefaad71ea318f8fa87cc57fe9355d6b5b817ff97f1a", "9876-54321")) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-database:some-tag", replaceImageTagOrDigestToTag("quay.io/mongodb/mongodb-enterprise-database:45678", "some-tag")) + assert.Equal(t, "quay.io:3000/mongodb/mongodb-enterprise-database:some-tag", replaceImageTagOrDigestToTag("quay.io:3000/mongodb/mongodb-enterprise-database:45678", "some-tag")) +} + +func TestContainerImage(t *testing.T) { + initDatabaseRelatedImageEnv1 := fmt.Sprintf("RELATED_IMAGE_%s_1_0_0", util.InitDatabaseImageUrlEnv) + initDatabaseRelatedImageEnv2 := fmt.Sprintf("RELATED_IMAGE_%s_12_0_4_7554_1", util.InitDatabaseImageUrlEnv) + initDatabaseRelatedImageEnv3 := fmt.Sprintf("RELATED_IMAGE_%s_2_0_0_b20220912000000", util.InitDatabaseImageUrlEnv) + + t.Setenv(util.InitDatabaseImageUrlEnv, "quay.io/mongodb/mongodb-enterprise-init-database") + t.Setenv(initDatabaseRelatedImageEnv1, "quay.io/mongodb/mongodb-enterprise-init-database@sha256:608daf56296c10c9bd02cc85bb542a849e9a66aff0697d6359b449540696b1fd") + t.Setenv(initDatabaseRelatedImageEnv2, "quay.io/mongodb/mongodb-enterprise-init-database@sha256:b631ee886bb49ba8d7b90bb003fe66051dadecbc2ac126ac7351221f4a7c377c") + t.Setenv(initDatabaseRelatedImageEnv3, "quay.io/mongodb/mongodb-enterprise-init-database@sha256:f1a7f49cd6533d8ca9425f25cdc290d46bb883997f07fac83b66cc799313adad") + + // there is no related image for 0.0.1 + imageUrls := LoadImageUrlsFromEnv() + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database:0.0.1", ContainerImage(imageUrls, util.InitDatabaseImageUrlEnv, "0.0.1")) + // for 10.2.25.6008-1 there is no RELATED_IMAGE variable set, so we use input instead of digest + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database:10.2.25.6008-1", ContainerImage(imageUrls, util.InitDatabaseImageUrlEnv, "10.2.25.6008-1")) + // for following versions we set RELATED_IMAGE_MONGODB_IMAGE_* env variables to sha256 digest + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database@sha256:608daf56296c10c9bd02cc85bb542a849e9a66aff0697d6359b449540696b1fd", ContainerImage(imageUrls, util.InitDatabaseImageUrlEnv, "1.0.0")) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database@sha256:b631ee886bb49ba8d7b90bb003fe66051dadecbc2ac126ac7351221f4a7c377c", ContainerImage(imageUrls, util.InitDatabaseImageUrlEnv, "12.0.4.7554-1")) + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-database@sha256:f1a7f49cd6533d8ca9425f25cdc290d46bb883997f07fac83b66cc799313adad", ContainerImage(imageUrls, util.InitDatabaseImageUrlEnv, "2.0.0-b20220912000000")) + + // env var has input already, so it is replaced + t.Setenv(util.InitAppdbImageUrlEnv, "quay.io/mongodb/mongodb-enterprise-init-appdb:12.0.4.7554-1") + imageUrls = LoadImageUrlsFromEnv() + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-appdb:10.2.25.6008-1", ContainerImage(imageUrls, util.InitAppdbImageUrlEnv, "10.2.25.6008-1")) + + // env var has input already, but there is related image with this input + t.Setenv(fmt.Sprintf("RELATED_IMAGE_%s_12_0_4_7554_1", util.InitAppdbImageUrlEnv), "quay.io/mongodb/mongodb-enterprise-init-appdb@sha256:a48829ce36bf479dc25a4de79234c5621b67beee62ca98a099d0a56fdb04791c") + imageUrls = LoadImageUrlsFromEnv() + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-appdb@sha256:a48829ce36bf479dc25a4de79234c5621b67beee62ca98a099d0a56fdb04791c", ContainerImage(imageUrls, util.InitAppdbImageUrlEnv, "12.0.4.7554-1")) + + t.Setenv(util.InitAppdbImageUrlEnv, "quay.io/mongodb/mongodb-enterprise-init-appdb@sha256:608daf56296c10c9bd02cc85bb542a849e9a66aff0697d6359b449540696b1fd") + imageUrls = LoadImageUrlsFromEnv() + // env var has input already as digest, but there is related image with this input + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-appdb@sha256:a48829ce36bf479dc25a4de79234c5621b67beee62ca98a099d0a56fdb04791c", ContainerImage(imageUrls, util.InitAppdbImageUrlEnv, "12.0.4.7554-1")) + // env var has input already as digest, there is no related image with this input, so we use input instead of digest + assert.Equal(t, "quay.io/mongodb/mongodb-enterprise-init-appdb:1.2.3", ContainerImage(imageUrls, util.InitAppdbImageUrlEnv, "1.2.3")) + + t.Setenv(util.OpsManagerImageUrl, "quay.io:3000/mongodb/ops-manager-kubernetes") + imageUrls = LoadImageUrlsFromEnv() + assert.Equal(t, "quay.io:3000/mongodb/ops-manager-kubernetes:1.2.3", ContainerImage(imageUrls, util.OpsManagerImageUrl, "1.2.3")) + + t.Setenv(util.OpsManagerImageUrl, "localhost/mongodb/ops-manager-kubernetes") + imageUrls = LoadImageUrlsFromEnv() + assert.Equal(t, "localhost/mongodb/ops-manager-kubernetes:1.2.3", ContainerImage(imageUrls, util.OpsManagerImageUrl, "1.2.3")) + + t.Setenv(util.OpsManagerImageUrl, "mongodb") + imageUrls = LoadImageUrlsFromEnv() + assert.Equal(t, "mongodb:1.2.3", ContainerImage(imageUrls, util.OpsManagerImageUrl, "1.2.3")) +} + +func TestGetAppDBImage(t *testing.T) { + // Note: if no construct.DefaultImageType is given, we will default to ubi8 + tests := []struct { + name string + input string + annotations map[string]string + want string + setupEnvs func(t *testing.T) + }{ + { + name: "Getting official image", + input: "4.2.11-ubi8", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + }, + }, + { + name: "Getting official image without suffix", + input: "4.2.11", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + }, + }, + { + name: "Getting official image keep suffix", + input: "4.2.11-something", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-something", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + }, + }, + { + name: "Getting official image with legacy suffix", + input: "4.2.11-ent", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + }, + }, + { + name: "Getting official image with legacy image", + input: "4.2.11-ent", + want: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.11-ent", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.DeprecatedImageAppdbUbiUrl) + }, + }, + { + name: "Getting official image with related image from deprecated URL", + input: "4.2.11-ubi8", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8", + setupEnvs: func(t *testing.T) { + t.Setenv("RELATED_IMAGE_MONGODB_IMAGE_4_2_11_ubi8", "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8") + t.Setenv(construct.MongoDBImageTypeEnv, "ubi8") + t.Setenv(construct.MongodbImageEnv, util.DeprecatedImageAppdbUbiUrl) + t.Setenv(construct.MongodbRepoUrlEnv, construct.OfficialMongodbRepoUrls[1]) + }, + }, + { + name: "Getting official image with related image with ent suffix even if old related image exists", + input: "4.2.11-ent", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8", + setupEnvs: func(t *testing.T) { + t.Setenv("RELATED_IMAGE_MONGODB_IMAGE_4_2_11_ubi8", "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8") + t.Setenv("RELATED_IMAGE_MONGODB_IMAGE_4_2_11_ent", "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ent") + t.Setenv(construct.MongoDBImageTypeEnv, "ubi8") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + t.Setenv(construct.MongodbRepoUrlEnv, construct.OfficialMongodbRepoUrls[1]) + }, + }, + { + name: "Getting deprecated image with related image from official URL. We do not replace -ent because the url is not a deprecated one we want to replace", + input: "4.2.11-ent", + want: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.11-ent", + setupEnvs: func(t *testing.T) { + t.Setenv("RELATED_IMAGE_MONGODB_IMAGE_4_2_11_ubi8", "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi8") + t.Setenv(construct.MongodbImageEnv, util.DeprecatedImageAppdbUbiUrl) + t.Setenv(construct.MongodbRepoUrlEnv, construct.OfficialMongodbRepoUrls[1]) + }, + }, + { + name: "Getting official image with legacy suffix but stopping migration", + input: "4.2.11-ent", + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ent", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + t.Setenv(util.MdbAppdbAssumeOldFormat, "true") + }, + }, + { + name: "Getting official image with legacy suffix on static architecture", + input: "4.2.11-ent", + annotations: map[string]string{ + "mongodb.com/v1.architecture": string(architectures.Static), + }, + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi9", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + }, + }, + { + name: "Getting official ubi9 image with ubi8 suffix on static architecture", + input: "4.2.11-ubi8", + annotations: map[string]string{ + "mongodb.com/v1.architecture": string(architectures.Static), + }, + want: "quay.io/mongodb/mongodb-enterprise-server:4.2.11-ubi9", + setupEnvs: func(t *testing.T) { + t.Setenv(construct.MongodbRepoUrlEnv, "quay.io/mongodb") + t.Setenv(construct.MongodbImageEnv, util.OfficialEnterpriseServerImageUrl) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setupEnvs(t) + imageUrlsMock := LoadImageUrlsFromEnv() + assert.Equalf(t, tt.want, GetOfficialImage(imageUrlsMock, tt.input, tt.annotations), "getOfficialImage(%v)", tt.input) + }) + } +} + +func TestIsEnterpriseImage(t *testing.T) { + tests := []struct { + name string + imageURL string + expectedResult bool + }{ + { + name: "Enterprise Image", + imageURL: "myregistry.com/mongo/mongodb-enterprise-server:latest", + expectedResult: true, + }, + { + name: "Community Image", + imageURL: "myregistry.com/mongo/mongodb-community-server:latest", + expectedResult: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IsEnterpriseImage(tt.imageURL) + + if result != tt.expectedResult { + t.Errorf("expected %v, got %v", tt.expectedResult, result) + } + }) + } +} diff --git a/pkg/kube/kube.go b/pkg/kube/kube.go new file mode 100644 index 000000000..1863ee80e --- /dev/null +++ b/pkg/kube/kube.go @@ -0,0 +1,32 @@ +package kube + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/10gen/ops-manager-kubernetes/api/v1" +) + +func ObjectKey(namespace, name string) client.ObjectKey { + return types.NamespacedName{Name: name, Namespace: namespace} +} + +func ObjectKeyFromApiObject(obj metav1.Object) client.ObjectKey { + return ObjectKey(obj.GetNamespace(), obj.GetName()) +} + +func BaseOwnerReference(owner v1.CustomResourceReadWriter) []metav1.OwnerReference { + if owner == nil { + return []metav1.OwnerReference{} + } + return []metav1.OwnerReference{ + *metav1.NewControllerRef(owner, schema.GroupVersionKind{ + Group: v1.SchemeGroupVersion.Group, + Version: v1.SchemeGroupVersion.Version, + Kind: owner.GetObjectKind().GroupVersionKind().Kind, + }), + } +} diff --git a/pkg/kube/service/service.go b/pkg/kube/service/service.go index abb749acf..2088124bb 100644 --- a/pkg/kube/service/service.go +++ b/pkg/kube/service/service.go @@ -2,45 +2,100 @@ package service import ( "context" + "fmt" + + "k8s.io/apimachinery/pkg/types" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/service" + corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + apiErrors "k8s.io/apimachinery/pkg/api/errors" ) -type Getter interface { - GetService(ctx context.Context, objectKey client.ObjectKey) (corev1.Service, error) +func DeleteServiceIfItExists(ctx context.Context, getterDeleter service.GetDeleter, serviceName types.NamespacedName) error { + _, err := getterDeleter.GetService(ctx, serviceName) + if err != nil { + // If it is not found return + if apiErrors.IsNotFound(err) { + return nil + } + // Otherwise we got an error when trying to get it + return fmt.Errorf("can't get service %s: %s", serviceName, err) + } + return getterDeleter.DeleteService(ctx, serviceName) } -type Updater interface { - UpdateService(ctx context.Context, service corev1.Service) error -} +// Merge merges `source` into `dest`. Both arguments will remain unchanged +// a new service will be created and returned. +// The "merging" process is arbitrary and it only handle specific attributes +func Merge(dest corev1.Service, source corev1.Service) corev1.Service { + if dest.Annotations == nil { + dest.Annotations = map[string]string{} + } + for k, v := range source.Annotations { + dest.Annotations[k] = v + } -type Creator interface { - CreateService(ctx context.Context, service corev1.Service) error -} + if dest.Labels == nil { + dest.Labels = map[string]string{} + } -type Deleter interface { - DeleteService(ctx context.Context, objectKey client.ObjectKey) error -} + for k, v := range source.Labels { + dest.Labels[k] = v + } -type GetDeleter interface { - Getter - Deleter -} + if dest.Spec.Selector == nil { + dest.Spec.Selector = map[string]string{} + } -type GetUpdater interface { - Getter - Updater -} + for k, v := range source.Spec.Selector { + dest.Spec.Selector[k] = v + } + + cachedNodePorts := map[int32]int32{} + for _, port := range dest.Spec.Ports { + cachedNodePorts[port.Port] = port.NodePort + } -type GetUpdateCreator interface { - Getter - Updater - Creator + if len(source.Spec.Ports) > 0 { + portCopy := make([]corev1.ServicePort, len(source.Spec.Ports)) + copy(portCopy, source.Spec.Ports) + dest.Spec.Ports = portCopy + + for i := range dest.Spec.Ports { + // Source might not specify NodePort and we shouldn't override existing NodePort value + if dest.Spec.Ports[i].NodePort == 0 { + dest.Spec.Ports[i].NodePort = cachedNodePorts[dest.Spec.Ports[i].Port] + } + } + } + + dest.Spec.Type = source.Spec.Type + dest.Spec.LoadBalancerIP = source.Spec.LoadBalancerIP + dest.Spec.ExternalTrafficPolicy = source.Spec.ExternalTrafficPolicy + return dest } -type GetUpdateCreateDeleter interface { - Getter - Updater - Creator - Deleter +// CreateOrUpdateService will create or update a service in Kubernetes. +func CreateOrUpdateService(ctx context.Context, getUpdateCreator service.GetUpdateCreator, desiredService corev1.Service) error { + namespacedName := types.NamespacedName{Namespace: desiredService.Namespace, Name: desiredService.Name} + existingService, err := getUpdateCreator.GetService(ctx, namespacedName) + + if err != nil { + if apiErrors.IsNotFound(err) { + err = getUpdateCreator.CreateService(ctx, desiredService) + if err != nil { + return err + } + } else { + return err + } + } else { + mergedService := Merge(existingService, desiredService) + err = getUpdateCreator.UpdateService(ctx, mergedService) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/kube/service/service_test.go b/pkg/kube/service/service_test.go new file mode 100644 index 000000000..6c5a66388 --- /dev/null +++ b/pkg/kube/service/service_test.go @@ -0,0 +1,195 @@ +package service + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" +) + +func TestService_merge0(t *testing.T) { + dst := corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}} + src := corev1.Service{} + dst = Merge(dst, src) + assert.Equal(t, "my-service", dst.Name) + assert.Equal(t, "my-namespace", dst.Namespace) + + // Name and Namespace will not be copied over. + src = corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "new-service", Namespace: "new-namespace"}} + dst = Merge(dst, src) + assert.Equal(t, "my-service", dst.Name) + assert.Equal(t, "my-namespace", dst.Namespace) +} + +func TestService_NodePortIsNotOverwritten(t *testing.T) { + dst := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{NodePort: 30030}}}, + } + src := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{}, + } + + dst = Merge(dst, src) + assert.Equal(t, int32(30030), dst.Spec.Ports[0].NodePort) +} + +func TestCreateOrUpdateService_NodePortsArePreservedWhenThereIsMoreThanOnePortDefined(t *testing.T) { + ctx := context.Background() + port1 := corev1.ServicePort{ + Name: "port1", + Port: 1000, + TargetPort: intstr.IntOrString{IntVal: 1001}, + NodePort: 30030, + } + + port2 := corev1.ServicePort{ + Name: "port2", + Port: 2000, + TargetPort: intstr.IntOrString{IntVal: 2001}, + NodePort: 40040, + } + + fakeClient, _ := mock.NewDefaultFakeClient() + existingService := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{port1, port2}}, + } + + err := CreateOrUpdateService(ctx, fakeClient, existingService) + assert.NoError(t, err) + + port1WithNodePortZero := port1 + port1WithNodePortZero.NodePort = 0 + port2WithNodePortZero := port2 + port2WithNodePortZero.NodePort = 0 + + newServiceWithoutNodePorts := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{port1WithNodePortZero, port2WithNodePortZero}}, + } + + err = CreateOrUpdateService(ctx, fakeClient, newServiceWithoutNodePorts) + assert.NoError(t, err) + + changedService, err := fakeClient.GetService(ctx, types.NamespacedName{Name: "my-service", Namespace: "my-namespace"}) + require.NoError(t, err) + require.NotNil(t, changedService) + require.Len(t, changedService.Spec.Ports, 2) + + assert.Equal(t, port1.NodePort, changedService.Spec.Ports[0].NodePort) + assert.Equal(t, port2.NodePort, changedService.Spec.Ports[1].NodePort) +} + +func TestService_NodePortIsNotOverwrittenIfNoNodePortIsSpecified(t *testing.T) { + dst := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{NodePort: 30030}}}, + } + src := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{}}}, + } + + dst = Merge(dst, src) + assert.Equal(t, int32(30030), dst.Spec.Ports[0].NodePort) +} + +func TestService_NodePortIsKeptWhenChangingServiceType(t *testing.T) { + dst := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 30030}}, + Type: corev1.ServiceTypeLoadBalancer, + }, + } + src := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 30099}}, + Type: corev1.ServiceTypeNodePort, + }, + } + dst = Merge(dst, src) + assert.Equal(t, int32(30099), dst.Spec.Ports[0].NodePort) + + src = corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "my-service", Namespace: "my-namespace"}, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{NodePort: 30011}}, + Type: corev1.ServiceTypeLoadBalancer, + }, + } + + dst = Merge(dst, src) + assert.Equal(t, int32(30011), dst.Spec.Ports[0].NodePort) +} + +func TestService_mergeAnnotations(t *testing.T) { + // Annotations will be added + annotationsDest := make(map[string]string) + annotationsDest["annotation0"] = "value0" + annotationsDest["annotation1"] = "value1" + annotationsSrc := make(map[string]string) + annotationsSrc["annotation0"] = "valueXXXX" + annotationsSrc["annotation2"] = "value2" + + src := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotationsSrc, + }, + } + dst := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-service", Namespace: "my-namespace", + Annotations: annotationsDest, + }, + } + dst = Merge(dst, src) + assert.Len(t, dst.Annotations, 3) + assert.Equal(t, dst.Annotations["annotation0"], "valueXXXX") +} + +func TestService_mergeFieldsWhenDestFieldsAreNil(t *testing.T) { + annotationsSrc := make(map[string]string) + annotationsSrc["annotation0"] = "value0" + annotationsSrc["annotation1"] = "value1" + + labelsSrc := make(map[string]string) + labelsSrc["label0"] = "labelValue0" + labelsSrc["label1"] = "labelValue1" + + selectorsSrc := make(map[string]string) + selectorsSrc["sel0"] = "selValue0" + selectorsSrc["sel1"] = "selValue1" + + src := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotationsSrc, + Labels: labelsSrc, + }, + Spec: corev1.ServiceSpec{Selector: selectorsSrc}, + } + dst := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-service", + Namespace: "my-namespace", + Annotations: nil, + Labels: nil, + }, + } + dst = Merge(dst, src) + assert.Len(t, dst.Annotations, 2) + assert.Equal(t, dst.Annotations, annotationsSrc) + assert.Equal(t, dst.Labels, labelsSrc) + assert.Equal(t, dst.Spec.Selector, selectorsSrc) +} diff --git a/pkg/multicluster/failedcluster/failedcluster.go b/pkg/multicluster/failedcluster/failedcluster.go new file mode 100644 index 000000000..0a0b4391b --- /dev/null +++ b/pkg/multicluster/failedcluster/failedcluster.go @@ -0,0 +1,11 @@ +package failedcluster + +const ( + FailedClusterAnnotation = "failedClusters" + ClusterSpecOverrideAnnotation = "clusterSpecOverride" +) + +type FailedCluster struct { + ClusterName string + Members int +} diff --git a/pkg/multicluster/memberwatch/clusterhealth.go b/pkg/multicluster/memberwatch/clusterhealth.go new file mode 100644 index 000000000..fae26c686 --- /dev/null +++ b/pkg/multicluster/memberwatch/clusterhealth.go @@ -0,0 +1,104 @@ +package memberwatch + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "time" + + "github.com/hashicorp/go-retryablehttp" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +type ClusterHealthChecker interface { + IsClusterHealthy() bool +} + +type MemberHeathCheck struct { + Server string + Client *retryablehttp.Client + Token string +} + +var ( + DefaultRetryWaitMin = 1 * time.Second + DefaultRetryWaitMax = 3 * time.Second + DefaultRetryMax = 10 +) + +func NewMemberHealthCheck(server string, ca []byte, token string, log *zap.SugaredLogger) *MemberHeathCheck { + certpool := x509.NewCertPool() + certpool.AppendCertsFromPEM(ca) + + return &MemberHeathCheck{ + Server: server, + Client: &retryablehttp.Client{ + HTTPClient: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + MinVersion: tls.VersionTLS12, + }, + }, + Timeout: time.Duration(env.ReadIntOrDefault(multicluster.ClusterClientTimeoutEnv, 10)) * time.Second, // nolint:forbidigo + }, + RetryWaitMin: DefaultRetryWaitMin, + RetryWaitMax: DefaultRetryWaitMax, + RetryMax: DefaultRetryMax, + // Will retry on all errors + CheckRetry: retryablehttp.DefaultRetryPolicy, + // Exponential backoff based on the attempt number and limited by the provided minimum and maximum durations. + // We don't need Jitter here as we're the only client to the OM, so there's no risk + // of overwhelming it in a peek. + Backoff: retryablehttp.DefaultBackoff, + RequestLogHook: func(logger retryablehttp.Logger, request *http.Request, i int) { + if i > 0 { + log.Warnf("Retrying (#%d) failed health check to %s (%s)", i, server, request.URL) + } + }, + }, + Token: token, + } +} + +// IsMemberClusterHealthy checks if there are some member clusters that are not in a "healthy" state +// by curl "ing" the healthz endpoint of the clusters. +func (m *MemberHeathCheck) IsClusterHealthy(log *zap.SugaredLogger) bool { + statusCode, err := check(m.Client, m.Server, m.Token) + if err != nil { + log.Errorf("Error running healthcheck for server: %s, error: %v", m.Server, err) + } + + if err != nil || statusCode != http.StatusOK { + log.Debugf("Response code from cluster endpoint call: %d", statusCode) + return false + } + + return true +} + +// check pings the "/readyz" endpoint of a cluster's API server and checks if it is healthy +func check(client *retryablehttp.Client, server string, token string) (int, error) { + endPoint := fmt.Sprintf("%s/readyz", server) + req, err := retryablehttp.NewRequest("GET", endPoint, nil) + if err != nil { + return 0, err + } + + bearer := "Bearer " + token + req.Header.Add("Authorization", bearer) + + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + return resp.StatusCode, nil +} diff --git a/pkg/multicluster/memberwatch/clusterhealth_test.go b/pkg/multicluster/memberwatch/clusterhealth_test.go new file mode 100644 index 000000000..63d93aa32 --- /dev/null +++ b/pkg/multicluster/memberwatch/clusterhealth_test.go @@ -0,0 +1,51 @@ +package memberwatch + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func init() { + logger, _ := zap.NewDevelopment() + zap.ReplaceGlobals(logger) +} + +func TestIsMemberClusterHealthy(t *testing.T) { + // mark cluster as healthy because "200" status code + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.WriteHeader(200) + })) + + memberHealthCheck := NewMemberHealthCheck(server.URL, []byte("ca-data"), "bhjkb", zap.S()) + healthy := memberHealthCheck.IsClusterHealthy(zap.S()) + assert.Equal(t, true, healthy) + + // mark cluster unhealthy because != "200" status code + server = httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.WriteHeader(500) + })) + + // check retry mechanism + DefaultRetryWaitMin = time.Second * 1 + DefaultRetryWaitMax = time.Second * 1 + DefaultRetryMax = 2 + + startTime := time.Now() + memberHealthCheck = NewMemberHealthCheck(server.URL, []byte("ca-data"), "hhfhj", zap.S()) + healthy = memberHealthCheck.IsClusterHealthy(zap.S()) + endTime := time.Since(startTime) + + assert.Equal(t, false, healthy) + assert.GreaterOrEqual(t, endTime, DefaultRetryWaitMin*2) + assert.LessOrEqual(t, endTime, DefaultRetryWaitMax*2+time.Second) + + // mark cluster unhealthy because of error + memberHealthCheck = NewMemberHealthCheck("", []byte("ca-data"), "bhdjbh", zap.S()) + healthy = memberHealthCheck.IsClusterHealthy(zap.S()) + assert.Equal(t, false, healthy) +} diff --git a/pkg/multicluster/memberwatch/memberwatch.go b/pkg/multicluster/memberwatch/memberwatch.go new file mode 100644 index 000000000..849aaec32 --- /dev/null +++ b/pkg/multicluster/memberwatch/memberwatch.go @@ -0,0 +1,281 @@ +package memberwatch + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "math" + "time" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/annotations" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster/failedcluster" +) + +type MemberClusterHealthChecker struct { + Cache map[string]*MemberHeathCheck +} + +type ClusterCredentials struct { + Server string + CertificateAuthority []byte + Token string +} + +func getClusterCredentials(clustersMap map[string]cluster.Cluster, + kubeConfig multicluster.KubeConfigFile, + kubeContext multicluster.KubeConfigContextItem, +) (*ClusterCredentials, error) { + clusterName := kubeContext.Context.Cluster + if _, ok := clustersMap[clusterName]; !ok { + return nil, fmt.Errorf("cluster %s not found in clustersMap", clusterName) + } + + kubeCluster := getClusterFromContext(clusterName, kubeConfig.Clusters) + if kubeCluster == nil { + return nil, fmt.Errorf("failed to get cluster with clustername: %s, doesn't exists in Kubeconfig clusters", clusterName) + } + + certificateAuthority, err := base64.StdEncoding.DecodeString(kubeCluster.CertificateAuthority) + if err != nil { + return nil, fmt.Errorf("failed to decode certificate for cluster: %s, err: %s", clusterName, err) + } + + user := getUserFromContext(kubeContext.Context.User, kubeConfig.Users) + if user == nil { + return nil, fmt.Errorf("failed to get user with name: %s, doesn't exists in Kubeconfig users", kubeContext.Context.User) + } + + return &ClusterCredentials{ + Server: kubeCluster.Server, + CertificateAuthority: certificateAuthority, + Token: user.Token, + }, nil +} + +func (m *MemberClusterHealthChecker) populateCache(clustersMap map[string]cluster.Cluster, log *zap.SugaredLogger) { + kubeConfigFile, err := multicluster.NewKubeConfigFile(multicluster.GetKubeConfigPath()) + if err != nil { + log.Errorf("Failed to read KubeConfig file err: %s", err) + // we can't populate the client so just bail out here + return + } + + kubeConfig, err := kubeConfigFile.LoadKubeConfigFile() + if err != nil { + log.Errorf("Failed to load the kubeconfig file content err: %s", err) + return + } + + for n := range kubeConfig.Contexts { + kubeContext := kubeConfig.Contexts[n] + clusterName := kubeContext.Context.Cluster + credentials, err := getClusterCredentials(clustersMap, kubeConfig, kubeContext) + if err != nil { + log.Errorf("Skipping cluster %s: %v", clusterName, err) + continue + } + m.Cache[clusterName] = NewMemberHealthCheck(credentials.Server, credentials.CertificateAuthority, credentials.Token, log) + } +} + +// WatchMemberClusterHealth watches member clusters healthcheck. If a cluster fails healthcheck it re-enqueues the +// MongoDBMultiCluster resources. It is spun up in the mongodb multi reconciler as a go-routine, and is executed every 10 seconds. +func (m *MemberClusterHealthChecker) WatchMemberClusterHealth(ctx context.Context, log *zap.SugaredLogger, watchChannel chan event.GenericEvent, centralClient kubernetesClient.Client, clustersMap map[string]cluster.Cluster) { + // check if the local cache is populated if not let's do that + if len(m.Cache) == 0 { + m.populateCache(clustersMap, log) + } + + for { + log.Info("Running member cluster healthcheck") + mdbmList := &mdbmulti.MongoDBMultiClusterList{} + + err := centralClient.List(ctx, mdbmList, &client.ListOptions{Namespace: ""}) + if err != nil { + log.Errorf("Failed to fetch MongoDBMultiClusterList from Kubernetes: %s", err) + } + + // check the cluster health status corresponding to each member cluster + for k, v := range m.Cache { + if v.IsClusterHealthy(log) { + log.Infof("Cluster %s reported healthy", k) + continue + } + + log.Warnf("Cluster %s reported unhealthy", k) + // re-enqueue all the MDBMultis the operator is watching into the reconcile loop + for _, mdbm := range mdbmList.Items { + if shouldAddFailedClusterAnnotation(mdbm.Annotations, k) && multicluster.ShouldPerformFailover() { + log.Infof("Enqueuing resource: %s, because cluster %s has failed healthcheck", mdbm.Name, k) + err := AddFailoverAnnotation(ctx, mdbm, k, centralClient) + if err != nil { + log.Errorf("Failed to add failover annotation to the mdbmc resource: %s, error: %s", mdbm.Name, err) + } + watchChannel <- event.GenericEvent{Object: &mdbm} + } else if shouldAddFailedClusterAnnotation(mdbm.Annotations, k) { + log.Infof("Marking resource: %s, with failed cluster %s annotation", mdbm.Name, k) + err := addFailedClustersAnnotation(ctx, mdbm, k, centralClient) + if err != nil { + log.Errorf("Failed to add failed cluster annotation to the mdbmc resource: %s, error: %s", mdbm.Name, err) + } + } + } + } + time.Sleep(10 * time.Second) + } +} + +// shouldAddFailedClusterAnnotation checks if we should add this cluster in the failedCluster annotation, +// if it's already not present. +func shouldAddFailedClusterAnnotation(annotations map[string]string, clusterName string) bool { + failedclusters := readFailedClusterAnnotation(annotations) + if failedclusters == nil { + return true + } + + for _, c := range failedclusters { + if c.ClusterName == clusterName { + return false + } + } + return true +} + +// readFailedClusterAnnotation reads the current failed clusters from the annotation. +func readFailedClusterAnnotation(annotations map[string]string) []failedcluster.FailedCluster { + if val, ok := annotations[failedcluster.FailedClusterAnnotation]; ok { + var failedClusters []failedcluster.FailedCluster + + err := json.Unmarshal([]byte(val), &failedClusters) + if err != nil { + return nil + } + + return failedClusters + } + return nil +} + +// clusterWithMinimumMembers returns the index of the cluster with the minimum number of nodes. +func clusterWithMinimumMembers(clusters mdb.ClusterSpecList) int { + mini, index := math.MaxInt64, -1 + + for nn, c := range clusters { + if c.Members < mini { + mini = c.Members + index = nn + } + } + return index +} + +// distributeFailedMembers evenly distributes the failed cluster's members amongst the remaining healthy clusters. +func distributeFailedMembers(clusters mdb.ClusterSpecList, clustername string) mdb.ClusterSpecList { + // add the cluster override annotations. Get the current clusterspec list from the CR and + // increase the members of the first cluster by the number of failed nodes + membersToFailOver := 0 + + for n, c := range clusters { + if c.ClusterName == clustername { + membersToFailOver = c.Members + clusters = append(clusters[:n], clusters[n+1:]...) + break + } + } + + for membersToFailOver > 0 { + // pick the cluster with the minumum number of nodes currently and increament + // its count by 1. + nn := clusterWithMinimumMembers(clusters) + clusters[nn].Members += 1 + membersToFailOver -= 1 + } + + return clusters +} + +// AddFailoverAnnotation adds the failed cluster spec to the annotation of the MongoDBMultiCluster CR for it to be used +// while performing the reconcilliation +func AddFailoverAnnotation(ctx context.Context, mrs mdbmulti.MongoDBMultiCluster, clustername string, client kubernetesClient.Client) error { + if mrs.Annotations == nil { + mrs.Annotations = map[string]string{} + } + + err := addFailedClustersAnnotation(ctx, mrs, clustername, client) + if err != nil { + return err + } + + currentClusterSpecs := mrs.Spec.ClusterSpecList + currentClusterSpecs = distributeFailedMembers(currentClusterSpecs, clustername) + + updatedClusterSpec, err := json.Marshal(currentClusterSpecs) + if err != nil { + return err + } + + return annotations.SetAnnotations(ctx, &mrs, map[string]string{failedcluster.ClusterSpecOverrideAnnotation: string(updatedClusterSpec)}, client) +} + +func addFailedClustersAnnotation(ctx context.Context, mrs mdbmulti.MongoDBMultiCluster, clustername string, client kubernetesClient.Client) error { + if mrs.Annotations == nil { + mrs.Annotations = map[string]string{} + } + + // read the existing failed cliuster annotations + var clusterData []failedcluster.FailedCluster + failedclusters := readFailedClusterAnnotation(mrs.Annotations) + if failedclusters != nil { + clusterData = failedclusters + } + + clusterData = append(clusterData, failedcluster.FailedCluster{ + ClusterName: clustername, + Members: getClusterMembers(mrs.Spec.ClusterSpecList, clustername), + }) + + clusterDataBytes, err := json.Marshal(clusterData) + if err != nil { + return err + } + return annotations.SetAnnotations(ctx, &mrs, map[string]string{failedcluster.FailedClusterAnnotation: string(clusterDataBytes)}, client) +} + +func getClusterMembers(clusterSpecList mdb.ClusterSpecList, clusterName string) int { + for _, e := range clusterSpecList { + if e.ClusterName == clusterName { + return e.Members + } + } + return 0 +} + +func getClusterFromContext(clusterName string, clusters []multicluster.KubeConfigClusterItem) *multicluster.KubeConfigCluster { + for _, e := range clusters { + if e.Name == clusterName { + return &e.Cluster + } + } + return nil +} + +func getUserFromContext(userName string, users []multicluster.KubeConfigUserItem) *multicluster.KubeConfigUser { + for _, e := range users { + if e.Name == userName { + return &e.User + } + } + return nil +} diff --git a/pkg/multicluster/memberwatch/memberwatch_test.go b/pkg/multicluster/memberwatch/memberwatch_test.go new file mode 100644 index 000000000..934fe7d4c --- /dev/null +++ b/pkg/multicluster/memberwatch/memberwatch_test.go @@ -0,0 +1,342 @@ +package memberwatch + +import ( + "encoding/base64" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/cluster" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mc "github.com/10gen/ops-manager-kubernetes/pkg/multicluster" + "github.com/10gen/ops-manager-kubernetes/pkg/multicluster/failedcluster" +) + +func TestClusterWithMinimumNumber(t *testing.T) { + tests := []struct { + inp mdb.ClusterSpecList + out int + }{ + { + inp: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster2", Members: 1}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 1}, + }, + out: 1, + }, + { + inp: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 1}, + {ClusterName: "cluster2", Members: 2}, + {ClusterName: "cluster3", Members: 3}, + {ClusterName: "cluster4", Members: 4}, + }, + out: 0, + }, + } + + for _, tt := range tests { + assert.Equal(t, tt.out, clusterWithMinimumMembers(tt.inp)) + } +} + +func TestDistributeFailedMembers(t *testing.T) { + tests := []struct { + inp mdb.ClusterSpecList + clusterName string + out mdb.ClusterSpecList + }{ + { + inp: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster2", Members: 1}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 1}, + }, + clusterName: "cluster1", + out: mdb.ClusterSpecList{ + {ClusterName: "cluster2", Members: 2}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 2}, + }, + }, + { + inp: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster2", Members: 1}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 1}, + }, + clusterName: "cluster2", + out: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 2}, + }, + }, + { + inp: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster2", Members: 1}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 1}, + }, + clusterName: "cluster3", + out: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 3}, + {ClusterName: "cluster2", Members: 3}, + {ClusterName: "cluster4", Members: 2}, + }, + }, + { + inp: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster2", Members: 1}, + {ClusterName: "cluster3", Members: 4}, + {ClusterName: "cluster4", Members: 1}, + }, + clusterName: "cluster4", + out: mdb.ClusterSpecList{ + {ClusterName: "cluster1", Members: 2}, + {ClusterName: "cluster2", Members: 2}, + {ClusterName: "cluster3", Members: 4}, + }, + }, + } + + for _, tt := range tests { + assert.Equal(t, tt.out, distributeFailedMembers(tt.inp, tt.clusterName)) + } +} + +func getFailedClusterList(clusters []string) string { + failedClusters := make([]failedcluster.FailedCluster, len(clusters)) + + for n, c := range clusters { + failedClusters[n] = failedcluster.FailedCluster{ClusterName: c, Members: 2} + } + + failedClusterBytes, _ := json.Marshal(failedClusters) + return string(failedClusterBytes) +} + +func TestShouldAddFailedClusterAnnotation(t *testing.T) { + tests := []struct { + annotations map[string]string + clusterName string + out bool + }{ + { + annotations: nil, + clusterName: "cluster1", + out: true, + }, + { + annotations: map[string]string{failedcluster.FailedClusterAnnotation: getFailedClusterList([]string{"cluster1", "cluster2"})}, + clusterName: "cluster1", + out: false, + }, + { + annotations: map[string]string{failedcluster.FailedClusterAnnotation: getFailedClusterList([]string{"cluster1", "cluster2", "cluster4"})}, + clusterName: "cluster3", + out: true, + }, + } + + for _, tt := range tests { + assert.Equal(t, shouldAddFailedClusterAnnotation(tt.annotations, tt.clusterName), tt.out) + } +} + +func TestGetClusterCredentials(t *testing.T) { + validCertContent := "valid-cert" + validCert := base64.StdEncoding.EncodeToString([]byte(validCertContent)) + invalidCert := "invalid-base64!!!" + clusterName := "cluster1" + userToken := "abc123" + mockUserItemList := []mc.KubeConfigUserItem{ + {Name: "user1", User: mc.KubeConfigUser{Token: userToken}}, + } + mockKubeContext := mc.KubeConfigContextItem{ + Name: "context1", + Context: mc.KubeConfigContext{ + Cluster: clusterName, + User: "user1", + }, + } + kubeconfigServerURL := "https://example.com" + mockKubeConfig := mc.KubeConfigFile{ + Clusters: []mc.KubeConfigClusterItem{ + { + Name: clusterName, + Cluster: mc.KubeConfigCluster{ + Server: kubeconfigServerURL, + CertificateAuthority: validCert, + }, + }, + }, + Users: mockUserItemList, + } + + tests := []struct { + name string + clustersMap map[string]cluster.Cluster // Using as a set; the value is not used. + kubeConfig mc.KubeConfigFile + kubeContext mc.KubeConfigContextItem + wantErr bool + errContains string + expectedServer string + expectedToken string + expectedCA []byte + }{ + { + name: "Cluster not in clustersMap", + clustersMap: map[string]cluster.Cluster{}, // Empty map; cluster1 is missing. + kubeConfig: mockKubeConfig, + kubeContext: mockKubeContext, + wantErr: true, + errContains: "cluster cluster1 not found in clustersMap", + }, + { + name: "Cluster missing in kubeConfig.Clusters", + clustersMap: map[string]cluster.Cluster{ + clusterName: nil, + }, + kubeConfig: mc.KubeConfigFile{ + Clusters: []mc.KubeConfigClusterItem{}, // No cluster defined. + Users: mockUserItemList, + }, + kubeContext: mockKubeContext, + wantErr: true, + errContains: "failed to get cluster with clustername: cluster1", + }, + { + name: "Invalid certificate authority", + clustersMap: map[string]cluster.Cluster{ + clusterName: nil, + }, + kubeConfig: mc.KubeConfigFile{ + Clusters: []mc.KubeConfigClusterItem{ + { + Name: clusterName, + Cluster: mc.KubeConfigCluster{ + Server: kubeconfigServerURL, + CertificateAuthority: invalidCert, // The kubeConfig has an invalid CA + }, + }, + }, + Users: mockUserItemList, + }, + kubeContext: mockKubeContext, + wantErr: true, + errContains: "failed to decode certificate for cluster: cluster1", + }, + { + name: "User not found", + clustersMap: map[string]cluster.Cluster{ + clusterName: nil, + }, + kubeConfig: mc.KubeConfigFile{ + Clusters: []mc.KubeConfigClusterItem{ + { + Name: clusterName, + Cluster: mc.KubeConfigCluster{ + Server: kubeconfigServerURL, + CertificateAuthority: validCert, + }, + }, + }, + Users: []mc.KubeConfigUserItem{}, // No users defined. + }, + kubeContext: mc.KubeConfigContextItem{ + Name: "context1", + Context: mc.KubeConfigContext{ + Cluster: clusterName, + User: "user1", // User is not present. + }, + }, + wantErr: true, + errContains: "failed to get user with name: user1", + }, + { + name: "Successful extraction", + clustersMap: map[string]cluster.Cluster{ + clusterName: nil, + }, + kubeConfig: mockKubeConfig, + kubeContext: mockKubeContext, + wantErr: false, + expectedServer: kubeconfigServerURL, + expectedToken: userToken, + expectedCA: []byte(validCertContent), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + creds, err := getClusterCredentials(tc.clustersMap, tc.kubeConfig, tc.kubeContext) + if tc.wantErr { + assert.ErrorContains(t, err, tc.errContains) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedServer, creds.Server) + assert.Equal(t, tc.expectedToken, creds.Token) + assert.Equal(t, tc.expectedCA, creds.CertificateAuthority) + } + }) + } +} + +func TestGetUserFromContext(t *testing.T) { + tests := []struct { + name string + userName string + users []mc.KubeConfigUserItem + expectedUser *mc.KubeConfigUser + }{ + { + name: "User exists", + userName: "alice", + users: []mc.KubeConfigUserItem{ + {Name: "alice", User: mc.KubeConfigUser{Token: "alice-token"}}, + {Name: "bob", User: mc.KubeConfigUser{Token: "bob-token"}}, + }, + expectedUser: &mc.KubeConfigUser{Token: "alice-token"}, + }, + { + name: "User does not exist", + userName: "charlie", + users: []mc.KubeConfigUserItem{ + {Name: "alice", User: mc.KubeConfigUser{Token: "alice-token"}}, + {Name: "bob", User: mc.KubeConfigUser{Token: "bob-token"}}, + }, + expectedUser: nil, + }, + { + name: "Empty users slice", + userName: "alice", + users: []mc.KubeConfigUserItem{}, + expectedUser: nil, + }, + { + name: "Multiple users with same name, returns first match", + userName: "duplicated", + users: []mc.KubeConfigUserItem{ + {Name: "duplicated", User: mc.KubeConfigUser{Token: "first-token"}}, + {Name: "duplicated", User: mc.KubeConfigUser{Token: "second-token"}}, + }, + expectedUser: &mc.KubeConfigUser{Token: "first-token"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + user := getUserFromContext(tc.userName, tc.users) + assert.Equal(t, tc.expectedUser, user) + }) + } +} diff --git a/pkg/multicluster/mockedcluster.go b/pkg/multicluster/mockedcluster.go new file mode 100644 index 000000000..e75713f50 --- /dev/null +++ b/pkg/multicluster/mockedcluster.go @@ -0,0 +1,70 @@ +package multicluster + +import ( + "context" + "net/http" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +var _ cluster.Cluster = &MockedCluster{} + +type MockedCluster struct { + client client.Client +} + +func (m *MockedCluster) GetHTTPClient() *http.Client { + panic("implement me") +} + +func (m *MockedCluster) SetFields(interface{}) error { + return nil +} + +func (m *MockedCluster) GetConfig() *rest.Config { + return nil +} + +func (m *MockedCluster) GetScheme() *runtime.Scheme { + return nil +} + +func (m *MockedCluster) GetClient() client.Client { + return m.client +} + +func (m *MockedCluster) GetFieldIndexer() client.FieldIndexer { + return nil +} + +func (m *MockedCluster) GetCache() cache.Cache { + return nil +} + +func (m *MockedCluster) GetEventRecorderFor(name string) record.EventRecorder { + return nil +} + +func (m *MockedCluster) GetRESTMapper() meta.RESTMapper { + return nil +} + +func (m *MockedCluster) GetAPIReader() client.Reader { + return nil +} + +func (m *MockedCluster) Start(ctx context.Context) error { + return nil +} + +func New(client client.Client) *MockedCluster { + return &MockedCluster{ + client: client, + } +} diff --git a/pkg/multicluster/multicluster.go b/pkg/multicluster/multicluster.go new file mode 100644 index 000000000..3139859d2 --- /dev/null +++ b/pkg/multicluster/multicluster.go @@ -0,0 +1,281 @@ +package multicluster + +import ( + "fmt" + "io" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/ghodss/yaml" + "golang.org/x/xerrors" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + restclient "k8s.io/client-go/rest" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/secrets" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + intp "github.com/10gen/ops-manager-kubernetes/pkg/util/int" +) + +const ( + // kubeconfig path holding the credentials for different member clusters + DefaultKubeConfigPath = "/etc/config/kubeconfig/kubeconfig" + KubeConfigPathEnv = "KUBE_CONFIG_PATH" + ClusterClientTimeoutEnv = "CLUSTER_CLIENT_TIMEOUT" +) + +type KubeConfig struct { + Reader io.Reader +} + +func NewKubeConfigFile(kubeConfigPath string) (KubeConfig, error) { + file, err := os.Open(kubeConfigPath) + if err != nil { + return KubeConfig{}, err + } + return KubeConfig{Reader: file}, nil +} + +func GetKubeConfigPath() string { + return env.ReadOrDefault(KubeConfigPathEnv, DefaultKubeConfigPath) // nolint:forbidigo +} + +// LoadKubeConfigFile returns the KubeConfig file containing the multi cluster context. +func (k KubeConfig) LoadKubeConfigFile() (KubeConfigFile, error) { + kubeConfigBytes, err := io.ReadAll(k.Reader) + if err != nil { + return KubeConfigFile{}, err + } + + kubeConfig := KubeConfigFile{} + if err := yaml.Unmarshal(kubeConfigBytes, &kubeConfig); err != nil { + return KubeConfigFile{}, err + } + return kubeConfig, nil +} + +// CreateMemberClusterClients creates a client(map of cluster-name to client) to talk to the API-Server corresponding to each member clusters. +func CreateMemberClusterClients(clusterNames []string, kubeConfigPath string) (map[string]*restclient.Config, error) { + clusterClientsMap := map[string]*restclient.Config{} + + for _, c := range clusterNames { + clientset, err := getClient(c, kubeConfigPath) + if err != nil { + return nil, xerrors.Errorf("failed to create clientset map: %w", err) + } + if clientset == nil { + return nil, xerrors.Errorf("failed to get clientset for cluster: %s", c) + } + clientset.Timeout = time.Duration(env.ReadIntOrDefault(ClusterClientTimeoutEnv, 10)) * time.Second // nolint:forbidigo + clusterClientsMap[c] = clientset + } + return clusterClientsMap, nil +} + +// getClient returns a kubernetes.Clientset using the given context from the +// specified KubeConfig filepath. +func getClient(context, kubeConfigPath string) (*restclient.Config, error) { + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeConfigPath}, + &clientcmd.ConfigOverrides{ + CurrentContext: context, + }).ClientConfig() + if err != nil { + return nil, xerrors.Errorf("failed to create client config: %w", err) + } + + return config, nil +} + +// shouldPerformFailover checks if the operator is configured to perform automatic failover +// of the MongoDB Replicaset members spread over multiple Kubernetes clusters. +func ShouldPerformFailover() bool { + str := os.Getenv("PERFORM_FAILOVER") // nolint:forbidigo + val, err := strconv.ParseBool(str) + if err != nil { + return false + } + return val +} + +// KubeConfigFile represents the contents of a KubeConfig file. +type KubeConfigFile struct { + Contexts []KubeConfigContextItem `json:"contexts"` + Clusters []KubeConfigClusterItem `json:"clusters"` + Users []KubeConfigUserItem `json:"users"` +} + +type KubeConfigClusterItem struct { + Name string `json:"name"` + Cluster KubeConfigCluster `json:"cluster"` +} + +type KubeConfigCluster struct { + CertificateAuthority string `json:"certificate-authority-data"` + Server string `json:"server"` +} + +type KubeConfigUserItem struct { + User KubeConfigUser `json:"user"` + Name string `json:"name"` +} + +type KubeConfigUser struct { + Token string `json:"token"` +} +type KubeConfigContextItem struct { + Name string `json:"name"` + Context KubeConfigContext `json:"context"` +} + +type KubeConfigContext struct { + Cluster string `json:"cluster"` + Namespace string `json:"namespace"` + User string `json:"user"` +} + +// GetMemberClusterNamespace returns the namespace that will be used for all member clusters. +func (k KubeConfigFile) GetMemberClusterNamespace() string { + return k.Contexts[0].Context.Namespace +} + +func (k KubeConfigFile) GetMemberClusterNames() []string { + clusterNames := make([]string, len(k.Contexts)) + + for n, e := range k.Contexts { + clusterNames[n] = e.Context.Cluster + } + return clusterNames +} + +// MustGetClusterNumFromMultiStsName parses the statefulset object name and returns the cluster number where it is created +func MustGetClusterNumFromMultiStsName(name string) int { + ss := strings.Split(name, "-") + + n, err := strconv.Atoi(ss[len(ss)-1]) + if err != nil { + panic(err) + } + return n +} + +// GetRsNamefromMultiStsName parses the statefulset object name and returns the name of MongoDBMultiCluster object name +func GetRsNamefromMultiStsName(name string) string { + ss := strings.Split(name, "-") + if len(ss) <= 1 || ss[0] == "" { + panic(fmt.Sprintf("invalid statefulset name: %s", name)) + } + return strings.Join(ss[:len(ss)-1], "-") +} + +func ClustersMapToClientMap(clusterMap map[string]cluster.Cluster) map[string]client.Client { + clientMap := map[string]client.Client{} + for memberClusterName, memberCluster := range clusterMap { + clientMap[memberClusterName] = memberCluster.GetClient() + } + return clientMap +} + +// MemberCluster is a wrapper type containing basic information about member cluster in one place. +// It is used to simplify reconciliation process and to ensure deterministic iteration over member clusters. +type MemberCluster struct { + Name string + Index int + Replicas int + Client kubernetesClient.Client + SecretClient secrets.SecretClient + // Active marks a cluster as a member holding database nodes. The flag is useful for only relying on active clusters when reading + // information about the topology of the multi-cluster MongoDB or AppDB resource. This could mean automation config or cluster specific configuration. + Active bool + // Healthy marks if we have connection to the cluster. + Healthy bool + // Legacy if set to true, marks this cluster to use the old naming conventions (without the cluster index) + Legacy bool +} + +// LegacyCentralClusterName is a cluster name for simulating multi-cluster mode when running in legacy single-cluster mode +// With the deployment state in config maps and multi-cluster-first we might store this dummy cluster name in the state config map. +// We cannot change this name from now on. +const LegacyCentralClusterName = "__default" + +// GetLegacyCentralMemberCluster returns a legacy central member cluster for unit tests. +// Such member cluster is created in the reconcile loop in SingleCluster topology +// in order to simulate multi-cluster deployment on one member cluster that has legacy naming conventions enabled. +func GetLegacyCentralMemberCluster(replicas int, index int, client kubernetesClient.Client, secretClient secrets.SecretClient) MemberCluster { + return MemberCluster{ + Name: LegacyCentralClusterName, + Index: index, + Replicas: replicas, + Client: client, + SecretClient: secretClient, + Active: true, + Healthy: true, + Legacy: true, + } +} + +// CreateMapWithUpdatedMemberClusterIndexes returns a new mapping for memberClusterNames. +// It maintains previously existing mappings and assigns new indexes for new cluster names. +func AssignIndexesForMemberClusterNames(existingMapping map[string]int, memberClusterNames []string) map[string]int { + newMapping := map[string]int{} + for k, v := range existingMapping { + newMapping[k] = v + } + + for _, clusterName := range memberClusterNames { + if _, ok := newMapping[clusterName]; !ok { + newMapping[clusterName] = getNextIndex(newMapping) + } + } + + return newMapping +} + +func getNextIndex(m map[string]int) int { + maxi := -1 + + for _, val := range m { + maxi = intp.Max(maxi, val) + } + return maxi + 1 +} + +var memberClusterMapMutex sync.Mutex + +// IsMemberClusterMapInitializedForMultiCluster checks if global member cluster map +// is properly initialized for multi-cluster workloads. The assumption is that if the map +// contains only __default cluster, that means it's not configured for multi-cluster. +func IsMemberClusterMapInitializedForMultiCluster(memberClusterMap map[string]client.Client) bool { + memberClusterMapMutex.Lock() + defer memberClusterMapMutex.Unlock() + + if len(memberClusterMap) == 0 { + return false + } else if len(memberClusterMap) == 1 { + if _, ok := memberClusterMap[LegacyCentralClusterName]; ok { + return false + } + } + + return true +} + +func InitializeGlobalMemberClusterMapForSingleCluster(globalMemberClustersMap map[string]client.Client, defaultKubeClient client.Client) map[string]client.Client { + memberClusterMapMutex.Lock() + defer memberClusterMapMutex.Unlock() + + if _, ok := globalMemberClustersMap[LegacyCentralClusterName]; !ok { + if globalMemberClustersMap == nil { + globalMemberClustersMap = map[string]client.Client{} + } + globalMemberClustersMap[LegacyCentralClusterName] = defaultKubeClient + } + + return globalMemberClustersMap +} diff --git a/pkg/multicluster/multicluster_test.go b/pkg/multicluster/multicluster_test.go new file mode 100644 index 000000000..ede2f46a2 --- /dev/null +++ b/pkg/multicluster/multicluster_test.go @@ -0,0 +1,135 @@ +package multicluster + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetRsNamefromMultiStsName(t *testing.T) { + tests := []struct { + inp string + want string + }{ + { + inp: "foo-bar-test-1", + want: "foo-bar-test", + }, + { + inp: "foo-0", + want: "foo", + }, + { + inp: "foo-bar-1-2", + want: "foo-bar-1", + }, + } + + for _, tt := range tests { + got := GetRsNamefromMultiStsName(tt.inp) + assert.Equal(t, tt.want, got) + } +} + +func TestGetRsNamefromMultiStsNamePanic(t *testing.T) { + tests := []struct { + inp string + }{ + { + inp: "", + }, + { + inp: "-1", + }, + } + + for _, tt := range tests { + assert.Panics(t, func() { GetRsNamefromMultiStsName(tt.inp) }, "The code did not panic") + } +} + +func TestLoadKubeConfigFile(t *testing.T) { + inp := []struct { + server string + name string + ca string + cluster string + }{ + { + server: "https://api.e2e.cluster1.mongokubernetes.com", + name: "e2e.cluster1.mongokubernetes.com", + ca: "abcdbbd", + }, + { + server: "https://api.e2e.cluster2.mongokubernetes.com", + name: "e2e.cluster2.mongokubernetes.com", + ca: "njcdkn", + }, + { + server: "https://api.e2e.cluster3.mongokubernetes.com", + name: "e2e.cluster3.mongokubernetes.com", + ca: "nxjknjk", + }, + } + + str := fmt.Sprintf( + ` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: %s + server: %s + name: %s +- cluster: + certificate-authority-data: %s + server: %s + name: %s +- cluster: + certificate-authority-data: %s + server: %s + name: %s +contexts: +- context: + cluster: e2e.cluster1.mongokubernetes.com + namespace: a-1661872869-pq35wlt3zzz + user: e2e.cluster1.mongokubernetes.com + name: e2e.cluster1.mongokubernetes.com +- context: + cluster: e2e.cluster2.mongokubernetes.com + namespace: a-1661872869-pq35wlt3zzz + user: e2e.cluster2.mongokubernetes.com + name: e2e.cluster2.mongokubernetes.com +- context: + cluster: e2e.cluster3.mongokubernetes.com + namespace: a-1661872869-pq35wlt3zzz + user: e2e.cluster3.mongokubernetes.com + name: e2e.cluster3.mongokubernetes.com +kind: Config +users: +- name: e2e.cluster1.mongokubernetes.com + user: + token: eyJhbGciOi +- name: e2e.cluster2.mongokubernetes.com + user: + token: eyJhbGc +- name: e2e.cluster3.mongokubernetes.com + user: + token: njncdnjn`, inp[0].ca, inp[0].server, inp[0].name, inp[1].ca, + inp[1].server, inp[1].name, inp[2].ca, inp[1].server, inp[1].name) + + k := KubeConfig{ + Reader: strings.NewReader(str), + } + + arr, err := k.LoadKubeConfigFile() + if err != nil { + t.Error(err) + } + + for n, e := range arr.Contexts { + assert.Equal(t, inp[n].name, e.Name) + assert.Equal(t, inp[n].name, e.Context.Cluster) + } +} diff --git a/pkg/passwordhash/passwordhash.go b/pkg/passwordhash/passwordhash.go new file mode 100644 index 000000000..3f2c734be --- /dev/null +++ b/pkg/passwordhash/passwordhash.go @@ -0,0 +1,29 @@ +package passwordhash + +import ( + "crypto/sha256" + "encoding/base64" + + "golang.org/x/crypto/pbkdf2" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/generate" +) + +const ( + HashIterations = 256 + HashLength = 32 +) + +// GenerateHashAndSaltForPassword returns a `hash` and `salt` for this password. +func GenerateHashAndSaltForPassword(password string) (string, string) { + salt, err := generate.GenerateRandomBytes(8) + if err != nil { + return "", "" + } + + // The implementation at + // https://github.com/10gen/mms-automation/blob/76078d46d56a91a7ca2edc91b811ee87682b24b6/go_planner/src/com.tengen/cm/metrics/prometheus/server.go#L207 + // is the counterpart of this code. + hash := pbkdf2.Key([]byte(password), salt, HashIterations, HashLength, sha256.New) + return base64.StdEncoding.EncodeToString(hash), base64.StdEncoding.EncodeToString(salt) +} diff --git a/pkg/placeholders/replacer.go b/pkg/placeholders/replacer.go new file mode 100644 index 000000000..b3304d547 --- /dev/null +++ b/pkg/placeholders/replacer.go @@ -0,0 +1,85 @@ +package placeholders + +import ( + "fmt" + "regexp" + "sort" + "strings" + + "golang.org/x/xerrors" +) + +// New returns a new instance of Replacer initialized with values for placeholders (it may be nil or empty). +// Keys in placeholderValues map are just bare placeholder names, without curly braces. +// Example: +// +// replacer := New(map[string]string{"value1": "v1"}) +// out, replacedFlag, _ := replacer.Process("value1={value1}") +// // out contains "value1=v1", replacedFlag is true +func New(placeholderValues map[string]string) *Replacer { + replacer := &Replacer{placeholders: map[string]string{}} + replacer.addPlaceholderValues(placeholderValues) + + return replacer +} + +// Replacer helps in replacing {placeholders} with concrete values in strings. +// It is immutable and thread safe. +type Replacer struct { + // maps placeholder key to its value + // keys are stored internally with curly braces {key} + placeholders map[string]string +} + +var placeholderRegex = regexp.MustCompile(`{(\w+)}`) + +// addPlaceholderValues is intentionally private to guarantee immutability. +func (r *Replacer) addPlaceholderValues(placeholderValues map[string]string) { + for placeholder, value := range placeholderValues { + r.placeholders[fmt.Sprintf("{%s}", placeholder)] = value + } +} + +// Process replaces all {placeholders} in str. +// All placeholders in the string must be replaced. Error is returned if there is no value for a placeholder registered, but the placeholder value may be empty. +// It always returns a copy of str, even if there is no placeholders present. +func (r *Replacer) Process(str string) (string, bool, error) { + notReplacedKeys := map[string]struct{}{} + replaceFunc := func(key string) string { + if value, ok := r.placeholders[key]; ok { + return value + } + notReplacedKeys[key] = struct{}{} + return "" + } + + replacedStr := placeholderRegex.ReplaceAllStringFunc(str, replaceFunc) + + if len(notReplacedKeys) > 0 { + var keys []string + for key := range notReplacedKeys { + keys = append(keys, key) + } + sort.Strings(keys) + return "", false, xerrors.Errorf("missing values for the following placeholders: %s", strings.Join(keys, ", ")) + } + + return replacedStr, str != replacedStr, nil +} + +// ProcessMap returns a copy of strMap with all values passed through Process. +// If any of the values fail to process, the error is returned immediately. +func (r *Replacer) ProcessMap(strMap map[string]string) (map[string]string, bool, error) { + newMap := map[string]string{} + mapWasModified := false + for key, value := range strMap { + processedValue, modified, err := r.Process(value) + if err != nil { + return nil, false, xerrors.Errorf("error replacing placeholders in map with key=%s, value=%s: %w", key, value, err) + } + newMap[key] = processedValue + mapWasModified = mapWasModified || modified + } + + return newMap, mapWasModified, nil +} diff --git a/pkg/placeholders/replacer_test.go b/pkg/placeholders/replacer_test.go new file mode 100644 index 000000000..b1de6b41d --- /dev/null +++ b/pkg/placeholders/replacer_test.go @@ -0,0 +1,165 @@ +package placeholders + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReplacer_Process(t *testing.T) { + tests := []struct { + name string + placeholders map[string]string + input string + expectedReplacedFlag bool + expectedOutput string + expectedErrMsg string + }{ + { + name: "All placeholders replaced", + placeholders: map[string]string{"val1": "1", "val2": "12"}, + input: "this is {val1} and {val2}", + expectedOutput: "this is 1 and 12", + expectedReplacedFlag: true, + expectedErrMsg: "", + }, + { + name: "All placeholders replaced, some multiple times", + placeholders: map[string]string{"val1": "v1", "val2": "v2", "number": "3"}, + input: "this is {val1} and {val2},{val2},{val2} (repeated {number} times)", + expectedOutput: "this is v1 and v2,v2,v2 (repeated 3 times)", + expectedReplacedFlag: true, + expectedErrMsg: "", + }, + { + name: "No changes when no placeholders in the input", + placeholders: map[string]string{"val1": "v1"}, + input: "no placeholders here", + expectedOutput: "no placeholders here", + expectedReplacedFlag: false, + expectedErrMsg: "", + }, + { + name: "Works for empty strings", + placeholders: map[string]string{"val1": "v1"}, + input: "", + expectedOutput: "", + expectedReplacedFlag: false, + expectedErrMsg: "", + }, + { + name: "Works for empty strings with empty placeholder values", + placeholders: map[string]string{}, + input: "", + expectedOutput: "", + expectedReplacedFlag: false, + expectedErrMsg: "", + }, + { + name: "Invalid placeholders are ignored", + placeholders: map[string]string{}, + input: `{inv@lid},{inv alid},{inv-alid}, {"json": {"string": }}, {{{{`, + expectedOutput: `{inv@lid},{inv alid},{inv-alid}, {"json": {"string": }}, {{{{`, + expectedReplacedFlag: false, + expectedErrMsg: "", + }, + { + name: "Empty string values are allowed", + placeholders: map[string]string{"empty1": "", "empty2": ""}, + input: "{empty1}{empty2}", + expectedOutput: "", + expectedReplacedFlag: true, + expectedErrMsg: "", + }, + { + name: "Missing placeholder values are not allowed", + placeholders: map[string]string{"val1": "v1", "val3": "v3", "val5": "v5"}, + input: "val6={val6},val1={val1},val2={val2},val3={val3},val4={val4},val5={val5},val6={val6}", + expectedOutput: "", + expectedReplacedFlag: false, + expectedErrMsg: "{val2}, {val4}, {val6}", + }, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + replacer := New(testCase.placeholders) + actualValue, actualReplacedFlag, err := replacer.Process(testCase.input) + if testCase.expectedErrMsg != "" { + assert.Error(t, err) + assert.ErrorContains(t, err, testCase.expectedErrMsg) + } else { + assert.NoError(t, err) + assert.Equal(t, testCase.expectedOutput, actualValue) + assert.Equal(t, testCase.expectedReplacedFlag, actualReplacedFlag) + } + }) + } +} + +func TestReplacer_ProcessMap(t *testing.T) { + tests := []struct { + name string + placeholders map[string]string + input map[string]string + expectedOutput map[string]string + expectedReplacedFlag bool + expectedErrMsg1 string + expectedErrMsg2 string + }{ + { + name: "All placeholders in all values replaced", + placeholders: map[string]string{"val1": "v1", "val2": "v2"}, + input: map[string]string{"key1": "val1={val1}", "key2": "val2={val2}", "key3": "val3=v3"}, + expectedOutput: map[string]string{"key1": "val1=v1", "key2": "val2=v2", "key3": "val3=v3"}, + expectedReplacedFlag: true, + expectedErrMsg1: "", + expectedErrMsg2: "", + }, + { + name: "No placeholders, no changes", + placeholders: map[string]string{}, + input: map[string]string{"key1": "val1=v1", "key2": "val2=v2", "key3": "val3=v3"}, + expectedOutput: map[string]string{"key1": "val1=v1", "key2": "val2=v2", "key3": "val3=v3"}, + expectedReplacedFlag: false, + expectedErrMsg1: "", + expectedErrMsg2: "", + }, + { + name: "Missing placeholder value returns error msg with key, value and missing placeholders", + placeholders: map[string]string{"val1": "v1"}, + input: map[string]string{"key1": "val1={val1}", "key2": "val2={missing1}{missing2}"}, + expectedOutput: nil, + expectedReplacedFlag: false, + expectedErrMsg1: "{missing1}, {missing2}", + expectedErrMsg2: "key=key2, value=val2={missing1}{missing2}", + }, + { + name: "empty map is ok", + placeholders: map[string]string{}, + input: map[string]string{}, + expectedOutput: map[string]string{}, + expectedReplacedFlag: false, + expectedErrMsg1: "", + expectedErrMsg2: "", + }, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + replacer := New(testCase.placeholders) + actualValue, actualReplacedFlag, err := replacer.ProcessMap(testCase.input) + if testCase.expectedErrMsg1 != "" || testCase.expectedErrMsg2 != "" { + assert.Error(t, err) + if testCase.expectedErrMsg1 != "" { + assert.ErrorContains(t, err, testCase.expectedErrMsg1) + } + if testCase.expectedErrMsg2 != "" { + assert.ErrorContains(t, err, testCase.expectedErrMsg2) + } + } else { + assert.NoError(t, err) + assert.Equal(t, testCase.expectedOutput, actualValue) + assert.Equal(t, testCase.expectedReplacedFlag, actualReplacedFlag) + } + }) + } +} diff --git a/pkg/statefulset/statefulset_test.go b/pkg/statefulset/statefulset_test.go new file mode 100644 index 000000000..a3c1ba4ae --- /dev/null +++ b/pkg/statefulset/statefulset_test.go @@ -0,0 +1,578 @@ +package statefulset + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + TestNamespace = "test-ns" + TestName = "test-name" +) + +func int64Ref(i int64) *int64 { + return &i +} + +func TestGetContainerIndexByName(t *testing.T) { + containers := []corev1.Container{ + { + Name: "container-0", + }, + { + Name: "container-1", + }, + { + Name: "container-2", + }, + } + + stsBuilder := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers(containers)) + idx, err := stsBuilder.GetContainerIndexByName("container-0") + + assert.NoError(t, err) + assert.NotEqual(t, -1, idx) + assert.Equal(t, 0, idx) + + idx, err = stsBuilder.GetContainerIndexByName("container-1") + + assert.NoError(t, err) + assert.NotEqual(t, -1, idx) + assert.Equal(t, 1, idx) + + idx, err = stsBuilder.GetContainerIndexByName("container-2") + + assert.NoError(t, err) + assert.NotEqual(t, -1, idx) + assert.Equal(t, 2, idx) + + idx, err = stsBuilder.GetContainerIndexByName("doesnt-exist") + + assert.Error(t, err) + assert.Equal(t, -1, idx) +} + +func TestAddVolumeAndMount(t *testing.T) { + var stsBuilder *statefulset.Builder + var sts appsv1.StatefulSet + var err error + vmd := statefulset.VolumeMountData{ + MountPath: "mount-path", + Name: "mount-name", + ReadOnly: true, + Volume: statefulset.CreateVolumeFromConfigMap("mount-name", "config-map"), + } + + stsBuilder = defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-name"}})).AddVolumeAndMount(vmd, "container-name") + sts, err = stsBuilder.Build() + + // assert container was correctly updated with the volumes + assert.NoError(t, err, "volume should successfully mount when the container exists") + assert.Len(t, sts.Spec.Template.Spec.Containers[0].VolumeMounts, 1, "volume mount should have been added to the container in the stateful set") + assert.Equal(t, sts.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name, "mount-name") + assert.Equal(t, sts.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath, "mount-path") + + // assert the volumes were added to the podspec template + assert.Len(t, sts.Spec.Template.Spec.Volumes, 1) + assert.Equal(t, sts.Spec.Template.Spec.Volumes[0].Name, "mount-name") + assert.NotNil(t, sts.Spec.Template.Spec.Volumes[0].ConfigMap, "volume should have been configured from a config map source") + assert.Nil(t, sts.Spec.Template.Spec.Volumes[0].Secret, "volume should not have been configured from a secret source") + + stsBuilder = defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-0"}, {Name: "container-1"}})).AddVolumeAndMount(vmd, "container-0") + sts, err = stsBuilder.Build() + + assert.NoError(t, err, "volume should successfully mount when the container exists") + + secretVmd := statefulset.VolumeMountData{ + MountPath: "mount-path-secret", + Name: "mount-name-secret", + ReadOnly: true, + Volume: statefulset.CreateVolumeFromSecret("mount-name-secret", "secret"), + } + + // add a 2nd container to previously defined stsBuilder + sts, err = stsBuilder.AddVolumeAndMount(secretVmd, "container-1").Build() + + assert.NoError(t, err, "volume should successfully mount when the container exists") + assert.Len(t, sts.Spec.Template.Spec.Containers[1].VolumeMounts, 1, "volume mount should have been added to the container in the stateful set") + assert.Equal(t, sts.Spec.Template.Spec.Containers[1].VolumeMounts[0].Name, "mount-name-secret") + assert.Equal(t, sts.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPath, "mount-path-secret") + + assert.Len(t, sts.Spec.Template.Spec.Volumes, 2) + assert.Equal(t, sts.Spec.Template.Spec.Volumes[1].Name, "mount-name-secret") + assert.Nil(t, sts.Spec.Template.Spec.Volumes[1].ConfigMap, "volume should not have been configured from a config map source") + assert.NotNil(t, sts.Spec.Template.Spec.Volumes[1].Secret, "volume should have been configured from a secret source") +} + +func TestAddVolumeClaimTemplates(t *testing.T) { + claim := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "claim-0", + }, + } + mount := corev1.VolumeMount{ + Name: "mount-0", + } + sts, err := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-name"}})).AddVolumeClaimTemplates([]corev1.PersistentVolumeClaim{claim}).AddVolumeMounts("container-name", []corev1.VolumeMount{mount}).Build() + + assert.NoError(t, err) + assert.Len(t, sts.Spec.VolumeClaimTemplates, 1) + assert.Equal(t, sts.Spec.VolumeClaimTemplates[0].Name, "claim-0") + assert.Len(t, sts.Spec.Template.Spec.Containers[0].VolumeMounts, 1) + assert.Equal(t, sts.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name, "mount-0") +} + +func TestBuildStructImmutable(t *testing.T) { + labels := map[string]string{"label_1": "a", "label_2": "b"} + stsBuilder := defaultStatefulSetBuilder().SetLabels(labels).SetReplicas(2) + var sts appsv1.StatefulSet + var err error + sts, err = stsBuilder.Build() + assert.NoError(t, err) + assert.Len(t, sts.Labels, 2) + assert.Equal(t, *sts.Spec.Replicas, int32(2)) + + delete(labels, "label_2") + // checks that modifying the underlying object did not change the built statefulset + assert.Len(t, sts.Labels, 2) + assert.Equal(t, *sts.Spec.Replicas, int32(2)) + sts, err = stsBuilder.Build() + assert.NoError(t, err) + assert.Len(t, sts.Labels, 1) + assert.Equal(t, *sts.Spec.Replicas, int32(2)) +} + +func defaultStatefulSetBuilder() *statefulset.Builder { + return statefulset.NewBuilder(). + SetName(TestName). + SetNamespace(TestNamespace). + SetServiceName(fmt.Sprintf("%s-svc", TestName)). + SetLabels(map[string]string{}) +} + +func podTemplateWithContainers(containers []corev1.Container) corev1.PodTemplateSpec { + return corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: containers, + }, + } +} + +func TestBuildStatefulSet_SortedEnvVariables(t *testing.T) { + podTemplateSpec := podTemplateWithContainers([]corev1.Container{{Name: "container-name"}}) + podTemplateSpec.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "one", Value: "X"}, + {Name: "two", Value: "Y"}, + {Name: "three", Value: "Z"}, + } + sts, err := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateSpec).Build() + assert.NoError(t, err) + expectedVars := []corev1.EnvVar{ + {Name: "one", Value: "X"}, + {Name: "three", Value: "Z"}, + {Name: "two", Value: "Y"}, + } + assert.Equal(t, expectedVars, sts.Spec.Template.Spec.Containers[0].Env) +} + +func getDefaultContainer() corev1.Container { + return corev1.Container{ + Name: "container-0", + Image: "image-0", + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/foo", + }}, + PeriodSeconds: 10, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "container-0.volume-mount-0", + }, + }, + } +} + +func getCustomContainer() corev1.Container { + return corev1.Container{ + Name: "container-1", + Image: "image-1", + } +} + +func getDefaultPodSpec() corev1.PodTemplateSpec { + initContainer := getDefaultContainer() + initContainer.Name = "init-container-default" + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-default-name", + Namespace: "my-default-namespace", + Labels: map[string]string{"app": "operator"}, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-0": "node-0", + }, + ServiceAccountName: "my-default-service-account", + TerminationGracePeriodSeconds: int64Ref(12), + ActiveDeadlineSeconds: int64Ref(10), + Containers: []corev1.Container{getDefaultContainer()}, + InitContainers: []corev1.Container{initContainer}, + Affinity: affinity("hostname", "default"), + }, + } +} + +func affinity(antiAffinityKey, nodeAffinityKey string) *corev1.Affinity { + return &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ + PodAffinityTerm: corev1.PodAffinityTerm{ + TopologyKey: antiAffinityKey, + }, + }}, + }, + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchFields: []corev1.NodeSelectorRequirement{{ + Key: nodeAffinityKey, + }}, + }}}, + }, + } +} + +func getCustomPodSpec() corev1.PodTemplateSpec { + initContainer := getCustomContainer() + initContainer.Name = "init-container-custom" + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"custom": "some"}, + }, + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{ + "node-1": "node-1", + }, + ServiceAccountName: "my-service-account-override", + TerminationGracePeriodSeconds: int64Ref(11), + NodeName: "my-node-name", + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{getCustomContainer()}, + InitContainers: []corev1.Container{initContainer}, + Affinity: affinity("zone", "custom"), + }, + } +} + +func TestMergePodSpecsEmptyCustom(t *testing.T) { + defaultPodSpec := getDefaultPodSpec() + customPodSpecTemplate := corev1.PodTemplateSpec{} + + mergedPodTemplateSpec := merge.PodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + assert.Equal(t, "my-default-service-account", mergedPodTemplateSpec.Spec.ServiceAccountName) + assert.Equal(t, int64Ref(12), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) + + assert.Equal(t, "my-default-name", mergedPodTemplateSpec.Name) + assert.Equal(t, "my-default-namespace", mergedPodTemplateSpec.Namespace) + assert.Equal(t, int64Ref(10), mergedPodTemplateSpec.Spec.ActiveDeadlineSeconds) + + // ensure collections have been merged + assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-0") + assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 1) + assert.Equal(t, "container-0", mergedPodTemplateSpec.Spec.Containers[0].Name) + assert.Equal(t, "image-0", mergedPodTemplateSpec.Spec.Containers[0].Image) + assert.Equal(t, "container-0.volume-mount-0", mergedPodTemplateSpec.Spec.Containers[0].VolumeMounts[0].Name) + assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 1) + assert.Equal(t, "init-container-default", mergedPodTemplateSpec.Spec.InitContainers[0].Name) +} + +func TestMergePodSpecsEmptyDefault(t *testing.T) { + defaultPodSpec := corev1.PodTemplateSpec{} + customPodSpecTemplate := getCustomPodSpec() + + mergedPodTemplateSpec := merge.PodTemplateSpecs(customPodSpecTemplate, defaultPodSpec) + + assert.Equal(t, "my-service-account-override", mergedPodTemplateSpec.Spec.ServiceAccountName) + assert.Equal(t, int64Ref(11), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) + assert.Equal(t, "my-node-name", mergedPodTemplateSpec.Spec.NodeName) + assert.Equal(t, corev1.RestartPolicy("Always"), mergedPodTemplateSpec.Spec.RestartPolicy) + + assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 1) + assert.Equal(t, "container-1", mergedPodTemplateSpec.Spec.Containers[0].Name) + assert.Equal(t, "image-1", mergedPodTemplateSpec.Spec.Containers[0].Image) + assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 1) + assert.Equal(t, "init-container-custom", mergedPodTemplateSpec.Spec.InitContainers[0].Name) +} + +func TestMergePodSpecsBoth(t *testing.T) { + defaultPodSpec := getDefaultPodSpec() + customPodSpecTemplate := getCustomPodSpec() + + var mergedPodTemplateSpec corev1.PodTemplateSpec + + // multiple merges must give the same result + for i := 0; i < 3; i++ { + mergedPodTemplateSpec = merge.PodTemplateSpecs(defaultPodSpec, customPodSpecTemplate) + // ensure values that were specified in the custom pod spec template remain unchanged + assert.Equal(t, "my-service-account-override", mergedPodTemplateSpec.Spec.ServiceAccountName) + assert.Equal(t, int64Ref(11), mergedPodTemplateSpec.Spec.TerminationGracePeriodSeconds) + assert.Equal(t, "my-node-name", mergedPodTemplateSpec.Spec.NodeName) + assert.Equal(t, corev1.RestartPolicy("Always"), mergedPodTemplateSpec.Spec.RestartPolicy) + + // ensure values from the default pod spec template have been merged in + assert.Equal(t, "my-default-name", mergedPodTemplateSpec.Name) + assert.Equal(t, "my-default-namespace", mergedPodTemplateSpec.Namespace) + assert.Equal(t, int64Ref(10), mergedPodTemplateSpec.Spec.ActiveDeadlineSeconds) + + // ensure collections have been merged + assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-0") + assert.Contains(t, mergedPodTemplateSpec.Spec.NodeSelector, "node-1") + assert.Len(t, mergedPodTemplateSpec.Spec.Containers, 2) + assert.Equal(t, "container-0", mergedPodTemplateSpec.Spec.Containers[0].Name) + assert.Equal(t, "image-0", mergedPodTemplateSpec.Spec.Containers[0].Image) + assert.Equal(t, "container-0.volume-mount-0", mergedPodTemplateSpec.Spec.Containers[0].VolumeMounts[0].Name) + assert.Equal(t, "container-1", mergedPodTemplateSpec.Spec.Containers[1].Name) + assert.Equal(t, "image-1", mergedPodTemplateSpec.Spec.Containers[1].Image) + assert.Len(t, mergedPodTemplateSpec.Spec.InitContainers, 2) + assert.Equal(t, "init-container-custom", mergedPodTemplateSpec.Spec.InitContainers[0].Name) + assert.Equal(t, "init-container-default", mergedPodTemplateSpec.Spec.InitContainers[1].Name) + + // ensure labels were appended + assert.Len(t, mergedPodTemplateSpec.Labels, 2) + assert.Contains(t, mergedPodTemplateSpec.Labels, "app") + assert.Contains(t, mergedPodTemplateSpec.Labels, "custom") + + // ensure the pointers are not the same + assert.NotEqual(t, mergedPodTemplateSpec.Spec.Affinity, defaultPodSpec.Spec.Affinity) + + // ensure the affinity rules slices were overridden + assert.Equal(t, affinity("zone", "custom"), mergedPodTemplateSpec.Spec.Affinity) + } +} + +func TestMergeSpec(t *testing.T) { + t.Run("Add Container to PodSpecTemplate", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().Build() + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-0"}})).Build() + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Contains(t, mergedSpec.Template.Spec.Containers, corev1.Container{Name: "container-0"}) + }) + t.Run("Change terminationGracePeriodSeconds", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().Build() + assert.NoError(t, err) + sts.Spec.Template.Spec.TerminationGracePeriodSeconds = int64Ref(30) + customSts, err := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-0"}})).Build() + sts.Spec.Template.Spec.TerminationGracePeriodSeconds = int64Ref(600) + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Contains(t, mergedSpec.Template.Spec.Containers, corev1.Container{Name: "container-0"}) + assert.Equal(t, mergedSpec.Template.Spec.TerminationGracePeriodSeconds, int64Ref(600)) + }) + t.Run("Containers are added to existing list", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-0"}})).Build() + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().SetPodTemplateSpec(podTemplateWithContainers([]corev1.Container{{Name: "container-1"}})).Build() + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Len(t, mergedSpec.Template.Spec.Containers, 2) + assert.Contains(t, mergedSpec.Template.Spec.Containers, corev1.Container{Name: "container-0"}) + assert.Contains(t, mergedSpec.Template.Spec.Containers, corev1.Container{Name: "container-1"}) + }) + t.Run("Cannot change fields in the StatefulSet outside of the spec", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().Build() + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().Build() + assert.NoError(t, err) + customSts.Annotations = map[string]string{ + "some-annotation": "some-value", + } + mergedSts := merge.StatefulSets(sts, customSts) + assert.NotContains(t, mergedSts.Annotations, "some-annotation") + }) + t.Run("change fields in the StatefulSet the Operator doesn't touch", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().Build() + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().AddVolumeClaimTemplates([]corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-volume-claim", + }, + }}).Build() + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Len(t, mergedSpec.VolumeClaimTemplates, 1) + assert.Equal(t, "my-volume-claim", mergedSpec.VolumeClaimTemplates[0].Name) + }) + t.Run("Volume Claim Templates are added to existing StatefulSet", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().AddVolumeClaimTemplates([]corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-volume-claim-0", + }, + }}).Build() + + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().AddVolumeClaimTemplates([]corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-volume-claim-1", + }, + }}).Build() + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Len(t, mergedSpec.VolumeClaimTemplates, 2) + assert.Equal(t, "my-volume-claim-0", mergedSpec.VolumeClaimTemplates[0].Name) + assert.Equal(t, "my-volume-claim-1", mergedSpec.VolumeClaimTemplates[1].Name) + }) + + t.Run("Volume Claim Templates are changed by name", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().AddVolumeClaimTemplates([]corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-volume-claim-0", + Namespace: "first-ns", + }, + }}).Build() + + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().AddVolumeClaimTemplates([]corev1.PersistentVolumeClaim{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-volume-claim-0", + Namespace: "new-ns", + }, + }}).Build() + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Len(t, mergedSpec.VolumeClaimTemplates, 1) + assert.Equal(t, "my-volume-claim-0", mergedSpec.VolumeClaimTemplates[0].Name) + assert.Equal(t, "new-ns", mergedSpec.VolumeClaimTemplates[0].Namespace) + }) + + t.Run("Volume Claims are added", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder(). + SetPodTemplateSpec(getDefaultPodSpec()). + AddVolumeAndMount(statefulset.VolumeMountData{ + MountPath: "path", + Name: "vol-0", + ReadOnly: false, + Volume: corev1.Volume{ + Name: "vol-0", + }, + }, "container-0").Build() + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder(). + SetPodTemplateSpec(getDefaultPodSpec()). + AddVolumeAndMount(statefulset.VolumeMountData{ + MountPath: "path-1", + Name: "vol-1", + ReadOnly: false, + Volume: corev1.Volume{ + Name: "vol-1", + }, + }, "container-0"). + AddVolumeAndMount(statefulset.VolumeMountData{ + MountPath: "path-2", + Name: "vol-2", + ReadOnly: false, + Volume: corev1.Volume{ + Name: "vol-2", + }, + }, "container-0").Build() + assert.NoError(t, err) + + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + + assert.Len(t, mergedSpec.Template.Spec.Volumes, 3) + for i, vol := range mergedSpec.Template.Spec.Volumes { + assert.Equal(t, fmt.Sprintf("vol-%d", i), vol.Name) + } + }) + + t.Run("Custom StatefulSet zero values don't override operator configured ones", func(t *testing.T) { + sts, err := defaultStatefulSetBuilder().SetServiceName("service-name").Build() + assert.NoError(t, err) + customSts, err := defaultStatefulSetBuilder().SetServiceName("").Build() + assert.NoError(t, err) + mergedSpec := merge.StatefulSetSpecs(sts.Spec, customSts.Spec) + assert.Equal(t, mergedSpec.ServiceName, "service-name") + }) +} + +func TestMergingVolumeMounts(t *testing.T) { + container0 := corev1.Container{ + Name: "container-0", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "database-scripts", + MountPath: "/opt/scripts", + SubPath: "", + }, + { + Name: "data", + MountPath: "/data", + SubPath: "data", + }, + { + Name: "data", + MountPath: "/journal", + SubPath: "journal", + }, + { + Name: "data", + MountPath: "/var/log/mongodb-mms-automation", + SubPath: "logs", + }, + }, + } + + container1 := corev1.Container{ + Name: "container-0", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test-volume", + MountPath: "/somewhere", + SubPath: "", + }, + }, + } + + podSpec0 := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + container0, + }, + }, + } + + podSpec1 := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + container1, + }, + }, + } + + merged := merge.PodTemplateSpecs(podSpec0, podSpec1) + + assert.Len(t, merged.Spec.Containers, 1) + mounts := merged.Spec.Containers[0].VolumeMounts + + assert.Equal(t, container0.VolumeMounts[1], mounts[0]) + assert.Equal(t, container0.VolumeMounts[2], mounts[1]) + assert.Equal(t, container0.VolumeMounts[3], mounts[2]) + assert.Equal(t, container0.VolumeMounts[0], mounts[3]) + assert.Equal(t, container1.VolumeMounts[0], mounts[4]) +} diff --git a/pkg/statefulset/statefulset_util.go b/pkg/statefulset/statefulset_util.go new file mode 100644 index 000000000..b0b4e60b9 --- /dev/null +++ b/pkg/statefulset/statefulset_util.go @@ -0,0 +1,197 @@ +package statefulset + +import ( + "cmp" + "context" + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + + "github.com/google/go-cmp/cmp/cmpopts" + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset" + + gocmp "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/10gen/ops-manager-kubernetes/controllers/operator/certs" + "github.com/10gen/ops-manager-kubernetes/pkg/kube" +) + +const PVCSizeAnnotation = "mongodb.com/storageSize" + +// isVolumeClaimEqualOnForbiddenFields takes two sts PVCs +// and returns whether we are allowed to update the first one to the second one. +func isVolumeClaimEqualOnForbiddenFields(existing, desired corev1.PersistentVolumeClaim) bool { + oldSpec := existing.Spec + newSpec := desired.Spec + + if !gocmp.Equal(oldSpec.AccessModes, newSpec.AccessModes, cmpopts.EquateEmpty()) { + return false + } + + if newSpec.Selector != nil && !gocmp.Equal(oldSpec.Selector, newSpec.Selector, cmpopts.EquateEmpty()) { + return false + } + + // using api-machinery here for semantic equality + if !apiEquality.Semantic.DeepEqual(oldSpec.Resources, newSpec.Resources) { + return false + } + + if newSpec.VolumeName != "" && newSpec.VolumeName != oldSpec.VolumeName { + return false + } + + if newSpec.StorageClassName != nil && !reflect.DeepEqual(oldSpec.StorageClassName, newSpec.StorageClassName) { + return false + } + + if newSpec.VolumeMode != nil && !reflect.DeepEqual(newSpec.VolumeMode, oldSpec.VolumeMode) { + return false + } + + if newSpec.DataSource != nil && !reflect.DeepEqual(newSpec.DataSource, oldSpec.DataSource) { + return false + } + + return true +} + +// isStatefulSetEqualOnForbiddenFields takes two statefulsets +// and returns whether we are allowed to update the first one to the second one. +// This is decided on equality on forbidden fields. +func isStatefulSetEqualOnForbiddenFields(existing, desired appsv1.StatefulSet) bool { + // We are using cmp equal on purpose to enforce equality between nil and [] + selectorsEqual := desired.Spec.Selector == nil || gocmp.Equal(existing.Spec.Selector, desired.Spec.Selector, cmpopts.EquateEmpty()) + serviceNamesEqual := existing.Spec.ServiceName == desired.Spec.ServiceName + podMgmtEqual := desired.Spec.PodManagementPolicy == "" || desired.Spec.PodManagementPolicy == existing.Spec.PodManagementPolicy + revHistoryLimitEqual := desired.Spec.RevisionHistoryLimit == nil || reflect.DeepEqual(desired.Spec.RevisionHistoryLimit, existing.Spec.RevisionHistoryLimit) + + if len(existing.Spec.VolumeClaimTemplates) != len(desired.Spec.VolumeClaimTemplates) { + return false + } + + // VolumeClaimTemplates must be checked one-by-one, to deal with empty string, nil pointers + for index, existingClaim := range existing.Spec.VolumeClaimTemplates { + if !isVolumeClaimEqualOnForbiddenFields(existingClaim, desired.Spec.VolumeClaimTemplates[index]) { + return false + } + } + + return selectorsEqual && serviceNamesEqual && podMgmtEqual && revHistoryLimitEqual +} + +// StatefulSetCantBeUpdatedError is returned when we are trying to update immutable fields on a sts. +type StatefulSetCantBeUpdatedError struct { + msg string +} + +func (s StatefulSetCantBeUpdatedError) Error() string { + return s.msg +} + +// CreateOrUpdateStatefulset will create or update a StatefulSet in Kubernetes. +// +// The method has to be flexible (create/update) as there are cases when custom resource is created but statefulset - not +// Service named "serviceName" is created optionally (it may already exist - created by either user or by operator before) +// Note the logic for "exposeExternally" parameter: if it is true then the second service is created of type "NodePort" +// (the random port will be allocated by Kubernetes) otherwise only one service of type "ClusterIP" is created and it +// won't be connectible from external (unless pods in statefulset expose themselves to outside using "hostNetwork: true") +// Function returns the service port number assigned +func CreateOrUpdateStatefulset(ctx context.Context, getUpdateCreator statefulset.GetUpdateCreator, ns string, log *zap.SugaredLogger, statefulSetToCreate *appsv1.StatefulSet) (*appsv1.StatefulSet, error) { + log = log.With("statefulset", kube.ObjectKey(ns, statefulSetToCreate.Name)) + existingStatefulSet, err := getUpdateCreator.GetStatefulSet(ctx, kube.ObjectKey(ns, statefulSetToCreate.Name)) + if err != nil { + if apiErrors.IsNotFound(err) { + if err = getUpdateCreator.CreateStatefulSet(ctx, *statefulSetToCreate); err != nil { + return nil, err + } + } else { + return nil, err + } + log.Debug("Created StatefulSet") + return statefulSetToCreate, nil + } + // preserve existing certificate hash if new one is not statefulSetToCreate + existingCertHash, okExisting := existingStatefulSet.Spec.Template.Annotations[certs.CertHashAnnotationKey] + if newCertHash, okNew := statefulSetToCreate.Spec.Template.Annotations[certs.CertHashAnnotationKey]; existingCertHash != "" && newCertHash == "" && okExisting && okNew { + if statefulSetToCreate.Spec.Template.Annotations == nil { + statefulSetToCreate.Spec.Template.Annotations = map[string]string{} + } + statefulSetToCreate.Spec.Template.Annotations[certs.CertHashAnnotationKey] = existingCertHash + } + + // there already exists a pvc size annotation, that means we did resize at least once + // we need to make sure to keep the annotation. + if _, okExisting := existingStatefulSet.Spec.Template.Annotations[PVCSizeAnnotation]; okExisting { + if err := AddPVCAnnotation(statefulSetToCreate); err != nil { + return nil, err + } + } + + log.Debug("Checking if we can update the current statefulset") + if !isStatefulSetEqualOnForbiddenFields(existingStatefulSet, *statefulSetToCreate) { + // Running into this code means we have updated sts fields which are not allowed to be changed. + log.Debug("Can't update the stateful set") + return nil, StatefulSetCantBeUpdatedError{ + msg: "can't execute update on forbidden fields", + } + } + + updatedSts, err := getUpdateCreator.UpdateStatefulSet(ctx, *statefulSetToCreate) + if err != nil { + return nil, err + } + + return &updatedSts, nil +} + +// AddPVCAnnotation adds pvc annotation to the statefulset.template, this can either trigger a rolling restart +// if the template has changed is a noop for an already existing one. +func AddPVCAnnotation(statefulSetToCreate *appsv1.StatefulSet) error { + type pvcSizes struct { + Name string + Size string + } + if statefulSetToCreate.Spec.Template.Annotations == nil { + statefulSetToCreate.Spec.Template.Annotations = map[string]string{} + } + var p []pvcSizes + for _, template := range statefulSetToCreate.Spec.VolumeClaimTemplates { + p = append(p, pvcSizes{ + Name: template.Name, + Size: template.Spec.Resources.Requests.Storage().String(), + }) + } + + // ensure a strict order to not have unnecessary restarts + slices.SortFunc(p, func(a, b pvcSizes) int { + return cmp.Compare(strings.ToLower(a.Name), strings.ToLower(b.Name)) + }) + + jsonString, err := json.Marshal(p) + if err != nil { + return err + } + statefulSetToCreate.Spec.Template.Annotations[PVCSizeAnnotation] = string(jsonString) + return nil +} + +// GetFilePathFromAnnotationOrDefault returns a concatenation of a default path and an annotation, or a default value +// if the annotation is not present. +func GetFilePathFromAnnotationOrDefault(sts appsv1.StatefulSet, key string, path string, defaultValue string) string { + val, ok := sts.Annotations[key] + + if ok { + return fmt.Sprintf("%s/%s", path, val) + } + + return defaultValue +} diff --git a/pkg/statefulset/statefulset_util_test.go b/pkg/statefulset/statefulset_util_test.go new file mode 100644 index 000000000..e33496fa6 --- /dev/null +++ b/pkg/statefulset/statefulset_util_test.go @@ -0,0 +1,205 @@ +package statefulset + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/api/resource" + + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIsStatefulSetUpdatableTo(t *testing.T) { + tests := []struct { + name string + existing v1.StatefulSet + desired v1.StatefulSet + want bool + }{ + { + name: "empty", + existing: v1.StatefulSet{}, + desired: v1.StatefulSet{}, + want: true, + }, + { + name: "TypeMeta: unequal", + existing: v1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "123", + }, + }, + desired: v1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "something else", + }, + }, + want: true, + }, + { + name: "Selector: unequal", + existing: v1.StatefulSet{ + Spec: v1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"a": "b"}, + }, + }, + }, + desired: v1.StatefulSet{ + Spec: v1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"c": "d"}, + }, + }, + }, + want: false, + }, + { + name: "Selector: equal nil and empty", + existing: v1.StatefulSet{ + Spec: v1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchExpressions: nil, + MatchLabels: nil, + }, + }, + }, + desired: v1.StatefulSet{ + Spec: v1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{}, + MatchLabels: map[string]string{}, + }, + }, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, isStatefulSetEqualOnForbiddenFields(tt.existing, tt.desired), "isStatefulSetEqualOnForbiddenFields(%v, %v)", tt.existing, tt.desired) + }) + } +} + +func TestIsVolumeClaimUpdatableTo(t *testing.T) { + tests := []struct { + name string + existing corev1.PersistentVolumeClaim + desired corev1.PersistentVolumeClaim + want bool + }{ + { + name: "empty", + existing: corev1.PersistentVolumeClaim{}, + desired: corev1.PersistentVolumeClaim{}, + want: true, + }, + { + name: "TypeMeta: unequal", + existing: corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "123", + }, + }, + desired: corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "abc", + }, + }, + want: true, + }, + { + name: "AccessModes: equal nil and empty", + existing: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: nil, + }, + }, + desired: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{}, + }, + }, + want: true, + }, + { + name: "AccessModes: unequal", + existing: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"a"}, + }, + }, + desired: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{"b"}, + }, + }, + want: false, + }, + { + name: "Storage: unequal", + existing: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, + desired: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("2Gi")}, + }, + }, + }, + want: false, + }, + { + name: "Selector: equal nil and empty", + existing: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Selector: &metav1.LabelSelector{ + MatchExpressions: nil, + MatchLabels: nil, + }, + }, + }, + desired: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Selector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{}, + MatchLabels: map[string]string{}, + }, + }, + }, + want: true, + }, + { + // CLOUDP-275888 + name: "Storage: fractional value that needs canonicalizing", + existing: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("966367641600m")}, + }, + }, + }, + desired: corev1.PersistentVolumeClaim{ + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("0.9Gi")}, + }, + }, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, isVolumeClaimEqualOnForbiddenFields(tt.existing, tt.desired), "isVolumeClaimEqualOnForbiddenFields(%v, %v)", tt.existing, tt.desired) + }) + } +} diff --git a/pkg/telemetry/client.go b/pkg/telemetry/client.go new file mode 100644 index 000000000..4ff8000f5 --- /dev/null +++ b/pkg/telemetry/client.go @@ -0,0 +1,124 @@ +package telemetry + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" + "go.uber.org/zap" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + + atlas "go.mongodb.org/atlas/mongodbatlas" +) + +type Client struct { + atlasClient *atlas.Client + maxRetries int + initialBackoff time.Duration +} + +const ( + telemetryTimeout = 1 * time.Second + keepAlive = 30 * time.Second + maxIdleConns = 5 + maxIdleConnsPerHost = 4 + idleConnTimeout = 30 * time.Second + expectContinueTimeout = 1 * time.Second + initialBackoff = 5 * time.Second + maxRetries = 5 + defaultRetryWaitMin = 1 * time.Second + defaultRetryWaitMax = 10 * time.Second +) + +// zapAdapter wraps zap.Logger to implement retryablehttp.Logger interface +type zapAdapter struct { + logger *zap.SugaredLogger +} + +func (z *zapAdapter) Printf(format string, v ...interface{}) { + z.logger.Infof(format, v...) +} + +var ( + telemetryTransport = newTelemetryTransport() + defaultRetryClient = &retryablehttp.Client{ + HTTPClient: &http.Client{Transport: telemetryTransport}, + RetryWaitMin: defaultRetryWaitMin, + RetryWaitMax: defaultRetryWaitMax, + RetryMax: maxRetries, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.LinearJitterBackoff, + Logger: &zapAdapter{logger: Logger}, + } +) + +func newTelemetryTransport() *http.Transport { + return &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: telemetryTimeout, + KeepAlive: keepAlive, + }).DialContext, + MaxIdleConns: maxIdleConns, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + Proxy: http.ProxyFromEnvironment, + IdleConnTimeout: idleConnTimeout, + ExpectContinueTimeout: expectContinueTimeout, + } +} + +func NewClient(retryClient *retryablehttp.Client) (*Client, error) { + if retryClient == nil { + retryClient = defaultRetryClient + } + + c := atlas.NewClient( + &http.Client{Transport: &retryablehttp.RoundTripper{Client: retryClient}}, + ) + + if urlStr := envvar.GetEnvOrDefault(BaseUrl, ""); urlStr != "" { + Logger.Debugf("Using different base url configured for atlasclient: %s", urlStr) + parsed, err := url.Parse(urlStr) + if err != nil { + return nil, fmt.Errorf("invalid base URL for atlas client: %w", err) + } + c.BaseURL = parsed + } + + c.OnResponseProcessed(func(resp *atlas.Response) { + respHeaders := "" + for key, value := range resp.Header { + respHeaders += fmt.Sprintf("%v: %v\n", key, strings.Join(value, " ")) + } + + Logger.Debugf(`request: +%v %v +response: +%v %v +%v +%v +`, resp.Request.Method, resp.Request.URL.String(), resp.Proto, resp.Status, respHeaders, string(resp.Raw)) + }) + + return &Client{ + atlasClient: c, + maxRetries: maxRetries, + initialBackoff: initialBackoff, + }, nil +} + +// SendEventWithRetry sends an HTTP request with retries on transient failures. +func (c *Client) SendEventWithRetry(ctx context.Context, body []Event) error { + atlasClient := c.atlasClient + request, err := atlasClient.NewRequest(ctx, http.MethodPost, "api/private/unauth/telemetry/events", body) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = atlasClient.Do(ctx, request, nil) + return err +} diff --git a/pkg/telemetry/client_test.go b/pkg/telemetry/client_test.go new file mode 100644 index 000000000..dae7ac162 --- /dev/null +++ b/pkg/telemetry/client_test.go @@ -0,0 +1,117 @@ +package telemetry + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/hashicorp/go-retryablehttp" + "github.com/stretchr/testify/assert" +) + +const ( + // baseURLPath is a non-empty Client.BaseURL path to use during tests, + // to ensure relative URLs are used for all endpoints. + baseURLPath = "/api-v1" +) + +// mockServerHandler simulates different HTTP responses for testing retry behavior. +type mockServerHandler struct { + attempts int + maxAttempts int + statusCode int + retryAfter string // Used for simulating 429 Retry-After header + wantRetry bool +} + +func (h *mockServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.attempts++ + + // Simulate a successful response after maxAttempts has passed + if h.attempts >= h.maxAttempts && h.wantRetry { + w.WriteHeader(http.StatusOK) + return + } + + // Simulate EOF by closing the connection immediately after hijacking + if h.statusCode == -1 { // Use a special status code to indicate EOF + if hj, ok := w.(http.Hijacker); ok { + conn, _, err := hj.Hijack() + if err != nil { + return + } + conn.Close() // Close the connection to simulate EOF + return + } + } + + // Simulate a retryable error (500, 502, 503, 504) + if h.statusCode == 429 && h.retryAfter != "" { + w.Header().Set("Retry-After", h.retryAfter) + } + w.WriteHeader(h.statusCode) +} + +func TestSendEventWithRetry(t *testing.T) { + // Override the sleep function to prevent actual delays + ctx := context.Background() + + tests := []struct { + name string + statusCode int + maxAttempts int + retryAfter string + wantRetry bool + expectError bool + }{ + {"Retry on 500, then success", 500, 3, "", true, false}, + {"Retry on 502, then success", 502, 3, "", true, false}, + {"Retry on 503, then success", 503, 3, "", true, false}, + {"Retry on 504, then success", 504, 3, "", true, false}, + {"Handle 429 with Retry-After", 429, 3, "0", true, false}, + {"No retry on 400", 400, 1, "", false, true}, + {"No retry on 403", 403, 1, "", false, true}, + {"No retry on 404", 404, 1, "", false, true}, + {"Retry on EOF, then success", -1, 3, "", true, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := &mockServerHandler{ + attempts: 0, + maxAttempts: tt.maxAttempts, + statusCode: tt.statusCode, + retryAfter: tt.retryAfter, + wantRetry: tt.wantRetry, + } + + server := httptest.NewServer(handler) + defer server.Close() + retryableClient := retryablehttp.NewClient() + retryableClient.RetryWaitMin = 0 + retryableClient.RetryWaitMax = 0 + + client, err := NewClient(retryableClient) + assert.NoError(t, err) + u, _ := url.Parse(server.URL + baseURLPath + "/") + client.atlasClient.BaseURL = u + // Call the function under test + err = client.SendEventWithRetry(ctx, []Event{}) + + // Check expectations + if tt.expectError && err == nil { + t.Errorf("Expected an error but got nil") + } + if !tt.expectError && err != nil { + t.Errorf("Did not expect an error, but got: %v", err) + } + + // Ensure retries were performed correctly + if handler.attempts != tt.maxAttempts { + t.Errorf("Expected %d attempts, but got %d", tt.maxAttempts, handler.attempts) + } + }) + } +} diff --git a/pkg/telemetry/cluster.go b/pkg/telemetry/cluster.go new file mode 100644 index 000000000..ead0c4c44 --- /dev/null +++ b/pkg/telemetry/cluster.go @@ -0,0 +1,157 @@ +package telemetry + +import ( + "context" + "strings" + "time" + + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + + corev1 "k8s.io/api/core/v1" + kubeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + unknown = "Unknown" + eks = "AWS (EKS)" + vmware = "VmWare" + gke = "Google (GKE)" + aks = "Azure (AKS)" + openshift = "Openshift" + rke = "RKE" + rke2 = "RKE2" +) + +var kubernetesFlavourLabelsMapping = map[string]string{ + "eks.amazonaws.com/nodegroup": eks, + "cloud.google.com/gke-nodepool": gke, + "kubernetes.azure.com/agentpool": aks, + "node.openshift.io/os_id": openshift, +} + +var kubernetesFlavourAnnotationsMapping = map[string]string{ + "rke.cattle.io/external-ip": rke, + "rke.cattle.io/internal-ip": rke, +} + +// detectClusterInfo detects the Kubernetes version and cluster flavor +func detectClusterInfos(ctx context.Context, memberClusterMap map[string]ConfigClient) []KubernetesClusterUsageSnapshotProperties { + var clusterProperties []KubernetesClusterUsageSnapshotProperties + + for _, mgr := range memberClusterMap { + discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + if err != nil { + Logger.Debugf("failed to create discovery client: %s, sending Unknown as version", err) + } + clusterProperty := getKubernetesClusterProperty(ctx, discoveryClient, mgr.GetAPIReader()) + clusterProperties = append(clusterProperties, clusterProperty) + } + + return clusterProperties +} + +// getKubernetesClusterProperty returns cluster properties like: +// - kubernetes server version +// - cloud provider (openshift, eks ...) +// - kubernetes cluster uid +// Notes: We are using a non-cached client to ensure we are properly timing out in case we don't have the +// necessary RBACs. +func getKubernetesClusterProperty(ctx context.Context, discoveryClient discovery.DiscoveryInterface, uncachedClient kubeclient.Reader) KubernetesClusterUsageSnapshotProperties { + kubernetesAPIVersion := unknown + kubeClusterUUID := getKubernetesClusterUUID(ctx, uncachedClient) + + if discoveryClient != nil { + if versionInfo := getServerVersion(discoveryClient); versionInfo != nil { + kubernetesAPIVersion = versionInfo.GitVersion + } + } + + kubernetesFlavour := detectKubernetesFlavour(ctx, uncachedClient, kubernetesAPIVersion) + + property := KubernetesClusterUsageSnapshotProperties{ + KubernetesClusterID: kubeClusterUUID, + KubernetesFlavour: kubernetesFlavour, + KubernetesAPIVersion: kubernetesAPIVersion, + } + + return property +} + +func getServerVersion(discoveryClient discovery.DiscoveryInterface) *version.Info { + versionInfo, err := discoveryClient.ServerVersion() + if err != nil { + Logger.Debugf("Failed to fetch server version: %s", err) + return nil + } + return versionInfo +} + +// detectKubernetesFlavour detects the cloud provider based on node labels. +func detectKubernetesFlavour(ctx context.Context, uncachedClient kubeclient.Reader, kubeGitApiVersion string) string { + // Check Kubernetes API version for known cloud providers + switch { + case strings.Contains(kubeGitApiVersion, "+rke2"): + return rke2 + case strings.Contains(kubeGitApiVersion, "-gke"): + return gke + case strings.Contains(kubeGitApiVersion, "-eks"): + return eks + case strings.Contains(kubeGitApiVersion, "+vmware"): + return vmware + } + + // Limit is propagated to the apiserver which propagates to etcd as it is. Thus, there is not a lot of + // work required on the APIServer and ETCD to retrieve that node even in large clusters + nodes := &corev1.NodeList{} + if err := uncachedClient.List(ctx, nodes, &kubeclient.ListOptions{Limit: 1}); err != nil { + Logger.Debugf("Failed to fetch node to detect the cloud provider: %s", err) + return unknown + } + if len(nodes.Items) == 0 { + Logger.Debugf("No nodes found, returning Unknown") + return unknown + } + + node := nodes.Items[0] + labels, annotations := node.Labels, node.Annotations + + for key, provider := range kubernetesFlavourLabelsMapping { + if _, exists := labels[key]; exists { + return provider + } + } + + for key, provider := range kubernetesFlavourAnnotationsMapping { + if _, exists := annotations[key]; exists { + return provider + } + } + + return unknown +} + +// getKubernetesClusterUUID retrieves the UUID from the kube-system namespace. +// We are using a non-cached client to ensure we are properly timing out in case we don't have the +// necessary RBACs. +func getKubernetesClusterUUID(ctx context.Context, uncachedClient kubeclient.Reader) string { + timeoutLengthStr := envvar.GetEnvOrDefault(KubeTimeout, "5m") + duration, err := time.ParseDuration(timeoutLengthStr) + if err != nil { + Logger.Warnf("Failed converting %s to a duration, using default 5m", KubeTimeout) + duration = 5 * time.Minute + } + nonCachedClientTimeout, cancel := context.WithTimeout(ctx, duration) + defer cancel() + + namespace := &corev1.Namespace{} + err = uncachedClient.Get(nonCachedClientTimeout, kubeclient.ObjectKey{Name: "kube-system"}, namespace) + if err != nil { + Logger.Debugf("failed to fetch kube-system namespace: %s", err) + return unknown + } + + return string(namespace.UID) +} diff --git a/pkg/telemetry/cluster_test.go b/pkg/telemetry/cluster_test.go new file mode 100644 index 000000000..e3ddb7fdf --- /dev/null +++ b/pkg/telemetry/cluster_test.go @@ -0,0 +1,229 @@ +package telemetry + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fake2 "k8s.io/client-go/discovery/fake" + clientFake "k8s.io/client-go/kubernetes/fake" + kubeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TestGetKubernetesClusterProperty - Unit tests for getKubernetesClusterProperty +func TestGetKubernetesClusterProperty(t *testing.T) { + ctx := context.Background() + + // Define test cases + tests := []struct { + name string + discoveryClient discovery.DiscoveryInterface + uncachedClient kubeclient.Reader + expectedClusterID string + expectedAPIVersion string + expectedFlavour string + }{ + { + name: "Valid EKS Cluster", + discoveryClient: func() *fake2.FakeDiscovery { + fakeDiscovery := clientFake.NewSimpleClientset().Discovery().(*fake2.FakeDiscovery) + fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: "v1.21.0"} + return fakeDiscovery + }(), + uncachedClient: fake.NewFakeClient( + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "mock-cluster-uuid", + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "eks.amazonaws.com/nodegroup": "default", + }, + }, + }, + ), + expectedClusterID: "mock-cluster-uuid", + expectedAPIVersion: "v1.21.0", + expectedFlavour: eks, + }, + { + name: "Missing Discovery Client", + discoveryClient: nil, + uncachedClient: fake.NewFakeClient( + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "mock-cluster-uuid", + }, + }, + ), + expectedClusterID: "mock-cluster-uuid", + expectedAPIVersion: unknown, // Should default to "unknown" + expectedFlavour: unknown, // No node labels, should be unknown + }, + { + name: "GKE Cluster", + discoveryClient: func() *fake2.FakeDiscovery { + fakeDiscovery := clientFake.NewSimpleClientset().Discovery().(*fake2.FakeDiscovery) + fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: "v1.24.3"} + return fakeDiscovery + }(), + uncachedClient: fake.NewFakeClient( + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "mock-cluster-uuid-gke", + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "cloud.google.com/gke-nodepool": "default-pool", + }, + }, + }, + ), + expectedClusterID: "mock-cluster-uuid-gke", + expectedAPIVersion: "v1.24.3", + expectedFlavour: gke, + }, + { + name: "AKS Cluster", + discoveryClient: func() *fake2.FakeDiscovery { + fakeDiscovery := clientFake.NewSimpleClientset().Discovery().(*fake2.FakeDiscovery) + fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: "v1.24.3"} + return fakeDiscovery + }(), + uncachedClient: fake.NewFakeClient( + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "mock-cluster-uuid-aks", + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "kubernetes.azure.com/agentpool": "default-pool", + }, + }, + }, + ), + expectedClusterID: "mock-cluster-uuid-aks", + expectedAPIVersion: "v1.24.3", + expectedFlavour: aks, + }, + { + name: "OpenShift Cluster", + discoveryClient: func() *fake2.FakeDiscovery { + fakeDiscovery := clientFake.NewSimpleClientset().Discovery().(*fake2.FakeDiscovery) + fakeDiscovery.FakedServerVersion = &version.Info{GitVersion: "v4.9.0"} + return fakeDiscovery + }(), + uncachedClient: fake.NewFakeClient( + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "mock-cluster-uuid-openshift", + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + "node.openshift.io/os_id": "rhcos", + }, + }, + }, + ), + expectedClusterID: "mock-cluster-uuid-openshift", + expectedAPIVersion: "v4.9.0", + expectedFlavour: openshift, + }, + } + + // Run test cases + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + property := getKubernetesClusterProperty(ctx, tt.discoveryClient, tt.uncachedClient) + + assert.Equal(t, tt.expectedClusterID, property.KubernetesClusterID, "Cluster ID mismatch") + assert.Equal(t, tt.expectedAPIVersion, property.KubernetesAPIVersion, "API version mismatch") + assert.Equal(t, tt.expectedFlavour, property.KubernetesFlavour, "Kubernetes flavour mismatch") + }) + } +} + +func TestDetectKubernetesFlavour(t *testing.T) { + ctx := context.Background() + + // Define test cases + tests := []struct { + name string + labels map[string]string + gitVersion string + expectedFlavour string + }{ + { + name: "eks", + labels: map[string]string{"eks.amazonaws.com/nodegroup": "default", "eks.amazonaws.com/cluster": "default"}, + expectedFlavour: eks, + }, + { + name: "unknown", + labels: map[string]string{"something": "default"}, + expectedFlavour: unknown, + }, + { + name: "based on gitversion", + labels: map[string]string{"something": "default"}, + gitVersion: "v123-gke", + expectedFlavour: gke, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + Labels: tt.labels, + }, + } + + fakeClient := fake.NewClientBuilder().WithObjects(node).Build() + cloudProvider := detectKubernetesFlavour(ctx, fakeClient, tt.gitVersion) + + assert.Equal(t, tt.expectedFlavour, cloudProvider) + }) + } +} + +func TestGetKubernetesClusterUUID(t *testing.T) { + ctx := context.Background() + + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-system", + UID: "fake-uuid", + }, + } + + fakeClient := fake.NewClientBuilder().WithObjects(namespace).Build() + uuid := getKubernetesClusterUUID(ctx, fakeClient) + + assert.Equal(t, "fake-uuid", uuid) +} diff --git a/pkg/telemetry/collector.go b/pkg/telemetry/collector.go new file mode 100644 index 000000000..4634c456f --- /dev/null +++ b/pkg/telemetry/collector.go @@ -0,0 +1,484 @@ +package telemetry + +import ( + "context" + "slices" + "strings" + "time" + + "go.uber.org/zap" + "golang.org/x/xerrors" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + + kubeclient "sigs.k8s.io/controller-runtime/pkg/client" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + mdbmultiv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/pkg/images" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" + "github.com/10gen/ops-manager-kubernetes/pkg/util/versionutil" +) + +// Logger should default to the global default from zap. Running into the main function of this package +// should reconfigure zap. +var Logger = zap.S() + +func ConfigureLogger() { + Logger = zap.S().With("module", "Telemetry") +} + +type ConfigClient interface { + GetConfig() *rest.Config + GetAPIReader() kubeclient.Reader +} + +type LeaderRunnable struct { + memberClusterObjectsMap map[string]cluster.Cluster + operatorMgr manager.Manager + atlasClient *Client + currentNamespace string + mongodbImage string + databaseNonStaticImage string + configuredOperatorEnv util.OperatorEnvironment +} + +func (l *LeaderRunnable) NeedLeaderElection() bool { + return true +} + +func NewLeaderRunnable(operatorMgr manager.Manager, memberClusterObjectsMap map[string]cluster.Cluster, currentNamespace, mongodbImage, databaseNonStaticImage string, operatorEnv util.OperatorEnvironment) (*LeaderRunnable, error) { + atlasClient, err := NewClient(nil) + if err != nil { + return nil, xerrors.Errorf("Failed creating atlas telemetry client: %w", err) + } + return &LeaderRunnable{ + atlasClient: atlasClient, + operatorMgr: operatorMgr, + memberClusterObjectsMap: memberClusterObjectsMap, + currentNamespace: currentNamespace, + configuredOperatorEnv: operatorEnv, + + mongodbImage: mongodbImage, + databaseNonStaticImage: databaseNonStaticImage, + }, nil +} + +func (l *LeaderRunnable) Start(ctx context.Context) error { + Logger.Debug("Starting leader-only telemetry goroutine") + RunTelemetry(ctx, l.mongodbImage, l.databaseNonStaticImage, l.currentNamespace, l.operatorMgr, l.memberClusterObjectsMap, l.atlasClient, l.configuredOperatorEnv) + + return nil +} + +type snapshotCollector func(ctx context.Context, memberClusterMap map[string]ConfigClient, operatorClusterMgr manager.Manager, operatorUUID, mongodbImage, databaseNonStaticImage string) []Event + +// RunTelemetry lists the specified CRDs and sends them as events to Segment +func RunTelemetry(ctx context.Context, mongodbImage, databaseNonStaticImage, namespace string, operatorClusterMgr manager.Manager, clusterMap map[string]cluster.Cluster, atlasClient *Client, configuredOperatorEnv util.OperatorEnvironment) { + Logger.Debug("sending telemetry!") + + intervalStr := envvar.GetEnvOrDefault(CollectionFrequency, DefaultCollectionFrequencyStr) + duration, err := time.ParseDuration(intervalStr) + if err != nil || duration < time.Minute { + Logger.Warn("Failed converting %s to a duration or value is too small (minimum is one minute), using default 1h", CollectionFrequency) + duration = DefaultCollectionFrequency + } + Logger.Debugf("%s is set to: %s", CollectionFrequency, duration) + + // converting to a smaller interface for better testing and clearer responsibilities + cc := map[string]ConfigClient{} + for s, c := range clusterMap { + cc[s] = c + } + + // Mapping of snapshot types to their respective collector functions + // The functions are not 100% identical, this map takes care of that + snapshotCollectors := map[EventType]func(ctx context.Context, memberClusterMap map[string]ConfigClient, operatorClusterMgr manager.Manager, operatorUUID, mongodbImage, databaseNonStaticImage string) []Event{ + Operators: collectOperatorSnapshot, + Clusters: func(ctx context.Context, cc map[string]ConfigClient, operatorClusterMgr manager.Manager, _, _, _ string) []Event { + return collectClustersSnapshot(ctx, cc, operatorClusterMgr) + }, + Deployments: func(ctx context.Context, _ map[string]ConfigClient, operatorClusterMgr manager.Manager, operatorUUID, mongodbImage, databaseNonStaticImage string) []Event { + return collectDeploymentsSnapshot(ctx, operatorClusterMgr, operatorUUID, mongodbImage, databaseNonStaticImage) + }, + } + + collectFunc := func() { + Logger.Debug("Collecting data") + + // we are calling this per "ticker" as customers might enable RBACs after the operator has been deployed + operatorUUID := getOrGenerateOperatorUUID(ctx, operatorClusterMgr.GetClient(), namespace) + + for eventType, f := range snapshotCollectors { + collectAndSendSnapshot(ctx, eventType, f, cc, operatorClusterMgr, operatorUUID, mongodbImage, databaseNonStaticImage, namespace, atlasClient, configuredOperatorEnv) + } + } + + collectFunc() + + ticker := time.NewTicker(duration) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + Logger.Debug("Received Shutdown; shutting down") + return + case <-ticker.C: + collectFunc() + } + } +} + +func collectAndSendSnapshot(ctx context.Context, eventType EventType, cf snapshotCollector, memberClusterMap map[string]ConfigClient, operatorClusterMgr manager.Manager, operatorUUID, mongodbImage, databaseNonStaticImage, namespace string, atlasClient *Client, configuredOperatorEnv util.OperatorEnvironment) { + telemetryIsEnabled := ReadBoolWithTrueAsDefault(EventTypeMappingToEnvVar[eventType]) + if !telemetryIsEnabled { + return + } + + Logger.Debugf("Collecting %s events!", eventType) + + events := cf(ctx, memberClusterMap, operatorClusterMgr, operatorUUID, mongodbImage, databaseNonStaticImage) + for _, event := range events { + event.Properties["operatorEnvironment"] = configuredOperatorEnv.String() + } + + handleEvents(ctx, atlasClient, events, eventType, namespace, operatorClusterMgr.GetClient()) +} + +func collectOperatorSnapshot(ctx context.Context, memberClusterMap map[string]ConfigClient, operatorClusterMgr manager.Manager, operatorUUID, _, _ string) []Event { + var kubeClusterUUIDList []string + uncachedClient := operatorClusterMgr.GetAPIReader() + kubeClusterOperatorUUID := getKubernetesClusterUUID(ctx, uncachedClient) + + // in single cluster we don't fill the memberClusterMap + if len(memberClusterMap) == 0 { + memberClusterMap["single"] = operatorClusterMgr + } + + for _, c := range memberClusterMap { + uncachedClient := c.GetAPIReader() + uid := getKubernetesClusterUUID(ctx, uncachedClient) + kubeClusterUUIDList = append(kubeClusterUUIDList, uid) + } + + slices.Sort(kubeClusterUUIDList) + + operatorEvent := OperatorUsageSnapshotProperties{ + KubernetesClusterID: kubeClusterOperatorUUID, + KubernetesClusterIDs: kubeClusterUUIDList, + OperatorID: operatorUUID, + OperatorVersion: versionutil.StaticContainersOperatorVersion(), + OperatorType: MEKO, + } + operatorProperties, err := maputil.StructToMap(operatorEvent) + if err != nil { + Logger.Debugf("failed converting properties to map: %s", err) + return nil + } + + return []Event{ + { + Timestamp: time.Now(), + Source: Operators, + Properties: operatorProperties, + }, + } +} + +func collectDeploymentsSnapshot(ctx context.Context, operatorClusterMgr manager.Manager, operatorUUID, mongodbImage, databaseNonStaticImage string) []Event { + var events []Event + operatorClusterClient := operatorClusterMgr.GetClient() + if operatorClusterClient == nil { + Logger.Debug("No operatorClusterClient available, not collecting Deployments!") + return nil + } + + now := time.Now() + events = append(events, getMdbEvents(ctx, operatorClusterClient, operatorUUID, mongodbImage, databaseNonStaticImage, now)...) + events = append(events, addMultiEvents(ctx, operatorClusterClient, operatorUUID, mongodbImage, databaseNonStaticImage, now)...) + // No need to pass databaseNonStaticImage because it is for sure not enterprise image + events = append(events, addOmEvents(ctx, operatorClusterClient, operatorUUID, mongodbImage, now)...) + return events +} + +func getMdbEvents(ctx context.Context, operatorClusterClient kubeclient.Client, operatorUUID, mongodbImage, databaseNonStaticImage string, now time.Time) []Event { + var events []Event + mdbList := &mdbv1.MongoDBList{} + + if err := operatorClusterClient.List(ctx, mdbList); err != nil { + Logger.Warnf("failed to fetch MongoDBList from Kubernetes: %v", err) + } else { + for _, item := range mdbList.Items { + imageURL := mongodbImage + if !architectures.IsRunningStaticArchitecture(item.Annotations) { + imageURL = databaseNonStaticImage + } + + numberOfClustersUsed := getMaxNumberOfClustersSCIsDeployedOn(item) + properties := DeploymentUsageSnapshotProperties{ + DeploymentUID: string(item.UID), + OperatorID: operatorUUID, + Architecture: string(architectures.GetArchitecture(item.Annotations)), + IsMultiCluster: item.Spec.IsMultiCluster(), + Type: string(item.Spec.GetResourceType()), + IsRunningEnterpriseImage: images.IsEnterpriseImage(imageURL), + ExternalDomains: getExternalDomainProperty(item), + } + + if numberOfClustersUsed > 0 { + properties.DatabaseClusters = ptr.To(numberOfClustersUsed) + } + + if event := createEvent(properties, now, Deployments); event != nil { + events = append(events, *event) + } + } + } + return events +} + +func addMultiEvents(ctx context.Context, operatorClusterClient kubeclient.Client, operatorUUID, mongodbImage, databaseNonStaticImage string, now time.Time) []Event { + var events []Event + + mdbMultiList := &mdbmultiv1.MongoDBMultiClusterList{} + if err := operatorClusterClient.List(ctx, mdbMultiList); err != nil { + Logger.Warnf("failed to fetch MongoDBMultiList from Kubernetes: %v", err) + } + for _, item := range mdbMultiList.Items { + imageURL := mongodbImage + if !architectures.IsRunningStaticArchitecture(item.Annotations) { + imageURL = databaseNonStaticImage + } + + clusters := len(item.Spec.ClusterSpecList) + + properties := DeploymentUsageSnapshotProperties{ + DatabaseClusters: ptr.To(clusters), // cannot be null in mdbmulti + DeploymentUID: string(item.UID), + OperatorID: operatorUUID, + Architecture: string(architectures.GetArchitecture(item.Annotations)), + IsMultiCluster: true, + Type: string(item.Spec.GetResourceType()), + IsRunningEnterpriseImage: images.IsEnterpriseImage(imageURL), + ExternalDomains: getExternalDomainPropertyForMongoDBMulti(item), + } + + if event := createEvent(properties, now, Deployments); event != nil { + events = append(events, *event) + } + } + + return events +} + +func addOmEvents(ctx context.Context, operatorClusterClient kubeclient.Client, operatorUUID, mongodbImage string, now time.Time) []Event { + var events []Event + omList := &omv1.MongoDBOpsManagerList{} + + if err := operatorClusterClient.List(ctx, omList); err != nil { + Logger.Warnf("failed to fetch OMList from Kubernetes: %v", err) + } else { + for _, item := range omList.Items { + // Detect enterprise + omClusters := len(item.Spec.ClusterSpecList) + appDBClusters := len(item.Spec.AppDB.ClusterSpecList) + properties := DeploymentUsageSnapshotProperties{ + DeploymentUID: string(item.UID), + OperatorID: operatorUUID, + Architecture: string(architectures.GetArchitecture(item.Annotations)), + IsMultiCluster: item.Spec.IsMultiCluster(), + Type: "OpsManager", + IsRunningEnterpriseImage: images.IsEnterpriseImage(mongodbImage), + ExternalDomains: getExternalDomainPropertyForOpsManager(item), + } + + if omClusters > 0 { + properties.OmClusters = ptr.To(omClusters) + } + + if appDBClusters > 0 { + properties.AppDBClusters = ptr.To(appDBClusters) + } + + if event := createEvent(properties, now, Deployments); event != nil { + events = append(events, *event) + } + } + } + return events +} + +func createEvent(properties any, now time.Time, eventType EventType) *Event { + convertedProperties, err := maputil.StructToMap(properties) + if err != nil { + Logger.Debugf("failed to parse %s properties: %v", eventType, err) + return nil + } + + return &Event{ + Timestamp: now, + Source: eventType, + Properties: convertedProperties, + } +} + +func getMaxNumberOfClustersSCIsDeployedOn(item mdbv1.MongoDB) int { + var numberOfClustersUsed int + if item.Spec.ConfigSrvSpec != nil { + numberOfClustersUsed = len(item.Spec.ConfigSrvSpec.ClusterSpecList) + } + if item.Spec.MongosSpec != nil { + numberOfClustersUsed = max(numberOfClustersUsed, len(item.Spec.MongosSpec.ClusterSpecList)) + } + if item.Spec.ShardSpec != nil { + numberOfClustersUsed = max(numberOfClustersUsed, len(item.Spec.ShardSpec.ClusterSpecList)) + } + return numberOfClustersUsed +} + +func ReadBoolWithTrueAsDefault(envVarName string) bool { + envVar := envvar.GetEnvOrDefault(envVarName, "true") + return strings.TrimSpace(strings.ToLower(envVar)) == "true" +} + +func handleEvents(ctx context.Context, atlasClient *Client, events []Event, eventType EventType, namespace string, operatorClusterClient kubeclient.Client) { + if err := updateTelemetryConfigMapPayload(ctx, operatorClusterClient, events, namespace, OperatorConfigMapTelemetryConfigMapName, eventType); err != nil { + Logger.Debugf("Failed to save last collected events: %s. Not sending data", err) + return + } + + if sendTelemetry := ReadBoolWithTrueAsDefault(SendEnabled); !sendTelemetry { + Logger.Debugf("Telemetry deactivated, not sending it for eventType: %s", string(eventType)) + return + } + + isOlder, err := isTimestampOlderThanConfiguredFrequency(ctx, operatorClusterClient, namespace, OperatorConfigMapTelemetryConfigMapName, eventType) + if err != nil { + Logger.Debugf("Failed to check for timestamp in configmap; not sending data: %s", err) + return + } + + if !isOlder { + Logger.Debugf("Not older than the configured collection interval, not sending telemetry!") + return + } + + err = atlasClient.SendEventWithRetry(ctx, events) + if err == nil { + if err := updateTelemetryConfigMapTimeStamp(ctx, operatorClusterClient, namespace, OperatorConfigMapTelemetryConfigMapName, eventType); err != nil { + Logger.Debugf("Failed saving timestamp of successful sending of data for type: %s with error: %s", eventType, err) + } + } else { + Logger.Debugf("Encountered error while trying to send payload to atlas; err: %s", err) + } +} + +func collectClustersSnapshot(ctx context.Context, memberClusterMap map[string]ConfigClient, operatorClusterMgr manager.Manager) []Event { + allClusterDetails := getClusterProperties(ctx, memberClusterMap, operatorClusterMgr) + now := time.Now() + + var events []Event + + for _, properties := range allClusterDetails { + if event := createEvent(properties, now, Clusters); event != nil { + events = append(events, *event) + } + } + return events +} + +func getClusterProperties(ctx context.Context, memberClusterMap map[string]ConfigClient, operatorClusterMgr manager.Manager) []KubernetesClusterUsageSnapshotProperties { + operatorMemberClusterProperties := detectClusterInfos(ctx, map[string]ConfigClient{"operator": operatorClusterMgr}) + memberClustersProperties := detectClusterInfos(ctx, memberClusterMap) + + uniqueProperties := make(map[string]KubernetesClusterUsageSnapshotProperties) + for _, property := range append(operatorMemberClusterProperties, memberClustersProperties...) { + uniqueProperties[property.KubernetesClusterID] = property + } + + allClusterDetails := make([]KubernetesClusterUsageSnapshotProperties, 0, len(uniqueProperties)) + for _, properties := range uniqueProperties { + allClusterDetails = append(allClusterDetails, properties) + } + + return allClusterDetails +} + +const ( + ExternalDomainMixed = "Mixed" + ExternalDomainClusterSpecific = "ClusterSpecific" + ExternalDomainUniform = "Uniform" + ExternalDomainNone = "None" +) + +func getExternalDomainProperty(mdb mdbv1.MongoDB) string { + isUniformExternalDomainSpecified := mdb.Spec.GetExternalDomain() != nil + + isClusterSpecificExternalDomainSpecified := isExternalDomainSpecifiedInAnyShardedClusterSpec(mdb) + + return mapExternalDomainConfigurationToEnum(isUniformExternalDomainSpecified, isClusterSpecificExternalDomainSpecified) +} + +func getExternalDomainPropertyForMongoDBMulti(mdb mdbmultiv1.MongoDBMultiCluster) string { + isUniformExternalDomainSpecified := mdb.Spec.GetExternalDomain() != nil + + isClusterSpecificExternalDomainSpecified := isExternalDomainSpecifiedInClusterSpecList(mdb.Spec.ClusterSpecList) + + return mapExternalDomainConfigurationToEnum(isUniformExternalDomainSpecified, isClusterSpecificExternalDomainSpecified) +} + +func getExternalDomainPropertyForOpsManager(om omv1.MongoDBOpsManager) string { + isUniformExternalDomainSpecified := om.Spec.AppDB.GetExternalDomain() != nil + + isClusterSpecificExternalDomainSpecified := isExternalDomainSpecifiedInClusterSpecList(om.Spec.AppDB.ClusterSpecList) + + return mapExternalDomainConfigurationToEnum(isUniformExternalDomainSpecified, isClusterSpecificExternalDomainSpecified) +} + +func mapExternalDomainConfigurationToEnum(isUniformExternalDomainSpecified bool, isClusterSpecificExternalDomainSpecified bool) string { + if isUniformExternalDomainSpecified && isClusterSpecificExternalDomainSpecified { + return ExternalDomainMixed + } + + if isClusterSpecificExternalDomainSpecified { + return ExternalDomainClusterSpecific + } + + if isUniformExternalDomainSpecified { + return ExternalDomainUniform + } + + return ExternalDomainNone +} + +func isExternalDomainSpecifiedInAnyShardedClusterSpec(mdb mdbv1.MongoDB) bool { + isExternalDomainSpecifiedForShard := isExternalDomainSpecifiedInShardedClusterSpec(mdb.Spec.ShardSpec) + isExternalDomainSpecifiedForMongos := isExternalDomainSpecifiedInShardedClusterSpec(mdb.Spec.MongosSpec) + isExternalDomainSpecifiedForConfigSrv := isExternalDomainSpecifiedInShardedClusterSpec(mdb.Spec.ConfigSrvSpec) + + return isExternalDomainSpecifiedForShard || isExternalDomainSpecifiedForMongos || isExternalDomainSpecifiedForConfigSrv +} + +func isExternalDomainSpecifiedInShardedClusterSpec(shardedSpec *mdbv1.ShardedClusterComponentSpec) bool { + if shardedSpec == nil { + return false + } + + return isExternalDomainSpecifiedInClusterSpecList(shardedSpec.ClusterSpecList) +} + +func isExternalDomainSpecifiedInClusterSpecList(clusterSpecList mdbv1.ClusterSpecList) bool { + if len(clusterSpecList) == 0 { + return false + } + + return clusterSpecList.IsExternalDomainSpecifiedInClusterSpecList() +} diff --git a/pkg/telemetry/collector_test.go b/pkg/telemetry/collector_test.go new file mode 100644 index 000000000..d47a5e2ae --- /dev/null +++ b/pkg/telemetry/collector_test.go @@ -0,0 +1,837 @@ +package telemetry + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + mockClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/mdbmulti" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/util/architectures" +) + +const ( + testOperatorUUID = "d05f3e84-8eb3-4eb4-a42a-9698a3350d46" + testDatabaseStaticImage = "static-image-mongodb-enterprise-server" + testDatabaseNonStaticImage = "non-static-image" +) + +func TestCollectDeploymentsSnapshot(t *testing.T) { + tests := map[string]struct { + objects []client.Object + expectedEventsWithProperties []map[string]any + }{ + "empty object list": { + objects: []client.Object{}, + expectedEventsWithProperties: nil, + }, + "single basic replicaset": { + objects: []client.Object{ + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "60c005b9-1a87-49de-b7d6-5ef9382d808f", + }, + }, + }, + expectedEventsWithProperties: []map[string]any{ + { + "deploymentUID": "60c005b9-1a87-49de-b7d6-5ef9382d808f", + "operatorID": testOperatorUUID, + "architecture": string(architectures.NonStatic), + "isMultiCluster": false, + "type": "ReplicaSet", + "IsRunningEnterpriseImage": false, + "externalDomains": ExternalDomainNone, + }, + }, + }, + "single basic multicluster replicaset": { + objects: []client.Object{ + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "d5a53056-157a-4cda-96e9-fe48a9732990", + }, + }, + }, + expectedEventsWithProperties: []map[string]any{ + { + "databaseClusters": float64(0), + "deploymentUID": "d5a53056-157a-4cda-96e9-fe48a9732990", + "operatorID": testOperatorUUID, + "architecture": string(architectures.NonStatic), + "isMultiCluster": true, + "type": "ReplicaSet", + "IsRunningEnterpriseImage": false, + "externalDomains": ExternalDomainNone, + }, + }, + }, + "single basic opsmanager": { + objects: []client.Object{ + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{}, + Topology: "Single", + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "7c338e30-8681-443d-aef4-ae4b17eb3a97", + }, + }, + }, + expectedEventsWithProperties: []map[string]any{ + { + "deploymentUID": "7c338e30-8681-443d-aef4-ae4b17eb3a97", + "operatorID": testOperatorUUID, + "architecture": string(architectures.NonStatic), + "isMultiCluster": false, + "type": "OpsManager", + "IsRunningEnterpriseImage": true, + "externalDomains": ExternalDomainNone, + }, + }, + }, + "architecture annotation test": { + objects: []client.Object{ + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "c20a7cf1-a12d-4cee-a87e-7f61aa2bd878", + Name: "test-rs-static", + Annotations: map[string]string{ + architectures.ArchitectureAnnotation: string(architectures.Static), + }, + }, + }, + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "97822e48-fb51-4ba5-9993-26841b44a7a3", + Name: "test-rs-non-static", + Annotations: map[string]string{ + architectures.ArchitectureAnnotation: string(architectures.NonStatic), + }, + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "71368077-ea95-4564-acd6-09ec573fdf61", + Name: "test-mrs-static", + Annotations: map[string]string{ + architectures.ArchitectureAnnotation: string(architectures.Static), + }, + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "a8a28c8a-6226-44fc-a8cd-e66a6942ffbd", + Name: "test-mrs-non-static", + Annotations: map[string]string{ + architectures.ArchitectureAnnotation: string(architectures.NonStatic), + }, + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{}, + Topology: "Single", + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "0d76d2c9-98cd-4a80-a565-ba038d223ed0", + Name: "test-om-static", + Annotations: map[string]string{ + architectures.ArchitectureAnnotation: string(architectures.Static), + }, + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{}, + Topology: "Single", + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "399680c7-e929-44f6-8b82-9be96a5e5533", + Name: "test-om-non-static", + Annotations: map[string]string{ + architectures.ArchitectureAnnotation: string(architectures.NonStatic), + }, + }, + }, + }, + expectedEventsWithProperties: []map[string]any{ + { + "deploymentUID": "c20a7cf1-a12d-4cee-a87e-7f61aa2bd878", + "architecture": string(architectures.Static), + "IsRunningEnterpriseImage": true, + }, + { + "deploymentUID": "97822e48-fb51-4ba5-9993-26841b44a7a3", + "architecture": string(architectures.NonStatic), + "IsRunningEnterpriseImage": false, + }, + { + "deploymentUID": "71368077-ea95-4564-acd6-09ec573fdf61", + "architecture": string(architectures.Static), + "IsRunningEnterpriseImage": true, + }, + { + "deploymentUID": "a8a28c8a-6226-44fc-a8cd-e66a6942ffbd", + "architecture": string(architectures.NonStatic), + "IsRunningEnterpriseImage": false, + }, + { + "deploymentUID": "0d76d2c9-98cd-4a80-a565-ba038d223ed0", + "architecture": string(architectures.Static), + "IsRunningEnterpriseImage": true, + }, + { + "deploymentUID": "399680c7-e929-44f6-8b82-9be96a5e5533", + "architecture": string(architectures.NonStatic), + "IsRunningEnterpriseImage": true, + }, + }, + }, + "multicluster test": { + objects: []client.Object{ + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + Topology: mdbv1.ClusterTopologyMultiCluster, + }, + ShardedClusterSpec: mdbv1.ShardedClusterSpec{ + ConfigSrvSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + }, + }, + MongosSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 2, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + }, + }, + ShardSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 1, + }, + { + ClusterName: "cluster3", + Members: 1, + }, + { + ClusterName: "cluster4", + Members: 1, + }, + }, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "1a58636d-6c10-49c9-a9ee-7c0fe80ac80c", + Name: "test-msc", + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + { + ClusterName: "cluster3", + Members: 3, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "a31ab7a8-e5bd-480b-afcc-ac2eec9ce348", + Name: "test-mrs", + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 3, + }, + { + ClusterName: "cluster2", + Members: 2, + }, + { + ClusterName: "cluster3", + Members: 2, + }, + }, + Topology: omv1.ClusterTopologyMultiCluster, + }, + Topology: omv1.ClusterTopologyMultiCluster, + ClusterSpecList: []omv1.ClusterSpecOMItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 1, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "2b138678-4e4c-4be4-9877-16e6eaae279b", + Name: "test-om-multi", + }, + }, + }, + expectedEventsWithProperties: []map[string]any{ + { + "deploymentUID": "1a58636d-6c10-49c9-a9ee-7c0fe80ac80c", + "databaseClusters": float64(4), + "isMultiCluster": true, + }, + { + "deploymentUID": "a31ab7a8-e5bd-480b-afcc-ac2eec9ce348", + "databaseClusters": float64(3), + "isMultiCluster": true, + }, + { + "deploymentUID": "2b138678-4e4c-4be4-9877-16e6eaae279b", + "OmClusters": float64(2), + "appDBClusters": float64(3), + "isMultiCluster": true, + }, + }, + }, + "external domains test": { + objects: []client.Object{ + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + Topology: mdbv1.ClusterTopologySingleCluster, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "2c60ec7b-b233-4d98-97e6-b7c423c19e24", + Name: "test-msc-external-domains-none", + }, + }, + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + Topology: mdbv1.ClusterTopologySingleCluster, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.default.domain"), + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "c7ccb57f-abd1-4944-8a99-02e5a79acf75", + Name: "test-msc-external-domains-uniform", + }, + }, + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + Topology: mdbv1.ClusterTopologyMultiCluster, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.default.domain"), + }, + }, + ShardedClusterSpec: mdbv1.ShardedClusterSpec{ + ConfigSrvSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster1.domain"), + }, + }, + { + ClusterName: "cluster2", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + }, + }, + }, + }, + MongosSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 2, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + }, + }, + ShardSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + }, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "7eed85ce-7a38-43ea-a338-6d959339c146", + Name: "test-msc-external-domains-mixed", + }, + }, + &mdbv1.MongoDB{ + Spec: mdbv1.MongoDbSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + Topology: mdbv1.ClusterTopologyMultiCluster, + }, + ShardedClusterSpec: mdbv1.ShardedClusterSpec{ + ConfigSrvSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster1.domain"), + }, + }, + { + ClusterName: "cluster2", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + }, + }, + }, + }, + MongosSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 2, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + }, + }, + ShardSpec: &mdbv1.ShardedClusterComponentSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + }, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "584515da-e797-48af-af7f-6561812c15f4", + Name: "test-msc-external-domains-cluster-specific", + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + { + ClusterName: "cluster3", + Members: 3, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "27b3d7cf-1f8b-434d-a002-ce85f7313507", + Name: "test-mrs-external-domains-none", + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.default.domain"), + }, + }, + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 3, + }, + { + ClusterName: "cluster3", + Members: 3, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "b050040e-7b53-4991-bae4-69663a523804", + Name: "test-mrs-external-domains-uniform", + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.default.domain"), + }, + }, + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster1.domain"), + }, + }, + { + ClusterName: "cluster2", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + }, + }, + { + ClusterName: "cluster3", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster3.domain"), + }, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "54427a32-1799-4a1b-b03f-a50484c09d2c", + Name: "test-mrs-external-domains-mixed", + }, + }, + &mdbmulti.MongoDBMultiCluster{ + Spec: mdbmulti.MongoDBMultiSpec{ + DbCommonSpec: mdbv1.DbCommonSpec{ + ResourceType: mdbv1.ReplicaSet, + }, + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 1, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster1.domain"), + }, + }, + { + ClusterName: "cluster2", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + }, + }, + { + ClusterName: "cluster3", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster3.domain"), + }, + }, + }, + }, ObjectMeta: metav1.ObjectMeta{ + UID: "fe6b6fad-51f2-4f98-8ddd-54ae24143ea6", + Name: "test-mrs-external-domains-cluster-specific", + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{}, + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "5999bccb-d17d-4657-9ea6-ee9fa264d749", + Name: "test-om-external-domains-none", + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{ + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.custom.domain"), + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "95808355-09d8-4a50-909a-e96c91c99665", + Name: "test-om-external-domains-uniform", + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{ + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("some.custom.domain"), + }, + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster1.domain"), + }, + }, + { + ClusterName: "cluster2", + Members: 2, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + }, + }, + { + ClusterName: "cluster3", + Members: 2, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster3.domain"), + }, + }, + }, + Topology: omv1.ClusterTopologyMultiCluster, + }, + Topology: omv1.ClusterTopologyMultiCluster, + ClusterSpecList: []omv1.ClusterSpecOMItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 1, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "34daced5-b4ae-418b-bf38-034667e676ca", + Name: "test-om-external-domains-mixed", + }, + }, + &omv1.MongoDBOpsManager{ + Spec: omv1.MongoDBOpsManagerSpec{ + AppDB: omv1.AppDBSpec{ + ClusterSpecList: []mdbv1.ClusterSpecItem{ + { + ClusterName: "cluster1", + Members: 3, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster1.domain"), + }, + }, + { + ClusterName: "cluster2", + Members: 2, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster2.domain"), + }, + }, + { + ClusterName: "cluster3", + Members: 2, + ExternalAccessConfiguration: &mdbv1.ExternalAccessConfiguration{ + ExternalDomain: ptr.To("cluster3.domain"), + }, + }, + }, + Topology: omv1.ClusterTopologyMultiCluster, + }, + Topology: omv1.ClusterTopologyMultiCluster, + ClusterSpecList: []omv1.ClusterSpecOMItem{ + { + ClusterName: "cluster1", + Members: 1, + }, + { + ClusterName: "cluster2", + Members: 1, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + UID: "cb365427-27d7-46dd-af31-cec0dad21bf0", + Name: "test-om-external-domains-cluster-specific", + }, + }, + }, + expectedEventsWithProperties: []map[string]any{ + { + "deploymentUID": "2c60ec7b-b233-4d98-97e6-b7c423c19e24", + "externalDomains": ExternalDomainNone, + "isMultiCluster": false, + }, + { + "deploymentUID": "c7ccb57f-abd1-4944-8a99-02e5a79acf75", + "externalDomains": ExternalDomainUniform, + "isMultiCluster": false, + }, + { + "deploymentUID": "7eed85ce-7a38-43ea-a338-6d959339c146", + "externalDomains": ExternalDomainMixed, + "isMultiCluster": true, + }, + { + "deploymentUID": "584515da-e797-48af-af7f-6561812c15f4", + "externalDomains": ExternalDomainClusterSpecific, + "isMultiCluster": true, + }, + { + "deploymentUID": "27b3d7cf-1f8b-434d-a002-ce85f7313507", + "externalDomains": ExternalDomainNone, + "isMultiCluster": true, + }, + { + "deploymentUID": "b050040e-7b53-4991-bae4-69663a523804", + "externalDomains": ExternalDomainUniform, + "isMultiCluster": true, + }, + { + "deploymentUID": "54427a32-1799-4a1b-b03f-a50484c09d2c", + "externalDomains": ExternalDomainMixed, + "isMultiCluster": true, + }, + { + "deploymentUID": "fe6b6fad-51f2-4f98-8ddd-54ae24143ea6", + "externalDomains": ExternalDomainClusterSpecific, + "isMultiCluster": true, + }, + { + "deploymentUID": "5999bccb-d17d-4657-9ea6-ee9fa264d749", + "externalDomains": ExternalDomainNone, + "isMultiCluster": false, + }, + { + "deploymentUID": "95808355-09d8-4a50-909a-e96c91c99665", + "externalDomains": ExternalDomainUniform, + "isMultiCluster": false, + }, + { + "deploymentUID": "34daced5-b4ae-418b-bf38-034667e676ca", + "externalDomains": ExternalDomainMixed, + "isMultiCluster": true, + }, + { + "deploymentUID": "cb365427-27d7-46dd-af31-cec0dad21bf0", + "externalDomains": ExternalDomainClusterSpecific, + "isMultiCluster": true, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + k8sClient := mock.NewEmptyFakeClientBuilder().WithObjects(test.objects...).Build() + mgr := mockClient.NewManagerWithClient(k8sClient) + + ctx := context.Background() + + beforeCallTimestamp := time.Now() + events := collectDeploymentsSnapshot(ctx, mgr, testOperatorUUID, testDatabaseStaticImage, testDatabaseNonStaticImage) + afterCallTimestamp := time.Now() + + require.Len(t, events, len(test.expectedEventsWithProperties), "expected and collected events count don't match") + for _, expectedEventWithProperties := range test.expectedEventsWithProperties { + deploymentUID := expectedEventWithProperties["deploymentUID"] + event := findEventWithDeploymentUID(events, deploymentUID.(string)) + require.NotNil(t, "could not find event with deploymentUID %s", deploymentUID) + + assert.Equal(t, Deployments, event.Source) + require.NotNilf(t, event.Timestamp, "event timestamp is nil for %s deployment", deploymentUID) + assert.LessOrEqual(t, beforeCallTimestamp, event.Timestamp) + assert.GreaterOrEqual(t, afterCallTimestamp, event.Timestamp) + + for key, value := range expectedEventWithProperties { + assert.Equal(t, value, event.Properties[key], "failed assertion for %s property for %s deployment", key, deploymentUID) + } + } + }) + } +} + +func findEventWithDeploymentUID(events []Event, deploymentUID string) *Event { + for _, event := range events { + if event.Properties["deploymentUID"] == deploymentUID { + return &event + } + } + + return nil +} diff --git a/pkg/telemetry/config.go b/pkg/telemetry/config.go new file mode 100644 index 000000000..1a2bc48be --- /dev/null +++ b/pkg/telemetry/config.go @@ -0,0 +1,29 @@ +package telemetry + +import "time" + +// Helm Chart Settings +const ( + Enabled = "MDB_OPERATOR_TELEMETRY_ENABLED" + BaseUrl = "MDB_OPERATOR_TELEMETRY_SEND_BASEURL" + KubeTimeout = "MDB_OPERATOR_TELEMETRY_KUBE_TIMEOUT" + CollectionFrequency = "MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY" + SendEnabled = "MDB_OPERATOR_TELEMETRY_SEND_ENABLED" + SendFrequency = "MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY" +) + +// Default Settings +const ( + DefaultCollectionFrequency = 1 * time.Hour + DefaultCollectionFrequencyStr = "1h" + DefaultSendFrequencyStr = "168h" + DefaultSendFrequency = time.Hour * 168 +) + +const ( + OperatorConfigMapTelemetryConfigMapName = "mongodb-enterprise-operator-telemetry" +) + +func IsTelemetryActivated() bool { + return ReadBoolWithTrueAsDefault(Enabled) +} diff --git a/pkg/telemetry/configmap.go b/pkg/telemetry/configmap.go new file mode 100644 index 000000000..5eb98e265 --- /dev/null +++ b/pkg/telemetry/configmap.go @@ -0,0 +1,200 @@ +package telemetry + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/google/uuid" + "k8s.io/apimachinery/pkg/types" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/envvar" + + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + v2 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + TimestampKey = "lastSendTimestamp" + TimestampInitialValue = "initialValue" + LastSendPayloadKey = "lastSendPayload" + UUIDKey = "operatorUUID" +) + +func getOrGenerateOperatorUUID(ctx context.Context, k8sClient kubeclient.Client, namespace string) string { + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, kubeclient.ObjectKey{Namespace: namespace, Name: OperatorConfigMapTelemetryConfigMapName}, configMap) + if err != nil { + if apiErrors.IsNotFound(err) { + // ConfigMap does not exist -> Create a new one + Logger.Debugf("ConfigMap %s not found. Creating a new one.", OperatorConfigMapTelemetryConfigMapName) + return createNewConfigMap(ctx, k8sClient, namespace) + } + + // Any other error (e.g., network issue, permissions issue) + Logger.Errorf("Failed to get ConfigMap %s: %v", OperatorConfigMapTelemetryConfigMapName, err) + return unknown + } + + // If ConfigMap exists, check for UUID and required keys + if existingUUID, exists := configMap.Data[UUIDKey]; exists { + // Ensure all event timestamps are present + if addMissingKeys(configMap) { + if updateErr := k8sClient.Update(ctx, configMap); updateErr != nil { + Logger.Debugf("Failed to update ConfigMap %s with missing keys", OperatorConfigMapTelemetryConfigMapName) + return unknown + } + } + return existingUUID + } + + // UUID is missing; generate and update + Logger.Debugf("ConfigMap %s exists but lacks a UUID, generating one", OperatorConfigMapTelemetryConfigMapName) + return updateConfigMapWithNewUUID(ctx, k8sClient, namespace) +} + +// Adds missing timestamp keys to the ConfigMap. Returns true if updates were made. +func addMissingKeys(configMap *corev1.ConfigMap) bool { + updated := false + for _, et := range AllEventTypes { + key := et.GetTimeStampKey() + if _, exists := configMap.Data[key]; !exists { + configMap.Data[key] = TimestampKey + updated = true + } + } + return updated +} + +// Updates an existing ConfigMap with a new UUID +func updateConfigMapWithNewUUID(ctx context.Context, k8sClient kubeclient.Client, namespace string) string { + newUUID, newConfigMap := createInitialConfigmap(namespace) + if err := k8sClient.Update(ctx, newConfigMap); err != nil { + Logger.Debugf("Failed to update ConfigMap %s with new UUID", OperatorConfigMapTelemetryConfigMapName) + return unknown + } + return newUUID +} + +// Creates a new ConfigMap with a generated UUID +func createNewConfigMap(ctx context.Context, k8sClient kubeclient.Client, namespace string) string { + newUUID, newConfigMap := createInitialConfigmap(namespace) + if err := k8sClient.Create(ctx, newConfigMap); err != nil { + Logger.Debugf("Failed to create ConfigMap %s: %s", OperatorConfigMapTelemetryConfigMapName, err) + return unknown + } + Logger.Debugf("Created ConfigMap %s with UUID %s", OperatorConfigMapTelemetryConfigMapName, newUUID) + return newUUID +} + +func createInitialConfigmap(namespace string) (string, *corev1.ConfigMap) { + // ConfigMap does not exist or does not contain uuid; create one + newUUID := uuid.NewString() + newConfigMap := &corev1.ConfigMap{ + ObjectMeta: v2.ObjectMeta{ + Name: OperatorConfigMapTelemetryConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + UUIDKey: newUUID, + }, + } + + for _, eventType := range AllEventTypes { + newConfigMap.Data[eventType.GetTimeStampKey()] = TimestampInitialValue + } + + return newUUID, newConfigMap +} + +// isTimestampOlderThanConfiguredFrequency is used to get the timestamp from the ConfigMap and check whether it's time to +// send the data to atlas. +func isTimestampOlderThanConfiguredFrequency(ctx context.Context, k8sClient kubeclient.Client, namespace string, OperatorConfigMapTelemetryConfigMapName string, et EventType) (bool, error) { + durationStr := envvar.GetEnvOrDefault(SendFrequency, DefaultSendFrequencyStr) + duration, err := time.ParseDuration(durationStr) + if err != nil || duration < 10*time.Minute { + Logger.Warn("Failed to parse or given durationString: %s too low (min: 10 minutes), defaulting to one week", durationStr) + duration = DefaultSendFrequency + } + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: OperatorConfigMapTelemetryConfigMapName, Namespace: namespace}, cm) + if err != nil { + return false, fmt.Errorf("failed to get ConfigMap: %w", err) + } + timestampStr, exists := cm.Data[et.GetTimeStampKey()] + + if !exists { + return false, fmt.Errorf("timestamp key: %s not found in ConfigMap", et.GetTimeStampKey()) + } + + // We are running this the first time, thus we are "older" than a week + if timestampStr == TimestampInitialValue { + return true, nil + } + + timestamp, err := strconv.ParseInt(timestampStr, 10, 64) + if err != nil { + return false, fmt.Errorf("invalid timestamp format: %w", err) + } + + timestampTime := time.Unix(timestamp, 0) + cutOffTime := time.Now().Add(-duration) + + isOlder := timestampTime.Before(cutOffTime) + + return isOlder, nil +} + +// updateTelemetryConfigMapPayload updates the configmap with the current collected telemetry data +func updateTelemetryConfigMapPayload(ctx context.Context, k8sClient kubeclient.Client, events []Event, namespace string, OperatorConfigMapTelemetryConfigMapName string, eventType EventType) error { + cm := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: OperatorConfigMapTelemetryConfigMapName, Namespace: namespace}, cm) + if err != nil { + return fmt.Errorf("failed to get ConfigMap: %w", err) + } + + if cm.Data == nil { + cm.Data = map[string]string{} + } + marshal, err := json.Marshal(events) + if err != nil { + return fmt.Errorf("failed to marshal payload: %w", err) + } + + cm.Data[eventType.GetPayloadKey()] = string(marshal) + + err = k8sClient.Update(ctx, cm) + if err != nil { + return fmt.Errorf("failed to update ConfigMap: %w", err) + } + + return nil +} + +// updateTelemetryConfigMapTimeStamp updates the configmap with the current timestamp +func updateTelemetryConfigMapTimeStamp(ctx context.Context, k8sClient kubeclient.Client, namespace string, OperatorConfigMapTelemetryConfigMapName string, eventType EventType) error { + cm := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: OperatorConfigMapTelemetryConfigMapName, Namespace: namespace}, cm) + if err != nil { + return fmt.Errorf("failed to get ConfigMap: %w", err) + } + + currentTimestamp := strconv.FormatInt(time.Now().Unix(), 10) + if cm.Data == nil { + cm.Data = map[string]string{} + } + + cm.Data[eventType.GetTimeStampKey()] = currentTimestamp + + err = k8sClient.Update(ctx, cm) + if err != nil { + return fmt.Errorf("failed to update ConfigMap: %w", err) + } + + return nil +} diff --git a/pkg/telemetry/configmap_test.go b/pkg/telemetry/configmap_test.go new file mode 100644 index 000000000..e5ce8b070 --- /dev/null +++ b/pkg/telemetry/configmap_test.go @@ -0,0 +1,335 @@ +package telemetry + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + testNamespace = "test-namespace" + testUUID = "123e4567-e89b-12d3-a456-426614174000" +) + +type failingFakeClient struct { + kubeclient.Client +} + +func (f *failingFakeClient) Get(ctx context.Context, key kubeclient.ObjectKey, obj kubeclient.Object, opts ...kubeclient.GetOption) error { + return fmt.Errorf("simulated API failure") +} + +type failingUpdateFakeClient struct { + kubeclient.Client +} + +func (f *failingUpdateFakeClient) Update(ctx context.Context, obj kubeclient.Object, opts ...kubeclient.UpdateOption) error { + return fmt.Errorf("simulated update failure") +} + +type failingCreateFakeClient struct { + kubeclient.Client +} + +func (f *failingCreateFakeClient) Create(ctx context.Context, obj kubeclient.Object, opts ...kubeclient.CreateOption) error { + return fmt.Errorf("simulated create failure") +} + +func TestGetOrGenerateOperatorUUID(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + ctx := context.Background() + + t.Run("should return existing UUID from ConfigMap", func(t *testing.T) { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorConfigMapTelemetryConfigMapName, + Namespace: testNamespace, + }, + Data: map[string]string{ + UUIDKey: testUUID, + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingCM).Build() + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.Equal(t, testUUID, result, "Expected to return the existing UUID") + }) + + t.Run("should create ConfigMap if not present", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(scheme).Build() + + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.NotEmpty(t, result, "Expected a new UUID to be generated") + + createdCM := &corev1.ConfigMap{} + err := client.Get(ctx, kubeclient.ObjectKey{Namespace: testNamespace, Name: OperatorConfigMapTelemetryConfigMapName}, createdCM) + assert.NoError(t, err, "Expected the ConfigMap to be created") + assert.Equal(t, result, createdCM.Data[UUIDKey], "Expected the new UUID to be stored in the ConfigMap") + }) + + t.Run("should update ConfigMap if required fields are missing", func(t *testing.T) { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorConfigMapTelemetryConfigMapName, + Namespace: testNamespace, + }, + Data: map[string]string{ + UUIDKey: testUUID, + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingCM).Build() + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.Equal(t, testUUID, result, "Expected to return the existing UUID") + + // Verify ConfigMap has been updated with missing fields + updatedCM := &corev1.ConfigMap{} + err := client.Get(ctx, kubeclient.ObjectKey{Namespace: testNamespace, Name: OperatorConfigMapTelemetryConfigMapName}, updatedCM) + assert.NoError(t, err, "Expected the ConfigMap to be updated") + assert.Contains(t, updatedCM.Data, Operators.GetTimeStampKey(), "Expected missing keys to be added") + assert.Contains(t, updatedCM.Data, Deployments.GetTimeStampKey(), "Expected missing keys to be added") + assert.Contains(t, updatedCM.Data, Clusters.GetTimeStampKey(), "Expected missing keys to be added") + }) + + t.Run("should generate and update UUID if missing", func(t *testing.T) { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorConfigMapTelemetryConfigMapName, + Namespace: testNamespace, + }, + Data: map[string]string{}, // No UUID key present + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingCM).Build() + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.NotEmpty(t, result, "Expected a new UUID to be generated") + + // Verify UUID is now stored in the ConfigMap + updatedCM := &corev1.ConfigMap{} + err := client.Get(ctx, kubeclient.ObjectKey{Namespace: testNamespace, Name: OperatorConfigMapTelemetryConfigMapName}, updatedCM) + assert.NoError(t, err, "Expected the ConfigMap to be updated") + assert.Equal(t, result, updatedCM.Data[UUIDKey], "Expected the new UUID to be stored in the ConfigMap") + }) + + t.Run("should return unknown if ConfigMap retrieval fails", func(t *testing.T) { + client := &failingFakeClient{} // Simulate API failure + + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.Equal(t, unknown, result, "Expected 'unknown' when ConfigMap retrieval fails") + }) + + t.Run("should return unknown if updating ConfigMap with missing keys fails", func(t *testing.T) { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorConfigMapTelemetryConfigMapName, + Namespace: testNamespace, + }, + Data: map[string]string{ + UUIDKey: testUUID, + }, + } + + client := &failingUpdateFakeClient{fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingCM).Build()} // Simulate update failure + + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.Equal(t, unknown, result, "Expected 'unknown' when ConfigMap update fails") + }) + + t.Run("should return unknown if creating a new ConfigMap fails", func(t *testing.T) { + client := &failingCreateFakeClient{fake.NewClientBuilder().WithScheme(scheme).Build()} // Simulate create failure + + result := getOrGenerateOperatorUUID(ctx, client, testNamespace) + + assert.Equal(t, unknown, result, "Expected 'unknown' when ConfigMap creation fails") + }) +} + +func TestUpdateTelemetryConfigMap(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + + ctx := context.Background() + eventType := Deployments + testEvents := []Event{{ + Timestamp: time.Now(), + Source: Deployments, + Properties: map[string]any{"test": "b"}, + }} + + t.Run("should update existing ConfigMap with timestamp", func(t *testing.T) { + existingCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorConfigMapTelemetryConfigMapName, + Namespace: testNamespace, + }, + Data: map[string]string{}, + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(existingCM).Build() + + err := updateTelemetryConfigMapTimeStamp(ctx, client, testNamespace, OperatorConfigMapTelemetryConfigMapName, eventType) + assert.NoError(t, err, "Expected update to succeed") + err = updateTelemetryConfigMapPayload(ctx, client, testEvents, testNamespace, OperatorConfigMapTelemetryConfigMapName, eventType) + assert.NoError(t, err, "Expected update to succeed") + + updatedCM := &corev1.ConfigMap{} + err = client.Get(ctx, kubeclient.ObjectKey{Name: OperatorConfigMapTelemetryConfigMapName, Namespace: testNamespace}, updatedCM) + assert.NoError(t, err, "Expected to retrieve updated ConfigMap") + + timestamp, err := strconv.ParseInt(updatedCM.Data[Deployments.GetTimeStampKey()], 10, 64) + assert.NoError(t, err, "Expected timestamp to be a valid integer") + assert.Greater(t, timestamp, int64(0), "Expected a non-zero timestamp") + + var storedEvents []Event + err = json.Unmarshal([]byte(updatedCM.Data[Deployments.GetPayloadKey()]), &storedEvents) + assert.NoError(t, err, "Expected payload to be valid JSON") + assert.Equal(t, testEvents[0].Properties, storedEvents[0].Properties, "Expected stored events to match") + assert.Equal(t, testEvents[0].Source, storedEvents[0].Source, "Expected stored events to match") + }) + + t.Run("should fail if ConfigMap does not exist", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(scheme).Build() + + err := updateTelemetryConfigMapTimeStamp(ctx, client, testNamespace, OperatorConfigMapTelemetryConfigMapName, eventType) + assert.Error(t, err, "Expected error when ConfigMap does not exist") + assert.Contains(t, err.Error(), "failed to get ConfigMap", "Expected get ConfigMap error message") + }) +} + +func TestIsTimestampOlderThanConfiguredFrequency(t *testing.T) { + ctx := context.Background() + + namespace := "test-namespace" + configMapName := "test-configmap" + + et := Deployments + + tests := []struct { + name string + frequencySetting string + configMapData map[string]string + shouldCollect bool + expectedErr bool + description string + }{ + { + name: "Default one-week check - outdated", + configMapData: map[string]string{ + et.GetTimeStampKey(): strconv.FormatInt(time.Now().Add(-8*24*time.Hour).Unix(), 10), // 8 days ago + }, + shouldCollect: true, + expectedErr: false, + description: "Timestamp is older than one week", + }, + { + name: "Default one-week check - recent", + configMapData: map[string]string{ + et.GetTimeStampKey(): strconv.FormatInt(time.Now().Add(-6*24*time.Hour).Unix(), 10), // 6 days ago + }, + shouldCollect: false, + expectedErr: false, + description: "Timestamp is within one week", + }, + { + name: "Custom duration from env - below minimum, will default to 1h", + frequencySetting: "5m", + configMapData: map[string]string{ + et.GetTimeStampKey(): strconv.FormatInt(time.Now().Add(-10*time.Minute).Unix(), 10), + }, + shouldCollect: false, + expectedErr: false, + description: "Timestamp is older than configured 5-minute threshold", + }, + { + name: "Custom duration from env - recent", + frequencySetting: "30m", + configMapData: map[string]string{ + et.GetTimeStampKey(): strconv.FormatInt(time.Now().Add(-10*time.Minute).Unix(), 10), + }, + shouldCollect: false, + expectedErr: false, + description: "Timestamp is within configured 30-minute threshold", + }, + { + name: "Invalid duration format", + frequencySetting: "invalid", + configMapData: map[string]string{ + et.GetTimeStampKey(): strconv.FormatInt(time.Now().Add(-10*time.Minute).Unix(), 10), + }, + shouldCollect: false, + expectedErr: false, + description: "Should default to 168h", + }, + { + name: "Missing timestamp key", + configMapData: map[string]string{ + "someOtherKey": "1650000000", + }, + shouldCollect: false, + expectedErr: true, + description: "Should return error due to missing timestamp key", + }, + { + name: "Initial timestamp value", + configMapData: map[string]string{ + et.GetTimeStampKey(): TimestampInitialValue, + }, + shouldCollect: true, + expectedErr: false, + description: "Should return true for initial timestamp", + }, + { + name: "Invalid timestamp format", + configMapData: map[string]string{ + et.GetTimeStampKey(): "invalid_timestamp", + }, + shouldCollect: false, + expectedErr: true, + description: "Should return error due to invalid timestamp format", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Setenv(SendFrequency, test.frequencySetting) + + fakeClient := fake.NewClientBuilder(). + WithObjects(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + }, + Data: test.configMapData, + }). + Build() + + result, err := isTimestampOlderThanConfiguredFrequency(ctx, fakeClient, namespace, configMapName, et) + + assert.Equal(t, test.shouldCollect, result, test.description) + + if test.expectedErr { + assert.Error(t, err, test.description) + } else { + assert.NoError(t, err, test.description) + } + }) + } +} diff --git a/pkg/telemetry/types.go b/pkg/telemetry/types.go new file mode 100644 index 000000000..b57e4d260 --- /dev/null +++ b/pkg/telemetry/types.go @@ -0,0 +1,79 @@ +package telemetry + +import ( + "fmt" + "time" +) + +// OperatorUsageSnapshotProperties represents the structure for tracking Kubernetes operator usage events. +type OperatorUsageSnapshotProperties struct { + KubernetesClusterID string `json:"kubernetesClusterID"` // Kubernetes cluster ID where the operator is running + KubernetesClusterIDs []string `json:"kubernetesClusterIDs"` // Sorted Kubernetes cluster IDs the operator is managing + OperatorID string `json:"operatorID"` // Operator UUID + OperatorVersion string `json:"operatorVersion"` // Version of the operator + OperatorType OperatorType `json:"operatorType"` // MEKO, MCK, MCO (here meko) +} + +// KubernetesClusterUsageSnapshotProperties represents the structure for tracking Kubernetes cluster usage events. +type KubernetesClusterUsageSnapshotProperties struct { + KubernetesClusterID string `json:"kubernetesClusterID"` // Kubernetes cluster ID where the operator is running + KubernetesAPIVersion string `json:"kubernetesAPIVersion"` + KubernetesFlavour string `json:"kubernetesFlavour"` +} + +// DeploymentUsageSnapshotProperties represents the structure for tracking Deployment events. +type DeploymentUsageSnapshotProperties struct { + DatabaseClusters *int `json:"databaseClusters,omitempty"` // pointers allow us to not send that value if it's not set. + AppDBClusters *int `json:"appDBClusters,omitempty"` + OmClusters *int `json:"OmClusters,omitempty"` + DeploymentUID string `json:"deploymentUID"` + OperatorID string `json:"operatorID"` + Architecture string `json:"architecture"` + IsMultiCluster bool `json:"isMultiCluster"` + Type string `json:"type"` // RS, SC, OM, Single + IsRunningEnterpriseImage bool `json:"IsRunningEnterpriseImage"` + ExternalDomains string `json:"externalDomains"` // None, Uniform, ClusterSpecific, Mixed +} + +type Event struct { + Timestamp time.Time `json:"timestamp"` + Source EventType `json:"source"` + Properties map[string]any `json:"properties"` +} + +type OperatorType string + +const ( + MCK OperatorType = "MCK" + MCO OperatorType = "MCO" + MEKO OperatorType = "MEKO" +) + +type EventType string + +const ( + Deployments EventType = "Deployments" + Operators EventType = "Operators" + Clusters EventType = "Clusters" +) + +var AllEventTypes = []EventType{ + Deployments, + Operators, + Clusters, +} + +var EventTypeMappingToEnvVar = map[EventType]string{ + Deployments: "MDB_OPERATOR_TELEMETRY_COLLECTION_DEPLOYMENTS_ENABLED", + Clusters: "MDB_OPERATOR_TELEMETRY_COLLECTION_CLUSTERS_ENABLED", + Operators: "MDB_OPERATOR_TELEMETRY_COLLECTION_OPERATORS_ENABLED", +} + +func (e EventType) GetPayloadKey() string { + return fmt.Sprintf("%s%s", LastSendPayloadKey, e) +} + +func (e EventType) GetTimeStampKey() string { + tsKey := fmt.Sprintf("%s%s", TimestampKey, e) + return tsKey +} diff --git a/pkg/telemetry/types_test.go b/pkg/telemetry/types_test.go new file mode 100644 index 000000000..dc0747a75 --- /dev/null +++ b/pkg/telemetry/types_test.go @@ -0,0 +1,34 @@ +package telemetry + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventTypeMethods(t *testing.T) { + tests := []struct { + eventType EventType + expectedPayload string + expectedTimeKey string + }{ + { + eventType: Deployments, + expectedTimeKey: "lastSendTimestampDeployments", + }, + { + eventType: Operators, + expectedTimeKey: "lastSendTimestampOperators", + }, + { + eventType: Clusters, + expectedTimeKey: "lastSendTimestampClusters", + }, + } + + for _, tt := range tests { + t.Run(string(tt.eventType), func(t *testing.T) { + assert.Equal(t, tt.expectedTimeKey, tt.eventType.GetTimeStampKey(), "GetTimeStampKey() mismatch") + }) + } +} diff --git a/pkg/test/external_domains.go b/pkg/test/external_domains.go new file mode 100644 index 000000000..29eb90e43 --- /dev/null +++ b/pkg/test/external_domains.go @@ -0,0 +1,36 @@ +package test + +type ClusterDomains struct { + MongosExternalDomain string + ConfigServerExternalDomain string + ShardsExternalDomain string + SingleClusterDomain string +} + +var ( + ClusterLocalDomains = ClusterDomains{ + MongosExternalDomain: "cluster.local", + ConfigServerExternalDomain: "cluster.local", + ShardsExternalDomain: "cluster.local", + SingleClusterDomain: "cluster.local", + } + ExampleExternalClusterDomains = ClusterDomains{ + MongosExternalDomain: "mongos.mongodb.com", + ConfigServerExternalDomain: "config.mongodb.com", + ShardsExternalDomain: "shards.mongodb.com", + SingleClusterDomain: "single.mongodb.com", + } + ExampleAccessWithNoExternalDomain = ClusterDomains{ + MongosExternalDomain: "my-namespace.svc.cluster.local", + ConfigServerExternalDomain: "my-namespace.svc.cluster.local", + ShardsExternalDomain: "my-namespace.svc.cluster.local", + SingleClusterDomain: "my-namespace.svc.cluster.local", + } + SingleExternalClusterDomains = ClusterDomains{ + MongosExternalDomain: "single.mongodb.com", + ConfigServerExternalDomain: "single.mongodb.com", + ShardsExternalDomain: "single.mongodb.com", + SingleClusterDomain: "single.mongodb.com", + } + NoneExternalClusterDomains = ClusterDomains{} +) diff --git a/pkg/test/member_clusters.go b/pkg/test/member_clusters.go new file mode 100644 index 000000000..f68975140 --- /dev/null +++ b/pkg/test/member_clusters.go @@ -0,0 +1,51 @@ +package test + +type MemberClusterDetails struct { + ClusterName string + ShardMap []int + NumberOfConfigServers int + NumberOfMongoses int +} + +type MemberClusters struct { + ClusterNames []string + ShardDistribution []map[string]int + ConfigServerDistribution map[string]int + MongosDistribution map[string]int +} + +func NewMemberClusters(memberClusterDetails ...MemberClusterDetails) MemberClusters { + ret := MemberClusters{ + ClusterNames: make([]string, 0), + ShardDistribution: make([]map[string]int, 0), + ConfigServerDistribution: make(map[string]int), + MongosDistribution: make(map[string]int), + } + for _, detail := range memberClusterDetails { + ret.ClusterNames = append(ret.ClusterNames, detail.ClusterName) + ret.ConfigServerDistribution[detail.ClusterName] = detail.NumberOfConfigServers + ret.MongosDistribution[detail.ClusterName] = detail.NumberOfMongoses + } + + // TODO: Think if shards map shouldn't be an argument of this method? We're always using 0 index? + for range memberClusterDetails[0].ShardMap { + shardDistribution := map[string]int{} + for clusterDetailIndex, clusterDetails := range memberClusterDetails { + shardDistribution[clusterDetails.ClusterName] = memberClusterDetails[0].ShardMap[clusterDetailIndex] + } + ret.ShardDistribution = append(ret.ShardDistribution, shardDistribution) + } + + return ret +} + +func (clusters MemberClusters) ShardCount() int { + ignoredValue := 1 + distinctShardNumbers := make(map[int]int) + for _, shardDistribution := range clusters.ShardDistribution { + for _, shardNumber := range shardDistribution { + distinctShardNumbers[shardNumber] = ignoredValue + } + } + return len(distinctShardNumbers) +} diff --git a/pkg/test/placeholders.go b/pkg/test/placeholders.go new file mode 100644 index 000000000..8766a21b1 --- /dev/null +++ b/pkg/test/placeholders.go @@ -0,0 +1,27 @@ +package test + +import "github.com/10gen/ops-manager-kubernetes/controllers/operator/create" + +var MultiClusterAnnotationsWithPlaceholders = map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongodProcessDomain: "{mongodProcessDomain}", + create.PlaceholderMongodProcessFQDN: "{mongodProcessFQDN}", + create.PlaceholderClusterName: "{clusterName}", + create.PlaceholderClusterIndex: "{clusterIndex}", +} + +var SingleClusterAnnotationsWithPlaceholders = map[string]string{ + create.PlaceholderPodIndex: "{podIndex}", + create.PlaceholderNamespace: "{namespace}", + create.PlaceholderResourceName: "{resourceName}", + create.PlaceholderPodName: "{podName}", + create.PlaceholderStatefulSetName: "{statefulSetName}", + create.PlaceholderExternalServiceName: "{externalServiceName}", + create.PlaceholderMongosProcessDomain: "{mongosProcessDomain}", + create.PlaceholderMongosProcessFQDN: "{mongosProcessFQDN}", +} diff --git a/pkg/test/sharded_cluster_builder.go b/pkg/test/sharded_cluster_builder.go new file mode 100644 index 000000000..15e92fca3 --- /dev/null +++ b/pkg/test/sharded_cluster_builder.go @@ -0,0 +1,363 @@ +package test + +import ( + v12 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + "github.com/10gen/ops-manager-kubernetes/api/v1/status" + "github.com/10gen/ops-manager-kubernetes/controllers/operator/mock" + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +const SCBuilderDefaultName = "slaney" + +type ClusterBuilder struct { + *mdb.MongoDB +} + +func DefaultClusterBuilder() *ClusterBuilder { + sizeConfig := status.MongodbShardedClusterSizeConfig{ + ShardCount: 2, + MongodsPerShardCount: 3, + ConfigServerCount: 3, + MongosCount: 4, + } + + status := mdb.MongoDbStatus{ + MongodbShardedClusterSizeConfig: sizeConfig, + } + + spec := mdb.MongoDbSpec{ + DbCommonSpec: mdb.DbCommonSpec{ + Persistent: util.BooleanRef(false), + ConnectionSpec: mdb.ConnectionSpec{ + SharedConnectionSpec: mdb.SharedConnectionSpec{ + OpsManagerConfig: &mdb.PrivateCloudConfig{ + ConfigMapRef: mdb.ConfigMapRef{ + Name: mock.TestProjectConfigMapName, + }, + }, + }, + Credentials: mock.TestCredentialsSecretName, + }, + Version: "3.6.4", + ResourceType: mdb.ShardedCluster, + + Security: &mdb.Security{ + TLSConfig: &mdb.TLSConfig{}, + Authentication: &mdb.Authentication{ + Modes: []mdb.AuthMode{}, + }, + }, + }, + MongodbShardedClusterSizeConfig: sizeConfig, + ShardedClusterSpec: mdb.ShardedClusterSpec{ + ConfigSrvSpec: &mdb.ShardedClusterComponentSpec{}, + MongosSpec: &mdb.ShardedClusterComponentSpec{}, + ShardSpec: &mdb.ShardedClusterComponentSpec{}, + ConfigSrvPodSpec: mdb.NewMongoDbPodSpec(), + ShardPodSpec: mdb.NewMongoDbPodSpec(), + }, + } + + resource := &mdb.MongoDB{ + ObjectMeta: v1.ObjectMeta{Name: SCBuilderDefaultName, Namespace: mock.TestNamespace}, + Status: status, + Spec: spec, + } + + return &ClusterBuilder{resource} +} + +func (b *ClusterBuilder) SetName(name string) *ClusterBuilder { + b.Name = name + return b +} + +func (b *ClusterBuilder) SetVersion(version string) *ClusterBuilder { + b.Spec.Version = version + return b +} + +func (b *ClusterBuilder) SetShardCountSpec(count int) *ClusterBuilder { + b.Spec.ShardCount = count + return b +} + +func (b *ClusterBuilder) SetMongodsPerShardCountSpec(count int) *ClusterBuilder { + b.Spec.MongodsPerShardCount = count + return b +} + +func (b *ClusterBuilder) SetConfigServerCountSpec(count int) *ClusterBuilder { + b.Spec.ConfigServerCount = count + return b +} + +func (b *ClusterBuilder) SetMongosCountSpec(count int) *ClusterBuilder { + b.Spec.MongosCount = count + return b +} + +func (b *ClusterBuilder) SetShardCountStatus(count int) *ClusterBuilder { + b.Status.ShardCount = count + return b +} + +func (b *ClusterBuilder) SetMongodsPerShardCountStatus(count int) *ClusterBuilder { + b.Status.MongodsPerShardCount = count + return b +} + +func (b *ClusterBuilder) SetConfigServerCountStatus(count int) *ClusterBuilder { + b.Status.ConfigServerCount = count + return b +} + +func (b *ClusterBuilder) SetMongosCountStatus(count int) *ClusterBuilder { + b.Status.MongosCount = count + return b +} + +func (b *ClusterBuilder) SetSecurity(security mdb.Security) *ClusterBuilder { + b.Spec.Security = &security + return b +} + +func (b *ClusterBuilder) EnableTLS() *ClusterBuilder { + if b.Spec.Security == nil || b.Spec.Security.TLSConfig == nil { + return b.SetSecurity(mdb.Security{TLSConfig: &mdb.TLSConfig{Enabled: true}}) + } + b.Spec.Security.TLSConfig.Enabled = true + return b +} + +func (b *ClusterBuilder) SetTLSCA(ca string) *ClusterBuilder { + if b.Spec.Security == nil || b.Spec.Security.TLSConfig == nil { + b.SetSecurity(mdb.Security{TLSConfig: &mdb.TLSConfig{}}) + } + b.Spec.Security.TLSConfig.CA = ca + return b +} + +func (b *ClusterBuilder) SetTLSConfig(tlsConfig mdb.TLSConfig) *ClusterBuilder { + if b.Spec.Security == nil { + b.Spec.Security = &mdb.Security{} + } + b.Spec.Security.TLSConfig = &tlsConfig + return b +} + +func (b *ClusterBuilder) EnableX509() *ClusterBuilder { + b.Spec.Security.Authentication.Enabled = true + b.Spec.Security.Authentication.Modes = append(b.Spec.Security.Authentication.Modes, util.X509) + return b +} + +func (b *ClusterBuilder) EnableSCRAM() *ClusterBuilder { + b.Spec.Security.Authentication.Enabled = true + b.Spec.Security.Authentication.Modes = append(b.Spec.Security.Authentication.Modes, util.SCRAM) + return b +} + +func (b *ClusterBuilder) RemoveAuth() *ClusterBuilder { + b.Spec.Security.Authentication = nil + + return b +} + +func (b *ClusterBuilder) EnableAuth() *ClusterBuilder { + b.Spec.Security.Authentication.Enabled = true + return b +} + +func (b *ClusterBuilder) SetAuthModes(modes []mdb.AuthMode) *ClusterBuilder { + b.Spec.Security.Authentication.Modes = modes + return b +} + +func (b *ClusterBuilder) EnableX509InternalClusterAuth() *ClusterBuilder { + b.Spec.Security.Authentication.InternalCluster = util.X509 + return b +} + +func (b *ClusterBuilder) SetShardPodSpec(spec v12.PodTemplateSpec) *ClusterBuilder { + if b.Spec.ShardPodSpec == nil { + b.Spec.ShardPodSpec = &mdb.MongoDbPodSpec{} + } + b.Spec.ShardPodSpec.PodTemplateWrapper.PodTemplate = &spec + return b +} + +func (b *ClusterBuilder) SetPodConfigSvrSpecTemplate(spec v12.PodTemplateSpec) *ClusterBuilder { + if b.Spec.ConfigSrvPodSpec == nil { + b.Spec.ConfigSrvPodSpec = &mdb.MongoDbPodSpec{} + } + b.Spec.ConfigSrvPodSpec.PodTemplateWrapper.PodTemplate = &spec + return b +} + +func (b *ClusterBuilder) SetMongosPodSpecTemplate(spec v12.PodTemplateSpec) *ClusterBuilder { + if b.Spec.MongosPodSpec == nil { + b.Spec.MongosPodSpec = &mdb.MongoDbPodSpec{} + } + b.Spec.MongosPodSpec.PodTemplateWrapper.PodTemplate = &spec + return b +} + +func (b *ClusterBuilder) SetShardSpecificPodSpecTemplate(specs []v12.PodTemplateSpec) *ClusterBuilder { + if b.Spec.ShardSpecificPodSpec == nil { + b.Spec.ShardSpecificPodSpec = make([]mdb.MongoDbPodSpec, 0) + } + + mongoDBPodSpec := make([]mdb.MongoDbPodSpec, len(specs)) + + for n, e := range specs { + mongoDBPodSpec[n] = mdb.MongoDbPodSpec{PodTemplateWrapper: mdb.PodTemplateSpecWrapper{ + PodTemplate: &e, + }} + } + + b.Spec.ShardSpecificPodSpec = mongoDBPodSpec + return b +} + +func (b *ClusterBuilder) SetAnnotations(annotations map[string]string) *ClusterBuilder { + b.Annotations = annotations + return b +} + +func (b *ClusterBuilder) SetTopology(topology string) *ClusterBuilder { + b.Spec.Topology = topology + return b +} + +func (b *ClusterBuilder) SetConfigSrvClusterSpec(clusterSpecList mdb.ClusterSpecList) *ClusterBuilder { + b.Spec.ConfigSrvSpec.ClusterSpecList = clusterSpecList + return b +} + +func (b *ClusterBuilder) SetMongosClusterSpec(clusterSpecList mdb.ClusterSpecList) *ClusterBuilder { + b.Spec.MongosSpec.ClusterSpecList = clusterSpecList + return b +} + +func (b *ClusterBuilder) SetShardClusterSpec(clusterSpecList mdb.ClusterSpecList) *ClusterBuilder { + b.Spec.ShardSpec.ClusterSpecList = clusterSpecList + return b +} + +func (b *ClusterBuilder) SetShardOverrides(override []mdb.ShardOverride) *ClusterBuilder { + b.Spec.ShardOverrides = override + return b +} + +func (b *ClusterBuilder) SetOpsManagerConfigMapName(configMapName string) *ClusterBuilder { + b.Spec.OpsManagerConfig.ConfigMapRef.Name = configMapName + return b +} + +func (b *ClusterBuilder) SetExternalAccessDomain(externalDomains ClusterDomains) *ClusterBuilder { + if b.Spec.IsMultiCluster() { + for i := range b.Spec.ConfigSrvSpec.ClusterSpecList { + if b.Spec.ConfigSrvSpec.ClusterSpecList[i].ExternalAccessConfiguration == nil { + b.Spec.ConfigSrvSpec.ClusterSpecList[i].ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + if len(externalDomains.ConfigServerExternalDomain) > 0 { + b.Spec.ConfigSrvSpec.ClusterSpecList[i].ExternalAccessConfiguration.ExternalDomain = &externalDomains.ConfigServerExternalDomain + } + } + for i := range b.Spec.MongosSpec.ClusterSpecList { + if b.Spec.MongosSpec.ClusterSpecList[i].ExternalAccessConfiguration == nil { + b.Spec.MongosSpec.ClusterSpecList[i].ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + if len(externalDomains.MongosExternalDomain) > 0 { + b.Spec.MongosSpec.ClusterSpecList[i].ExternalAccessConfiguration.ExternalDomain = &externalDomains.MongosExternalDomain + } + } + for i := range b.Spec.ShardSpec.ClusterSpecList { + if b.Spec.ShardSpec.ClusterSpecList[i].ExternalAccessConfiguration == nil { + b.Spec.ShardSpec.ClusterSpecList[i].ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + if len(externalDomains.ShardsExternalDomain) > 0 { + b.Spec.ShardSpec.ClusterSpecList[i].ExternalAccessConfiguration.ExternalDomain = &externalDomains.ShardsExternalDomain + } + } + } else { + if b.Spec.ExternalAccessConfiguration == nil { + b.Spec.ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + b.Spec.ExternalAccessConfiguration.ExternalDomain = &externalDomains.SingleClusterDomain + } + return b +} + +func (b *ClusterBuilder) SetExternalAccessDomainAnnotations(annotationWithPlaceholders map[string]string) *ClusterBuilder { + if b.Spec.IsMultiCluster() { + for i := range b.Spec.ConfigSrvSpec.ClusterSpecList { + if b.Spec.ConfigSrvSpec.ClusterSpecList[i].ExternalAccessConfiguration == nil { + b.Spec.ConfigSrvSpec.ClusterSpecList[i].ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + b.Spec.ConfigSrvSpec.ClusterSpecList[i].ExternalAccessConfiguration.ExternalService.Annotations = annotationWithPlaceholders + } + for i := range b.Spec.MongosSpec.ClusterSpecList { + if b.Spec.MongosSpec.ClusterSpecList[i].ExternalAccessConfiguration == nil { + b.Spec.MongosSpec.ClusterSpecList[i].ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + b.Spec.MongosSpec.ClusterSpecList[i].ExternalAccessConfiguration.ExternalService.Annotations = annotationWithPlaceholders + + } + for i := range b.Spec.ShardSpec.ClusterSpecList { + if b.Spec.ShardSpec.ClusterSpecList[i].ExternalAccessConfiguration == nil { + b.Spec.ShardSpec.ClusterSpecList[i].ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + b.Spec.ShardSpec.ClusterSpecList[i].ExternalAccessConfiguration.ExternalService.Annotations = annotationWithPlaceholders + } + } else { + if b.Spec.ExternalAccessConfiguration == nil { + b.Spec.ExternalAccessConfiguration = &mdb.ExternalAccessConfiguration{} + } + b.Spec.ExternalAccessConfiguration.ExternalService.Annotations = annotationWithPlaceholders + } + return b +} + +func (b *ClusterBuilder) WithMultiClusterSetup(memberClusters MemberClusters) *ClusterBuilder { + b.SetTopology(mdb.ClusterTopologyMultiCluster) + b.SetShardCountSpec(memberClusters.ShardCount()) + + // The below parameters should be ignored when a clusterSpecList is configured/for multiClusterTopology + b.SetMongodsPerShardCountSpec(0) + b.SetConfigServerCountSpec(0) + b.SetMongosCountSpec(0) + + b.SetShardClusterSpec(CreateClusterSpecList(memberClusters.ClusterNames, memberClusters.ShardDistribution[0])) + b.SetConfigSrvClusterSpec(CreateClusterSpecList(memberClusters.ClusterNames, memberClusters.ConfigServerDistribution)) + b.SetMongosClusterSpec(CreateClusterSpecList(memberClusters.ClusterNames, memberClusters.MongosDistribution)) + + return b +} + +func (b *ClusterBuilder) Build() *mdb.MongoDB { + b.Spec.ResourceType = mdb.ShardedCluster + b.InitDefaults() + return b.MongoDB +} + +// Creates a list of ClusterSpecItems based on names and distribution +// The two input list must have the same size +func CreateClusterSpecList(clusterNames []string, memberCounts map[string]int) mdb.ClusterSpecList { + specList := make(mdb.ClusterSpecList, 0) + for _, clusterName := range clusterNames { + if _, ok := memberCounts[clusterName]; !ok { + continue + } + + specList = append(specList, mdb.ClusterSpecItem{ + ClusterName: clusterName, + Members: memberCounts[clusterName], + }) + } + return specList +} diff --git a/pkg/tls/tls.go b/pkg/tls/tls.go new file mode 100644 index 000000000..98bc13d12 --- /dev/null +++ b/pkg/tls/tls.go @@ -0,0 +1,32 @@ +package tls + +import ( + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" +) + +type Mode string + +const ( + Disabled Mode = "disabled" + Require Mode = "requireTLS" + Prefer Mode = "preferTLS" + Allow Mode = "allowTLS" + ConfigMapVolumeCAName = "secret-ca" +) + +func GetTLSModeFromMongodConfig(config map[string]interface{}) Mode { + // spec.Security.TLSConfig.IsEnabled() is true -> requireSSLMode + if config == nil { + return Require + } + mode := maputil.ReadMapValueAsString(config, "net", "tls", "mode") + + if mode == "" { + mode = maputil.ReadMapValueAsString(config, "net", "ssl", "mode") + } + if mode == "" { + return Require + } + + return Mode(mode) +} diff --git a/pkg/util/architectures/static.go b/pkg/util/architectures/static.go new file mode 100644 index 000000000..b96538bc4 --- /dev/null +++ b/pkg/util/architectures/static.go @@ -0,0 +1,89 @@ +package architectures + +import ( + "strings" + + "k8s.io/utils/env" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type DefaultArchitecture string + +type ImageType string + +const ( + ImageTypeUBI8 ImageType = "ubi8" + ImageTypeUBI9 ImageType = "ubi9" + DefaultImageType ImageType = ImageTypeUBI8 +) + +func HasSupportedImageTypeSuffix(imageVersion string) (suffixFound bool, suffix string) { + if strings.HasSuffix(imageVersion, string(ImageTypeUBI8)) { + return true, string(ImageTypeUBI8) + } + if strings.HasSuffix(imageVersion, string(ImageTypeUBI9)) { + return true, string(ImageTypeUBI9) + } + return false, "" +} + +const ( + ArchitectureAnnotation = "mongodb.com/v1.architecture" + DefaultEnvArchitecture = "MDB_DEFAULT_ARCHITECTURE" + Static DefaultArchitecture = "static" + NonStatic DefaultArchitecture = "non-static" + // MdbAssumeEnterpriseImage allows the customer to override the version image detection used by the operator to + // set up the automation config. + // true: always append the -ent suffix and assume enterprise + // false: do not append the -ent suffix and assume community + // default: false + MdbAssumeEnterpriseImage = "MDB_ASSUME_ENTERPRISE_IMAGE" + // MdbAgentImageRepo contains the repository containing the agent image for the database + MdbAgentImageRepo = "MDB_AGENT_IMAGE_REPOSITORY" + MdbAgentImageRepoDefault = "quay.io/mongodb/mongodb-agent-ubi" +) + +// IsRunningStaticArchitecture checks whether the operator is running in static or non-static mode. +// This is either decided via an annotation per resource or per operator level. +// The resource annotation takes precedence. +// A nil map is equivalent to an empty map except that no elements may be added. +func IsRunningStaticArchitecture(annotations map[string]string) bool { + if annotations != nil { + if architecture, ok := annotations[ArchitectureAnnotation]; ok { + if architecture == string(Static) { + return true + } + if architecture == string(NonStatic) { + return false + } + } + } + + operatorEnv := env.GetString(DefaultEnvArchitecture, string(NonStatic)) + return operatorEnv == string(Static) +} + +func GetArchitecture(annotations map[string]string) DefaultArchitecture { + if IsRunningStaticArchitecture(annotations) { + return Static + } + return NonStatic +} + +// GetMongoVersionForAutomationConfig returns the required version with potentially the suffix -ent. +// If we are in static containers architecture, we need the -ent suffix in case we are running the ea image. +// If not, the agent will try to change the version to reflect the non-enterprise image. +func GetMongoVersionForAutomationConfig(mongoDBImage, version string, forceEnterprise bool, architecture DefaultArchitecture) string { + if architecture != Static { + return version + } + // the image repo should be either mongodb / mongodb-enterprise-server or mongodb / mongodb-community-server + if strings.Contains(mongoDBImage, util.OfficialEnterpriseServerImageUrl) || forceEnterprise { + if !strings.HasSuffix(version, "-ent") { + version = version + "-ent" + } + } + + return version +} diff --git a/pkg/util/architectures/static_test.go b/pkg/util/architectures/static_test.go new file mode 100644 index 000000000..2f11a0015 --- /dev/null +++ b/pkg/util/architectures/static_test.go @@ -0,0 +1,121 @@ +package architectures + +import ( + "testing" +) + +func TestIsRunningStaticArchitecture(t *testing.T) { + tests := []struct { + name string + annotations map[string]string + want bool + envFunc func(t *testing.T) + }{ + { + name: "no annotation and no env", + annotations: nil, + want: false, + envFunc: nil, + }, + { + name: "only env and is static", + want: true, + envFunc: func(t *testing.T) { + t.Setenv(DefaultEnvArchitecture, string(Static)) + }, + }, + { + name: "only env and is non static", + want: false, + envFunc: func(t *testing.T) { + t.Setenv(DefaultEnvArchitecture, string(NonStatic)) + }, + }, + { + name: "only annotation and is static", + annotations: map[string]string{ArchitectureAnnotation: string(Static)}, + want: true, + }, + { + name: "only annotation and is not static", + annotations: map[string]string{ArchitectureAnnotation: string(NonStatic)}, + + want: false, + }, + { + name: "only annotation and is wrong value", + annotations: map[string]string{ArchitectureAnnotation: "brokenValue"}, + want: false, + }, + { + name: "env and annotation differ. Annotations has precedence", + annotations: map[string]string{ArchitectureAnnotation: string(Static)}, + envFunc: func(t *testing.T) { + t.Setenv(DefaultEnvArchitecture, string(NonStatic)) + }, + want: true, + }, + } + for _, tt := range tests { + testConfig := tt + t.Run(testConfig.name, func(t *testing.T) { + if testConfig.envFunc != nil { + testConfig.envFunc(t) + } + if got := IsRunningStaticArchitecture(testConfig.annotations); got != testConfig.want { + t.Errorf("IsRunningStaticArchitecture() = %v, want %v", got, testConfig.want) + } + }) + } +} + +func TestGetMongoVersion(t *testing.T) { + tests := []struct { + name string + mongoDBImage string + version string + forceEnterprise bool + architecture DefaultArchitecture + want string + }{ + { + name: "nothing", + mongoDBImage: "", + version: "8.0.0", + forceEnterprise: false, + architecture: NonStatic, + want: "8.0.0", + }, + { + name: "enterprise repo", + mongoDBImage: "quay.io/mongodb/mongodb-enterprise-server", + version: "8.0.0", + forceEnterprise: false, + architecture: Static, + want: "8.0.0-ent", + }, + { + name: "community repo", + mongoDBImage: "quay.io/mongodb/mongodb-community-server", + version: "8.0.0", + forceEnterprise: false, + architecture: NonStatic, + want: "8.0.0", + }, + { + name: "enterprise repo forced", + mongoDBImage: "quay.io/mongodb/mongodb-private-server", + version: "8.0.0", + forceEnterprise: true, + architecture: Static, + want: "8.0.0-ent", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetMongoVersionForAutomationConfig(tt.mongoDBImage, tt.version, tt.forceEnterprise, tt.architecture); got != tt.want { + t.Errorf("GetMongoVersionForAutomationConfig() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/util/constants.go b/pkg/util/constants.go new file mode 100644 index 000000000..1353a79fb --- /dev/null +++ b/pkg/util/constants.go @@ -0,0 +1,334 @@ +package util + +import ( + "strings" + "time" +) + +const ( + // MongoDbStandaloneController name of the Standalone controller + MongoDbStandaloneController = "mongodbstandalone-controller" + + // MongoDbReplicaSetController name of the ReplicaSet controller + MongoDbReplicaSetController = "mongodbreplicaset-controller" + + // MongoDbMultiClusterController is the name of the MongoDB Multi controller + MongoDbMultiClusterController = "mongodbmulticluster-controller" + + // MongoDbMultiReplicaSetController name of the multi-cluster ReplicaSet controller + MongoDbMultiReplicaSetController = "mongodbmultireplicaset-controller" + + // MongoDbShardedClusterController name of the ShardedCluster controller + MongoDbShardedClusterController = "mongodbshardedcluster-controller" + + // MongoDbUserController name of the MongoDBUser controller + MongoDbUserController = "mongodbuser-controller" + + // MongoDbOpsManagerController name of the OpsManager controller + MongoDbOpsManagerController = "opsmanager-controller" + + // Ops manager config map and secret variables + OmBaseUrl = "baseUrl" + OmOrgId = "orgId" + OmProjectName = "projectName" + OldOmPublicApiKey = "publicApiKey" + OldOmUser = "user" + OmPublicApiKey = "publicKey" + OmPrivateKey = "privateKey" + OmAgentApiKey = "agentApiKey" + OmCredentials = "credentials" + + // SSLRequireValidMMSServerCertificates points at the string name of the + // same name variable in OM configuration passed in the "Project" config + SSLRequireValidMMSServerCertificates = "sslRequireValidMMSServerCertificates" + + // SSLTrustedMMSServerCertificate points at the string name of the + // same name variable in OM configuration passed in the "Project" config + SSLTrustedMMSServerCertificate = "sslTrustedMMSServerCertificate" + + // SSLMMSCAConfigMap indicates the name of the ConfigMap that stores the + // CA certificate used to sign the MMS TLS certificate + SSLMMSCAConfigMap = "sslMMSCAConfigMap" + + // UseCustomCAConfigMap flags the operator to try to generate certificates + // (if false) or to not generate them (if true). + UseCustomCAConfigMap = "useCustomCA" + + // CaCertMMS is the name of the CA file provided for MMS. + CaCertMMS = "mms-ca.crt" + + // Env variables names for pods + EnvVarBaseUrl = "BASE_URL" + EnvVarProjectId = "GROUP_ID" + EnvVarUser = "USER_LOGIN" + EnvVarLogLevel = "LOG_LEVEL" + + // EnvVarDebug is used to decide whether we want to start the agent in debug mode + EnvVarDebug = "MDB_AGENT_DEBUG" + EnvVarAgentVersion = "MDB_AGENT_VERSION" + EnvVarMultiClusterMode = "MULTI_CLUSTER_MODE" + + // EnvVarSSLRequireValidMMSCertificates bla bla + EnvVarSSLRequireValidMMSCertificates = "SSL_REQUIRE_VALID_MMS_CERTIFICATES" + + // EnvVarSSLTrustedMMSServerCertificate env variable will point to where the CA cert is mounted. + EnvVarSSLTrustedMMSServerCertificate = "SSL_TRUSTED_MMS_SERVER_CERTIFICATE" + + // Pod/StatefulSet specific constants + OperatorName = "mongodb-enterprise-operator" + MultiClusterOperatorName = "mongodb-enterprise-operator-multi-cluster" + OpsManagerContainerName = "mongodb-ops-manager" + BackupDaemonContainerName = "mongodb-backup-daemon" + DatabaseContainerName = "mongodb-enterprise-database" + AgentContainerName = "mongodb-agent" + InitOpsManagerContainerName = "mongodb-enterprise-init-ops-manager" + PvcNameData = "data" + PvcMountPathData = "/data" + PvcNameJournal = "journal" + PvcMountPathJournal = "/journal" + PvcNameLogs = "logs" + PvcMountPathLogs = "/var/log/mongodb-mms-automation" + PvcNameHeadDb = "head" + PvcMountPathHeadDb = "/head/" + PvcNameTmp = "tmp" + PvcMountPathTmp = "/tmp" + PvcMmsHome = "mongodb-automation" + PvcMmsHomeMountPath = "/mongodb-automation" + CAFilePathInContainer = PvcMmsHomeMountPath + "/ca.pem" + PEMKeyFilePathInContainer = PvcMmsHomeMountPath + "/server.pem" + KMIPSecretsHome = "/mongodb-ops-manager/kmip" //nolint + KMIPServerCAHome = KMIPSecretsHome + "/server" + KMIPClientSecretsHome = KMIPSecretsHome + "/client" + KMIPServerCAName = "kmip-server" + KMIPClientSecretNamePrefix = "kmip-client-" //nolint + KMIPCAFileInContainer = KMIPServerCAHome + "/ca.pem" + PvcMms = "mongodb-mms-automation" + PvcMmsMountPath = "/var/lib/mongodb-mms-automation" + PvMms = "agent" + AgentDownloadsDir = PvcMmsMountPath + "/downloads" + AgentAuthenticationKeyfilePath = PvcMmsMountPath + "/keyfile" + AutomationConfigFilePath = PvcMountPathData + "/automation-mongod.conf" + MongosConfigFileDirPath = PvcMmsMountPath + "/workspace" + + MmsPemKeyFileDirInContainer = "/opt/mongodb/mms/secrets" + AppDBMmsCaFileDirInContainer = "/opt/mongodb/mms/ca/" + + AutomationAgentName = "mms-automation-agent" + AutomationAgentPemSecretKey = AutomationAgentName + "-pem" + AutomationAgentPemFilePath = PvcMmsHomeMountPath + "/" + AgentSecretName + "/" + AutomationAgentPemSecretKey + + // Key used in concatenated pem secrets to denote the hash of the latest certificate + LatestHashSecretKey = "latestHash" + PreviousHashSecretKey = "previousHash" + + RunAsUser = 2000 + FsGroup = 2000 + + // Service accounts + OpsManagerServiceAccount = "mongodb-enterprise-ops-manager" + MongoDBServiceAccount = "mongodb-enterprise-database-pods" + + // Authentication + AgentSecretName = "agent-certs" + AutomationConfigX509Option = "MONGODB-X509" + AutomationConfigLDAPOption = "PLAIN" + AutomationConfigScramSha256Option = "SCRAM-SHA-256" + AutomationConfigScramSha1Option = "MONGODB-CR" + AutomationAgentUserName = "mms-automation-agent" + RequireClientCertificates = "REQUIRE" + OptionalClientCertficates = "OPTIONAL" + ClusterFileName = "clusterfile" + InternalClusterAuthMountPath = PvcMmsHomeMountPath + "/cluster-auth/" + DefaultUserDatabase = "admin" + X509 = "X509" + SCRAM = "SCRAM" + SCRAMSHA1 = "SCRAM-SHA-1" + MONGODBCR = "MONGODB-CR" + SCRAMSHA256 = "SCRAM-SHA-256" + LDAP = "LDAP" + MinimumScramSha256MdbVersion = "4.0.0" + + // these were historically used and constituted a security issue—if set they should be changed + InvalidKeyFileContents = "DUMMYFILE" + InvalidAutomationAgentPassword = "D9XK2SfdR2obIevI9aKsYlVH" //nolint //Part of the algorithm + + // AutomationAgentWindowsKeyFilePath is the default path for the windows key file. This is never + // used, but we want to keep it the default value so it is is possible to add new users without modifying + // it. Ops Manager will attempt to reset this value to the default if new MongoDB users are added + // when x509 auth is enabled + AutomationAgentWindowsKeyFilePath = "%SystemDrive%\\MMSAutomation\\versions\\keyfile" + + // AutomationAgentKeyFilePathInContainer is the default path of the keyfile and should be + // kept as is for the same reason as above + AutomationAgentKeyFilePathInContainer = PvcMmsMountPath + "/keyfile" + + // Operator Env configuration properties. Please note that when adding environment variables to this list, + // make sure you append them to util.go:PrintEnvVars function's `printableEnvPrefixes` if you need the + // new variable to be printed at operator start. + OpsManagerImageUrl = "OPS_MANAGER_IMAGE_REPOSITORY" + InitOpsManagerImageUrl = "INIT_OPS_MANAGER_IMAGE_REPOSITORY" + InitOpsManagerVersion = "INIT_OPS_MANAGER_VERSION" + InitAppdbImageUrlEnv = "INIT_APPDB_IMAGE_REPOSITORY" + InitDatabaseImageUrlEnv = "INIT_DATABASE_IMAGE_REPOSITORY" + OpsManagerPullPolicy = "OPS_MANAGER_IMAGE_PULL_POLICY" + NonStaticDatabaseEnterpriseImage = "MONGODB_ENTERPRISE_DATABASE_IMAGE" + AutomationAgentImagePullPolicy = "IMAGE_PULL_POLICY" + ImagePullSecrets = "IMAGE_PULL_SECRETS" //nolint + OmOperatorEnv = "OPERATOR_ENV" + MemberListConfigMapName = "mongodb-enterprise-operator-member-list" + BackupDisableWaitSecondsEnv = "BACKUP_WAIT_SEC" + BackupDisableWaitRetriesEnv = "BACKUP_WAIT_RETRIES" + ManagedSecurityContextEnv = "MANAGED_SECURITY_CONTEXT" + CurrentNamespace = "NAMESPACE" + WatchNamespace = "WATCH_NAMESPACE" + OpsManagerMonitorAppDB = "OPS_MANAGER_MONITOR_APPDB" + + MdbWebhookRegisterConfigurationEnv = "MDB_WEBHOOK_REGISTER_CONFIGURATION" + MdbWebhookPortEnv = "MDB_WEBHOOK_PORT" + + MaxConcurrentReconcilesEnv = "MDB_MAX_CONCURRENT_RECONCILES" + + // Different default configuration values + DefaultMongodStorageSize = "16G" + DefaultConfigSrvStorageSize = "5G" + DefaultJournalStorageSize = "1G" // maximum size for single journal file is 100Mb, journal files are removed soon after checkpoints + DefaultLogsStorageSize = "3G" + DefaultHeadDbStorageSize = "32G" + DefaultMemoryAppDB = "500M" + DefaultMemoryOpsManager = "5G" + DefaultAntiAffinityTopologyKey = "kubernetes.io/hostname" + MongoDbDefaultPort = 27017 + OpsManagerDefaultPortHTTP = 8080 + OpsManagerDefaultPortHTTPS = 8443 + DefaultBackupDisableWaitSeconds = "3" + DefaultBackupDisableWaitRetries = "30" // 30 * 3 = 90 seconds, should be ok for backup job to terminate + DefaultPodTerminationPeriodSeconds = 600 // 10 min. Keep this in sync with 'cleanup()' function in agent-launcher-lib.sh + DefaultK8sCacheRefreshTimeSeconds = 2 + OpsManagerMonitorAppDBDefault = true + + // S3 constants + S3AccessKey = "accessKey" + S3SecretKey = "secretKey" + DefaultS3MaxConnections = 50 + + // Ops Manager related constants + OmPropertyPrefix = "OM_PROP_" + MmsJvmParamEnvVar = "CUSTOM_JAVA_MMS_UI_OPTS" + BackupDaemonJvmParamEnvVar = "CUSTOM_JAVA_DAEMON_OPTS" + GenKeyPath = "/mongodb-ops-manager/.mongodb-mms" + LatestOmVersion = "5.0" + AppDBAutomationConfigKey = "cluster-config.json" + AppDBMonitoringAutomationConfigKey = "monitoring-cluster-config.json" + DefaultAppDbPasswordKey = "password" + AppDbConnectionStringKey = "connectionString" + AppDbProjectIdKey = "projectId" + + // Below is a list of non-persistent PV and PVCs for OpsManager + OpsManagerPvcNameData = "data" + OpsManagerPvcNameConf = "conf" + OpsManagerPvcMountPathConf = "/mongodb-ops-manager/conf" + OpsManagerPvcNameLogs = "logs" + OpsManagerPvcMountPathLogs = "/mongodb-ops-manager/logs" + OpsManagerPvcNameTmp = "tmp-ops-manager" + OpsManagerPvcMountPathTmp = "/mongodb-ops-manager/tmp" + OpsManagerPvcNameDownloads = "mongodb-releases" + OpsManagerPvcMountDownloads = "/mongodb-ops-manager/mongodb-releases" + OpsManagerPvcNameEtc = "etc-ops-manager" + OpsManagerPvcMountPathEtc = "/etc/mongodb-mms" + + OpsManagerPvcLogBackNameVolume = "logback-volume" + OpsManagerPvcLogbackMountPath = "/mongodb-ops-manager/conf-template/logback.xml" + OpsManagerPvcLogbackSubPath = "logback.xml" + + OpsManagerPvcLogBackAccessNameVolume = "logback-access-volume" + OpsManagerPvcLogbackAccessMountPath = "/mongodb-ops-manager/conf-template/logback-access.xml" + OpsManagerPvcLogbackAccessSubPath = "logback-access.xml" + + // Ops Manager configuration properties + MmsCentralUrlPropKey = "mms.centralUrl" + MmsMongoUri = "mongo.mongoUri" + MmsMongoSSL = "mongo.ssl" + MmsMongoCA = "mongodb.ssl.CAFile" + MmsFeatureControls = "mms.featureControls.enable" + MmsVersionsDirectory = "automation.versions.directory" + MmsPEMKeyFile = "mms.https.PEMKeyFile" + BrsQueryablePem = "brs.queryable.pem" + + // SecretVolumeMountPath defines where in the Pod will be the secrets + // object mounted. + SecretVolumeMountPath = "/var/lib/mongodb-automation/secrets" //nolint + + SecretVolumeMountPathPrometheus = SecretVolumeMountPath + "/prometheus" + + TLSCertMountPath = PvcMmsHomeMountPath + "/tls" + TLSCaMountPath = PvcMmsHomeMountPath + "/tls/ca" + + // TODO: remove this from here and move it to the certs package + // This currently creates an import cycle + InternalCertAnnotationKey = "internalCertHash" + LastAchievedSpec = "mongodb.com/v1.lastSuccessfulConfiguration" + + // SecretVolumeName is the name of the volume resource. + SecretVolumeName = "secret-certs" + + // PrometheusSecretVolumeName + PrometheusSecretVolumeName = "prometheus-certs" + + // ConfigMapVolumeCAMountPath defines where CA root certs will be + // mounted in the pod + ConfigMapVolumeCAMountPath = SecretVolumeMountPath + "/ca" + + // Ops Manager authentication constants + OpsManagerMongoDBUserName = "mongodb-ops-manager" + OpsManagerPasswordKey = "password" + + // Env variables used for testing mostly to decrease waiting time + PodWaitSecondsEnv = "POD_WAIT_SEC" + PodWaitRetriesEnv = "POD_WAIT_RETRIES" + + // All others + OmGroupExternallyManagedTag = "EXTERNALLY_MANAGED_BY_KUBERNETES" + GenericErrorMessage = "Something went wrong validating your Automation Config" + MethodNotAllowed = "405 (Method Not Allowed)" + + RetryTimeSec = 10 + + DeprecatedImageAppdbUbiUrl = "mongodb-enterprise-appdb-database-ubi" + + OfficialEnterpriseServerImageUrl = "mongodb-enterprise-server" + + MdbAppdbAssumeOldFormat = "MDB_APPDB_ASSUME_OLD_FORMAT" + + Finalizer = "mongodb.com/v1.userRemovalFinalizer" +) + +type OperatorEnvironment string + +func (o OperatorEnvironment) String() string { + return string(o) +} + +const ( + OperatorEnvironmentDev OperatorEnvironment = "dev" + OperatorEnvironmentProd OperatorEnvironment = "prod" + OperatorEnvironmentLocal OperatorEnvironment = "local" +) + +// ***** These variables are set at compile time + +// OperatorVersion is the version of the current Operator. Important: currently it's empty when the Operator is +// installed for development (using 'make') meaning the Ops Manager/AppDB images deployed won't have +// "operator specific" part of the version tag +var OperatorVersion string + +var LogAutomationConfigDiff string + +func ShouldLogAutomationConfigDiff() bool { + return strings.EqualFold(LogAutomationConfigDiff, "true") +} + +const ( + TWENTY_FOUR_HOURS = 24 * time.Hour +) + +const AlwaysMatchVersionFCV = "AlwaysMatchVersion" diff --git a/pkg/util/env/env.go b/pkg/util/env/env.go new file mode 100644 index 000000000..fae45ac28 --- /dev/null +++ b/pkg/util/env/env.go @@ -0,0 +1,131 @@ +package env + +import ( + "fmt" + "os" + "sort" + "strconv" + "strings" + + "github.com/spf13/cast" + "go.uber.org/zap" + + corev1 "k8s.io/api/core/v1" +) + +func Read(env string) (string, bool) { + return os.LookupEnv(env) +} + +func ReadBool(env string) (valueAsBool bool, isPresent bool) { + value, isPresent := Read(env) + if !isPresent { + return false, false + } + boolValue, err := strconv.ParseBool(value) + return boolValue, err == nil +} + +func ReadBoolOrDefault(key string, defaultValue bool) bool { + value, isPresent := ReadBool(key) + if isPresent { + return value + } + return defaultValue +} + +// EnsureVar tests the env variable and sets it if it doesn't exist. We tolerate any errors setting env variable and +// just log the warning +func EnsureVar(key, value string) { + if _, exist := Read(key); !exist { + if err := os.Setenv(key, value); err != nil { // nolint:forbidigo + zap.S().Warnf("Failed to set environment variable \"%s\" to \"%s\": %s", key, value, err) + } + } +} + +// PrintWithPrefix prints environment variables to the global SugaredLogger. It will only print the environment variables +// with a given prefix set inside the function. +func PrintWithPrefix(printableEnvPrefixes []string) { + zap.S().Info("Environment variables:") + envVariables := os.Environ() + sort.Strings(envVariables) + for _, e := range envVariables { + for _, prefix := range printableEnvPrefixes { + if strings.HasPrefix(e, prefix) { + zap.S().Infof("%s", e) + break + } + } + } +} + +func ReadOrPanic(key string) string { + value := os.Getenv(key) + if value == "" { + panic(fmt.Sprintf("%s environment variable is not set!", key)) + } + return value +} + +func ReadIntOrPanic(key string) int { + value := os.Getenv(key) + i, e := cast.ToIntE(value) + if e != nil { + panic(fmt.Sprintf("%s env variable is supposed to be of type int but the value is %s", key, value)) + } + return i +} + +func ReadOrDefault(key string, dflt string) string { + value, exists := os.LookupEnv(key) + if !exists || value == "" { + return dflt + } + return value +} + +func ReadIntOrDefault(key string, dflt int) int { + value := ReadOrDefault(key, strconv.Itoa(dflt)) + i, e := cast.ToIntE(value) + if e != nil { + return dflt + } + return i +} + +// PodEnvVars is a convenience struct to pass environment variables to Pods as needed. +// They are used by the automation agent to connect to Ops/Cloud Manager. +type PodEnvVars struct { + BaseURL string + ProjectID string + User string + AgentAPIKey string + LogLevel string + + // Related to MMS SSL configuration + SSLProjectConfig +} + +// SSLProjectConfig contains the configuration options that are relevant for MMS SSL configuration +type SSLProjectConfig struct { + // This is set to true if baseUrl is HTTPS + SSLRequireValidMMSServerCertificates bool + + // Name of a configmap containing a `mms-ca.crt` entry that will be mounted + // on every Pod. + SSLMMSCAConfigMap string + + // SSLMMSCAConfigMap will contain the CA cert, used to push multiple + SSLMMSCAConfigMapContents string +} + +// ToMap accepts a variable number of EnvVars and returns them as a map +// with the name as the key. +func ToMap(vars ...corev1.EnvVar) map[string]string { + variablesMap := map[string]string{} + for _, envVar := range vars { + variablesMap[envVar.Name] = envVar.Value + } + return variablesMap +} diff --git a/pkg/util/env/env_test.go b/pkg/util/env/env_test.go new file mode 100644 index 000000000..5490837e6 --- /dev/null +++ b/pkg/util/env/env_test.go @@ -0,0 +1,34 @@ +package env + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadBoolEnv(t *testing.T) { + t.Setenv("ENV_1", "true") + t.Setenv("ENV_2", "false") + t.Setenv("ENV_3", "TRUE") + t.Setenv("NOT_BOOL", "not-true") + + result, present := ReadBool("ENV_1") + assert.True(t, present) + assert.True(t, result) + + result, present = ReadBool("ENV_2") + assert.True(t, present) + assert.False(t, result) + + result, present = ReadBool("ENV_3") + assert.True(t, present) + assert.True(t, result) + + result, present = ReadBool("NOT_BOOL") + assert.False(t, present) + assert.False(t, result) + + result, present = ReadBool("NOT_HERE") + assert.False(t, present) + assert.False(t, result) +} diff --git a/pkg/util/generate/generate.go b/pkg/util/generate/generate.go index 338e0d1b8..c0714a4bf 100644 --- a/pkg/util/generate/generate.go +++ b/pkg/util/generate/generate.go @@ -2,87 +2,53 @@ package generate import ( "crypto/rand" - "crypto/sha1" // nolint - "crypto/sha256" "encoding/base64" - "hash" - "unicode" - - "github.com/mongodb/mongodb-kubernetes-operator/pkg/authentication/scramcredentials" + "math/big" ) +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123") + // final key must be between 6 and at most 1024 characters func KeyFileContents() (string, error) { return generateRandomString(500) } -// RandomValidDNS1123Label generates a random fixed-length string with characters in a certain range. -func RandomValidDNS1123Label(n int) (string, error) { - str, err := RandomFixedLengthStringOfSize(n) - if err != nil { - return "", err - } - - runes := []rune(str) - - // Make sure that any letters are lowercase and that if any non-alphanumeric characters appear they are set to '0'. - for i, r := range runes { - if unicode.IsLetter(r) { - runes[i] = unicode.ToLower(r) - } else if !unicode.IsNumber(r) { - runes[i] = rune('0') - } - } - - return string(runes), nil -} - func RandomFixedLengthStringOfSize(n int) (string, error) { - b, err := generateRandomBytes(n) + b, err := GenerateRandomBytes(n) return base64.URLEncoding.EncodeToString(b)[:n], err } -// Salts generates 2 different salts. The first is for the sha1 algorithm -// the second is for sha256 -func Salts() ([]byte, []byte, error) { - sha1Salt, err := salt(sha1.New) +func GenerateRandomBytes(size int) ([]byte, error) { + b := make([]byte, size) + _, err := rand.Read(b) if err != nil { - return nil, nil, err + return nil, err } - sha256Salt, err := salt(sha256.New) - if err != nil { - return nil, nil, err - } - return sha1Salt, sha256Salt, nil + return b, nil } -// salt will create a salt which can be used to compute Scram Sha credentials based on the given hashConstructor. -// sha1.New should be used for MONGODB-CR/SCRAM-SHA-1 and sha256.New should be used for SCRAM-SHA-256 -func salt(hashConstructor func() hash.Hash) ([]byte, error) { - saltSize := hashConstructor().Size() - scramcredentials.RFC5802MandatedSaltSize - salt, err := RandomFixedLengthStringOfSize(20) +func generateRandomString(numBytes int) (string, error) { + b, err := GenerateRandomBytes(numBytes) + return base64.StdEncoding.EncodeToString(b), err +} +func randSeq(n int) string { + maxRand := int64(len(letters)) + randomRune, err := rand.Int(rand.Reader, big.NewInt(maxRand)) if err != nil { - return nil, err + panic(err) } - shaBytes32 := sha256.Sum256([]byte(salt)) - // the algorithms expect a salt of a specific size. - return shaBytes32[:saltSize], nil -} + randomRuneAsInt := int(randomRune.Int64()) -func generateRandomBytes(size int) ([]byte, error) { - b := make([]byte, size) - _, err := rand.Read(b) - if err != nil { - return nil, err + b := make([]rune, n) + for i := range b { + b[i] = letters[randomRuneAsInt] } - - return b, nil + return string(b) } -func generateRandomString(numBytes int) (string, error) { - b, err := generateRandomBytes(numBytes) - return base64.StdEncoding.EncodeToString(b), err +func GenerateRandomPassword() string { + return randSeq(10) } diff --git a/pkg/util/identifiable/identifiable.go b/pkg/util/identifiable/identifiable.go new file mode 100644 index 000000000..f86691f01 --- /dev/null +++ b/pkg/util/identifiable/identifiable.go @@ -0,0 +1,90 @@ +package identifiable + +import ( + "reflect" +) + +// Identifiable is a simple interface wrapping any object which has some key field which can be used for later +// aggregation operations (grouping, intersection, difference etc) +type Identifiable interface { + Identifier() interface{} +} + +// SetDifference returns all 'Identifiable' elements that are in left slice and not in the right one +func SetDifference(left, right []Identifiable) []Identifiable { + result := make([]Identifiable, 0) + for _, l := range left { + found := false + for _, r := range right { + if r.Identifier() == l.Identifier() { + found = true + break + } + } + if !found { + result = append(result, l) + } + } + return result +} + +// SetIntersection returns all 'Identifiable' elements from 'left' and 'right' slice that intersect by 'Identifier()' +// value. Each intersection is represented as a tuple of two elements - matching elements from 'left' and 'right' +func SetIntersection(left, right []Identifiable) [][]Identifiable { + result := make([][]Identifiable, 0) + for _, l := range left { + for _, r := range right { + if r.Identifier() == l.Identifier() { + result = append(result, []Identifiable{l, r}) + } + } + } + return result +} + +// SetDifferenceGeneric is a convenience function solving lack of covariance in Go: it allows to pass the arrays declared +// as some types implementing 'Identifiable' and find the difference between them +// Important: the arrays past must declare types implementing 'Identifiable'! +func SetDifferenceGeneric(left, right interface{}) []Identifiable { + leftIdentifiers := toIdentifiableSlice(left) + rightIdentifiers := toIdentifiableSlice(right) + + return SetDifference(leftIdentifiers, rightIdentifiers) +} + +// SetIntersectionGeneric is a convenience function solving lack of covariance in Go: it allows to pass the arrays declared +// as some types implementing 'Identifiable' and find the intersection between them +// Important: the arrays past must declare types implementing 'Identifiable'! +func SetIntersectionGeneric(left, right interface{}) [][]Identifiable { + leftIdentifiers := toIdentifiableSlice(left) + rightIdentifiers := toIdentifiableSlice(right) + + // check if there is a difference in the config with same ID + found := false + for _, l := range leftIdentifiers { + for _, r := range rightIdentifiers { + if l.Identifier() == r.Identifier() { + if l != r { + found = true + break + } + } + } + } + if !found { + return nil + } + + return SetIntersection(leftIdentifiers, rightIdentifiers) +} + +// toIdentifiableSlice uses reflection to cast the array +func toIdentifiableSlice(data interface{}) []Identifiable { + value := reflect.ValueOf(data) + + result := make([]Identifiable, value.Len()) + for i := 0; i < value.Len(); i++ { + result[i] = value.Index(i).Interface().(Identifiable) + } + return result +} diff --git a/pkg/util/int/int_util.go b/pkg/util/int/int_util.go new file mode 100644 index 000000000..a6338ff85 --- /dev/null +++ b/pkg/util/int/int_util.go @@ -0,0 +1,17 @@ +package int + +// Min returns the minimum of 2 integer arguments. +func Min(a, b int) int { + if a < b { + return a + } + + return b +} + +func Max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/pkg/util/int/int_util_test.go b/pkg/util/int/int_util_test.go new file mode 100644 index 000000000..c5ed17c25 --- /dev/null +++ b/pkg/util/int/int_util_test.go @@ -0,0 +1,20 @@ +package int + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIntMin(t *testing.T) { + var min int + + min = Min(1, 2) + assert.Equal(t, 1, min) + + min = Min(-1, -2) + assert.Equal(t, -2, min) + + min = Min(-2, 10) + assert.Equal(t, -2, min) +} diff --git a/pkg/util/maputil/mapmerge.go b/pkg/util/maputil/mapmerge.go new file mode 100644 index 000000000..0ee297f6e --- /dev/null +++ b/pkg/util/maputil/mapmerge.go @@ -0,0 +1,63 @@ +package maputil + +import ( + "github.com/spf13/cast" +) + +// MergeMaps is a simplified function to merge one generic map into another recursively. It doesn't use reflection. +// It has some known limitations triggered by use case mainly (as it is used for mongodb options which have quite simple +// structure): +// - slices are not copied recursively +// - pointers are overridden +// +// Dev note: it's used instead of 'mergo' as the latter one is very restrictive in merged types +// (float32 vs float64, string vs string type etc) so works poorly with merging in-memory maps to the unmarshalled ones +// Also it's possible to add visitors functionality for flexible merging if necessary +func MergeMaps(dst, src map[string]interface{}) { + if dst == nil { + return + } + if src == nil { + return + } + for key, srcValue := range src { + switch t := srcValue.(type) { + case map[string]interface{}: + { + if _, ok := dst[key]; !ok { + dst[key] = map[string]interface{}{} + } + // this will fall if the destination value is not map - this is fine + dstMap := dst[key].(map[string]interface{}) + MergeMaps(dstMap, t) + } + default: + { + dst[key] = castValue(dst[key], t) + } + } + } +} + +// Note, that currently slices will not be copied - this is OK for current use cases (mongodb options almost don't +// use arrays). All strings will be copied. +func castValue(dst interface{}, src interface{}) interface{} { + switch dst.(type) { + case int: + return cast.ToInt(src) + case int8: + return cast.ToInt8(src) + case int16: + return cast.ToInt16(src) + case int32: + return cast.ToInt32(src) + case int64: + return cast.ToInt64(src) + case float32: + return cast.ToFloat32(src) + case float64: + return cast.ToFloat64(src) + default: + return src + } +} diff --git a/pkg/util/maputil/mapmerge_test.go b/pkg/util/maputil/mapmerge_test.go new file mode 100644 index 000000000..95273a663 --- /dev/null +++ b/pkg/util/maputil/mapmerge_test.go @@ -0,0 +1,97 @@ +package maputil + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +type SomeType string + +const ( + TypeMongos SomeType = "mongos" +) + +func TestMergeMaps(t *testing.T) { + t.Run("Merge to empty map", func(t *testing.T) { + dst := map[string]interface{}{} + src := mapForTest() + MergeMaps(dst, src) + + assert.Equal(t, dst, src) + + // mutation of the initial map doesn't change the destination + ReadMapValueAsMap(src, "nestedMap")["key4"] = 80 + assert.Equal(t, int32(30), ReadMapValueAsInterface(dst, "nestedMap", "key4")) + }) + t.Run("Merge overrides only common fields", func(t *testing.T) { + dst := map[string]interface{}{ + "key1": "old value", // must be overridden + "key2": map[string]interface{}{"rubbish": "yes"}, // completely different type - will be overridden + "oldkey": "must retain!", + "nestedMap": map[string]interface{}{ + "key4": 100, // the destination type is int - will be cast from int32 + "nestedNestedMap": map[string]interface{}{ + "key7": float32(100.55), // the destination type is float32 - will be cast + "key8": "mongod", // will be overridden by TypeMongos + "key9": []string{"old"}, // must be overridden + "oldkey2": []int{1}, + "anotherNestedMap": map[string]interface{}{}, + }, + }, + } + src := mapForTest() + MergeMaps(dst, src) + + expected := mapForTest() + expected["oldkey"] = "must retain!" + ReadMapValueAsMap(expected, "nestedMap")["key4"] = 30 + ReadMapValueAsMap(expected, "nestedMap", "nestedNestedMap")["key7"] = float32(40.56) + ReadMapValueAsMap(expected, "nestedMap", "nestedNestedMap")["oldkey2"] = []int{1} + ReadMapValueAsMap(expected, "nestedMap", "nestedNestedMap")["anotherNestedMap"] = map[string]interface{}{} + + assert.Equal(t, expected, dst) + + // mutation of the initial map doesn't change the destination + src["nestedMap"].(map[string]interface{})["newkey"] = 80 + assert.Empty(t, ReadMapValueAsInterface(dst, "nestedMap", "newkey")) + }) + t.Run("Fails if destination is not map", func(t *testing.T) { + dst := map[string]interface{}{"nestedMap": "foo"} + src := mapForTest() + assert.Panics(t, func() { MergeMaps(dst, src) }) + }) + t.Run("Pointers are not copied", func(t *testing.T) { + dst := map[string]interface{}{} + src := mapForTest() + MergeMaps(dst, src) + + pointer := ReadMapValueAsInterface(src, "nestedMap", "nestedNestedMap", "key11").(*int32) + *pointer = 20 + + // destination map has changed as well as we don't copy pointers, just reassign them + assert.Equal(t, util.Int32Ref(20), ReadMapValueAsInterface(dst, "nestedMap", "nestedNestedMap", "key11")) + }) +} + +func mapForTest() map[string]interface{} { + return map[string]interface{}{ + "key1": "value1", + "key2": int8(10), + "key3": int16(20), + "nestedMap": map[string]interface{}{ + "key4": int32(30), + "key5": int64(40), + "nestedNestedMap": map[string]interface{}{ + "key6": float32(40.56), + "key7": float64(40.56), + "key8": TypeMongos, + "key9": []string{"one", "two"}, + "key10": true, + "key11": util.Int32Ref(10), + }, + }, + } +} diff --git a/pkg/util/maputil/maputil.go b/pkg/util/maputil/maputil.go new file mode 100644 index 000000000..6cb7b771f --- /dev/null +++ b/pkg/util/maputil/maputil.go @@ -0,0 +1,156 @@ +package maputil + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/spf13/cast" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// ReadMapValueAsInterface traverses the nested maps inside the 'm' map following the 'keys' path and returns the last element +// as an 'interface{}' +func ReadMapValueAsInterface(m map[string]interface{}, keys ...string) interface{} { + currentMap := m + for i, k := range keys { + if _, ok := currentMap[k]; !ok { + return nil + } + if i == len(keys)-1 { + return currentMap[k] + } + currentMap = currentMap[k].(map[string]interface{}) + } + return nil +} + +// ReadMapValueAsString traverses the nested maps inside the 'm' map following the 'keys' path and returns the last element +// as a 'string' +func ReadMapValueAsString(m map[string]interface{}, keys ...string) string { + res := ReadMapValueAsInterface(m, keys...) + + if res == nil { + return "" + } + return res.(string) +} + +func ReadMapValueAsInt(m map[string]interface{}, keys ...string) int { + res := ReadMapValueAsInterface(m, keys...) + if res == nil { + return 0 + } + return cast.ToInt(res) +} + +// ReadMapValueAsMap traverses the nested maps inside the 'm' map following the 'keys' path and returns the last element +// as a 'map[string]interface{}' +func ReadMapValueAsMap(m map[string]interface{}, keys ...string) map[string]interface{} { + res := ReadMapValueAsInterface(m, keys...) + + if res == nil { + return nil + } + return res.(map[string]interface{}) +} + +// ToFlatList returns all elements as a sorted list of string values. +// It performs a recursive traversal of maps and dumps the current config to the final list of configs +func ToFlatList(m map[string]interface{}) []string { + result := traverse(m, []string{}) + sort.Strings(result) + return result +} + +// SetMapValue traverses the nested maps inside the 'm' map following the 'keys' path and sets the value 'value' to the +// final key. The key -> nested map entry will be created if doesn't exist +func SetMapValue(m map[string]interface{}, value interface{}, keys ...string) { + current := m + for _, k := range keys[0 : len(keys)-1] { + if _, ok := current[k]; !ok { + current[k] = map[string]interface{}{} + } + current = current[k].(map[string]interface{}) + } + last := keys[len(keys)-1] + current[last] = value +} + +func DeleteMapValue(m map[string]interface{}, keys ...string) { + current := m + for _, k := range keys[0 : len(keys)-1] { + if _, ok := current[k]; !ok { + current[k] = map[string]interface{}{} + } + current = current[k].(map[string]interface{}) + } + delete(current, keys[len(keys)-1]) +} + +func traverse(currentValue interface{}, currentPath []string) []string { + switch v := currentValue.(type) { + case map[string]interface{}: + { + var allPaths []string + for key, value := range v { + allPaths = append(allPaths, traverse(value, append(currentPath, key))...) + } + return allPaths + } + default: + { + // We found the "terminal" node in the map - need to dump the current path + path := strings.Join(currentPath, ".") + return []string{path} + } + } +} + +// RemoveFieldsBasedOnDesiredAndPrevious returns a "currentMap" that has had fields removed based on what was in the previousMap +// and what is in the desiredMap. Any values that were there previously, but are no longer desired, will be removed and the +// resulting map will not contain them. +func RemoveFieldsBasedOnDesiredAndPrevious(currentMap, desiredMap, previousMap map[string]interface{}) map[string]interface{} { + if desiredMap == nil { + desiredMap = map[string]interface{}{} + } + + if previousMap == nil { + previousMap = map[string]interface{}{} + } + + desiredFlatList := ToFlatList(desiredMap) + previousFlatList := ToFlatList(previousMap) + + var itemsToRemove []string + for _, item := range previousFlatList { + // if an item was set previously, but is not set now + // it means we want to remove this item. + if !stringutil.Contains(desiredFlatList, item) { + itemsToRemove = append(itemsToRemove, item) + } + } + + for _, item := range itemsToRemove { + DeleteMapValue(currentMap, strings.Split(item, ".")...) + } + + return currentMap +} + +// StructToMap is a function to convert struct to map using JSON tags +func StructToMap(v interface{}) (map[string]interface{}, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + + var result map[string]interface{} + err = json.Unmarshal(data, &result) + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/pkg/util/maputil/maputil_test.go b/pkg/util/maputil/maputil_test.go new file mode 100644 index 000000000..4e0013269 --- /dev/null +++ b/pkg/util/maputil/maputil_test.go @@ -0,0 +1,80 @@ +package maputil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSetMapValue(t *testing.T) { + t.Run("Set to empty map", func(t *testing.T) { + dest := map[string]interface{}{} + SetMapValue(dest, 30, "one", "two", "three") + expectedMap := map[string]interface{}{ + "one": map[string]interface{}{ + "two": map[string]interface{}{ + "three": 30, + }, + }, + } + assert.Equal(t, expectedMap, dest) + }) + t.Run("Set to non-empty map", func(t *testing.T) { + dest := map[string]interface{}{ + "one": map[string]interface{}{ + "ten": "bar", + "two": map[string]interface{}{ + "three": 100, + "eleven": true, + }, + }, + } + SetMapValue(dest, 30, "one", "two", "three") + expectedMap := map[string]interface{}{ + "one": map[string]interface{}{ + "ten": "bar", + "two": map[string]interface{}{ + "three": 30, // this was changed + "eleven": true, + }, + }, + } + assert.Equal(t, expectedMap, dest) + }) +} + +func TestRemoveFieldsBasedOnDesiredAndPrevious(t *testing.T) { + p := map[string]interface{}{ + "one": "oneValue", + "two": map[string]interface{}{ + "three": "threeValue", + "four": "fourValue", + }, + } + + // we are removing the "two.three" entry in this case. + spec := map[string]interface{}{ + "one": "oneValue", + "two": map[string]interface{}{ + "four": "fourValue", + }, + } + + prev := map[string]interface{}{ + "one": "oneValue", + "two": map[string]interface{}{ + "three": "threeValue", + "four": "fourValue", + }, + } + + expected := map[string]interface{}{ + "one": "oneValue", + "two": map[string]interface{}{ + "four": "fourValue", + }, + } + + actual := RemoveFieldsBasedOnDesiredAndPrevious(p, spec, prev) + assert.Equal(t, expected, actual, "three was set previously, and so should have been removed.") +} diff --git a/pkg/util/mergo_utils.go b/pkg/util/mergo_utils.go new file mode 100644 index 000000000..d2fc900d8 --- /dev/null +++ b/pkg/util/mergo_utils.go @@ -0,0 +1,117 @@ +package util + +import ( + "encoding/json" + "reflect" + + "github.com/imdario/mergo" + "github.com/spf13/cast" +) + +// MergoDelete is a sentinel value that indicates a field is to be removed during the merging process +const MergoDelete = "MERGO_DELETE" + +// AutomationConfigTransformer when we want to delete the last element of a list, if we use the +// default behaviour we will still be left with the final element from the original map. Using this +// Transformer allows us to override that behaviour and perform the merging as we expect. +type AutomationConfigTransformer struct{} + +func isStringMap(elem interface{}) bool { + return reflect.TypeOf(elem) == reflect.TypeOf(make(map[string]interface{})) +} + +// withoutElementAtIndex returns the given slice without the element at the specified index +func withoutElementAtIndex(slice []interface{}, index int) []interface{} { + return append(slice[:index], slice[index+1:]...) // slice[i+1:] returns an empty slice if i >= len(slice) +} + +// mergeBoth is called when both maps have a common field +func mergeBoth(structAsMap map[string]interface{}, unmodifiedOriginalMap map[string]interface{}, key string, val interface{}) { + switch val.(type) { + case map[string]interface{}: + // we already know about the key, and it's a nested map so we can continue + merge(cast.ToStringMap(structAsMap[key]), cast.ToStringMap(unmodifiedOriginalMap[key])) + case []interface{}: + i, j := 0, 0 + for _, element := range cast.ToSlice(val) { + elementsFromStruct := cast.ToSlice(structAsMap[key]) + + if i >= len(elementsFromStruct) { + break + } + + // in the case of a nested map, we can continue the merging process + if isStringMap(element) { + // by marking an element as nil, we indicate that we want to delete this element + if cast.ToSlice(structAsMap[key])[i] == nil { + slice := cast.ToSlice(structAsMap[key]) + structAsMap[key] = withoutElementAtIndex(slice, i) + i-- // if we removed the element at a given position, we want to examine the same index again as the contents have shifted + } else { + merge(cast.ToStringMap(cast.ToSlice(structAsMap[key])[i]), cast.ToStringMap(cast.ToSlice(unmodifiedOriginalMap[key])[j])) + } + } + // we need to maintain 2 counters in order to prevent merging a map from "structAsMap" with a value from "unmodifiedOriginalMap" + // that doesn't correspond to the same logical value. + i++ + j++ + } + // for any other type, the value has been set by the operator, so we don't want to override + // a value from the existing Automation Config value in that case. + } +} + +// merge takes a map dst (serialized from a struct) and a map src (the map from an unmodified deployment) +// and merges them together based on a set of rules +func merge(structAsMap, unmodifiedOriginalMap map[string]interface{}) { + for key, val := range unmodifiedOriginalMap { + if _, ok := structAsMap[key]; !ok { + switch val.(type) { + case []interface{}: + structAsMap[key] = make([]interface{}, 0) + default: + // if we don't know about this value, then we can just accept the value coming from the Automation Config + structAsMap[key] = val + } + } else { // the value exists already in the map we have, we need to perform merge + mergeBoth(structAsMap, unmodifiedOriginalMap, key, val) + } + } + + // Delete any fields marked with "util.MergoDelete" + for key, val := range structAsMap { + // if we're explicitly sending a value of nil, it means we want to delete the corresponding entry. + // We don't want to ever send nil values. + if val == MergoDelete || val == nil { + delete(structAsMap, key) + } + } +} + +func (t AutomationConfigTransformer) Transformer(reflect.Type) func(dst, src reflect.Value) error { + return func(dst, src reflect.Value) error { + dstMap := cast.ToStringMap(dst.Interface()) + srcMap := cast.ToStringMap(src.Interface()) + merge(dstMap, srcMap) + return nil + } +} + +// MergeWith takes a structToMerge, a source map src, and returns the result of the merging, and an error +func MergeWith(structToMerge interface{}, src map[string]interface{}, transformers mergo.Transformers) (map[string]interface{}, error) { + bytes, err := json.Marshal(structToMerge) + if err != nil { + return nil, err + } + dst := make(map[string]interface{}) + err = json.Unmarshal(bytes, &dst) + if err != nil { + return nil, err + } + + if err := mergo.Merge(&dst, src, mergo.WithTransformers(transformers)); err != nil { + return nil, err + } + + return dst, nil +} diff --git a/pkg/util/stringutil/stringutil.go b/pkg/util/stringutil/stringutil.go new file mode 100644 index 000000000..b5375c285 --- /dev/null +++ b/pkg/util/stringutil/stringutil.go @@ -0,0 +1,93 @@ +package stringutil + +import "strings" + +// Ref is a convenience function which returns +// a reference to the provided string +func Ref(s string) *string { + return &s +} + +// Contains returns true if there is at least one string in `slice` +// that is equal to `s`. +func Contains(slice []string, s string) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// CheckCertificateAddresses determines if the provided FQDN can match any of the addresses or +// SubjectAltNames (SAN) in an array of FQDNs/wildcards/shortnames. +// Both the availableAddressNames and the testAddressName can contain wildcards, e.g. *.cluster-1.example.com +// Once a wildcard is found on any tested argument, only a domain-level comparison is checked. +func CheckCertificateAddresses(availableAddressNames []string, testAddressName string) bool { + checkedTestAddressName := CheckWithLevelDomain(testAddressName) + star := "*" + for _, availableAddress := range availableAddressNames { + // Determine if the certificate name is a wildcard, FQDN, unqualified domain name, or shortname + // Strip the first character from the wildcard and hostname to determine if they match + // (wildcards only work for one level of domain) + if availableAddress[0:1] == star { + checkAddress := CheckWithLevelDomain(availableAddress) + if checkAddress == checkedTestAddressName { + return true + } + } + if availableAddress == testAddressName { + return true + } + // This is the multi-cluster with an external domain case. + // We do not want to deal if this is per-member cert or a wildcard, that's why we will only + // compare the domains. + if testAddressName[0:1] == star { + domainOnlyTestAddress := CheckWithLevelDomain(testAddressName) + domainOnlyAvailableAddress := CheckWithLevelDomain(availableAddress) + + if domainOnlyAvailableAddress == domainOnlyTestAddress { + return true + } + } + } + return false +} + +// CheckWithLevelDomain determines if the address is a shortname/top level domain +// or FQDN/Unqualified Domain Name +func CheckWithLevelDomain(address string) string { + addressExploded := strings.Split(address, ".") + if len(addressExploded) < 2 { + return addressExploded[0] + } + return strings.Join(addressExploded[1:], ".") +} + +func ContainsAny(slice []string, ss ...string) bool { + for _, s := range ss { + if Contains(slice, s) { + return true + } + } + + return false +} + +func Remove(slice []string, s string) (result []string) { + for _, item := range slice { + if item == s { + continue + } + result = append(result, item) + } + return +} + +// UpperCaseFirstChar ensures the message first char is uppercased. +func UpperCaseFirstChar(msg string) string { + if msg == "" { + return "" + } + return strings.ToUpper(msg[:1]) + msg[1:] +} diff --git a/pkg/util/stringutil/stringutil_test.go b/pkg/util/stringutil/stringutil_test.go new file mode 100644 index 000000000..1ce59a5d0 --- /dev/null +++ b/pkg/util/stringutil/stringutil_test.go @@ -0,0 +1,28 @@ +package stringutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContainsAny(t *testing.T) { + assert.True(t, ContainsAny([]string{"one", "two"}, "one")) + assert.True(t, ContainsAny([]string{"one", "two"}, "two")) + assert.True(t, ContainsAny([]string{"one", "two"}, "one", "two")) + assert.True(t, ContainsAny([]string{"one", "two"}, "one", "two", "three")) + + assert.False(t, ContainsAny([]string{"one", "two"}, "three")) + assert.False(t, ContainsAny([]string{"one", "two"})) +} + +func TestCheckCertificateDomains(t *testing.T) { + assert.True(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com"}, "abc.cluster.local")) + assert.True(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com"}, "*.cluster.local")) + assert.True(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com"}, "abd.efg.com")) + assert.True(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com", "abcdefg"}, "abcdefg")) + + assert.False(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com"}, "abc.efg.com")) + assert.False(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com", "abcdef"}, "abdcdefg")) + assert.False(t, CheckCertificateAddresses([]string{"abd.efg.com", "*.cluster.local", "*.dev.local", "abc.mongodb.com", "abcdef"}, "*.somethingthatdoesntfit")) +} diff --git a/pkg/util/timeutil/timeutil.go b/pkg/util/timeutil/timeutil.go new file mode 100644 index 000000000..5b4ceaf2a --- /dev/null +++ b/pkg/util/timeutil/timeutil.go @@ -0,0 +1,8 @@ +package timeutil + +import "time" + +// Now returns the current time formatted with RFC3339 +func Now() string { + return time.Now().Format(time.RFC3339) +} diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..4fea1a0ae --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,186 @@ +package util + +import ( + "bytes" + "crypto/md5" //nolint //Part of the algorithm + "encoding/gob" + "encoding/hex" + "fmt" + "regexp" + "strings" + "time" + + "github.com/blang/semver" + "go.uber.org/zap" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/stringutil" +) + +// ************** This is a file containing any general "algorithmic" or "util" functions used by other packages + +// FindLeftDifference finds the difference between arrays of string - the elements that are present in "left" but absent +// in "right" array +func FindLeftDifference(left, right []string) []string { + ans := make([]string, 0) + for _, v := range left { + if !stringutil.Contains(right, v) { + ans = append(ans, v) + } + } + return ans +} + +// Int32Ref is required to return a *int32, which can't be declared as a literal. +func Int32Ref(i int32) *int32 { + return &i +} + +// Int64Ref is required to return a *int64, which can't be declared as a literal. +func Int64Ref(i int64) *int64 { + return &i +} + +// Float64Ref is required to return a *float64, which can't be declared as a literal. +func Float64Ref(i float64) *float64 { + return &i +} + +// BooleanRef is required to return a *bool, which can't be declared as a literal. +func BooleanRef(b bool) *bool { + return &b +} + +func StripEnt(version string) string { + return strings.Trim(version, "-ent") +} + +// DoAndRetry performs the task 'f' until it returns true or 'count' retrials are executed. Sleeps for 'interval' seconds +// between retries. String return parameter contains the fail message that is printed in case of failure. +func DoAndRetry(f func() (string, bool), log *zap.SugaredLogger, count, interval int) (bool, string) { + var ok bool + var msg string + for i := 0; i < count; i++ { + msg, ok = f() + if ok { + return true, msg + } + if msg != "" { + msg += "." + } + log.Debugf("%s Retrying %d/%d (waiting for %d more seconds)", msg, i+1, count, interval) + time.Sleep(time.Duration(interval) * time.Second) + } + return false, msg +} + +// MapDeepCopy is a quick implementation of deep copy mechanism for any Go structures, it uses Go serialization and +// deserialization mechanisms so will always be slower than any manual copy +// https://rosettacode.org/wiki/Deepcopy#Go +// TODO move to maputil +func MapDeepCopy(m map[string]interface{}) (map[string]interface{}, error) { + gob.Register(map[string]interface{}{}) + + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + dec := gob.NewDecoder(&buf) + err := enc.Encode(m) + if err != nil { + return nil, err + } + var copy map[string]interface{} + err = dec.Decode(©) + if err != nil { + return nil, err + } + return copy, nil +} + +func ReadOrCreateMap(m map[string]interface{}, key string) map[string]interface{} { + if _, ok := m[key]; !ok { + m[key] = make(map[string]interface{}) + } + return m[key].(map[string]interface{}) +} + +func CompareVersions(version1, version2 string) (int, error) { + v1, err := semver.Make(version1) + if err != nil { + return 0, err + } + v2, err := semver.Make(version2) + if err != nil { + return 0, err + } + return v1.Compare(v2), nil +} + +func MajorMinorVersion(version string) (string, semver.Version, error) { + v1, err := semver.Make(version) + if err != nil { + return "", v1, nil + } + return fmt.Sprintf("%d.%d", v1.Major, v1.Minor), v1, nil +} + +// ************ Different functions to work with environment variables ************** + +func MaxInt(x, y int) int { + if x > y { + return x + } + return y +} + +// ************ Different string/array functions ************** +// +// Helper functions to check and remove string from a slice of strings. +// + +// MD5Hex computes the MDB checksum of the given string as per https://golang.org/pkg/crypto/md5/ +func MD5Hex(s string) string { + h := md5.New() //nolint //This is part of the HTTP Digest Authentication mechanism. + h.Write([]byte(s)) + return hex.EncodeToString(h.Sum(nil)) +} + +// RedactMongoURI will strip the password out of the MongoURI and replace it with the text "" +func RedactMongoURI(uri string) string { + if !strings.Contains(uri, "@") { + return uri + } + re := regexp.MustCompile("(mongodb://.*:)(.*)(@.*:.*)") + return re.ReplaceAllString(uri, "$1$3") +} + +func Redact(toRedact interface{}) string { + if toRedact == nil { + return "nil" + } + return "" +} + +// Transform converts a slice of objects to a new slice containing objects returned from f. +// It is useful for simple slice transformations that otherwise require declaring a new slice var and for loop. +// +// Example: +// +// processHostnames := util.Transform(ac.Processes, func(obj automationconfig.Process) string { +// return obj.HostName +// }) +func Transform[T any, U any](objs []T, f func(obj T) U) []U { + result := make([]U, len(objs)) + for i := 0; i < len(objs); i++ { + result[i] = f(objs[i]) + } + return result +} + +// TransformToMap converts a slice of objects to a map with key values returned from f. +func TransformToMap[T any, K comparable, V any](objs []T, f func(obj T, idx int) (K, V)) map[K]V { + result := make(map[K]V, len(objs)) + for i := 0; i < len(objs); i++ { + k, v := f(objs[i], i) + result[k] = v + } + return result +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go new file mode 100644 index 000000000..d75dd4438 --- /dev/null +++ b/pkg/util/util_test.go @@ -0,0 +1,202 @@ +package util + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/identifiable" +) + +func TestCompareVersions(t *testing.T) { + i, e := CompareVersions("4.0.5", "4.0.4") + assert.NoError(t, e) + assert.Equal(t, 1, i) + + i, e = CompareVersions("4.0.0", "4.0.0") + assert.NoError(t, e) + assert.Equal(t, 0, i) + + i, e = CompareVersions("3.6.15", "4.1.0") + assert.NoError(t, e) + assert.Equal(t, -1, i) + + i, e = CompareVersions("3.6.2", "3.6.12") + assert.NoError(t, e) + assert.Equal(t, -1, i) + + i, e = CompareVersions("4.0.2-ent", "4.0.1") + assert.NoError(t, e) + assert.Equal(t, 1, i) +} + +func TestMajorMinorVersion(t *testing.T) { + s, _, e := MajorMinorVersion("3.6.12") + assert.NoError(t, e) + assert.Equal(t, "3.6", s) + + s, _, e = MajorMinorVersion("4.0.0") + assert.NoError(t, e) + assert.Equal(t, "4.0", s) + + s, _, e = MajorMinorVersion("4.2.12-ent") + assert.NoError(t, e) + assert.Equal(t, "4.2", s) +} + +func TestRedactURI(t *testing.T) { + uri := "mongo.mongoUri=mongodb://mongodb-ops-manager:my-scram-password@om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017/?connectTimeoutMS=20000&serverSelectionTimeoutMS=20000&authSource=admin&authMechanism=SCRAM-SHA-1" + expected := "mongo.mongoUri=mongodb://mongodb-ops-manager:@om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017/?connectTimeoutMS=20000&serverSelectionTimeoutMS=20000&authSource=admin&authMechanism=SCRAM-SHA-1" + assert.Equal(t, expected, RedactMongoURI(uri)) + + uri = "mongo.mongoUri=mongodb://mongodb-ops-manager:mongodb-ops-manager@om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017/?connectTimeoutMS=20000&serverSelectionTimeoutMS=20000" + expected = "mongo.mongoUri=mongodb://mongodb-ops-manager:@om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017/?connectTimeoutMS=20000&serverSelectionTimeoutMS=20000" + assert.Equal(t, expected, RedactMongoURI(uri)) + + // the password with '@' in it + uri = "mongo.mongoUri=mongodb://some-user:12345AllTheCharactersWith@SymbolToo@om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017" + expected = "mongo.mongoUri=mongodb://some-user:@om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017" + assert.Equal(t, expected, RedactMongoURI(uri)) + + // no authentication data + uri = "mongo.mongoUri=mongodb://om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017" + expected = "mongo.mongoUri=mongodb://om-scram-db-0.om-scram-db-svc.mongodb.svc.cluster.local:27017" + assert.Equal(t, expected, RedactMongoURI(uri)) +} + +type someId struct { + // name is a "key" field used for merging + name string + // some other property. Indicates which exactly object was returned by an aggregation operation + property string +} + +func newSome(name, property string) someId { + return someId{ + name: name, + property: property, + } +} + +func (s someId) Identifier() interface{} { + return s.name +} + +func TestSetDifference(t *testing.T) { + oneLeft := newSome("1", "left") + twoLeft := newSome("2", "left") + twoRight := newSome("2", "right") + threeRight := newSome("3", "right") + fourRight := newSome("4", "right") + + left := []identifiable.Identifiable{oneLeft, twoLeft} + right := []identifiable.Identifiable{twoRight, threeRight} + + assert.Equal(t, []identifiable.Identifiable{oneLeft}, identifiable.SetDifference(left, right)) + assert.Equal(t, []identifiable.Identifiable{threeRight}, identifiable.SetDifference(right, left)) + + left = []identifiable.Identifiable{oneLeft, twoLeft} + right = []identifiable.Identifiable{threeRight, fourRight} + assert.Equal(t, left, identifiable.SetDifference(left, right)) + + left = []identifiable.Identifiable{} + right = []identifiable.Identifiable{threeRight, fourRight} + assert.Empty(t, identifiable.SetDifference(left, right)) + assert.Equal(t, right, identifiable.SetDifference(right, left)) + + left = nil + right = []identifiable.Identifiable{threeRight, fourRight} + assert.Empty(t, identifiable.SetDifference(left, right)) + assert.Equal(t, right, identifiable.SetDifference(right, left)) + + // check reflection magic to solve lack of covariance in go. The arrays are declared as '[]someId' instead of + // '[]Identifiable' + leftNotIdentifiable := []someId{oneLeft, twoLeft} + rightNotIdentifiable := []someId{twoRight, threeRight} + + assert.Equal(t, []identifiable.Identifiable{oneLeft}, identifiable.SetDifferenceGeneric(leftNotIdentifiable, rightNotIdentifiable)) + assert.Equal(t, []identifiable.Identifiable{threeRight}, identifiable.SetDifferenceGeneric(rightNotIdentifiable, leftNotIdentifiable)) +} + +func TestSetIntersection(t *testing.T) { + oneLeft := newSome("1", "left") + oneRight := newSome("1", "right") + twoLeft := newSome("2", "left") + twoRight := newSome("2", "right") + threeRight := newSome("3", "right") + fourRight := newSome("4", "right") + + left := []identifiable.Identifiable{oneLeft, twoLeft} + right := []identifiable.Identifiable{twoRight, threeRight} + + assert.Equal(t, [][]identifiable.Identifiable{pair(twoLeft, twoRight)}, identifiable.SetIntersection(left, right)) + assert.Equal(t, [][]identifiable.Identifiable{pair(twoRight, twoLeft)}, identifiable.SetIntersection(right, left)) + + left = []identifiable.Identifiable{oneLeft, twoLeft} + right = []identifiable.Identifiable{threeRight, fourRight} + assert.Empty(t, identifiable.SetIntersection(left, right)) + assert.Empty(t, identifiable.SetIntersection(right, left)) + + left = []identifiable.Identifiable{} + right = []identifiable.Identifiable{threeRight, fourRight} + assert.Empty(t, identifiable.SetIntersection(left, right)) + assert.Empty(t, identifiable.SetIntersection(right, left)) + + left = nil + right = []identifiable.Identifiable{threeRight, fourRight} + assert.Empty(t, identifiable.SetIntersection(left, right)) + assert.Empty(t, identifiable.SetIntersection(right, left)) + + // check reflection magic to solve lack of covariance in go. The arrays are declared as '[]someId' instead of + // '[]Identifiable' + leftNotIdentifiable := []someId{oneLeft, twoLeft} + rightNotIdentifiable := []someId{oneRight, twoRight, threeRight} + + assert.Equal(t, [][]identifiable.Identifiable{pair(oneLeft, oneRight), pair(twoLeft, twoRight)}, identifiable.SetIntersectionGeneric(leftNotIdentifiable, rightNotIdentifiable)) + assert.Equal(t, [][]identifiable.Identifiable{pair(oneRight, oneLeft), pair(twoRight, twoLeft)}, identifiable.SetIntersectionGeneric(rightNotIdentifiable, leftNotIdentifiable)) + + leftNotIdentifiable = []someId{oneLeft, twoLeft} + rightNotIdentifiable = []someId{oneLeft, twoLeft} + + assert.Len(t, identifiable.SetIntersectionGeneric(leftNotIdentifiable, rightNotIdentifiable), 0) +} + +func TestTransform(t *testing.T) { + assert.Equal(t, []string{"1", "2", "3"}, Transform([]int{1, 2, 3}, func(v int) string { + return fmt.Sprintf("%d", v) + })) + + assert.Equal(t, []string{}, Transform([]int{}, func(v int) string { + return fmt.Sprintf("%d", v) + })) + + type tmpStruct struct { + str string + } + assert.Equal(t, []string{"a", "b", "c"}, Transform([]tmpStruct{{"a"}, {"b"}, {"c"}}, func(v tmpStruct) string { + return v.str + })) +} + +func TestTransformToMap(t *testing.T) { + assert.Equal(t, map[string]string{"0": "1", "1": "2", "2": "3"}, TransformToMap([]int{1, 2, 3}, func(v int, idx int) (string, string) { + return fmt.Sprintf("%d", idx), fmt.Sprintf("%d", v) + })) + + assert.Equal(t, map[string]int{}, TransformToMap([]string{}, func(v string, idx int) (string, int) { + return "", 0 + })) + + type tmpStruct struct { + str string + int int + } + assert.Equal(t, map[string]int{"a": 0, "b": 1, "c": 2}, TransformToMap([]tmpStruct{{"a", 0}, {"b", 1}, {"c", 2}}, func(v tmpStruct, idx int) (string, int) { + return v.str, v.int + })) +} + +func pair(left, right identifiable.Identifiable) []identifiable.Identifiable { + return []identifiable.Identifiable{left, right} +} diff --git a/pkg/util/versionutil/versionutil.go b/pkg/util/versionutil/versionutil.go new file mode 100644 index 000000000..e11158965 --- /dev/null +++ b/pkg/util/versionutil/versionutil.go @@ -0,0 +1,106 @@ +package versionutil + +import ( + "regexp" + "strings" + + "github.com/blang/semver" + "golang.org/x/xerrors" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" +) + +var semverRegex *regexp.Regexp + +// StringToSemverVersion returns semver.Version for the 'version' provided as a string. +// Important: this method is a bit hacky as ignores everything after patch and must be used only when needed +// (so far only for creating the semver for OM version as this was needed to support IBM) +func StringToSemverVersion(version string) (semver.Version, error) { + v, err := semver.Make(version) + if err != nil { + // Regex adapted from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + // but removing the parts after the patch + if semverRegex == nil { + semverRegex = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)?(-|$)`) + } + result := semverRegex.FindStringSubmatch(version) + if len(result) < 4 { + return semver.Version{}, xerrors.Errorf("Ops Manager Status spec.version %s is invalid", version) + } + // Concatenate Major.Minor.Patch + v, err = semver.Make(result[1] + "." + result[2] + "." + result[3]) + } + return v, err +} + +// StaticContainersOperatorVersion gets the Operator version for the Static Containers Architecture based on +// util.OperatorVersion variable which is set during the build time. For development, it's "latest". +func StaticContainersOperatorVersion() string { + if len(util.OperatorVersion) == 0 { + return "latest" + } + return util.OperatorVersion +} + +type OpsManagerVersion struct { + VersionString string +} + +func (v OpsManagerVersion) Semver() (semver.Version, error) { + if v.IsCloudManager() { + return semver.Version{}, nil + } + + versionParts := strings.Split(v.VersionString, ".") // [4 2 4 56729 20191105T2247Z] + if len(versionParts) < 3 { + return semver.Version{}, nil + } + + sv, err := semver.Make(strings.Join(versionParts[:3], ".")) + if err != nil { + return semver.Version{}, err + } + + return sv, nil +} + +func (v OpsManagerVersion) IsCloudManager() bool { + return strings.HasPrefix(strings.ToLower(v.VersionString), "v") +} + +func (v OpsManagerVersion) IsUnknown() bool { + return v.VersionString == "" +} + +func (v OpsManagerVersion) String() string { + return v.VersionString +} + +// GetVersionFromOpsManagerApiHeader returns the major, minor and patch version from the string +// which is returned in the header of all Ops Manager responses in the form of: +// gitHash=f7bdac406b7beceb1415fd32c81fc64501b6e031; versionString=4.2.4.56729.20191105T2247Z +func GetVersionFromOpsManagerApiHeader(versionString string) string { + if versionString == "" || !strings.Contains(versionString, "versionString=") { + return "" + } + + splitString := strings.Split(versionString, "versionString=") + + if len(splitString) == 2 { + return splitString[1] + } + + return "" +} + +func IsDowngrade(oldV string, currentV string) bool { + oldVersion, err := semver.Make(oldV) + if err != nil { + return false + } + currentVersion, err := semver.Make(currentV) + if err != nil { + return false + } + return oldVersion.GT(currentVersion) +} diff --git a/pkg/util/versionutil/versionutil_test.go b/pkg/util/versionutil/versionutil_test.go new file mode 100644 index 000000000..18565269e --- /dev/null +++ b/pkg/util/versionutil/versionutil_test.go @@ -0,0 +1,64 @@ +package versionutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetVersionString(t *testing.T) { + assert.Equal(t, "4.2.4.56729.20191105T2247Z", + GetVersionFromOpsManagerApiHeader("gitHash=f7bdac406b7beceb1415fd32c81fc64501b6e031; versionString=4.2.4.56729.20191105T2247Z")) + assert.Equal(t, "4.4.41.12345.20191105T2247Z", + GetVersionFromOpsManagerApiHeader("gitHash=f7bdac406b7beceb1415fd32c81fc64501b6e031; versionString=4.4.41.12345.20191105T2247Z")) + assert.Equal(t, "4.3.0.56729.DEFXYZ", + GetVersionFromOpsManagerApiHeader("gitHash=f7bdac406b7beceb1415fd32c81fc64501b6e031; versionString=4.3.0.56729.DEFXYZ")) + assert.Equal(t, "31.24.55.202056729.ABCXYZ", + GetVersionFromOpsManagerApiHeader("gitHash=f7bdac406b7beceb1415fd32c81fc64501b6e031; versionString=31.24.55.202056729.ABCXYZ")) +} + +func TestIsInDowngrade(t *testing.T) { + tests := []struct { + name string + specVersion string + lastSpecVersion string + expected bool + }{ + { + name: "No downgrade - current version greater", + specVersion: "4.4.0", + lastSpecVersion: "4.2.0", + expected: false, + }, + { + name: "Downgrade detected - current version smaller", + specVersion: "4.0.0", + lastSpecVersion: "4.2.0", + expected: true, + }, + { + name: "Same version - no downgrade", + specVersion: "4.2.0", + lastSpecVersion: "4.2.0", + expected: false, + }, + { + name: "Invalid current version", + specVersion: "invalid", + lastSpecVersion: "4.2.0", + expected: false, + }, + { + name: "Invalid last spec version", + specVersion: "4.2.0", + lastSpecVersion: "invalid", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, IsDowngrade(tt.lastSpecVersion, tt.specVersion), tt.expected) + }) + } +} diff --git a/pkg/vault/vault.go b/pkg/vault/vault.go new file mode 100644 index 000000000..f12ad1bbd --- /dev/null +++ b/pkg/vault/vault.go @@ -0,0 +1,577 @@ +package vault + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + "github.com/hashicorp/vault/api" + "golang.org/x/xerrors" + "k8s.io/client-go/kubernetes" + + "github.com/mongodb/mongodb-kubernetes-operator/pkg/util/merge" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" + "github.com/10gen/ops-manager-kubernetes/pkg/util/maputil" +) + +const ( + VaultBackend = "VAULT_BACKEND" + K8sSecretBackend = "K8S_SECRET_BACKEND" //nolint + + DEFAULT_OPERATOR_SECRET_PATH = "mongodbenterprise/operator" + DEFAULT_OPS_MANAGER_SECRET_PATH = "mongodbenterprise/opsmanager" + DEFAULT_DATABASE_SECRET_PATH = "mongodbenterprise/database" + DEFAULT_APPDB_SECRET_PATH = "mongodbenterprise/appdb" + + DEFAULT_VAULT_ADDRESS = "vault.vault.svc.cluster.local" + DEFAULT_VAULT_PORT = "8200" + + DatabaseVaultRoleName = "mongodbenterprisedatabase" + OpsManagerVaultRoleName = "mongodbenterpriseopsmanager" + AppDBVaultRoleName = "mongodbenterpriseappdb" + + VAULT_SERVER_ADDRESS = "VAULT_SERVER_ADDRESS" + OPERATOR_SECRET_BASE_PATH = "OPERATOR_SECRET_BASE_PATH" + TLS_SECRET_REF = "TLS_SECRET_REF" //nolint + OPS_MANAGER_SECRET_BASE_PATH = "OPS_MANAGER_SECRET_BASE_PATH" //nolint + DATABASE_SECRET_BASE_PATH = "DATABASE_SECRET_BASE_PATH" //nolint + APPDB_SECRET_BASE_PATH = "APPDB_SECRET_BASE_PATH" //nolint + + DEFAULT_AGENT_INJECT_TEMPLATE = `{{- with secret "%s" -}} + {{ index .Data.data "%s" }} + {{- end }}` + PREVIOUS_HASH_INJECT_COMMAND = `sh -c 'test -s %[1]s/%[2]s && tail -n+2 %[1]s/%[2]s > %[1]s/\$(head -n1 %[1]s/%[2]s) || true'` + PREVIOUS_HASH_INJECT_TEMPLATE = `{{- with secret "%s" -}} +{{- if .Data.data.%[2]s -}} +{{ .Data.data.%[2]s }} +{{ index .Data.data (.Data.data.%[2]s) }} +{{- end }} +{{- end }}` +) + +type DatabaseSecretsToInject struct { + AgentCerts string + AgentApiKey string + InternalClusterAuth string + InternalClusterHash string + MemberClusterAuth string + MemberClusterHash string + Config VaultConfiguration + Prometheus string + PrometheusTLSCertHash string +} + +type AppDBSecretsToInject struct { + AgentApiKey string + TLSSecretName string + TLSClusterHash string + + AutomationConfigSecretName string + AutomationConfigPath string + AgentType string + Config VaultConfiguration + + PrometheusTLSCertHash string + PrometheusTLSPath string +} + +type OpsManagerSecretsToInject struct { + TLSSecretName string + TLSHash string + GenKeyPath string + AppDBConnection string + AppDBConnectionVolume string + Config VaultConfiguration +} + +func IsVaultSecretBackend() bool { + return os.Getenv("SECRET_BACKEND") == VaultBackend // nolint:forbidigo +} + +type VaultConfiguration struct { + OperatorSecretPath string + DatabaseSecretPath string + OpsManagerSecretPath string + AppDBSecretPath string + VaultAddress string + TLSSecretRef string +} + +type VaultClient struct { + client *api.Client + VaultConfig VaultConfiguration +} + +func readVaultConfig(ctx context.Context, client *kubernetes.Clientset) VaultConfiguration { + cm, err := client.CoreV1().ConfigMaps(env.ReadOrPanic(util.CurrentNamespace)).Get(ctx, "secret-configuration", v1.GetOptions{}) // nolint:forbidigo + if err != nil { + panic(xerrors.Errorf("error reading vault configmap: %w", err)) + } + + config := VaultConfiguration{ + OperatorSecretPath: cm.Data[OPERATOR_SECRET_BASE_PATH], + VaultAddress: cm.Data[VAULT_SERVER_ADDRESS], + OpsManagerSecretPath: cm.Data[OPS_MANAGER_SECRET_BASE_PATH], + DatabaseSecretPath: cm.Data[DATABASE_SECRET_BASE_PATH], + AppDBSecretPath: cm.Data[APPDB_SECRET_BASE_PATH], + } + + if tlsRef, ok := cm.Data[TLS_SECRET_REF]; ok { + config.TLSSecretRef = tlsRef + } + + return config +} + +func setTLSConfig(ctx context.Context, config *api.Config, client *kubernetes.Clientset, tlsSecretRef string) error { + if tlsSecretRef == "" { + return nil + } + var secret *corev1.Secret + var err error + secret, err = client.CoreV1().Secrets(env.ReadOrPanic(util.CurrentNamespace)).Get(ctx, tlsSecretRef, v1.GetOptions{}) // nolint:forbidigo + if err != nil { + return xerrors.Errorf("can't read tls secret %s for vault: %w", tlsSecretRef, err) + } + + // Read the secret and write ca.crt to a temporary file + caData := secret.Data["ca.crt"] + f, err := os.CreateTemp("/tmp", "VaultCAData") + if err != nil { + return xerrors.Errorf("can't create temporary file for CA data: %w", err) + } + defer func(f *os.File) { + _ = f.Close() + }(f) + + _, err = f.Write(caData) + if err != nil { + return xerrors.Errorf("can't write caData to file %s: %w", f.Name(), err) + } + if err = f.Sync(); err != nil { + return xerrors.Errorf("can't call Sync on file %s: %w", f.Name(), err) + } + + return config.ConfigureTLS( + &api.TLSConfig{ + CACert: f.Name(), + }, + ) +} + +func InitVaultClient(ctx context.Context, client *kubernetes.Clientset) (*VaultClient, error) { + vaultConfig := readVaultConfig(ctx, client) + + config := api.DefaultConfig() + config.Address = vaultConfig.VaultAddress + + if err := setTLSConfig(ctx, config, client, vaultConfig.TLSSecretRef); err != nil { + return nil, err + } + + vclient, err := api.NewClient(config) + if err != nil { + return nil, err + } + + return &VaultClient{client: vclient, VaultConfig: vaultConfig}, nil +} + +func (v *VaultClient) Login() error { + // Read the service-account token from the path where the token's Kubernetes Secret is mounted. + jwt, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + if err != nil { + return xerrors.Errorf("unable to read file containing service account token: %w", err) + } + + params := map[string]interface{}{ + "jwt": string(jwt), + "role": "mongodbenterprise", // the name of the role in Vault that was created with this app's Kubernetes service account bound to it + } + + // log in to Vault's Kubernetes auth method + resp, err := v.client.Logical().Write("auth/kubernetes/login", params) + if err != nil { + return xerrors.Errorf("unable to log in with Kubernetes auth: %w", err) + } + + if resp == nil || resp.Auth == nil || resp.Auth.ClientToken == "" { + return xerrors.Errorf("login response did not return client token") + } + + // will use the resulting Vault token for making all future calls to Vault + v.client.SetToken(resp.Auth.ClientToken) + return nil +} + +func (v *VaultClient) PutSecret(path string, data map[string]interface{}) error { + if err := v.Login(); err != nil { + return xerrors.Errorf("unable to log in: %w", err) + } + _, err := v.client.Logical().Write(path, data) + if err != nil { + return err + } + return nil +} + +func (v *VaultClient) ReadSecretVersion(path string) (int, error) { + data, err := v.ReadSecret(path) + if err != nil { + return -1, err + } + current_version, err := data.Data["current_version"].(json.Number).Int64() + if err != nil { + // this shouldn't happen but if it does the caller should log it with error that + // secret rotation won't work + return -1, err + } + return int(current_version), nil +} + +func (v *VaultClient) ReadSecret(path string) (*api.Secret, error) { + if err := v.Login(); err != nil { + return nil, xerrors.Errorf("unable to log in: %w", err) + } + secret, err := v.client.Logical().Read(path) + if err != nil { + return nil, xerrors.Errorf("can't read secret from vault: %w", err) + } + if secret == nil { + return nil, xerrors.Errorf("secret not found at %s", path) + } + return secret, nil +} + +func (v *VaultClient) ReadSecretBytes(path string) (map[string][]byte, error) { + secret, err := v.ReadSecret(path) + if err != nil { + return map[string][]byte{}, err + } + + secrets := make(map[string][]byte) + for k, v := range maputil.ReadMapValueAsMap(secret.Data, "data") { + secrets[k] = []byte(fmt.Sprintf("%v", v)) + } + return secrets, nil +} + +func (v *VaultClient) ReadSecretString(path string) (map[string]string, error) { + secretBytes, err := v.ReadSecretBytes(path) + if err != nil { + return map[string]string{}, err + } + + secretString := map[string]string{} + for k, v := range secretBytes { + secretString[k] = string(v) + } + return secretString, nil +} + +func (v VaultConfiguration) TLSAnnotations() map[string]string { + if v.TLSSecretRef == "" { + return map[string]string{} + } + return map[string]string{ + "vault.hashicorp.com/tls-secret": v.TLSSecretRef, + "vault.hashicorp.com/ca-cert": "/vault/tls/ca.crt", + } +} + +func (v *VaultClient) OperatorSecretPath() string { + if v.VaultConfig.OperatorSecretPath != "" { + return fmt.Sprintf("/secret/data/%s", v.VaultConfig.OperatorSecretPath) + } + return fmt.Sprintf("/secret/data/%s", DEFAULT_OPERATOR_SECRET_PATH) +} + +func (v *VaultClient) OperatorScretMetadataPath() string { + if v.VaultConfig.OperatorSecretPath != "" { + return fmt.Sprintf("/secret/metadata/%s", v.VaultConfig.OperatorSecretPath) + } + return fmt.Sprintf("/secret/metadata/%s", DEFAULT_OPERATOR_SECRET_PATH) +} + +func (v *VaultClient) OpsManagerSecretPath() string { + if v.VaultConfig.OperatorSecretPath != "" { + return fmt.Sprintf("/secret/data/%s", v.VaultConfig.OpsManagerSecretPath) + } + return fmt.Sprintf("/secret/data/%s", DEFAULT_OPS_MANAGER_SECRET_PATH) +} + +func (v *VaultClient) OpsManagerSecretMetadataPath() string { + if v.VaultConfig.OpsManagerSecretPath != "" { + return fmt.Sprintf("/secret/metadata/%s", v.VaultConfig.OpsManagerSecretPath) + } + return fmt.Sprintf("/secret/metadata/%s", DEFAULT_OPS_MANAGER_SECRET_PATH) +} + +func (v *VaultClient) DatabaseSecretPath() string { + if v.VaultConfig.OperatorSecretPath != "" { + return fmt.Sprintf("/secret/data/%s", v.VaultConfig.DatabaseSecretPath) + } + return fmt.Sprintf("/secret/data/%s", DEFAULT_DATABASE_SECRET_PATH) +} + +func (v *VaultClient) DatabaseSecretMetadataPath() string { + if v.VaultConfig.OperatorSecretPath != "" { + return fmt.Sprintf("/secret/metadata/%s", v.VaultConfig.DatabaseSecretPath) + } + return fmt.Sprintf("/secret/metadata/%s", DEFAULT_DATABASE_SECRET_PATH) +} + +func (v *VaultClient) AppDBSecretPath() string { + if v.VaultConfig.AppDBSecretPath != "" { + return fmt.Sprintf("/secret/data/%s", v.VaultConfig.AppDBSecretPath) + } + return fmt.Sprintf("/secret/data/%s", APPDB_SECRET_BASE_PATH) +} + +func (v *VaultClient) AppDBSecretMetadataPath() string { + if v.VaultConfig.AppDBSecretPath != "" { + return fmt.Sprintf("/secret/metadata/%s", v.VaultConfig.AppDBSecretPath) + } + return fmt.Sprintf("/secret/metadata/%s", APPDB_SECRET_BASE_PATH) +} + +func (s OpsManagerSecretsToInject) OpsManagerAnnotations(namespace string) map[string]string { + var opsManagerSecretPath string + if s.Config.OpsManagerSecretPath != "" { + opsManagerSecretPath = fmt.Sprintf("/secret/data/%s", s.Config.OpsManagerSecretPath) + } else { + opsManagerSecretPath = fmt.Sprintf("/secret/metadata/%s", DEFAULT_OPS_MANAGER_SECRET_PATH) + } + + annotations := map[string]string{ + "vault.hashicorp.com/agent-inject": "true", + "vault.hashicorp.com/role": OpsManagerVaultRoleName, + "vault.hashicorp.com/preserve-secret-case": "true", + } + + annotations = merge.StringToStringMap(annotations, s.Config.TLSAnnotations()) + + if s.TLSSecretName != "" { + omTLSPath := fmt.Sprintf("%s/%s/%s", opsManagerSecretPath, namespace, s.TLSSecretName) + annotations["vault.hashicorp.com/agent-inject-secret-om-tls-cert-pem"] = omTLSPath + annotations["vault.hashicorp.com/agent-inject-file-om-tls-cert-pem"] = s.TLSHash + annotations["vault.hashicorp.com/secret-volume-path-om-tls-cert-pem"] = util.MmsPemKeyFileDirInContainer + annotations["vault.hashicorp.com/agent-inject-template-om-tls-cert-pem"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, omTLSPath, s.TLSHash) + + annotations["vault.hashicorp.com/agent-inject-secret-previous-om-tls-cert-pem"] = omTLSPath + annotations["vault.hashicorp.com/secret-volume-path-previous-om-tls-cert-pem"] = util.MmsPemKeyFileDirInContainer + annotations["vault.hashicorp.com/agent-inject-file-previous-om-tls-cert-pem"] = util.PreviousHashSecretKey + annotations["vault.hashicorp.com/agent-inject-template-previous-om-tls-cert-pem"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_TEMPLATE, omTLSPath, util.PreviousHashSecretKey) + annotations["vault.hashicorp.com/agent-inject-command-previous-om-tls-cert-pem"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_COMMAND, util.MmsPemKeyFileDirInContainer, util.PreviousHashSecretKey) + } + + if s.GenKeyPath != "" { + genKeyPath := fmt.Sprintf("%s/%s/%s", opsManagerSecretPath, namespace, s.GenKeyPath) + annotations["vault.hashicorp.com/agent-inject-secret-gen-key"] = genKeyPath + annotations["vault.hashicorp.com/agent-inject-file-gen-key"] = "gen.key" + annotations["vault.hashicorp.com/secret-volume-path-gen-key"] = util.GenKeyPath + annotations["vault.hashicorp.com/agent-inject-template-gen-key"] = fmt.Sprintf(`{{- with secret "%s" -}} + {{ range $k, $v := .Data.data }} + {{- base64Decode $v }} + {{- end }} + {{- end }}`, genKeyPath) + } + + // add appDB connection string + appDBConnPath := fmt.Sprintf("%s/%s/%s", opsManagerSecretPath, namespace, s.AppDBConnection) + annotations["vault.hashicorp.com/agent-inject-secret-appdb-connection-string"] = appDBConnPath + annotations["vault.hashicorp.com/agent-inject-file-appdb-connection-string"] = util.AppDbConnectionStringKey + annotations["vault.hashicorp.com/secret-volume-path-appdb-connection-string"] = s.AppDBConnectionVolume + annotations["vault.hashicorp.com/agent-inject-template-appdb-connection-string"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, appDBConnPath, util.AppDbConnectionStringKey) + return annotations +} + +func (s DatabaseSecretsToInject) DatabaseAnnotations(namespace string) map[string]string { + var databaseSecretPath string + if s.Config.DatabaseSecretPath != "" { + databaseSecretPath = fmt.Sprintf("/secret/data/%s", s.Config.DatabaseSecretPath) + } else { + databaseSecretPath = fmt.Sprintf("/secret/data/%s", DEFAULT_DATABASE_SECRET_PATH) + } + + apiKeySecretPath := fmt.Sprintf("%s/%s/%s", databaseSecretPath, namespace, s.AgentApiKey) + + agentAPIKeyTemplate := fmt.Sprintf(`{{- with secret "%s" -}} + {{ .Data.data.agentApiKey }} + {{- end }}`, apiKeySecretPath) + + annotations := map[string]string{ + "vault.hashicorp.com/agent-inject": "true", + "vault.hashicorp.com/agent-inject-secret-agentApiKey": apiKeySecretPath, + "vault.hashicorp.com/role": DatabaseVaultRoleName, + "vault.hashicorp.com/secret-volume-path-agentApiKey": "/mongodb-automation/agent-api-key", + "vault.hashicorp.com/preserve-secret-case": "true", + "vault.hashicorp.com/agent-inject-template-agentApiKey": agentAPIKeyTemplate, + } + + annotations = merge.StringToStringMap(annotations, s.Config.TLSAnnotations()) + + if s.AgentCerts != "" { + agentCertsPath := fmt.Sprintf("%s/%s/%s", databaseSecretPath, namespace, s.AgentCerts) + annotations["vault.hashicorp.com/agent-inject-secret-mms-automation-agent-pem"] = agentCertsPath + annotations["vault.hashicorp.com/secret-volume-path-mms-automation-agent-pem"] = "/mongodb-automation/agent-certs" + annotations["vault.hashicorp.com/agent-inject-template-mms-automation-agent-pem"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, agentCertsPath, util.AutomationAgentPemSecretKey) + } + if s.InternalClusterAuth != "" { + internalClusterPath := fmt.Sprintf("%s/%s/%s", databaseSecretPath, namespace, s.InternalClusterAuth) + + annotations["vault.hashicorp.com/agent-inject-secret-internal-cluster"] = internalClusterPath + annotations["vault.hashicorp.com/agent-inject-file-internal-cluster"] = s.InternalClusterHash + annotations["vault.hashicorp.com/secret-volume-path-internal-cluster"] = util.InternalClusterAuthMountPath + annotations["vault.hashicorp.com/agent-inject-template-internal-cluster"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, internalClusterPath, s.InternalClusterHash) + + annotations["vault.hashicorp.com/agent-inject-secret-previous-internal-cluster"] = internalClusterPath + annotations["vault.hashicorp.com/secret-volume-path-previous-internal-cluster"] = util.InternalClusterAuthMountPath + annotations["vault.hashicorp.com/agent-inject-file-previous-internal-cluster"] = util.PreviousHashSecretKey + annotations["vault.hashicorp.com/agent-inject-template-previous-internal-cluster"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_TEMPLATE, internalClusterPath, util.PreviousHashSecretKey) + annotations["vault.hashicorp.com/agent-inject-command-previous-internal-cluster"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_COMMAND, util.InternalClusterAuthMountPath, util.PreviousHashSecretKey) + } + if s.MemberClusterAuth != "" { + memberClusterPath := fmt.Sprintf("%s/%s/%s", databaseSecretPath, namespace, s.MemberClusterAuth) + + annotations["vault.hashicorp.com/agent-inject-secret-tls-certificate"] = memberClusterPath + annotations["vault.hashicorp.com/agent-inject-file-tls-certificate"] = s.MemberClusterHash + annotations["vault.hashicorp.com/secret-volume-path-tls-certificate"] = util.TLSCertMountPath + annotations["vault.hashicorp.com/agent-inject-template-tls-certificate"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, memberClusterPath, s.MemberClusterHash) + + annotations["vault.hashicorp.com/agent-inject-secret-previous-tls-certificate"] = memberClusterPath + annotations["vault.hashicorp.com/secret-volume-path-previous-tls-certificate"] = util.TLSCertMountPath + annotations["vault.hashicorp.com/agent-inject-file-previous-tls-certificate"] = util.PreviousHashSecretKey + annotations["vault.hashicorp.com/agent-inject-template-previous-tls-certificate"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_TEMPLATE, memberClusterPath, util.PreviousHashSecretKey) + annotations["vault.hashicorp.com/agent-inject-command-previous-tls-certificate"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_COMMAND, util.TLSCertMountPath, util.PreviousHashSecretKey) + } + + if s.Prometheus != "" { + promPath := fmt.Sprintf("%s/%s/%s", databaseSecretPath, namespace, s.Prometheus) + + annotations["vault.hashicorp.com/agent-inject-secret-prom-https-cert"] = promPath + annotations["vault.hashicorp.com/agent-inject-file-prom-https-cert"] = s.PrometheusTLSCertHash + annotations["vault.hashicorp.com/secret-volume-path-prom-https-cert"] = util.SecretVolumeMountPathPrometheus + annotations["vault.hashicorp.com/agent-inject-template-prom-https-cert"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, promPath, s.PrometheusTLSCertHash) + + annotations["vault.hashicorp.com/agent-inject-secret-previous-prom-https-cert"] = promPath + annotations["vault.hashicorp.com/secret-volume-path-previous-prom-https-cert"] = util.SecretVolumeMountPathPrometheus + annotations["vault.hashicorp.com/agent-inject-file-previous-prom-https-cert"] = util.PreviousHashSecretKey + annotations["vault.hashicorp.com/agent-inject-template-previous-prom-https-cert"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_TEMPLATE, promPath, util.PreviousHashSecretKey) + annotations["vault.hashicorp.com/agent-inject-command-previous-prom-https-cert"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_COMMAND, util.SecretVolumeMountPathPrometheus, util.PreviousHashSecretKey) + } + + return annotations +} + +func (v *VaultClient) GetSecretAnnotation(path string) map[string]string { + n, err := v.ReadSecretVersion(path) + if err != nil { + return map[string]string{} + } + + ss := strings.Split(path, "/") + secretName := ss[len(ss)-1] + + return map[string]string{ + secretName: strconv.FormatInt(int64(n), 10), + } +} + +func (a AppDBSecretsToInject) AppDBAnnotations(namespace string) map[string]string { + annotations := map[string]string{ + "vault.hashicorp.com/agent-inject": "true", + "vault.hashicorp.com/role": AppDBVaultRoleName, + "vault.hashicorp.com/preserve-secret-case": "true", + } + + annotations = merge.StringToStringMap(annotations, a.Config.TLSAnnotations()) + var appdbSecretPath string + if a.Config.AppDBSecretPath != "" { + appdbSecretPath = fmt.Sprintf("/secret/data/%s", a.Config.AppDBSecretPath) + } else { + appdbSecretPath = fmt.Sprintf("/secret/data/%s", APPDB_SECRET_BASE_PATH) + } + if a.AgentApiKey != "" { + + apiKeySecretPath := fmt.Sprintf("%s/%s/%s", appdbSecretPath, namespace, a.AgentApiKey) + agentAPIKeyTemplate := fmt.Sprintf(DEFAULT_AGENT_INJECT_TEMPLATE, apiKeySecretPath, util.OmAgentApiKey) + + annotations["vault.hashicorp.com/agent-inject-secret-agentApiKey"] = apiKeySecretPath + annotations["vault.hashicorp.com/secret-volume-path-agentApiKey"] = "/mongodb-automation/agent-api-key" + annotations["vault.hashicorp.com/agent-inject-template-agentApiKey"] = agentAPIKeyTemplate + } + + if a.TLSSecretName != "" { + memberClusterPath := fmt.Sprintf("%s/%s/%s", appdbSecretPath, namespace, a.TLSSecretName) + annotations["vault.hashicorp.com/agent-inject-secret-tls-certificate"] = memberClusterPath + annotations["vault.hashicorp.com/agent-inject-file-tls-certificate"] = a.TLSClusterHash + annotations["vault.hashicorp.com/secret-volume-path-tls-certificate"] = util.SecretVolumeMountPath + "/certs" + annotations["vault.hashicorp.com/agent-inject-template-tls-certificate"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, memberClusterPath, a.TLSClusterHash) + + annotations["vault.hashicorp.com/agent-inject-secret-previous-tls-certificate"] = memberClusterPath + annotations["vault.hashicorp.com/secret-volume-path-previous-tls-certificate"] = util.SecretVolumeMountPath + "/certs" + annotations["vault.hashicorp.com/agent-inject-file-previous-tls-certificate"] = util.PreviousHashSecretKey + annotations["vault.hashicorp.com/agent-inject-template-previous-tls-certificate"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_TEMPLATE, memberClusterPath, util.PreviousHashSecretKey) + annotations["vault.hashicorp.com/agent-inject-command-previous-tls-certificate"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_COMMAND, util.SecretVolumeMountPath+"/certs", util.PreviousHashSecretKey) + + } + + if a.AutomationConfigSecretName != "" { + // There are two different type of annotations here: for the automation agent + // and for the monitoring agent. + acSecretPath := fmt.Sprintf("%s/%s/%s", appdbSecretPath, namespace, a.AutomationConfigSecretName) + annotations["vault.hashicorp.com/agent-inject-secret-"+a.AgentType] = acSecretPath + annotations["vault.hashicorp.com/agent-inject-file-"+a.AgentType] = a.AutomationConfigPath + annotations["vault.hashicorp.com/secret-volume-path-"+a.AgentType] = "/var/lib/automation/config" + annotations["vault.hashicorp.com/agent-inject-template-"+a.AgentType] = fmt.Sprintf(`{{- with secret "%s" -}} + {{ range $k, $v := .Data.data }} + {{- $v }} + {{- end }} + {{- end }}`, acSecretPath) + } + + if a.PrometheusTLSCertHash != "" && a.PrometheusTLSPath != "" { + promPath := fmt.Sprintf("%s/%s/%s", appdbSecretPath, namespace, a.PrometheusTLSPath) + annotations["vault.hashicorp.com/agent-inject-secret-prom-https-cert"] = promPath + annotations["vault.hashicorp.com/agent-inject-file-prom-https-cert"] = a.PrometheusTLSCertHash + annotations["vault.hashicorp.com/secret-volume-path-prom-https-cert"] = util.SecretVolumeMountPathPrometheus + annotations["vault.hashicorp.com/agent-inject-template-prom-https-cert"] = fmt.Sprintf( + DEFAULT_AGENT_INJECT_TEMPLATE, promPath, a.PrometheusTLSCertHash) + + annotations["vault.hashicorp.com/agent-inject-secret-previous-prom-https-cert"] = promPath + annotations["vault.hashicorp.com/secret-volume-path-previous-prom-https-cert"] = util.SecretVolumeMountPathPrometheus + annotations["vault.hashicorp.com/agent-inject-file-previous-prom-https-cert"] = util.PreviousHashSecretKey + annotations["vault.hashicorp.com/agent-inject-template-previous-prom-https-cert"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_TEMPLATE, promPath, util.PreviousHashSecretKey) + annotations["vault.hashicorp.com/agent-inject-command-previous-prom-https-cert"] = fmt.Sprintf( + PREVIOUS_HASH_INJECT_COMMAND, util.SecretVolumeMountPathPrometheus, util.PreviousHashSecretKey) + } + + return annotations +} diff --git a/pkg/vault/vaultwatcher/vaultsecretwatch.go b/pkg/vault/vaultwatcher/vaultsecretwatch.go new file mode 100644 index 000000000..f9500332e --- /dev/null +++ b/pkg/vault/vaultwatcher/vaultsecretwatch.go @@ -0,0 +1,111 @@ +package vaultwatcher + +import ( + "context" + "fmt" + "strconv" + "time" + + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + + mdbv1 "github.com/10gen/ops-manager-kubernetes/api/v1/mdb" + omv1 "github.com/10gen/ops-manager-kubernetes/api/v1/om" + "github.com/10gen/ops-manager-kubernetes/pkg/vault" +) + +func WatchSecretChangeForMDB(ctx context.Context, log *zap.SugaredLogger, watchChannel chan event.GenericEvent, k8sClient kubernetesClient.Client, vaultClient *vault.VaultClient, resourceType mdbv1.ResourceType) { + for { + mdbList := &mdbv1.MongoDBList{} + err := k8sClient.List(ctx, mdbList, &client.ListOptions{Namespace: ""}) + if err != nil { + log.Errorf("failed to fetch MongoDBList from Kubernetes: %s", err) + } + + for n, mdb := range mdbList.Items { + // check if we care about the resource type, if not return early + if mdb.Spec.ResourceType != resourceType { + continue + } + // the credentials secret is mandatory and stored in a different path + path := fmt.Sprintf("%s/%s/%s", vaultClient.OperatorScretMetadataPath(), mdb.Namespace, mdb.Spec.Credentials) + latestResourceVersion, currentResourceVersion := getCurrentAndLatestVersion(vaultClient, path, mdb.Spec.Credentials, mdb.Annotations, log) + if latestResourceVersion > currentResourceVersion { + watchChannel <- event.GenericEvent{Object: &mdbList.Items[n]} + break + } + + for _, secretName := range mdb.GetSecretsMountedIntoDBPod() { + path := fmt.Sprintf("%s/%s/%s", vaultClient.DatabaseSecretMetadataPath(), mdb.Namespace, secretName) + latestResourceVersion, currentResourceVersion := getCurrentAndLatestVersion(vaultClient, path, secretName, mdb.Annotations, log) + + if latestResourceVersion > currentResourceVersion { + watchChannel <- event.GenericEvent{Object: &mdbList.Items[n]} + break + } + } + } + + time.Sleep(10 * time.Second) + } +} + +func WatchSecretChangeForOM(ctx context.Context, log *zap.SugaredLogger, watchChannel chan event.GenericEvent, k8sClient kubernetesClient.Client, vaultClient *vault.VaultClient) { + for { + omList := &omv1.MongoDBOpsManagerList{} + err := k8sClient.List(ctx, omList, &client.ListOptions{Namespace: ""}) + if err != nil { + log.Errorf("failed to fetch MongoDBOpsManagerList from Kubernetes: %s", err) + } + + triggeredReconciliation := false + for n, om := range omList.Items { + for _, secretName := range om.GetSecretsMountedIntoPod() { + path := fmt.Sprintf("%s/%s/%s", vaultClient.OpsManagerSecretMetadataPath(), om.Namespace, secretName) + latestResourceVersion, currentResourceVersion := getCurrentAndLatestVersion(vaultClient, path, secretName, om.Annotations, log) + + if latestResourceVersion > currentResourceVersion { + watchChannel <- event.GenericEvent{Object: &omList.Items[n]} + triggeredReconciliation = true + break + } + } + if triggeredReconciliation { + break + } + for _, secretName := range om.Spec.AppDB.GetSecretsMountedIntoPod() { + path := fmt.Sprintf("%s/%s/%s", vaultClient.AppDBSecretMetadataPath(), om.Namespace, secretName) + latestResourceVersion, currentResourceVersion := getCurrentAndLatestVersion(vaultClient, path, secretName, om.Annotations, log) + + if latestResourceVersion > currentResourceVersion { + watchChannel <- event.GenericEvent{Object: &omList.Items[n]} + break + } + } + } + + time.Sleep(10 * time.Second) + } +} + +func getCurrentAndLatestVersion(vaultClient *vault.VaultClient, path string, annotationKey string, annotations map[string]string, log *zap.SugaredLogger) (int, int) { + latestResourceVersion, err := vaultClient.ReadSecretVersion(path) + if err != nil { + log.Errorf("failed to fetch secret revision for the path %s, err: %v", path, err) + } + + // read the secret version from the annotation + currentResourceAnnotation := annotations[annotationKey] + + var currentResourceVersion int + if currentResourceAnnotation == "" { + currentResourceVersion = latestResourceVersion + } else { + currentResourceVersion, _ = strconv.Atoi(currentResourceAnnotation) + } + + return latestResourceVersion, currentResourceVersion +} diff --git a/pkg/webhook/certificates.go b/pkg/webhook/certificates.go new file mode 100644 index 000000000..55809c180 --- /dev/null +++ b/pkg/webhook/certificates.go @@ -0,0 +1,108 @@ +package webhook + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path" + "time" +) + +// createSelfSignedCert creates a self-signed certificate, valid for the +// specified hosts, and returns the certificate and key. +func createSelfSignedCert(hosts []string) (certBytes, privBytes []byte, err error) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + + privBytes, err = x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return nil, nil, err + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + + dnsNames := []string{} + ipAddresses := []net.IP{} + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + ipAddresses = append(ipAddresses, ip) + } else { + dnsNames = append(dnsNames, h) + } + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"MongoDB"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), // cert expires in 10 years + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: dnsNames, + IPAddresses: ipAddresses, + } + certBytes, err = x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, nil, err + } + + return certBytes, privBytes, nil +} + +// this whole file is largely cribbed from here: +// https://golang.org/src/crypto/tls/generate_cert.go + +func CreateCertFiles(hosts []string, directory string) error { + certBytes, privBytes, err := createSelfSignedCert(hosts) + if err != nil { + return err + } + + if err := os.MkdirAll(directory, 0o755); err != nil { + return err + } + + certOut, err := os.Create(path.Join(directory, "tls.crt")) + if err != nil { + return err + } + + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certBytes}); err != nil { + return err + } + + if err := certOut.Close(); err != nil { + return err + } + + keyOut, err := os.OpenFile(path.Join(directory, "tls.key"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return err + } + + if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil { + return err + } + + if err := keyOut.Close(); err != nil { + return err + } + + return nil +} diff --git a/pkg/webhook/setup.go b/pkg/webhook/setup.go new file mode 100644 index 000000000..69a8ebc36 --- /dev/null +++ b/pkg/webhook/setup.go @@ -0,0 +1,242 @@ +package webhook + +import ( + "context" + "os" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + kubernetesClient "github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/client" + admissionv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + apiErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mekoService "github.com/10gen/ops-manager-kubernetes/pkg/kube/service" + "github.com/10gen/ops-manager-kubernetes/pkg/util" + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +// This label must match the label used for Operator deployment +const controllerLabelName = "app.kubernetes.io/name" + +// createWebhookService creates a Kubernetes service for the webhook. +func createWebhookService(ctx context.Context, client client.Client, location types.NamespacedName, webhookPort int, multiClusterMode bool) error { + svcSelector := util.OperatorName + if multiClusterMode { + svcSelector = util.MultiClusterOperatorName + } + + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: location.Name, + Namespace: location.Namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "operator", + Port: 443, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(webhookPort), + }, + }, + Selector: map[string]string{ + controllerLabelName: svcSelector, + }, + }, + } + + // create the service if it doesn't already exist + existingService := &corev1.Service{} + err := client.Get(ctx, location, existingService) + if apiErrors.IsNotFound(err) { + return client.Create(ctx, &svc) + } else if err != nil { + return err + } + + // Update existing client with resource version and cluster IP + svc.ResourceVersion = existingService.ResourceVersion + svc.Spec.ClusterIP = existingService.Spec.ClusterIP + return client.Update(ctx, &svc) +} + +// GetWebhookConfig constructs a Kubernetes configuration resource for the +// validating admission webhook based on the name and namespace of the webhook +// service. +func GetWebhookConfig(serviceLocation types.NamespacedName) admissionv1.ValidatingWebhookConfiguration { + caBytes, err := os.ReadFile("/tmp/k8s-webhook-server/serving-certs/tls.crt") + if err != nil { + panic("could not read CA") + } + + // need to make variables as one can't take the address of a constant + scope := admissionv1.NamespacedScope + sideEffects := admissionv1.SideEffectClassNone + failurePolicy := admissionv1.Ignore + var port int32 = 443 + dbPath := "/validate-mongodb-com-v1-mongodb" + dbmultiPath := "/validate-mongodb-com-v1-mongodbmulticluster" + omPath := "/validate-mongodb-com-v1-mongodbopsmanager" + return admissionv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mdbpolicy.mongodb.com", + }, + Webhooks: []admissionv1.ValidatingWebhook{ + { + Name: "mdbpolicy.mongodb.com", + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Name: serviceLocation.Name, + Namespace: serviceLocation.Namespace, + Path: &dbPath, + // NOTE: port isn't supported in k8s 1.11 and lower. It works in + // 1.15 but I am unsure about the intervening versions. + Port: &port, + }, + CABundle: caBytes, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"mongodb.com"}, + APIVersions: []string{"*"}, + Resources: []string{"mongodb"}, + Scope: &scope, + }, + }, + }, + AdmissionReviewVersions: []string{"v1"}, + SideEffects: &sideEffects, + FailurePolicy: &failurePolicy, + }, + { + Name: "mdbmultipolicy.mongodb.com", + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Name: serviceLocation.Name, + Namespace: serviceLocation.Namespace, + Path: &dbmultiPath, + Port: &port, + }, + CABundle: caBytes, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"mongodb.com"}, + APIVersions: []string{"*"}, + Resources: []string{"mongodbmulticluster"}, + Scope: &scope, + }, + }, + }, + AdmissionReviewVersions: []string{"v1"}, + SideEffects: &sideEffects, + FailurePolicy: &failurePolicy, + }, + { + Name: "ompolicy.mongodb.com", + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ + Name: serviceLocation.Name, + Namespace: serviceLocation.Namespace, + Path: &omPath, + // NOTE: port isn't supported in k8s 1.11 and lower. It works in + // 1.15 but I am unsure about the intervening versions. + Port: &port, + }, + CABundle: caBytes, + }, + Rules: []admissionv1.RuleWithOperations{ + { + Operations: []admissionv1.OperationType{ + admissionv1.Create, + admissionv1.Update, + }, + Rule: admissionv1.Rule{ + APIGroups: []string{"mongodb.com"}, + APIVersions: []string{"*"}, + Resources: []string{"opsmanagers"}, + Scope: &scope, + }, + }, + }, + AdmissionReviewVersions: []string{"v1"}, + SideEffects: &sideEffects, + FailurePolicy: &failurePolicy, + }, + }, + } +} + +func shouldRegisterWebhookConfiguration() bool { + return env.ReadBoolOrDefault(util.MdbWebhookRegisterConfigurationEnv, true) // nolint:forbidigo +} + +func Setup(ctx context.Context, client client.Client, serviceLocation types.NamespacedName, certDirectory string, webhookPort int, multiClusterMode bool, log *zap.SugaredLogger) error { + if !shouldRegisterWebhookConfiguration() { + log.Debugf("Skipping configuration of ValidatingWebhookConfiguration") + // After upgrading OLM version after migrating to proper OLM webhooks we don't need that `operator-service` anymore. + // By default, the service is created by the operator in createWebhookService below + // It will also be useful here if someone decides to disable automatic webhook configuration by the operator. + if err := mekoService.DeleteServiceIfItExists(ctx, kubernetesClient.NewClient(client), serviceLocation); err != nil { + log.Warnf("Failed to delete webhook service %v: %w", serviceLocation, err) + // we don't want to fail the operator startup if we cannot do the cleanup + } + + webhookConfig := admissionv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mdbpolicy.mongodb.com", + }, + } + if err := client.Delete(ctx, &webhookConfig); err != nil { + if !apiErrors.IsNotFound(err) { + log.Warnf("Failed to perform cleanup of ValidatingWebhookConfiguration %s. The operator might not have necessary permissions. Remove the configuration manually. Error: %s", webhookConfig.Name, err) + // we don't want to fail the operator startup if we cannot do the cleanup + } + } + + return nil + } + + webhookServerHost := []string{serviceLocation.Name + "." + serviceLocation.Namespace + ".svc"} + if err := CreateCertFiles(webhookServerHost, certDirectory); err != nil { + return err + } + + if err := createWebhookService(ctx, client, serviceLocation, webhookPort, multiClusterMode); err != nil { + return err + } + + webhookConfig := GetWebhookConfig(serviceLocation) + err := client.Create(ctx, &webhookConfig) + if apiErrors.IsAlreadyExists(err) { + // client.Update results in internal K8s error "Invalid value: 0x0: must be specified for an update" + // (see https://github.com/kubernetes/kubernetes/issues/80515) + // this fixed in K8s 1.16.0+ + if err = client.Delete(context.Background(), &webhookConfig); err == nil { + err = client.Create(ctx, &webhookConfig) + } + } + if err != nil { + log.Warnf("Failed to configure admission webhooks. The operator might not have necessary permissions anymore. " + + "Admission webhooks might not work correctly. Ignore this error if the cluster role for the operator was removed deliberately.") + return nil + } + log.Debugf("Configured ValidatingWebhookConfiguration %s", webhookConfig.Name) + + return nil +} diff --git a/public/.evergreen.yml b/public/.evergreen.yml new file mode 100644 index 000000000..fddd5b6b4 --- /dev/null +++ b/public/.evergreen.yml @@ -0,0 +1,98 @@ +variables: + - &go_env + XDG_CONFIG_HOME: ${go_base_path}${workdir} + GO111MODULE: "on" + GOROOT: "/opt/golang/go1.24" +functions: + "clone": + - command: subprocess.exec + type: setup + params: + command: "mkdir -p src/github.com/mongodb" + - command: git.get_project + type: setup + params: + directory: src/github.com/mongodb/mongodb-enterprise-kubernetes + + "install goreleaser": + - command: shell.exec + type: setup + include_expansions_in_env: + - goreleaser_pro_tar_gz + params: + script: | + set -Eeu pipefail + curl -fL "${goreleaser_pro_tar_gz}" --output goreleaser_Linux_x86_64.tar.gz + tar -xf goreleaser_Linux_x86_64.tar.gz + chmod 755 ./goreleaser + + "install macos notarization service": + - command: shell.exec + type: setup + params: + include_expansions_in_env: + - notary_service_url + script: | + set -Eeu pipefail + + curl "${notary_service_url}" --output macos-notary.zip + unzip -u macos-notary.zip + chmod 755 ./linux_amd64/macnotary + "release": + - command: github.generate_token + params: + expansion_name: generated_token + - command: shell.exec + type: setup + params: + working_dir: src/github.com/mongodb/mongodb-enterprise-kubernetes/tools/multicluster + include_expansions_in_env: + - GRS_USERNAME + - GRS_PASSWORD + - PKCS11_URI + - ARTIFACTORY_URL + - ARTIFACTORY_PASSWORD + - SIGNING_IMAGE_URI + - macos_notary_keyid + - macos_notary_secret + - workdir + - triggered_by_git_tag + env: + <<: *go_env + MACOS_NOTARY_KEY: ${macos_notary_keyid} + MACOS_NOTARY_SECRET: ${macos_notary_secret} + GORELEASER_CURRENT_TAG: ${triggered_by_git_tag} + # shell.exec EVG Task doesn't have add_to_path, so we need to explicitly add the path export below. + script: | + set -Eeu pipefail + + export PATH=$GOROOT/bin:$PATH + export GITHUB_TOKEN=${generated_token} + ${workdir}/goreleaser release --rm-dist + +tasks: + - name: package_goreleaser + allowed_requesters: ["patch", "github_tag"] + tags: ["packaging"] + commands: + - func: "clone" + - func: "install goreleaser" + - func: "install macos notarization service" + - func: "release" + # add a noop task because if the only task in a variant is git_tag_only: true Evergreen doesn't start it at all + - name: noop + commands: + - command: shell.exec + params: + shell: bash + script: echo "this is the noop task" + +buildvariants: + # This variant is run when a new tag is out similar to github actions. + - name: release_mcli + display_name: Release Go multi-cluster binary + run_on: + - ubuntu2204-small + tasks: + - name: package_goreleaser + - name: noop diff --git a/public/.github/ISSUE_TEMPLATE/bug_report.md b/public/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..c442f56fd --- /dev/null +++ b/public/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,46 @@ +--- +name: Bug report +about: File a report about a problem with the Operator +title: '' +labels: '' +assignees: '' + +--- +**What did you do to encounter the bug?** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**What did you expect?** +A clear and concise description of what you expected to happen. + +**What happened instead?** +A clear and concise description of what happened instead + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Operator Information** + - Operator Version + - Base Images used (ubuntu or UBI) + +**Ops Manager Information** + - Ops Manager Version + - Is Ops Manager managed by the Operator or not? + - Is Ops Manager in Local Mode? + +**Kubernetes Cluster Information** + - Distribution: + - Version: + - Image Registry location (quay, or an internal registry) + +**Additional context** +Add any other context about the problem here. + +If possible, please include: + - `kubectl describe` output + - yaml definitions for your objects + - log files for the operator, database pods and Ops Manager + - An [Ops Manager Diagnostic Archive](https://docs.opsmanager.mongodb.com/current/tutorial/retrieve-debug-diagnostics) diff --git a/public/.github/ISSUE_TEMPLATE/config.yml b/public/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..ea996e736 --- /dev/null +++ b/public/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: MongoDB Support + url: https://support.mongodb.com + about: Please use the Support Center to receive official support within a timeline. Use this for urgent requests. + - name: MongoDB Feedback + url: https://feedback.mongodb.com/forums/924355-ops-tools + about: Use our Feedback page for making feature requests. diff --git a/public/.github/PULL_REQUEST_TEMPLATE.md b/public/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..4f3aec72a --- /dev/null +++ b/public/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,7 @@ +### All Submissions: + +* [ ] Have you opened an Issue before filing this PR? +* [ ] Have you signed our [CLA](https://www.mongodb.com/legal/contributor-agreement)? +* [ ] Have you checked to ensure there aren't other open [Pull Requests](../../../pulls) for the same update/change? +* [ ] Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes (if such). + diff --git a/public/.gitignore b/public/.gitignore new file mode 100644 index 000000000..7b4c90068 --- /dev/null +++ b/public/.gitignore @@ -0,0 +1,4 @@ +.idea +*.iml +.DS_Store +tools/multicluster/linux_amd64/* diff --git a/public/LICENSE b/public/LICENSE new file mode 100644 index 000000000..d6af3a584 --- /dev/null +++ b/public/LICENSE @@ -0,0 +1,3 @@ +Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. + +https://www.mongodb.com/customer-agreement/ diff --git a/public/README.md b/public/README.md new file mode 100644 index 000000000..7ce31378d --- /dev/null +++ b/public/README.md @@ -0,0 +1,200 @@ +# MongoDB Enterprise Kubernetes Operator # + +Welcome to the MongoDB Enterprise Kubernetes Operator. The Operator enables easy deploy of the following applications into Kubernetes clusters: +* MongoDB - Replica Sets, Sharded Clusters and Standalones - with authentication, TLS and many more options. +* Ops Manager - our enterprise management, monitoring and backup platform for MongoDB. The Operator can install and manage Ops Manager in Kubernetes for you. Ops Manager can manage MongoDB instances both inside and outside Kubernetes. + +The Operator requires access to one of our database management tools - Ops Manager or Cloud Manager - to deploy MongoDB instances. You may run Ops Manager either inside or outside Kubernetes, or may use Cloud Manager (cloud.mongodb.com) instead. + +This is an Enterprise product, available under the Enterprise Advanced license. +We also have a [Community Operator](https://github.com/mongodb/mongodb-kubernetes-operator). + +## Support, Feature Requests and Community ## + +The Enterprise Operator is supported by the [MongoDB Support Team](https://support.mongodb.com/). If you need help, please file a support ticket. +If you have a feature request, you can make one on our [Feedback Site](https://feedback.mongodb.com/forums/924355-ops-tools) + +You can discuss this integration in our new [Community Forum](https://developer.mongodb.com/community/forums/) - please use the tag [kubernetes-operator](https://developer.mongodb.com/community/forums/tag/kubernetes-operator) + +## Videos ## + +Here are some talks from MongoDB Live 2020 about the Operator: +* [Kubernetes, MongoDB, and Your MongoDB Data Platform](https://www.youtube.com/watch?v=o1fUPIOdKeU) +* [Run it in Kubernetes! Community and Enterprise MongoDB in Containers](https://www.youtube.com/watch?v=2Xszdg-4T6A) + +## Documentation ## + +[Install Kubernetes Operator](https://docs.opsmanager.mongodb.com/current/tutorial/install-k8s-operator) + +[Deploy MongoDB](https://docs.mongodb.com/kubernetes-operator/stable/mdb-resources/) + +[Deploy Ops Manager](https://docs.mongodb.com/kubernetes-operator/stable/om-resources/) + +[MongoDB Resource Specification](https://docs.opsmanager.mongodb.com/current/reference/k8s-operator-specification) + +[Ops Manager Resource Specification](https://docs.mongodb.com/kubernetes-operator/stable/reference/k8s-operator-om-specification/) + +[Troubleshooting Kubernetes Operator](https://docs.opsmanager.mongodb.com/current/reference/troubleshooting/k8s/) + +[Known Issues for Kubernetes Operator](https://docs.mongodb.com/kubernetes-operator/stable/reference/known-issues/) + +## Requirements ## + +Please refer to the [Installation Instructions](https://docs.mongodb.com/kubernetes-operator/stable/tutorial/plan-k8s-operator-install/) +to see which Kubernetes and Openshift versions the Operator is compatible with + +To work with MongoDB resource this Operator requires [Ops Manager](https://docs.opsmanager.mongodb.com/current/) (Ops Manager can +be installed into the same Kubernetes cluster by the Operator or installed outside of the cluster manually) +or [Cloud Manager](https://cloud.mongodb.com/user#/cloud/login). +> If this is your first time trying the Operator, Cloud Manager is easier to get started. Log in, and create 'Cloud Manager' Organizations and Projects to use with the Operator. + + +## Installation + +### Create Kubernetes Namespace + +The Mongodb Enterprise Operator is installed, into the `mongodb` namespace by default, but this namespace is not created automatically. To create this namespace you should execute: + + kubectl create namespace mongodb + +To use a different namespace, update the yaml files' `metadata.namespace` attribute to point to your preferred namespace. If using `helm` you need to override the `namespace` attribute with `--set namespace=<..>` during helm installation. + +### Installation using yaml files + +#### Create CustomResourceDefinitions + +`CustomResourceDefinition`s (or `CRDs`) are Kubernetes Objects which can be used to instruct the Operators to perform operations on your Kubernetes cluster. Our CRDs control MongoDB and Ops Manager deployments. They should be installed before installing the Operator. +CRDs are defined cluster-wide, so to install them, you must have Cluster-level access. However, once the CRDs are installed, MongoDB instances can be deployed with namespace-level access only. + + kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-enterprise-kubernetes/master/crds.yaml + +#### Operator Installation + +> In order to install the Operator in OpenShift, please follow [these](openshift-install.md) instructions instead. + +To install the Operator using yaml files, you may apply the config directly from github; + + kubectl apply -f https://raw.githubusercontent.com/mongodb/mongodb-enterprise-kubernetes/master/mongodb-enterprise.yaml + +or can clone this repo, make any edits you need, and apply it from disk: + + kubectl apply -f mongodb-enterprise.yaml + +### Installation using the Helm Chart + +MongoDB's official Helm Charts are hosted at https://github.com/mongodb/helm-charts + +## MongoDB Resource ## + +*This section describes how to deploy MongoDB instances. This requires a working Ops or Cloud Manager installation. See below for instructions on how to configure Ops Manager.* + +### Adding Ops Manager Credentials ### + +For the Operator to work, you will need the following information: + +* Base URL - the URL of an Ops Manager instance (for Cloud Manager use `https://cloud.mongodb.com`) +* (optional) Project Name - the name of an Ops Manager Project for MongoDB instances to be deployed into. This project will be created by the Operator if it doesn't exist. We recommend that you allow the Operator to create and manage the projects it uses. By default, the Operator will use the name of the MongoDB resource as the project name. +* (optional) Organization ID - the ID of the Organization which the Project belongs to. By default, the Operator will create an Organization with the same name as the Project. +* API Credentials. This can be any pair of: + * Public and Private Programmatic API keys. They correspond to `user` and `publicApiKey` fields in the Secret storing +credentials. More information about the way to create them using Ops Manager UI can be found +[here](https://docs.opsmanager.mongodb.com/current/tutorial/configure-public-api-access/#programmatic-api-keys) + * Username and Public API key. More information about the way to create them using Ops Manager UI can be found + [here](https://docs.opsmanager.mongodb.com/current/tutorial/configure-public-api-access/#personal-api-keys-deprecated) + +Note: When creating API credentials, you must allow the Pod IP range of your Kubernetes cluster to use the credentials - otherwise, API requests from the Operator to Ops Manager will be rejected. +You can get the Pod IP range of your kubernetes cluster by executing the command: ```kubectl cluster-info dump | grep -m 1 cluster-cidr``` + +This is documented in greater detail in our [installation guide](https://docs.opsmanager.mongodb.com/current/tutorial/install-k8s-operator) + + +### Projects ### + +A `Project` object is a Kubernetes `ConfigMap` that points to an Ops Manager installation and a `Project`. This `ConfigMap` has the following structure: + +``` +$ cat my-project.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-project + namespace: mongodb +data: + projectName: myProjectName # this is an optional parameter + orgId: 5b890e0feacf0b76ff3e7183 # this is an optional parameter + baseUrl: https://my-ops-manager-or-cloud-manager-url +``` +> `projectName` is optional, and the value of `metadata.name` will be used if it is not defined. +> `orgId` is required. + +Apply this file to create the new `Project`: + + kubectl apply -f my-project.yaml + +### Credentials ### + +For a user to be able to create or update objects in this Ops Manager Project they need either a Public API Key or a +Programmatic API Key. These will be held by Kubernetes as a `Secret` object. You can create this Secret with the following command: + +``` bash +$ kubectl -n mongodb create secret generic my-credentials --from-literal="user=my-public-api-key" --from-literal="publicApiKey=my-private-api-key" +``` + +### Creating a MongoDB Resource ### + +A MongoDB resource in Kubernetes is a MongoDB. We are going to create a replica set to test that everything is working as expected. There is a MongoDB replica set yaml file in `samples/mongodb/minimal/replica-set.yaml`. + +If you have a Project with the name `my-project` and Credentials stored in a secret called `my-credentials`, then after applying this file everything should be running and a new Replica Set with 3 members should soon appear in Ops Manager UI. + + kubectl apply -f samples/mongodb/minimal/replica-set.yaml -n mongodb + +## MongoDBOpsManager Resource ## + +This section describes how to create the Ops Manager Custom Resource in Kubernetes. Note, that this requires all +the CRDs and the Operator application to be installed as described above. + +### Create Admin Credentials Secret ### + +Before creating the Ops Manager resource you need to prepare the information about the admin user which will be +created automatically in Ops Manager. You can use the following command to do it: + +```bash +$ kubectl create secret generic ops-manager-admin-secret --from-literal=Username="user.name@example.com" --from-literal=Password="Passw0rd." --from-literal=FirstName="User" --from-literal=LastName="Name" -n +``` + +Note, that the secret is needed only during the initialization of the Ops Manager object - you can remove it or +change the password using Ops Manager UI after the Ops Manager object is created. + +### Create MongoDBOpsManager Resource ### + +Use the file `samples/ops-manager/ops-manager.yaml`. Edit the fields and create the object in Kubernetes: + +```bash +$ kubectl apply -f samples/ops-manager/ops-manager.yaml -n +``` + +Note, that it can take up to 8 minutes to initialize the Application Database and start Ops Manager. + +## Accessing the Ops Manager UI using your web browser + +In order to access the Ops Manager UI from outside the Kubernetes cluster, you must enable `spec.externalConnectivity` in the Ops Manager resource definition. The easiest approach is by configuring the LoadBalancer service type. + +You will be able to fetch the URL to connect to Ops Manager UI from the `Service` object created by the Operator. + +## Removing the Operator, Databases and Ops Manager from your Kubernetes cluster ## + +As the Operator manages MongoDB and Ops Manager resources, if you want to remove them from your Kubernetes cluster, database instances and Ops Manager must be removed before removing the Operator. Removing the Operator first, or deleting the namespace will cause delays or stall the removal process of MongoDB objects, requiring manual intervention. + +Here is the correct order to completely remove the Operator and the services managed by it: + +* Remove all database clusters managed by the Operator +* Remove Ops Manager +* Remove the Operator +* Remove the CRDs + +## Contributing + +For PRs to be accepted, all contributors must sign our [CLA](https://www.mongodb.com/legal/contributor-agreement). + +Reviewers, please ensure that the CLA has been signed by referring to [the contributors tool](https://contributors.corp.mongodb.com/) (internal link). diff --git a/public/architectures/mongodb-replicaset-mc-no-mesh/code_snippets/1050_generate_certs.sh b/public/architectures/mongodb-replicaset-mc-no-mesh/code_snippets/1050_generate_certs.sh new file mode 100644 index 000000000..ba87ae92a --- /dev/null +++ b/public/architectures/mongodb-replicaset-mc-no-mesh/code_snippets/1050_generate_certs.sh @@ -0,0 +1,20 @@ +kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - < certs/ca.crt + +mongosh --host "${external_ip}" --username rs-user --password password --tls --tlsCAFile certs/ca.crt --tlsAllowInvalidHostnames --eval "db.runCommand({connectionStatus : 1})" diff --git a/public/architectures/mongodb-replicaset-mc-no-mesh/code_snippets/9000_delete_resources.sh b/public/architectures/mongodb-replicaset-mc-no-mesh/code_snippets/9000_delete_resources.sh new file mode 100644 index 000000000..5f4021e70 --- /dev/null +++ b/public/architectures/mongodb-replicaset-mc-no-mesh/code_snippets/9000_delete_resources.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete mdbu/rs-user + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete "mdbmc/${RS_RESOURCE_NAME}" diff --git a/public/architectures/mongodb-replicaset-mc-no-mesh/env_variables.sh b/public/architectures/mongodb-replicaset-mc-no-mesh/env_variables.sh new file mode 100755 index 000000000..4740dda18 --- /dev/null +++ b/public/architectures/mongodb-replicaset-mc-no-mesh/env_variables.sh @@ -0,0 +1,15 @@ +# This script builds on top of the environment configured in the setup guides. +# It depends (uses) the following env variables defined there to work correctly. +# If you don't use the setup guide to bootstrap the environment, then define them here. +# ${K8S_CLUSTER_0_CONTEXT_NAME} +# ${K8S_CLUSTER_1_CONTEXT_NAME} +# ${K8S_CLUSTER_2_CONTEXT_NAME} +# ${MDB_NAMESPACE} +# ${CUSTOM_DOMAIN} + +export RS_RESOURCE_NAME=mdb +export MONGODB_VERSION="8.0.5-ent" + +export MDB_CLUSTER_0_EXTERNAL_DOMAIN="${K8S_CLUSTER_0}.${CUSTOM_DOMAIN}" +export MDB_CLUSTER_1_EXTERNAL_DOMAIN="${K8S_CLUSTER_1}.${CUSTOM_DOMAIN}" +export MDB_CLUSTER_2_EXTERNAL_DOMAIN="${K8S_CLUSTER_2}.${CUSTOM_DOMAIN}" diff --git a/public/architectures/mongodb-replicaset-mc-no-mesh/output/1210_verify_mongosh_connection.out b/public/architectures/mongodb-replicaset-mc-no-mesh/output/1210_verify_mongosh_connection.out new file mode 100644 index 000000000..2771f435b --- /dev/null +++ b/public/architectures/mongodb-replicaset-mc-no-mesh/output/1210_verify_mongosh_connection.out @@ -0,0 +1,15 @@ +{ + authInfo: { + authenticatedUsers: [ { user: 'rs-user', db: 'admin' } ], + authenticatedUserRoles: [ { role: 'root', db: 'admin' } ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1743589744, i: 1 }), + signature: { + hash: Binary.createFromBase64('fiBrPX9aaxTmMmLb1K2q6d4/XfQ=', 0), + keyId: Long('7488660369775263749') + } + }, + operationTime: Timestamp({ t: 1743589744, i: 1 }) +} diff --git a/public/architectures/mongodb-replicaset-mc-no-mesh/teardown.sh b/public/architectures/mongodb-replicaset-mc-no-mesh/teardown.sh new file mode 100755 index 000000000..dd5dd71a8 --- /dev/null +++ b/public/architectures/mongodb-replicaset-mc-no-mesh/teardown.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9000_delete_resources.sh + +popd diff --git a/public/architectures/mongodb-replicaset-mc-no-mesh/test.sh b/public/architectures/mongodb-replicaset-mc-no-mesh/test.sh new file mode 100755 index 000000000..4b7622391 --- /dev/null +++ b/public/architectures/mongodb-replicaset-mc-no-mesh/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 1050_generate_certs.sh +run 1100_mongodb_replicaset_multi_cluster.sh +run 1110_mongodb_replicaset_multi_cluster_wait_for_running_state.sh + +run 1200_create_mongodb_user.sh +sleep 10 +run_for_output 1210_verify_mongosh_connection.sh + +popd diff --git a/public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1050_generate_certs.sh b/public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1050_generate_certs.sh new file mode 100644 index 000000000..a8b8327ae --- /dev/null +++ b/public/architectures/mongodb-replicaset-multi-cluster/code_snippets/1050_generate_certs.sh @@ -0,0 +1,18 @@ +kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - < certs/ca.crt + +mongosh --host "${external_ip}" --username rs-user --password password --tls --tlsCAFile certs/ca.crt --tlsAllowInvalidHostnames --eval "db.runCommand({connectionStatus : 1})" diff --git a/public/architectures/mongodb-replicaset-multi-cluster/code_snippets/9000_delete_resources.sh b/public/architectures/mongodb-replicaset-multi-cluster/code_snippets/9000_delete_resources.sh new file mode 100644 index 000000000..5f4021e70 --- /dev/null +++ b/public/architectures/mongodb-replicaset-multi-cluster/code_snippets/9000_delete_resources.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete mdbu/rs-user + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete "mdbmc/${RS_RESOURCE_NAME}" diff --git a/public/architectures/mongodb-replicaset-multi-cluster/env_variables.sh b/public/architectures/mongodb-replicaset-multi-cluster/env_variables.sh new file mode 100755 index 000000000..3a83024b5 --- /dev/null +++ b/public/architectures/mongodb-replicaset-multi-cluster/env_variables.sh @@ -0,0 +1,10 @@ +# This script builds on top of the environment configured in the setup guides. +# It depends (uses) the following env variables defined there to work correctly. +# If you don't use the setup guide to bootstrap the environment, then define them here. +# ${K8S_CLUSTER_0_CONTEXT_NAME} +# ${K8S_CLUSTER_1_CONTEXT_NAME} +# ${K8S_CLUSTER_2_CONTEXT_NAME} +# ${MDB_NAMESPACE} + +export RS_RESOURCE_NAME=mdb +export MONGODB_VERSION="8.0.5-ent" diff --git a/public/architectures/mongodb-replicaset-multi-cluster/output/1210_verify_mongosh_connection.out b/public/architectures/mongodb-replicaset-multi-cluster/output/1210_verify_mongosh_connection.out new file mode 100644 index 000000000..36220b9a3 --- /dev/null +++ b/public/architectures/mongodb-replicaset-multi-cluster/output/1210_verify_mongosh_connection.out @@ -0,0 +1,15 @@ +{ + authInfo: { + authenticatedUsers: [ { user: 'rs-user', db: 'admin' } ], + authenticatedUserRoles: [ { role: 'root', db: 'admin' } ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1741701953, i: 1 }), + signature: { + hash: Binary.createFromBase64('uhYReuUiWNWP6m1lZ5umgDVgO48=', 0), + keyId: Long('7480552820140146693') + } + }, + operationTime: Timestamp({ t: 1741701953, i: 1 }) +} diff --git a/public/architectures/mongodb-replicaset-multi-cluster/teardown.sh b/public/architectures/mongodb-replicaset-multi-cluster/teardown.sh new file mode 100755 index 000000000..dd5dd71a8 --- /dev/null +++ b/public/architectures/mongodb-replicaset-multi-cluster/teardown.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9000_delete_resources.sh + +popd diff --git a/public/architectures/mongodb-replicaset-multi-cluster/test.sh b/public/architectures/mongodb-replicaset-multi-cluster/test.sh new file mode 100755 index 000000000..4b7622391 --- /dev/null +++ b/public/architectures/mongodb-replicaset-multi-cluster/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 1050_generate_certs.sh +run 1100_mongodb_replicaset_multi_cluster.sh +run 1110_mongodb_replicaset_multi_cluster_wait_for_running_state.sh + +run 1200_create_mongodb_user.sh +sleep 10 +run_for_output 1210_verify_mongosh_connection.sh + +popd diff --git a/public/architectures/mongodb-sharded-mc-no-mesh/code_snippets/2050_generate_certs.sh b/public/architectures/mongodb-sharded-mc-no-mesh/code_snippets/2050_generate_certs.sh new file mode 100644 index 000000000..87be35a7c --- /dev/null +++ b/public/architectures/mongodb-sharded-mc-no-mesh/code_snippets/2050_generate_certs.sh @@ -0,0 +1,115 @@ +kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - < certs/ca.crt + +mongosh --host "${external_ip}" --username sc-user --password password --tls --tlsCAFile certs/ca.crt --tlsAllowInvalidHostnames --eval "db.runCommand({connectionStatus : 1})" diff --git a/public/architectures/mongodb-sharded-mc-no-mesh/code_snippets/9000_delete_resources.sh b/public/architectures/mongodb-sharded-mc-no-mesh/code_snippets/9000_delete_resources.sh new file mode 100644 index 000000000..8072e72cb --- /dev/null +++ b/public/architectures/mongodb-sharded-mc-no-mesh/code_snippets/9000_delete_resources.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete mdbu/sc-user + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete "mdb/${SC_RESOURCE_NAME}" diff --git a/public/architectures/mongodb-sharded-mc-no-mesh/env_variables.sh b/public/architectures/mongodb-sharded-mc-no-mesh/env_variables.sh new file mode 100755 index 000000000..1df2c3b69 --- /dev/null +++ b/public/architectures/mongodb-sharded-mc-no-mesh/env_variables.sh @@ -0,0 +1,15 @@ +# This script builds on top of the environment configured in the setup guides. +# It depends (uses) the following env variables defined there to work correctly. +# If you don't use the setup guide to bootstrap the environment, then define them here. +# ${K8S_CLUSTER_0_CONTEXT_NAME} +# ${K8S_CLUSTER_1_CONTEXT_NAME} +# ${K8S_CLUSTER_2_CONTEXT_NAME} +# ${MDB_NAMESPACE} +# ${CUSTOM_DOMAIN} + +export SC_RESOURCE_NAME=mdb-sh +export MONGODB_VERSION="8.0.5-ent" + +export MDB_CLUSTER_0_EXTERNAL_DOMAIN="${K8S_CLUSTER_0}.${CUSTOM_DOMAIN}" +export MDB_CLUSTER_1_EXTERNAL_DOMAIN="${K8S_CLUSTER_1}.${CUSTOM_DOMAIN}" +export MDB_CLUSTER_2_EXTERNAL_DOMAIN="${K8S_CLUSTER_2}.${CUSTOM_DOMAIN}" diff --git a/public/architectures/mongodb-sharded-mc-no-mesh/output/2210_verify_mongosh_connection.out b/public/architectures/mongodb-sharded-mc-no-mesh/output/2210_verify_mongosh_connection.out new file mode 100644 index 000000000..6f5868691 --- /dev/null +++ b/public/architectures/mongodb-sharded-mc-no-mesh/output/2210_verify_mongosh_connection.out @@ -0,0 +1,15 @@ +{ + authInfo: { + authenticatedUsers: [ { user: 'sc-user', db: 'admin' } ], + authenticatedUserRoles: [ { role: 'root', db: 'admin' } ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1743590424, i: 1 }), + signature: { + hash: Binary.createFromBase64('1+SD+TJDayNhxsFsJzaGb2mtd+c=', 0), + keyId: Long('7488663363367469079') + } + }, + operationTime: Timestamp({ t: 1743590424, i: 1 }) +} diff --git a/public/architectures/mongodb-sharded-mc-no-mesh/teardown.sh b/public/architectures/mongodb-sharded-mc-no-mesh/teardown.sh new file mode 100755 index 000000000..dd5dd71a8 --- /dev/null +++ b/public/architectures/mongodb-sharded-mc-no-mesh/teardown.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9000_delete_resources.sh + +popd diff --git a/public/architectures/mongodb-sharded-mc-no-mesh/test.sh b/public/architectures/mongodb-sharded-mc-no-mesh/test.sh new file mode 100755 index 000000000..d45db935c --- /dev/null +++ b/public/architectures/mongodb-sharded-mc-no-mesh/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 2050_generate_certs.sh +run 2100_mongodb_sharded_multi_cluster.sh +run 2110_mongodb_sharded_multi_cluster_wait_for_running_state.sh + +run 2200_create_mongodb_user.sh +sleep 10 +run_for_output 2210_verify_mongosh_connection.sh + +popd diff --git a/public/architectures/mongodb-sharded-multi-cluster/code_snippets/2050_generate_certs.sh b/public/architectures/mongodb-sharded-multi-cluster/code_snippets/2050_generate_certs.sh new file mode 100644 index 000000000..18d2f5f00 --- /dev/null +++ b/public/architectures/mongodb-sharded-multi-cluster/code_snippets/2050_generate_certs.sh @@ -0,0 +1,103 @@ +kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" -f - < certs/ca.crt + +mongosh --host "${external_ip}" --username sc-user --password password --tls --tlsCAFile certs/ca.crt --tlsAllowInvalidHostnames --eval "db.runCommand({connectionStatus : 1})" diff --git a/public/architectures/mongodb-sharded-multi-cluster/code_snippets/9000_delete_resources.sh b/public/architectures/mongodb-sharded-multi-cluster/code_snippets/9000_delete_resources.sh new file mode 100644 index 000000000..8072e72cb --- /dev/null +++ b/public/architectures/mongodb-sharded-multi-cluster/code_snippets/9000_delete_resources.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete mdbu/sc-user + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" delete "mdb/${SC_RESOURCE_NAME}" diff --git a/public/architectures/mongodb-sharded-multi-cluster/env_variables.sh b/public/architectures/mongodb-sharded-multi-cluster/env_variables.sh new file mode 100755 index 000000000..1ade0e1c4 --- /dev/null +++ b/public/architectures/mongodb-sharded-multi-cluster/env_variables.sh @@ -0,0 +1,10 @@ +# This script builds on top of the environment configured in the setup guides. +# It depends (uses) the following env variables defined there to work correctly. +# If you don't use the setup guide to bootstrap the environment, then define them here. +# ${K8S_CLUSTER_0_CONTEXT_NAME} +# ${K8S_CLUSTER_1_CONTEXT_NAME} +# ${K8S_CLUSTER_2_CONTEXT_NAME} +# ${MDB_NAMESPACE} + +export SC_RESOURCE_NAME=mdb-sh +export MONGODB_VERSION="8.0.5-ent" diff --git a/public/architectures/mongodb-sharded-multi-cluster/output/2210_verify_mongosh_connection.out b/public/architectures/mongodb-sharded-multi-cluster/output/2210_verify_mongosh_connection.out new file mode 100644 index 000000000..2c19b62f7 --- /dev/null +++ b/public/architectures/mongodb-sharded-multi-cluster/output/2210_verify_mongosh_connection.out @@ -0,0 +1,15 @@ +{ + authInfo: { + authenticatedUsers: [ { user: 'sc-user', db: 'admin' } ], + authenticatedUserRoles: [ { role: 'root', db: 'admin' } ] + }, + ok: 1, + '$clusterTime': { + clusterTime: Timestamp({ t: 1741702735, i: 1 }), + signature: { + hash: Binary.createFromBase64('kVqqNDHTI1zxYrPsU0QaYqyksJA=', 0), + keyId: Long('7480555706358169606') + } + }, + operationTime: Timestamp({ t: 1741702735, i: 1 }) +} diff --git a/public/architectures/mongodb-sharded-multi-cluster/teardown.sh b/public/architectures/mongodb-sharded-multi-cluster/teardown.sh new file mode 100755 index 000000000..dd5dd71a8 --- /dev/null +++ b/public/architectures/mongodb-sharded-multi-cluster/teardown.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9000_delete_resources.sh + +popd diff --git a/public/architectures/mongodb-sharded-multi-cluster/test.sh b/public/architectures/mongodb-sharded-multi-cluster/test.sh new file mode 100755 index 000000000..d45db935c --- /dev/null +++ b/public/architectures/mongodb-sharded-multi-cluster/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 2050_generate_certs.sh +run 2100_mongodb_sharded_multi_cluster.sh +run 2110_mongodb_sharded_multi_cluster_wait_for_running_state.sh + +run 2200_create_mongodb_user.sh +sleep 10 +run_for_output 2210_verify_mongosh_connection.sh + +popd diff --git a/public/architectures/ops-manager-mc-no-mesh/code_snippets/0100_generate_certs.sh b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0100_generate_certs.sh new file mode 100644 index 000000000..b426e0f63 --- /dev/null +++ b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0100_generate_certs.sh @@ -0,0 +1,37 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OM_NAMESPACE}" apply -f - < certs/tls.crt +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OM_NAMESPACE}" get secret cert-prefix-om-cert -o jsonpath="{.data['tls\.key']}" | base64 --decode > certs/tls.key + +gcloud compute ssl-certificates create om-certificate --certificate=certs/tls.crt --private-key=certs/tls.key diff --git a/public/architectures/ops-manager-mc-no-mesh/code_snippets/0150_om_load_balancer.sh b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0150_om_load_balancer.sh new file mode 100755 index 000000000..7dede2b50 --- /dev/null +++ b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0150_om_load_balancer.sh @@ -0,0 +1,27 @@ +gcloud compute firewall-rules create fw-ops-manager-hc \ + --action=allow \ + --direction=ingress \ + --target-tags=mongodb \ + --source-ranges=130.211.0.0/22,35.191.0.0/16 \ + --rules=tcp:8443 + +gcloud compute health-checks create https om-healthcheck \ + --use-serving-port \ + --request-path=/monitor/health + +gcloud compute backend-services create om-backend-service \ + --protocol HTTPS \ + --health-checks om-healthcheck \ + --global + +gcloud compute url-maps create om-url-map \ + --default-service om-backend-service + +gcloud compute target-https-proxies create om-lb-proxy \ + --url-map om-url-map \ + --ssl-certificates=om-certificate + +gcloud compute forwarding-rules create om-forwarding-rule \ + --global \ + --target-https-proxy=om-lb-proxy \ + --ports=443 diff --git a/public/architectures/ops-manager-mc-no-mesh/code_snippets/0160_add_dns_record.sh b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0160_add_dns_record.sh new file mode 100644 index 000000000..e814cb8ca --- /dev/null +++ b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0160_add_dns_record.sh @@ -0,0 +1,3 @@ +ip_address=$(gcloud compute forwarding-rules describe om-forwarding-rule --global --format="get(IPAddress)") + +gcloud dns record-sets create "${OPS_MANAGER_EXTERNAL_DOMAIN}" --zone="${DNS_ZONE}" --type="A" --ttl="300" --rrdatas="${ip_address}" diff --git a/public/architectures/ops-manager-mc-no-mesh/code_snippets/0300_ops_manager_create_admin_credentials.sh b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0300_ops_manager_create_admin_credentials.sh new file mode 100755 index 000000000..88d14f73a --- /dev/null +++ b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0300_ops_manager_create_admin_credentials.sh @@ -0,0 +1,5 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" --namespace "${OM_NAMESPACE}" create secret generic om-admin-user-credentials \ + --from-literal=Username="admin" \ + --from-literal=Password="Passw0rd@" \ + --from-literal=FirstName="Jane" \ + --from-literal=LastName="Doe" diff --git a/public/architectures/ops-manager-mc-no-mesh/code_snippets/0320_ops_manager_no_mesh.sh b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0320_ops_manager_no_mesh.sh new file mode 100755 index 000000000..625bfa9be --- /dev/null +++ b/public/architectures/ops-manager-mc-no-mesh/code_snippets/0320_ops_manager_no_mesh.sh @@ -0,0 +1,55 @@ +kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OM_NAMESPACE}" -f - <out}' + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" get secret root-secret -n cert-manager -o jsonpath="{.data['ca\.crt']}" | base64 --decode > certs/ca.crt +cat certs/ca.crt certs/cert2.crt certs/cert3.crt >> certs/mms-ca.crt + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create cm ca-issuer -n "${MDB_NAMESPACE}" --from-file=ca-pem=certs/mms-ca.crt --from-file=mms-ca.crt=certs/mms-ca.crt +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create cm ca-issuer -n "${OM_NAMESPACE}" --from-file=ca-pem=certs/mms-ca.crt --from-file=mms-ca.crt=certs/mms-ca.crt diff --git a/public/architectures/setup-multi-cluster/setup-cert-manager/output/0215_helm_configure_repo.out b/public/architectures/setup-multi-cluster/setup-cert-manager/output/0215_helm_configure_repo.out new file mode 100644 index 000000000..eb8c697bd --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-cert-manager/output/0215_helm_configure_repo.out @@ -0,0 +1 @@ +"jetstack" has been added to your repositories diff --git a/public/architectures/setup-multi-cluster/setup-cert-manager/output/0216_helm_install_cert_manager.out b/public/architectures/setup-multi-cluster/setup-cert-manager/output/0216_helm_install_cert_manager.out new file mode 100644 index 000000000..bd370a980 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-cert-manager/output/0216_helm_install_cert_manager.out @@ -0,0 +1,23 @@ +Release "cert-manager" does not exist. Installing it now. +NAME: cert-manager +LAST DEPLOYED: Wed Apr 2 18:07:45 2025 +NAMESPACE: cert-manager +STATUS: deployed +REVISION: 1 +TEST SUITE: None +NOTES: +cert-manager v1.17.1 has been deployed successfully! + +In order to begin issuing certificates, you will need to set up a ClusterIssuer +or Issuer resource (for example, by creating a 'letsencrypt-staging' issuer). + +More information on the different types of issuers and how to configure them +can be found in our documentation: + +https://cert-manager.io/docs/configuration/ + +For information on how to configure cert-manager to automatically provision +Certificates for Ingress resources, take a look at the `ingress-shim` +documentation: + +https://cert-manager.io/docs/usage/ingress/ diff --git a/public/architectures/setup-multi-cluster/setup-cert-manager/output/0221_verify_issuer.out b/public/architectures/setup-multi-cluster/setup-cert-manager/output/0221_verify_issuer.out new file mode 100644 index 000000000..045b0fa2b --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-cert-manager/output/0221_verify_issuer.out @@ -0,0 +1,3 @@ +certificate.cert-manager.io/test-selfsigned-cert created +certificate.cert-manager.io/test-selfsigned-cert condition met +certificate.cert-manager.io "test-selfsigned-cert" deleted diff --git a/public/architectures/setup-multi-cluster/setup-cert-manager/test.sh b/public/architectures/setup-multi-cluster/setup-cert-manager/test.sh new file mode 100755 index 000000000..6019bab95 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-cert-manager/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run_for_output 0215_helm_configure_repo.sh +run_for_output 0216_helm_install_cert_manager.sh +run 0220_create_issuer.sh +run_for_output 0221_verify_issuer.sh +run 0225_create_ca_configmap.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0100_create_gke_sa.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0100_create_gke_sa.sh new file mode 100644 index 000000000..67736f802 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0100_create_gke_sa.sh @@ -0,0 +1 @@ +gcloud iam service-accounts create "${DNS_SA_NAME}" --display-name "${DNS_SA_NAME}" diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0120_add_role_to_sa.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0120_add_role_to_sa.sh new file mode 100644 index 000000000..73c3ccc4f --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0120_add_role_to_sa.sh @@ -0,0 +1 @@ +gcloud projects add-iam-policy-binding "${MDB_GKE_PROJECT}" --member serviceAccount:"${DNS_SA_EMAIL}" --role roles/dns.admin diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0130_create_sa_key.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0130_create_sa_key.sh new file mode 100644 index 000000000..af41bc117 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0130_create_sa_key.sh @@ -0,0 +1,3 @@ +mkdir -p secrets + +gcloud iam service-accounts keys create secrets/external-dns-sa-key.json --iam-account="${DNS_SA_EMAIL}" diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0140_create_namespaces.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0140_create_namespaces.sh new file mode 100644 index 000000000..424490fe2 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0140_create_namespaces.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create ns external-dns +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" create ns external-dns +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" create ns external-dns diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0150_create_sa_secrets.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0150_create_sa_secrets.sh new file mode 100644 index 000000000..6635976b3 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0150_create_sa_secrets.sh @@ -0,0 +1,4 @@ +# create secret with service account key +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n external-dns create secret generic external-dns-sa-secret --from-file credentials.json=secrets/external-dns-sa-key.json +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n external-dns create secret generic external-dns-sa-secret --from-file credentials.json=secrets/external-dns-sa-key.json +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n external-dns create secret generic external-dns-sa-secret --from-file credentials.json=secrets/external-dns-sa-key.json diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0200_install_externaldns.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0200_install_externaldns.sh new file mode 100644 index 000000000..18cc5a6d9 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0200_install_externaldns.sh @@ -0,0 +1,3 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n external-dns apply -f yamls/externaldns.yaml +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n external-dns apply -f yamls/externaldns.yaml +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n external-dns apply -f yamls/externaldns.yaml diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0300_setup_dns_zone.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0300_setup_dns_zone.sh new file mode 100644 index 000000000..20e153e78 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/0300_setup_dns_zone.sh @@ -0,0 +1,9 @@ +FQ_CLUSTER_0="projects/${MDB_GKE_PROJECT}/locations/${K8S_CLUSTER_0_ZONE}/clusters/${K8S_CLUSTER_0}" +FQ_CLUSTER_1="projects/${MDB_GKE_PROJECT}/locations/${K8S_CLUSTER_1_ZONE}/clusters/${K8S_CLUSTER_1}" +FQ_CLUSTER_2="projects/${MDB_GKE_PROJECT}/locations/${K8S_CLUSTER_2_ZONE}/clusters/${K8S_CLUSTER_2}" + +gcloud dns managed-zones create "${DNS_ZONE}" \ + --description="" \ + --dns-name="${CUSTOM_DOMAIN}" \ + --visibility="private" \ + --gkeclusters="${FQ_CLUSTER_0}","${FQ_CLUSTER_1}","${FQ_CLUSTER_2}" diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9000_delete_sa.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9000_delete_sa.sh new file mode 100644 index 000000000..582252272 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9000_delete_sa.sh @@ -0,0 +1,3 @@ +gcloud projects remove-iam-policy-binding "${MDB_GKE_PROJECT}" --member serviceAccount:"${DNS_SA_EMAIL}" --role roles/dns.admin + +gcloud iam service-accounts delete "${DNS_SA_EMAIL}" -q diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9050_delete_namespace.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9050_delete_namespace.sh new file mode 100644 index 000000000..490e05598 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9050_delete_namespace.sh @@ -0,0 +1,4 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" delete ns "external-dns" & +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" delete ns "external-dns" & +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" delete ns "external-dns" & +wait diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9100_delete_dns_zone.sh b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9100_delete_dns_zone.sh new file mode 100644 index 000000000..7773d6e8b --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/code_snippets/9100_delete_dns_zone.sh @@ -0,0 +1,10 @@ +gcloud dns record-sets list --zone="${DNS_ZONE}" --format=json | jq -c '.[]' | while read -r record; do + NAME=$(echo "${record}" | jq -r '.name') + TYPE=$(echo "${record}" | jq -r '.type') + + if [[ "${TYPE}" == "A" || "${TYPE}" == "TXT" ]]; then + gcloud dns record-sets delete "${NAME}" --zone="${DNS_ZONE}" --type="${TYPE}" + fi +done + +gcloud dns managed-zones delete "${DNS_ZONE}" -q diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/env_variables.sh b/public/architectures/setup-multi-cluster/setup-externaldns/env_variables.sh new file mode 100644 index 000000000..7d21abe03 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/env_variables.sh @@ -0,0 +1,19 @@ +# This script builds on top of the environment configured in the setup guides. +# It depends (uses) the following env variables defined there to work correctly. +# If you don't use the setup guide to bootstrap the environment, then define them here. +# ${K8S_CLUSTER_0} +# ${K8S_CLUSTER_1} +# ${K8S_CLUSTER_2} +# ${K8S_CLUSTER_0_ZONE} +# ${K8S_CLUSTER_1_ZONE} +# ${K8S_CLUSTER_2_ZONE} +# ${K8S_CLUSTER_0_CONTEXT_NAME} +# ${K8S_CLUSTER_1_CONTEXT_NAME} +# ${K8S_CLUSTER_2_CONTEXT_NAME} +# ${MDB_GKE_PROJECT} + +export DNS_SA_NAME="external-dns-sa" +export DNS_SA_EMAIL="${DNS_SA_NAME}@${MDB_GKE_PROJECT}.iam.gserviceaccount.com" + +export CUSTOM_DOMAIN="mongodb.custom" +export DNS_ZONE="mongodb" diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/teardown.sh b/public/architectures/setup-multi-cluster/setup-externaldns/teardown.sh new file mode 100755 index 000000000..f6fb4ad2e --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/teardown.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9000_delete_sa.sh +run 9050_delete_namespace.sh +run 9100_delete_dns_zone.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/test.sh b/public/architectures/setup-multi-cluster/setup-externaldns/test.sh new file mode 100755 index 000000000..be99a98ac --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 0100_create_gke_sa.sh +# need to wait as the SA is not immediately available +sleep 10 +run 0120_add_role_to_sa.sh +run 0130_create_sa_key.sh +run 0140_create_namespaces.sh +run 0150_create_sa_secrets.sh +run 0200_install_externaldns.sh +run 0300_setup_dns_zone.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-externaldns/yamls/externaldns.yaml b/public/architectures/setup-multi-cluster/setup-externaldns/yamls/externaldns.yaml new file mode 100644 index 000000000..98ca9cf83 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-externaldns/yamls/externaldns.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: external-dns + labels: + app.kubernetes.io/name: external-dns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: external-dns + labels: + app.kubernetes.io/name: external-dns +rules: + - apiGroups: [""] + resources: ["services","endpoints","pods","nodes"] + verbs: ["get","watch","list"] + - apiGroups: ["extensions","networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get","watch","list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: external-dns-viewer + labels: + app.kubernetes.io/name: external-dns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-dns +subjects: + - kind: ServiceAccount + name: external-dns + namespace: external-dns +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: external-dns + labels: + app.kubernetes.io/name: external-dns +spec: + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: external-dns + template: + metadata: + labels: + app.kubernetes.io/name: external-dns + spec: + serviceAccountName: external-dns + containers: + - name: external-dns + image: registry.k8s.io/external-dns/external-dns:v0.16.1 + args: + - --source=service + - --source=ingress + - --provider=google + - --log-format=json # google cloud logs parses severity of the "text" log format incorrectly + - --interval=10s + - --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization + - --registry=txt + # # uncomment below if static credentials are used + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/secrets/service-account/credentials.json + volumeMounts: + - name: google-service-account + mountPath: /etc/secrets/service-account/ + volumes: + - name: google-service-account + secret: + secretName: external-dns-sa-secret diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0005_gcloud_set_current_project.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0005_gcloud_set_current_project.sh new file mode 100644 index 000000000..48cf31e52 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0005_gcloud_set_current_project.sh @@ -0,0 +1 @@ +gcloud config set project "${MDB_GKE_PROJECT}" diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_0.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_0.sh new file mode 100644 index 000000000..70ed570fa --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_0.sh @@ -0,0 +1,6 @@ +gcloud container clusters create "${K8S_CLUSTER_0}" \ + --zone="${K8S_CLUSTER_0_ZONE}" \ + --num-nodes="${K8S_CLUSTER_0_NUMBER_OF_NODES}" \ + --machine-type "${K8S_CLUSTER_0_MACHINE_TYPE}" \ + --tags=mongodb \ + "${GKE_SPOT_INSTANCES_SWITCH:-""}" diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_1.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_1.sh new file mode 100644 index 000000000..5dec0e2ff --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_1.sh @@ -0,0 +1,6 @@ +gcloud container clusters create "${K8S_CLUSTER_1}" \ + --zone="${K8S_CLUSTER_1_ZONE}" \ + --num-nodes="${K8S_CLUSTER_1_NUMBER_OF_NODES}" \ + --machine-type "${K8S_CLUSTER_1_MACHINE_TYPE}" \ + --tags=mongodb \ + "${GKE_SPOT_INSTANCES_SWITCH:-""}" diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_2.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_2.sh new file mode 100644 index 000000000..294836111 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0010_create_gke_cluster_2.sh @@ -0,0 +1,6 @@ +gcloud container clusters create "${K8S_CLUSTER_2}" \ + --zone="${K8S_CLUSTER_2_ZONE}" \ + --num-nodes="${K8S_CLUSTER_2_NUMBER_OF_NODES}" \ + --machine-type "${K8S_CLUSTER_2_MACHINE_TYPE}" \ + --tags=mongodb \ + "${GKE_SPOT_INSTANCES_SWITCH:-""}" diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0020_get_gke_credentials.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0020_get_gke_credentials.sh new file mode 100644 index 000000000..58dbe0e82 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0020_get_gke_credentials.sh @@ -0,0 +1,3 @@ +gcloud container clusters get-credentials "${K8S_CLUSTER_0}" --zone="${K8S_CLUSTER_0_ZONE}" +gcloud container clusters get-credentials "${K8S_CLUSTER_1}" --zone="${K8S_CLUSTER_1_ZONE}" +gcloud container clusters get-credentials "${K8S_CLUSTER_2}" --zone="${K8S_CLUSTER_2_ZONE}" diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0030_verify_access_to_clusters.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0030_verify_access_to_clusters.sh new file mode 100644 index 000000000..e8d4b58eb --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/0030_verify_access_to_clusters.sh @@ -0,0 +1,6 @@ +echo "Nodes in cluster ${K8S_CLUSTER_0_CONTEXT_NAME}" +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" get nodes +echo; echo "Nodes in cluster ${K8S_CLUSTER_1_CONTEXT_NAME}" +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" get nodes +echo; echo "Nodes in cluster ${K8S_CLUSTER_2_CONTEXT_NAME}" +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" get nodes diff --git a/public/architectures/setup-multi-cluster/setup-gke/code_snippets/9010_delete_gke_clusters.sh b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/9010_delete_gke_clusters.sh new file mode 100755 index 000000000..2f6da78e1 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/code_snippets/9010_delete_gke_clusters.sh @@ -0,0 +1,4 @@ +yes | gcloud container clusters delete "${K8S_CLUSTER_0}" --zone="${K8S_CLUSTER_0_ZONE}" & +yes | gcloud container clusters delete "${K8S_CLUSTER_1}" --zone="${K8S_CLUSTER_1_ZONE}" & +yes | gcloud container clusters delete "${K8S_CLUSTER_2}" --zone="${K8S_CLUSTER_2_ZONE}" & +wait diff --git a/public/architectures/setup-multi-cluster/setup-gke/env_variables.sh b/public/architectures/setup-multi-cluster/setup-gke/env_variables.sh new file mode 100755 index 000000000..09ae179c6 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/env_variables.sh @@ -0,0 +1,26 @@ +# GCP project name - this is the only parameter that is mandatory to change here +export MDB_GKE_PROJECT="${MDB_GKE_PROJECT:="### Set your GKE project name here ###"}" + +# Adjust the values for each Kubernetes cluster in your deployment. +# The deployment script references the following variables to get values for each cluster. +export K8S_CLUSTER_0="k8s-mdb-0${K8S_CLUSTER_SUFFIX:-""}" +export K8S_CLUSTER_0_ZONE="europe-central2-a" +export K8S_CLUSTER_0_NUMBER_OF_NODES=3 +export K8S_CLUSTER_0_MACHINE_TYPE="e2-standard-4" +export K8S_CLUSTER_0_CONTEXT_NAME="gke_${MDB_GKE_PROJECT}_${K8S_CLUSTER_0_ZONE}_${K8S_CLUSTER_0}" + +export K8S_CLUSTER_1="k8s-mdb-1${K8S_CLUSTER_SUFFIX:-""}" +export K8S_CLUSTER_1_ZONE="europe-central2-b" +export K8S_CLUSTER_1_NUMBER_OF_NODES=3 +export K8S_CLUSTER_1_MACHINE_TYPE="e2-standard-4" +export K8S_CLUSTER_1_CONTEXT_NAME="gke_${MDB_GKE_PROJECT}_${K8S_CLUSTER_1_ZONE}_${K8S_CLUSTER_1}" + +export K8S_CLUSTER_2="k8s-mdb-2${K8S_CLUSTER_SUFFIX:-""}" +export K8S_CLUSTER_2_ZONE="europe-central2-c" +export K8S_CLUSTER_2_NUMBER_OF_NODES=1 +export K8S_CLUSTER_2_MACHINE_TYPE="e2-standard-4" +export K8S_CLUSTER_2_CONTEXT_NAME="gke_${MDB_GKE_PROJECT}_${K8S_CLUSTER_2_ZONE}_${K8S_CLUSTER_2}" + +# Comment out the following line so that the script does not create preemptible nodes. +# DO NOT USE preemptible nodes in production. +export GKE_SPOT_INSTANCES_SWITCH="--preemptible" diff --git a/public/architectures/setup-multi-cluster/setup-gke/output/0030_verify_access_to_clusters.out b/public/architectures/setup-multi-cluster/setup-gke/output/0030_verify_access_to_clusters.out new file mode 100644 index 000000000..7fd821bdc --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/output/0030_verify_access_to_clusters.out @@ -0,0 +1,15 @@ +Nodes in cluster gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a +NAME STATUS ROLES AGE VERSION +gke-k8s-mdb-0-67d0389d75-default-pool-bd2d7e42-99mr Ready 30s v1.31.5-gke.1233000 +gke-k8s-mdb-0-67d0389d75-default-pool-bd2d7e42-d3ql Ready 30s v1.31.5-gke.1233000 +gke-k8s-mdb-0-67d0389d75-default-pool-bd2d7e42-stxx Ready 29s v1.31.5-gke.1233000 + +Nodes in cluster gke_scratch-kubernetes-team_europe-central2-b_k8s-mdb-1-67d0389d75b70a0007e5894a +NAME STATUS ROLES AGE VERSION +gke-k8s-mdb-1-67d0389d75-default-pool-c4129558-10n1 Ready 76s v1.31.5-gke.1233000 +gke-k8s-mdb-1-67d0389d75-default-pool-c4129558-gdrg Ready 76s v1.31.5-gke.1233000 +gke-k8s-mdb-1-67d0389d75-default-pool-c4129558-jbgt Ready 76s v1.31.5-gke.1233000 + +Nodes in cluster gke_scratch-kubernetes-team_europe-central2-c_k8s-mdb-2-67d0389d75b70a0007e5894a +NAME STATUS ROLES AGE VERSION +gke-k8s-mdb-2-67d0389d75-default-pool-0f8822ab-f617 Ready 56s v1.31.5-gke.1233000 diff --git a/public/architectures/setup-multi-cluster/setup-gke/teardown.sh b/public/architectures/setup-multi-cluster/setup-gke/teardown.sh new file mode 100755 index 000000000..155327fc8 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/teardown.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9010_delete_gke_clusters.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-gke/test.sh b/public/architectures/setup-multi-cluster/setup-gke/test.sh new file mode 100755 index 000000000..040e2092a --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-gke/test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 0005_gcloud_set_current_project.sh +run 0010_create_gke_cluster_0.sh & +run 0010_create_gke_cluster_1.sh & +run 0010_create_gke_cluster_2.sh & +wait + +run 0020_get_gke_credentials.sh +run_for_output 0030_verify_access_to_clusters.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-istio/code_snippets/0040_install_istio.sh b/public/architectures/setup-multi-cluster/setup-istio/code_snippets/0040_install_istio.sh new file mode 100755 index 000000000..a27cc779e --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-istio/code_snippets/0040_install_istio.sh @@ -0,0 +1,5 @@ +CTX_CLUSTER1=${K8S_CLUSTER_0_CONTEXT_NAME} \ +CTX_CLUSTER2=${K8S_CLUSTER_1_CONTEXT_NAME} \ +CTX_CLUSTER3=${K8S_CLUSTER_2_CONTEXT_NAME} \ +ISTIO_VERSION="1.20.2" \ +./install_istio_separate_network.sh diff --git a/public/architectures/setup-multi-cluster/setup-istio/code_snippets/0050_label_namespaces.sh b/public/architectures/setup-multi-cluster/setup-istio/code_snippets/0050_label_namespaces.sh new file mode 100644 index 000000000..7238e305a --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-istio/code_snippets/0050_label_namespaces.sh @@ -0,0 +1,11 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" label namespace "${OPERATOR_NAMESPACE}" istio-injection=enabled --overwrite +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" label namespace "${OPERATOR_NAMESPACE}" istio-injection=enabled --overwrite +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" label namespace "${OPERATOR_NAMESPACE}" istio-injection=enabled --overwrite + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" label namespace "${OM_NAMESPACE}" istio-injection=enabled --overwrite +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" label namespace "${OM_NAMESPACE}" istio-injection=enabled --overwrite +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" label namespace "${OM_NAMESPACE}" istio-injection=enabled --overwrite + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" label namespace "${MDB_NAMESPACE}" istio-injection=enabled --overwrite +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" label namespace "${MDB_NAMESPACE}" istio-injection=enabled --overwrite +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" label namespace "${MDB_NAMESPACE}" istio-injection=enabled --overwrite diff --git a/public/architectures/setup-multi-cluster/setup-istio/install_istio_separate_network.sh b/public/architectures/setup-multi-cluster/setup-istio/install_istio_separate_network.sh new file mode 100755 index 000000000..12f063bc1 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-istio/install_istio_separate_network.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash + +# This script is an adjusted version of the official Istio guide: +# https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/ +# The script requires setting the following env variables: +# - CTX_CLUSTER1 +# - CTX_CLUSTER2 +# - CTX_CLUSTER3 + +set -eux + +export ISTIO_VERSION=${ISTIO_VERSION:-1.20.2} + +if [[ ! -d istio-${ISTIO_VERSION} ]]; then + # download Istio under the path + curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${ISTIO_VERSION} sh - +fi + +# checks if external IP has been assigned to a service object, in our case we are interested in east-west gateway +function_check_external_ip_assigned() { + while : ; do + ip=$(kubectl --context="$1" get svc istio-eastwestgateway -n istio-system --output jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [ -n "${ip}" ] + then + echo "external ip assigned ${ip}" + break + else + echo "waiting for external ip to be assigned" + fi + sleep 1 +done +} + +cd "istio-${ISTIO_VERSION}" + +bin/istioctl uninstall --context="${CTX_CLUSTER1}" --purge -y +bin/istioctl uninstall --context="${CTX_CLUSTER2}" --purge -y +bin/istioctl uninstall --context="${CTX_CLUSTER3}" --purge -y + +kubectl --context="${CTX_CLUSTER1}" delete ns istio-system || true +kubectl --context="${CTX_CLUSTER2}" delete ns istio-system || true +kubectl --context="${CTX_CLUSTER3}" delete ns istio-system || true + +mkdir -p certs +pushd certs + +# create root trust for the clusters +make -f ../tools/certs/Makefile.selfsigned.mk root-ca +make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER1}-cacerts" +make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER2}-cacerts" +make -f ../tools/certs/Makefile.selfsigned.mk "${CTX_CLUSTER3}-cacerts" + +kubectl --context="${CTX_CLUSTER1}" create ns istio-system +kubectl --context="${CTX_CLUSTER1}" create secret generic cacerts -n istio-system \ + --from-file="${CTX_CLUSTER1}/ca-cert.pem" \ + --from-file="${CTX_CLUSTER1}/ca-key.pem" \ + --from-file="${CTX_CLUSTER1}/root-cert.pem" \ + --from-file="${CTX_CLUSTER1}/cert-chain.pem" + +kubectl --context="${CTX_CLUSTER2}" create ns istio-system +kubectl --context="${CTX_CLUSTER2}" create secret generic cacerts -n istio-system \ + --from-file="${CTX_CLUSTER2}/ca-cert.pem" \ + --from-file="${CTX_CLUSTER2}/ca-key.pem" \ + --from-file="${CTX_CLUSTER2}/root-cert.pem" \ + --from-file="${CTX_CLUSTER2}/cert-chain.pem" + +kubectl --context="${CTX_CLUSTER3}" create ns istio-system +kubectl --context="${CTX_CLUSTER3}" create secret generic cacerts -n istio-system \ + --from-file="${CTX_CLUSTER3}/ca-cert.pem" \ + --from-file="${CTX_CLUSTER3}/ca-key.pem" \ + --from-file="${CTX_CLUSTER3}/root-cert.pem" \ + --from-file="${CTX_CLUSTER3}/cert-chain.pem" +popd + +# label namespace in cluster1 +kubectl --context="${CTX_CLUSTER1}" get namespace istio-system && \ + kubectl --context="${CTX_CLUSTER1}" label namespace istio-system topology.istio.io/network=network1 + +cat < cluster1.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + meshConfig: + defaultConfig: + terminationDrainDuration: 30s + proxyMetadata: + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + ISTIO_META_DNS_CAPTURE: "true" + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster1 + network: network1 +EOF +bin/istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml -y +samples/multicluster/gen-eastwest-gateway.sh \ + --mesh mesh1 --cluster cluster1 --network network1 | \ + bin/istioctl --context="${CTX_CLUSTER1}" install -y -f - + + +# check if external IP is assigned to east-west gateway in cluster1 +function_check_external_ip_assigned "${CTX_CLUSTER1}" + + +# expose services in cluster1 +kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ + samples/multicluster/expose-services.yaml + + +kubectl --context="${CTX_CLUSTER2}" get namespace istio-system && \ + kubectl --context="${CTX_CLUSTER2}" label namespace istio-system topology.istio.io/network=network2 + + +cat < cluster2.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + meshConfig: + defaultConfig: + terminationDrainDuration: 30s + proxyMetadata: + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + ISTIO_META_DNS_CAPTURE: "true" + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster2 + network: network2 +EOF + +bin/istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml -y + +samples/multicluster/gen-eastwest-gateway.sh \ + --mesh mesh1 --cluster cluster2 --network network2 | \ + bin/istioctl --context="${CTX_CLUSTER2}" install -y -f - + +# check if external IP is assigned to east-west gateway in cluster2 +function_check_external_ip_assigned "${CTX_CLUSTER2}" + +kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \ + samples/multicluster/expose-services.yaml + +# cluster3 +kubectl --context="${CTX_CLUSTER3}" get namespace istio-system && \ + kubectl --context="${CTX_CLUSTER3}" label namespace istio-system topology.istio.io/network=network3 + +cat < cluster3.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + meshConfig: + defaultConfig: + terminationDrainDuration: 30s + proxyMetadata: + ISTIO_META_DNS_AUTO_ALLOCATE: "true" + ISTIO_META_DNS_CAPTURE: "true" + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster3 + network: network3 +EOF + +bin/istioctl install --context="${CTX_CLUSTER3}" -f cluster3.yaml -y + +samples/multicluster/gen-eastwest-gateway.sh \ + --mesh mesh1 --cluster cluster3 --network network3 | \ + bin/istioctl --context="${CTX_CLUSTER3}" install -y -f - + + +# check if external IP is assigned to east-west gateway in cluster3 +function_check_external_ip_assigned "${CTX_CLUSTER3}" + +kubectl --context="${CTX_CLUSTER3}" apply -n istio-system -f \ + samples/multicluster/expose-services.yaml + + +# enable endpoint discovery +bin/istioctl create-remote-secret \ + --context="${CTX_CLUSTER1}" \ + -n istio-system \ + --name=cluster1 | \ + kubectl apply -f - --context="${CTX_CLUSTER2}" + +bin/istioctl create-remote-secret \ + --context="${CTX_CLUSTER1}" \ + -n istio-system \ + --name=cluster1 | \ + kubectl apply -f - --context="${CTX_CLUSTER3}" + +bin/istioctl create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + -n istio-system \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" + +bin/istioctl create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + -n istio-system \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER3}" + +bin/istioctl create-remote-secret \ + --context="${CTX_CLUSTER3}" \ + -n istio-system \ + --name=cluster3 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" + +bin/istioctl create-remote-secret \ + --context="${CTX_CLUSTER3}" \ + -n istio-system \ + --name=cluster3 | \ + kubectl apply -f - --context="${CTX_CLUSTER2}" + + # cleanup: delete the istio repo at the end +cd .. +#rm -r istio-${ISTIO_VERSION} +#rm -f cluster1.yaml cluster2.yaml cluster3.yaml diff --git a/public/architectures/setup-multi-cluster/setup-istio/test.sh b/public/architectures/setup-multi-cluster/setup-istio/test.sh new file mode 100755 index 000000000..9d874fdd0 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-istio/test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 0040_install_istio.sh +run 0050_label_namespaces.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0045_create_namespaces.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0045_create_namespaces.sh new file mode 100755 index 000000000..86155a11c --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0045_create_namespaces.sh @@ -0,0 +1,11 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create namespace "${OPERATOR_NAMESPACE}" +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" create namespace "${OPERATOR_NAMESPACE}" +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" create namespace "${OPERATOR_NAMESPACE}" + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create namespace "${OM_NAMESPACE}" +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" create namespace "${OM_NAMESPACE}" +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" create namespace "${OM_NAMESPACE}" + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create namespace "${MDB_NAMESPACE}" +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" create namespace "${MDB_NAMESPACE}" +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" create namespace "${MDB_NAMESPACE}" diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0046_create_image_pull_secrets.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0046_create_image_pull_secrets.sh new file mode 100755 index 000000000..f9065c000 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0046_create_image_pull_secrets.sh @@ -0,0 +1,15 @@ +mkdir -p secrets + +kubectl create secret generic "image-registries-secret" \ + --from-file=.dockerconfigjson="${HOME}/.docker/config.json" --type=kubernetes.io/dockerconfigjson \ + --dry-run=client -o yaml > secrets/image-registries-secret.yaml + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OPERATOR_NAMESPACE}" apply -f secrets/image-registries-secret.yaml + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OM_NAMESPACE}" apply -f secrets/image-registries-secret.yaml +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n "${OM_NAMESPACE}" apply -f secrets/image-registries-secret.yaml +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n "${OM_NAMESPACE}" apply -f secrets/image-registries-secret.yaml + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" apply -f secrets/image-registries-secret.yaml +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" apply -f secrets/image-registries-secret.yaml +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n "${MDB_NAMESPACE}" apply -f secrets/image-registries-secret.yaml diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0200_kubectl_mongodb_configure_multi_cluster.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0200_kubectl_mongodb_configure_multi_cluster.sh new file mode 100755 index 000000000..b307f93af --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0200_kubectl_mongodb_configure_multi_cluster.sh @@ -0,0 +1,18 @@ +kubectl mongodb multicluster setup \ + --central-cluster="${K8S_CLUSTER_0_CONTEXT_NAME}" \ + --member-clusters="${K8S_CLUSTER_0_CONTEXT_NAME},${K8S_CLUSTER_1_CONTEXT_NAME},${K8S_CLUSTER_2_CONTEXT_NAME}" \ + --member-cluster-namespace="${OM_NAMESPACE}" \ + --central-cluster-namespace="${OPERATOR_NAMESPACE}" \ + --create-service-account-secrets \ + --install-database-roles=true \ + --image-pull-secrets=image-registries-secret + +kubectl mongodb multicluster setup \ + --central-cluster="${K8S_CLUSTER_0_CONTEXT_NAME}" \ + --member-clusters="${K8S_CLUSTER_0_CONTEXT_NAME},${K8S_CLUSTER_1_CONTEXT_NAME},${K8S_CLUSTER_2_CONTEXT_NAME}" \ + --member-cluster-namespace="${MDB_NAMESPACE}" \ + --central-cluster-namespace="${OPERATOR_NAMESPACE}" \ + --create-service-account-secrets \ + --install-database-roles=true \ + --image-pull-secrets=image-registries-secret + diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0205_helm_configure_repo.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0205_helm_configure_repo.sh new file mode 100644 index 000000000..85b9d219f --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0205_helm_configure_repo.sh @@ -0,0 +1,3 @@ +helm repo add mongodb https://mongodb.github.io/helm-charts +helm repo update mongodb +helm search repo "${OFFICIAL_OPERATOR_HELM_CHART}" diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0210_helm_install_operator.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0210_helm_install_operator.sh new file mode 100755 index 000000000..b27894a0e --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0210_helm_install_operator.sh @@ -0,0 +1,15 @@ +helm upgrade --install \ + --debug \ + --kube-context "${K8S_CLUSTER_0_CONTEXT_NAME}" \ + mongodb-enterprise-operator-multi-cluster \ + "${OPERATOR_HELM_CHART}" \ + --namespace="${OPERATOR_NAMESPACE}" \ + --set namespace="${OPERATOR_NAMESPACE}" \ + --set operator.namespace="${OPERATOR_NAMESPACE}" \ + --set operator.watchNamespace="${OM_NAMESPACE}\,${MDB_NAMESPACE}" \ + --set operator.name=mongodb-enterprise-operator-multi-cluster \ + --set operator.createOperatorServiceAccount=false \ + --set operator.createResourcesServiceAccountsAndRoles=false \ + --set "multiCluster.clusters={${K8S_CLUSTER_0_CONTEXT_NAME},${K8S_CLUSTER_1_CONTEXT_NAME},${K8S_CLUSTER_2_CONTEXT_NAME}}" \ + --set "${OPERATOR_ADDITIONAL_HELM_VALUES:-"dummy=value"}" \ + --set operator.env=dev diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0211_check_operator_deployment.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0211_check_operator_deployment.sh new file mode 100755 index 000000000..553aabf8c --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/0211_check_operator_deployment.sh @@ -0,0 +1,5 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OPERATOR_NAMESPACE}" rollout status deployment/mongodb-enterprise-operator-multi-cluster +echo "Operator deployment in ${OPERATOR_NAMESPACE} namespace" +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OPERATOR_NAMESPACE}" get deployments +echo; echo "Operator pod in ${OPERATOR_NAMESPACE} namespace" +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "${OPERATOR_NAMESPACE}" get pods diff --git a/public/architectures/setup-multi-cluster/setup-operator/code_snippets/9000_delete_namespaces.sh b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/9000_delete_namespaces.sh new file mode 100755 index 000000000..560a62ae1 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/code_snippets/9000_delete_namespaces.sh @@ -0,0 +1,12 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" delete ns "${OM_NAMESPACE}" & +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" delete ns "${OM_NAMESPACE}" & +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" delete ns "${OM_NAMESPACE}" & + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" delete ns "${OPERATOR_NAMESPACE}" & +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" delete ns "${OPERATOR_NAMESPACE}" & +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" delete ns "${OPERATOR_NAMESPACE}" & + +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" delete ns "${MDB_NAMESPACE}" & +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" delete ns "${MDB_NAMESPACE}" & +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" delete ns "${MDB_NAMESPACE}" & +wait diff --git a/public/architectures/setup-multi-cluster/setup-operator/env_variables.sh b/public/architectures/setup-multi-cluster/setup-operator/env_variables.sh new file mode 100644 index 000000000..5b1546094 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/env_variables.sh @@ -0,0 +1,12 @@ +# Namespace in which Ops Manager and AppDB will be deployed +export OM_NAMESPACE="mongodb-om" +# Namespace in which the operator will be installed +export OPERATOR_NAMESPACE="mongodb-operator" +# Namespace in which MongoDB resources will be deployed +export MDB_NAMESPACE="mongodb" + +# comma-separated key=value pairs for additional parameters passed to the helm-chart installing the operator +export OPERATOR_ADDITIONAL_HELM_VALUES="${OPERATOR_ADDITIONAL_HELM_VALUES:-""}" + +export OFFICIAL_OPERATOR_HELM_CHART="mongodb/enterprise-operator" +export OPERATOR_HELM_CHART="${OPERATOR_HELM_CHART:-${OFFICIAL_OPERATOR_HELM_CHART}}" diff --git a/public/architectures/setup-multi-cluster/setup-operator/output/0200_kubectl_mongodb_configure_multi_cluster.out b/public/architectures/setup-multi-cluster/setup-operator/output/0200_kubectl_mongodb_configure_multi_cluster.out new file mode 100644 index 000000000..68afb67f1 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/output/0200_kubectl_mongodb_configure_multi_cluster.out @@ -0,0 +1,44 @@ + +Build: , +Ensured namespaces exist in all clusters. +creating central cluster roles in cluster: gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +creating member roles in cluster: gke_scratch-kubernetes-team_europe-central2-b_k8s-mdb-1-67d0389d75b70a0007e5894a +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +creating member roles in cluster: gke_scratch-kubernetes-team_europe-central2-c_k8s-mdb-2-67d0389d75b70a0007e5894a +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +Ensured ServiceAccounts and Roles. +Creating KubeConfig secret mongodb-operator/mongodb-enterprise-operator-multi-cluster-kubeconfig in cluster gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a +Ensured database Roles in member clusters. +Creating Member list Configmap mongodb-operator/mongodb-enterprise-operator-member-list in cluster gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a + +Build: , +Ensured namespaces exist in all clusters. +creating central cluster roles in cluster: gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +creating member roles in cluster: gke_scratch-kubernetes-team_europe-central2-b_k8s-mdb-1-67d0389d75b70a0007e5894a +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +creating member roles in cluster: gke_scratch-kubernetes-team_europe-central2-c_k8s-mdb-2-67d0389d75b70a0007e5894a +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +created clusterrole: mongodb-enterprise-operator-multi-cluster-role-telemetry +created clusterrolebinding: mongodb-enterprise-operator-multi-telemetry-cluster-role-binding +Ensured ServiceAccounts and Roles. +Creating KubeConfig secret mongodb-operator/mongodb-enterprise-operator-multi-cluster-kubeconfig in cluster gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a +Ensured database Roles in member clusters. +Creating Member list Configmap mongodb-operator/mongodb-enterprise-operator-member-list in cluster gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a diff --git a/public/architectures/setup-multi-cluster/setup-operator/output/0205_helm_configure_repo.out b/public/architectures/setup-multi-cluster/setup-operator/output/0205_helm_configure_repo.out new file mode 100644 index 000000000..b52c2e0ed --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/output/0205_helm_configure_repo.out @@ -0,0 +1,6 @@ +"mongodb" has been added to your repositories +Hang tight while we grab the latest from your chart repositories... +...Successfully got an update from the "mongodb" chart repository +Update Complete. ⎈Happy Helming!⎈ +NAME CHART VERSION APP VERSION DESCRIPTION +mongodb/enterprise-operator 1.32.0 MongoDB Kubernetes Enterprise Operator diff --git a/public/architectures/setup-multi-cluster/setup-operator/output/0210_helm_install_operator.out b/public/architectures/setup-multi-cluster/setup-operator/output/0210_helm_install_operator.out new file mode 100644 index 000000000..5b30862ed --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/output/0210_helm_install_operator.out @@ -0,0 +1,311 @@ +Release "mongodb-enterprise-operator-multi-cluster" does not exist. Installing it now. +NAME: mongodb-enterprise-operator-multi-cluster +LAST DEPLOYED: Tue Mar 11 13:36:49 2025 +NAMESPACE: mongodb-operator +STATUS: deployed +REVISION: 1 +TEST SUITE: None +USER-SUPPLIED VALUES: +dummy: value +multiCluster: + clusters: + - gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a + - gke_scratch-kubernetes-team_europe-central2-b_k8s-mdb-1-67d0389d75b70a0007e5894a + - gke_scratch-kubernetes-team_europe-central2-c_k8s-mdb-2-67d0389d75b70a0007e5894a +namespace: mongodb-operator +operator: + createOperatorServiceAccount: false + createResourcesServiceAccountsAndRoles: false + env: dev + mdbDefaultArchitecture: static + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb-operator + watchNamespace: mongodb-om,mongodb + +COMPUTED VALUES: +agent: + name: mongodb-agent-ubi + version: 108.0.2.8729-1 +database: + name: mongodb-enterprise-database-ubi + version: 1.32.0 +dummy: value +initAppDb: + name: mongodb-enterprise-init-appdb-ubi + version: 1.32.0 +initDatabase: + name: mongodb-enterprise-init-database-ubi + version: 1.32.0 +initOpsManager: + name: mongodb-enterprise-init-ops-manager-ubi + version: 1.32.0 +managedSecurityContext: false +mongodb: + appdbAssumeOldFormat: false + imageType: ubi8 + name: mongodb-enterprise-server + repo: quay.io/mongodb +mongodbLegacyAppDb: + name: mongodb-enterprise-appdb-database-ubi + repo: quay.io/mongodb +multiCluster: + clusterClientTimeout: 10 + clusters: + - gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a + - gke_scratch-kubernetes-team_europe-central2-b_k8s-mdb-1-67d0389d75b70a0007e5894a + - gke_scratch-kubernetes-team_europe-central2-c_k8s-mdb-2-67d0389d75b70a0007e5894a + kubeConfigSecretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + performFailOver: true +namespace: mongodb-operator +operator: + additionalArguments: [] + affinity: {} + createOperatorServiceAccount: false + createResourcesServiceAccountsAndRoles: false + deployment_name: mongodb-enterprise-operator + enablePVCResize: true + env: dev + maxConcurrentReconciles: 1 + mdbDefaultArchitecture: static + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb-operator + nodeSelector: {} + operator_image_name: mongodb-enterprise-operator-ubi + replicas: 1 + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + telemetry: + collection: + clusters: {} + deployments: {} + frequency: 1h + operators: {} + send: + frequency: 168h + tolerations: [] + vaultSecretBackend: + enabled: false + tlsSecretRef: "" + version: 1.32.0 + watchNamespace: mongodb-om,mongodb + watchedResources: + - mongodb + - opsmanagers + - mongodbusers + webhook: + installClusterRole: true + registerConfiguration: true +opsManager: + name: mongodb-enterprise-ops-manager-ubi +registry: + agent: quay.io/mongodb + appDb: quay.io/mongodb + database: quay.io/mongodb + imagePullSecrets: null + initAppDb: quay.io/mongodb + initDatabase: quay.io/mongodb + initOpsManager: quay.io/mongodb + operator: quay.io/mongodb + opsManager: quay.io/mongodb + pullPolicy: Always +subresourceEnabled: true + +HOOKS: +MANIFEST: +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster-mongodb-operator-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb-operator +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster-mongodb-operator-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-multi-cluster-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb-operator +--- +# Source: enterprise-operator/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb-operator +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator-multi-cluster + app.kubernetes.io/instance: mongodb-enterprise-operator-multi-cluster + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator-multi-cluster + app.kubernetes.io/instance: mongodb-enterprise-operator-multi-cluster + spec: + serviceAccountName: mongodb-enterprise-operator-multi-cluster + securityContext: + runAsNonRoot: true + runAsUser: 2000 + containers: + - name: mongodb-enterprise-operator-multi-cluster + image: "quay.io/mongodb/mongodb-enterprise-operator-ubi:1.32.0" + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + - -watch-resource=mongodbmulticluster + command: + - /usr/local/bin/mongodb-enterprise-operator + volumeMounts: + - mountPath: /etc/config/kubeconfig + name: kube-config-volume + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: dev + - name: MDB_DEFAULT_ARCHITECTURE + value: static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + value: "mongodb-om,mongodb" + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1h" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/mongodb-enterprise-database-ubi + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-database-ubi + - name: INIT_DATABASE_VERSION + value: 1.32.0 + - name: DATABASE_VERSION + value: 1.32.0 + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi + - name: INIT_OPS_MANAGER_VERSION + value: 1.32.0 + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-appdb-ubi + - name: INIT_APPDB_VERSION + value: 1.32.0 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent-ubi" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: "ubi9" + - name: PERFORM_FAILOVER + value: 'true' + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "1" + volumes: + - name: kube-config-volume + secret: + defaultMode: 420 + secretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + diff --git a/public/architectures/setup-multi-cluster/setup-operator/output/0211_check_operator_deployment.out b/public/architectures/setup-multi-cluster/setup-operator/output/0211_check_operator_deployment.out new file mode 100644 index 000000000..87ab6d3fa --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/output/0211_check_operator_deployment.out @@ -0,0 +1,9 @@ +Waiting for deployment "mongodb-enterprise-operator-multi-cluster" rollout to finish: 0 of 1 updated replicas are available... +deployment "mongodb-enterprise-operator-multi-cluster" successfully rolled out +Operator deployment in mongodb-operator namespace +NAME READY UP-TO-DATE AVAILABLE AGE +mongodb-enterprise-operator-multi-cluster 1/1 1 1 9s + +Operator pod in mongodb-operator namespace +NAME READY STATUS RESTARTS AGE +mongodb-enterprise-operator-multi-cluster-786c8fcd9b-9k465 2/2 Running 1 (3s ago) 10s diff --git a/public/architectures/setup-multi-cluster/setup-operator/teardown.sh b/public/architectures/setup-multi-cluster/setup-operator/teardown.sh new file mode 100755 index 000000000..faef13d92 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/teardown.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 9000_delete_namespaces.sh + +popd diff --git a/public/architectures/setup-multi-cluster/setup-operator/test.sh b/public/architectures/setup-multi-cluster/setup-operator/test.sh new file mode 100755 index 000000000..12cbd5f47 --- /dev/null +++ b/public/architectures/setup-multi-cluster/setup-operator/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 0045_create_namespaces.sh +run 0046_create_image_pull_secrets.sh + +run_for_output 0200_kubectl_mongodb_configure_multi_cluster.sh +run_for_output 0205_helm_configure_repo.sh +run_for_output 0210_helm_install_operator.sh +run_for_output 0211_check_operator_deployment.sh + +popd diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0045_create_connectivity_test_namespaces.sh b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0045_create_connectivity_test_namespaces.sh new file mode 100644 index 000000000..0702eba74 --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0045_create_connectivity_test_namespaces.sh @@ -0,0 +1,8 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" create namespace "connectivity-test" +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" label namespace "connectivity-test" istio-injection=enabled --overwrite + +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" create namespace "connectivity-test" +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" label namespace "connectivity-test" istio-injection=enabled --overwrite + +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" create namespace "connectivity-test" +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" label namespace "connectivity-test" istio-injection=enabled --overwrite diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0050_check_cluster_connectivity_create_sts_0.sh b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0050_check_cluster_connectivity_create_sts_0.sh new file mode 100755 index 000000000..e1adc750b --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0050_check_cluster_connectivity_create_sts_0.sh @@ -0,0 +1,22 @@ +kubectl apply --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "connectivity-test" -f - <&1); + +if grep "Hostname: ${target_pod}" &>/dev/null <<< "${out}" +then + echo "SUCCESS" +else + echo "ERROR: ${out}" + return 1 +fi diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.sh b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.sh new file mode 100755 index 000000000..c70f10b6e --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.sh @@ -0,0 +1,15 @@ +source_cluster=${K8S_CLUSTER_0_CONTEXT_NAME} +target_pod="echoserver1-0" +source_pod="echoserver0-0" +target_url="http://${target_pod}.connectivity-test.svc.cluster.local:8080" +echo "Checking cross-cluster DNS resolution and connectivity from ${source_pod} in ${source_cluster} to ${target_pod}" +out=$(kubectl exec --context "${source_cluster}" -n "connectivity-test" "${source_pod}" -- \ + /bin/bash -c "curl -v ${target_url}" 2>&1); + +if grep "Hostname: ${target_pod}" &>/dev/null <<< "${out}" +then + echo "SUCCESS" +else + echo "ERROR: ${out}" + return 1 +fi diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.sh b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.sh new file mode 100755 index 000000000..1c1e0b6b6 --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.sh @@ -0,0 +1,15 @@ +source_cluster=${K8S_CLUSTER_2_CONTEXT_NAME} +target_pod="echoserver1-0" +source_pod="echoserver2-0" +target_url="http://${target_pod}.connectivity-test.svc.cluster.local:8080" +echo "Checking cross-cluster DNS resolution and connectivity from ${source_pod} in ${source_cluster} to ${target_pod}" +out=$(kubectl exec --context "${source_cluster}" -n "connectivity-test" "${source_pod}" -- \ + /bin/bash -c "curl -v ${target_url}" 2>&1); + +if grep "Hostname: ${target_pod}" &>/dev/null <<< "${out}" +then + echo "SUCCESS" +else + echo "ERROR: ${out}" + return 1 +fi diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.sh b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.sh new file mode 100755 index 000000000..1b608fe0c --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.sh @@ -0,0 +1,15 @@ +source_cluster=${K8S_CLUSTER_0_CONTEXT_NAME} +target_pod="echoserver2-0" +source_pod="echoserver0-0" +target_url="http://${target_pod}.connectivity-test.svc.cluster.local:8080" +echo "Checking cross-cluster DNS resolution and connectivity from ${source_pod} in ${source_cluster} to ${target_pod}" +out=$(kubectl exec --context "${source_cluster}" -n "connectivity-test" "${source_pod}" -- \ + /bin/bash -c "curl -v ${target_url}" 2>&1); + +if grep "Hostname: ${target_pod}" &>/dev/null <<< "${out}" +then + echo "SUCCESS" +else + echo "ERROR: ${out}" + return 1 +fi diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0100_check_cluster_connectivity_cleanup.sh b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0100_check_cluster_connectivity_cleanup.sh new file mode 100755 index 000000000..d8966574d --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/code_snippets/0100_check_cluster_connectivity_cleanup.sh @@ -0,0 +1,12 @@ +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "connectivity-test" delete statefulset echoserver0 +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n "connectivity-test" delete statefulset echoserver1 +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n "connectivity-test" delete statefulset echoserver2 +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "connectivity-test" delete service echoserver +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n "connectivity-test" delete service echoserver +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n "connectivity-test" delete service echoserver +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" -n "connectivity-test" delete service echoserver0-0 +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" -n "connectivity-test" delete service echoserver1-0 +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" -n "connectivity-test" delete service echoserver2-0 +kubectl --context "${K8S_CLUSTER_0_CONTEXT_NAME}" delete ns "connectivity-test" +kubectl --context "${K8S_CLUSTER_1_CONTEXT_NAME}" delete ns "connectivity-test" +kubectl --context "${K8S_CLUSTER_2_CONTEXT_NAME}" delete ns "connectivity-test" diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_0_0_from_cluster_1.out b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_0_0_from_cluster_1.out new file mode 100644 index 000000000..1298e195b --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_0_0_from_cluster_1.out @@ -0,0 +1,2 @@ +Checking cross-cluster DNS resolution and connectivity from echoserver1-0 in gke_scratch-kubernetes-team_europe-central2-b_k8s-mdb-1-67d0389d75b70a0007e5894a to echoserver0-0 +SUCCESS diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.out b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.out new file mode 100644 index 000000000..c3abde152 --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.out @@ -0,0 +1,2 @@ +Checking cross-cluster DNS resolution and connectivity from echoserver0-0 in gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a to echoserver1-0 +SUCCESS diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.out b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.out new file mode 100644 index 000000000..2d957f000 --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.out @@ -0,0 +1,2 @@ +Checking cross-cluster DNS resolution and connectivity from echoserver2-0 in gke_scratch-kubernetes-team_europe-central2-c_k8s-mdb-2-67d0389d75b70a0007e5894a to echoserver1-0 +SUCCESS diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.out b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.out new file mode 100644 index 000000000..710747294 --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/output/0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.out @@ -0,0 +1,2 @@ +Checking cross-cluster DNS resolution and connectivity from echoserver0-0 in gke_scratch-kubernetes-team_europe-central2-a_k8s-mdb-0-67d0389d75b70a0007e5894a to echoserver2-0 +SUCCESS diff --git a/public/architectures/setup-multi-cluster/verify-connectivity/test.sh b/public/architectures/setup-multi-cluster/verify-connectivity/test.sh new file mode 100755 index 000000000..543ffab2e --- /dev/null +++ b/public/architectures/setup-multi-cluster/verify-connectivity/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -eou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source scripts/code_snippets/sample_test_runner.sh + +pushd "${script_dir}" + +prepare_snippets + +run 0045_create_connectivity_test_namespaces.sh + +run 0050_check_cluster_connectivity_create_sts_0.sh +run 0050_check_cluster_connectivity_create_sts_1.sh +run 0050_check_cluster_connectivity_create_sts_2.sh + +run 0060_check_cluster_connectivity_wait_for_sts.sh +run 0070_check_cluster_connectivity_create_pod_service_0.sh +run 0070_check_cluster_connectivity_create_pod_service_1.sh +run 0070_check_cluster_connectivity_create_pod_service_2.sh +run 0080_check_cluster_connectivity_create_round_robin_service_0.sh +run 0080_check_cluster_connectivity_create_round_robin_service_1.sh +run 0080_check_cluster_connectivity_create_round_robin_service_2.sh +run_for_output 0090_check_cluster_connectivity_verify_pod_0_0_from_cluster_1.sh +run_for_output 0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_0.sh +run_for_output 0090_check_cluster_connectivity_verify_pod_1_0_from_cluster_2.sh +run_for_output 0090_check_cluster_connectivity_verify_pod_2_0_from_cluster_0.sh +run 0100_check_cluster_connectivity_cleanup.sh + +popd diff --git a/public/crds.yaml b/public/crds.yaml new file mode 100644 index 000000000..b4ebac0c1 --- /dev/null +++ b/public/crds.yaml @@ -0,0 +1,5865 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodb.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDB + listKind: MongoDBList + plural: mongodb + shortNames: + - mdb + singular: mongodb + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment. + jsonPath: .status.phase + name: Phase + type: string + - description: Version of MongoDB server. + jsonPath: .status.version + name: Version + type: string + - description: The type of MongoDB deployment. One of 'ReplicaSet', 'ShardedCluster' + and 'Standalone'. + jsonPath: .spec.type + name: Type + type: string + - description: The time since the MongoDB resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for the + mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the mongodb + processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + backup: + description: |- + Backup contains configuration options for configuring + backup for this MongoDB resource + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + autoTerminateOnDeletion: + description: |- + AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + when the MongoDB CR is deleted + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + client: + description: KMIP Client configuration + properties: + clientCertificatePrefix: + description: |- + A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + The names are generated using the following pattern: + KMIP Client Certificate (TLS Secret): + --kmip-client + KMIP Client Certificate Password: + --kmip-client-password + The expected key inside is called "password". + type: string + type: object + required: + - client + type: object + type: object + mode: + enum: + - enabled + - disabled + - terminated + type: string + snapshotSchedule: + properties: + clusterCheckpointIntervalMin: + enum: + - 15 + - 30 + - 60 + type: integer + dailySnapshotRetentionDays: + description: Number of days to retain daily snapshots. Setting + 0 will disable this rule. + maximum: 365 + minimum: 0 + type: integer + fullIncrementalDayOfWeek: + description: Day of the week when Ops Manager takes a full + snapshot. This ensures a recent complete backup. Ops Manager + sets the default value to SUNDAY. + enum: + - SUNDAY + - MONDAY + - TUESDAY + - WEDNESDAY + - THURSDAY + - FRIDAY + - SATURDAY + type: string + monthlySnapshotRetentionMonths: + description: Number of months to retain weekly snapshots. + Setting 0 will disable this rule. + maximum: 36 + minimum: 0 + type: integer + pointInTimeWindowHours: + description: Number of hours in the past for which a point-in-time + snapshot can be created. + enum: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 15 + - 30 + - 60 + - 90 + - 120 + - 180 + - 360 + type: integer + referenceHourOfDay: + description: Hour of the day to schedule snapshots using a + 24-hour clock, in UTC. + maximum: 23 + minimum: 0 + type: integer + referenceMinuteOfHour: + description: Minute of the hour to schedule snapshots, in + UTC. + maximum: 59 + minimum: 0 + type: integer + snapshotIntervalHours: + description: Number of hours between snapshots. + enum: + - 6 + - 8 + - 12 + - 24 + type: integer + snapshotRetentionDays: + description: Number of days to keep recent snapshots. + maximum: 365 + minimum: 1 + type: integer + weeklySnapshotRetentionWeeks: + description: Number of weeks to retain weekly snapshots. Setting + 0 will disable this rule + maximum: 365 + minimum: 0 + type: integer + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + format: hostname + type: string + configServerCount: + type: integer + configSrv: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + configSrvPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + duplicateServiceObjects: + description: |- + In few service mesh options for ex: Istio, by default we would need to duplicate the + service objects created per pod in all the clusters to enable DNS resolution. Users can + however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + type: boolean + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing MongoDB + to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added to the + externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + logLevel: + enum: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities and + tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Amount of members for this MongoDB Replica Set + type: integer + mongodsPerShardCount: + type: integer + mongos: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + mongosCount: + type: integer + mongosPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + persistent: + type: boolean + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + prometheus: + description: Prometheus configurations. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth Password. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode that + the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with a + CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + service: + description: |- + DEPRECATED please use `spec.statefulSet.spec.serviceName` to provide a custom service name. + this is an optional service, it will get the name "-service" in case not provided + type: string + shard: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: |- + Configuring logRotation is not allowed under this section. + Please use the most top level resource fields for this; spec.Agent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + shardCount: + type: integer + shardOverrides: + description: |- + ShardOverrides allow for overriding the configuration of a specific shard. + It replaces deprecated spec.shard.shardSpecificPodSpec field. When spec.shard.shardSpecificPodSpec is still defined then + spec.shard.shardSpecificPodSpec is applied first to the particular shard and then spec.shardOverrides is applied on top + of that (if defined for the same shard). + items: + properties: + additionalMongodConfig: + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation + for the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + clusterSpecList: + items: + description: |- + ClusterSpecItemOverride is almost exact copy of ClusterSpecItem object. + The object is used in ClusterSpecList in ShardedClusterComponentOverrideSpec in shard overrides. + The difference lies in some fields being optional, e.g. Members to make it possible to NOT override fields and rely on + what was set in top level shard configuration. + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + type: array + memberConfig: + description: Process configuration override for this shard. + Used in SingleCluster only. The number of items specified + must be >= spec.mongodsPerShardCount or spec.shardOverride.members. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + members: + description: Number of member nodes in this shard. Used only + in SingleCluster. For MultiCluster the number of members is + specified in ShardOverride.ClusterSpecList. + type: integer + podSpec: + description: The following override fields work for SingleCluster + only. For MultiCluster - fields from specific clusters are + used. + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + shardNames: + items: + type: string + minItems: 1 + type: array + statefulSet: + description: Statefulset override for this particular shard. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - shardNames + type: object + type: array + shardPodSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + shardSpecificPodSpec: + description: |- + ShardSpecificPodSpec allows you to provide a Statefulset override per shard. + DEPRECATED please use spec.shard.shardOverrides instead + items: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array + statefulSet: + description: |- + StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + the global one + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of MongoDB resources + It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - credentials + - type + - version + type: object + x-kubernetes-preserve-unknown-fields: true + status: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + configServerCount: + type: integer + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + members: + type: integer + message: + type: string + mongodsPerShardCount: + type: integer + mongosCount: + type: integer + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + shardCount: + type: integer + sizeStatusInClusters: + description: MongodbShardedSizeStatusInClusters describes the number + and sizes of replica sets members deployed across member clusters + properties: + configServerMongodsInClusters: + additionalProperties: + type: integer + type: object + mongosCountInClusters: + additionalProperties: + type: integer + type: object + shardMongodsInClusters: + additionalProperties: + type: integer + type: object + shardOverridesInClusters: + additionalProperties: + additionalProperties: + type: integer + type: object + type: object + type: object + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodbmulticluster.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBMultiCluster + listKind: MongoDBMultiClusterList + plural: mongodbmulticluster + shortNames: + - mdbmc + singular: mongodbmulticluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Current state of the MongoDB deployment. + jsonPath: .status.phase + name: Phase + type: string + - description: The time since the MongoDBMultiCluster resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig is additional configuration that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file before + rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for the + mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the mongodb + processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the BackupAgent + processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + backup: + description: |- + Backup contains configuration options for configuring + backup for this MongoDB resource + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + autoTerminateOnDeletion: + description: |- + AutoTerminateOnDeletion indicates if the Operator should stop and terminate the Backup before the cleanup, + when the MongoDB CR is deleted + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + client: + description: KMIP Client configuration + properties: + clientCertificatePrefix: + description: |- + A prefix used to construct KMIP client certificate (and corresponding password) Secret names. + The names are generated using the following pattern: + KMIP Client Certificate (TLS Secret): + --kmip-client + KMIP Client Certificate Password: + --kmip-client-password + The expected key inside is called "password". + type: string + type: object + required: + - client + type: object + type: object + mode: + enum: + - enabled + - disabled + - terminated + type: string + snapshotSchedule: + properties: + clusterCheckpointIntervalMin: + enum: + - 15 + - 30 + - 60 + type: integer + dailySnapshotRetentionDays: + description: Number of days to retain daily snapshots. Setting + 0 will disable this rule. + maximum: 365 + minimum: 0 + type: integer + fullIncrementalDayOfWeek: + description: Day of the week when Ops Manager takes a full + snapshot. This ensures a recent complete backup. Ops Manager + sets the default value to SUNDAY. + enum: + - SUNDAY + - MONDAY + - TUESDAY + - WEDNESDAY + - THURSDAY + - FRIDAY + - SATURDAY + type: string + monthlySnapshotRetentionMonths: + description: Number of months to retain weekly snapshots. + Setting 0 will disable this rule. + maximum: 36 + minimum: 0 + type: integer + pointInTimeWindowHours: + description: Number of hours in the past for which a point-in-time + snapshot can be created. + enum: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 15 + - 30 + - 60 + - 90 + - 120 + - 180 + - 360 + type: integer + referenceHourOfDay: + description: Hour of the day to schedule snapshots using a + 24-hour clock, in UTC. + maximum: 23 + minimum: 0 + type: integer + referenceMinuteOfHour: + description: Minute of the hour to schedule snapshots, in + UTC. + maximum: 59 + minimum: 0 + type: integer + snapshotIntervalHours: + description: Number of hours between snapshots. + enum: + - 6 + - 8 + - 12 + - 24 + type: integer + snapshotRetentionDays: + description: Number of days to keep recent snapshots. + maximum: 365 + minimum: 1 + type: integer + weeklySnapshotRetentionWeeks: + description: Number of weeks to retain weekly snapshots. Setting + 0 will disable this rule + maximum: 365 + minimum: 0 + type: integer + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + format: hostname + type: string + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added + to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the name + "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + duplicateServiceObjects: + description: |- + In few service mesh options for ex: Istio, by default we would need to duplicate the + service objects created per pod in all the clusters to enable DNS resolution. Users can + however configure their ServiceMesh with DNS proxy(https://istio.io/latest/docs/ops/configuration/traffic-management/dns-proxy/) + enabled in which case the operator doesn't need to create the service objects per cluster. This options tells the operator + whether it should create the service objects in all the clusters or not. By default, if not specified the operator would create the duplicate svc objects. + type: boolean + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing MongoDB + to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added to the + externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + logLevel: + enum: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + type: string + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + persistent: + type: boolean + prometheus: + description: Prometheus configurations. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth Password. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this password. + Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this user's + password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode that + the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with a + CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + statefulSet: + description: |- + StatefulSetConfiguration provides the statefulset override for each of the cluster's statefulset + if "StatefulSetConfiguration" is specified at cluster level under "clusterSpecList" that takes precedence over + the global one + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of MongoDB resources + It defaults (if empty or not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - credentials + - type + - version + type: object + x-kubernetes-preserve-unknown-fields: true + status: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + clusterStatusList: + description: ClusterStatusList holds a list of clusterStatuses corresponding + to each cluster + properties: + clusterStatuses: + items: + description: |- + ClusterStatusItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + lastTransition: + type: string + members: + type: integer + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent + resource which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + warnings: + items: + type: string + type: array + required: + - phase + type: object + type: array + type: object + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: mongodbusers.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBUser + listKind: MongoDBUserList + plural: mongodbusers + shortNames: + - mdbu + singular: mongodbuser + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of the MongoDB User. + jsonPath: .status.phase + name: Phase + type: string + - description: The time since the MongoDB User resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + connectionStringSecretName: + type: string + db: + type: string + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + passwordSecretKeyRef: + description: |- + SecretKeyRef is a reference to a value in a given secret in the same + namespace. Based on: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#secretkeyselector-v1-core + properties: + key: + type: string + name: + type: string + required: + - name + type: object + roles: + items: + properties: + db: + type: string + name: + type: string + required: + - db + - name + type: object + type: array + username: + type: string + required: + - db + - username + type: object + status: + properties: + db: + type: string + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + project: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource which + is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes resource. + Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + roles: + items: + properties: + db: + type: string + name: + type: string + required: + - db + - name + type: object + type: array + username: + type: string + warnings: + items: + type: string + type: array + required: + - db + - phase + - project + - username + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: opsmanagers.mongodb.com +spec: + group: mongodb.com + names: + kind: MongoDBOpsManager + listKind: MongoDBOpsManagerList + plural: opsmanagers + shortNames: + - om + singular: opsmanager + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The number of replicas of MongoDBOpsManager. + jsonPath: .spec.replicas + name: Replicas + type: integer + - description: The version of MongoDBOpsManager. + jsonPath: .spec.version + name: Version + type: string + - description: The current state of the MongoDBOpsManager. + jsonPath: .status.opsManager.phase + name: State (OpsManager) + type: string + - description: The current state of the MongoDBOpsManager Application Database. + jsonPath: .status.applicationDatabase.phase + name: State (AppDB) + type: string + - description: The current state of the MongoDBOpsManager Backup Daemon. + jsonPath: .status.backup.phase + name: State (Backup) + type: string + - description: The time since the MongoDBOpsManager resource was created. + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Warnings. + jsonPath: .status.warnings + name: Warnings + type: string + name: v1 + schema: + openAPIV3Schema: + description: The MongoDBOpsManager resource allows you to deploy Ops Manager + within your Kubernetes cluster + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + adminCredentials: + description: |- + AdminSecret is the secret for the first admin user to create + has the fields: "Username", "Password", "FirstName", "LastName" + type: string + applicationDatabase: + properties: + additionalMongodConfig: + description: |- + AdditionalMongodConfig are additional configurations that can be passed to + each data-bearing mongod at runtime. Uses the same structure as the mongod + configuration file: + https://docs.mongodb.com/manual/reference/configuration-options/ + type: object + x-kubernetes-preserve-unknown-fields: true + agent: + description: specify configuration like startup flags and automation + config settings for the AutomationAgent and MonitoringAgent + properties: + backupAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + logLevel: + type: string + logRotate: + description: DEPRECATED please use mongod.logRotate + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + maxLogFileDurationHours: + type: integer + mongod: + description: AgentLoggingMongodConfig contain settings for + the mongodb processes configured by the agent + properties: + auditlogRotate: + description: LogRotate configures audit log rotation for + the mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + logRotate: + description: LogRotate configures log rotation for the + mongodb processes + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log file + before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + systemLog: + description: SystemLog configures system log of mongod + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + monitoringAgent: + properties: + logRotate: + description: LogRotate configures log rotation for the + BackupAgent processes + properties: + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + OM only supports ints + type: integer + timeThresholdHrs: + description: Number of hours after which this MongoDB + Agent rotates the log file. + type: integer + type: object + type: object + readinessProbe: + properties: + environmentVariables: + additionalProperties: + type: string + type: object + type: object + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + systemLog: + description: DEPRECATED please use mongod.systemLog + properties: + destination: + type: string + logAppend: + type: boolean + path: + type: string + required: + - destination + - logAppend + - path + type: object + type: object + automationConfig: + description: |- + AutomationConfigOverride holds any fields that will be merged on top of the Automation Config + that the operator creates for the AppDB. Currently only the process.disabled and logRotate field is recognized. + properties: + processes: + items: + description: OverrideProcess contains fields that we can + override on the AutomationConfig processes. + properties: + disabled: + type: boolean + logRotate: + description: CrdLogRotate is the crd definition of LogRotate + including fields in strings while the agent supports + them as float64 + properties: + includeAuditLogsWithMongoDBLogs: + description: |- + set to 'true' to have the Automation Agent rotate the audit files along + with mongodb log files + type: boolean + numTotal: + description: maximum number of log files to have + total + type: integer + numUncompressed: + description: maximum number of log files to leave + uncompressed + type: integer + percentOfDiskspace: + description: |- + Maximum percentage of the total disk space these log files should take up. + The string needs to be able to be converted to float64 + type: string + sizeThresholdMB: + description: |- + Maximum size for an individual log file before rotation. + The string needs to be able to be converted to float64. + Fractional values of MB are supported. + type: string + timeThresholdHrs: + description: maximum hours for an individual log + file before rotation + type: integer + required: + - sizeThresholdMB + - timeThresholdHrs + type: object + name: + type: string + required: + - disabled + - name + type: object + type: array + replicaSet: + properties: + settings: + description: |- + MapWrapper is a wrapper for a map to be used by other structs. + The CRD generator does not support map[string]interface{} + on the top level and hence we need to work around this with + a wrapping struct. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + cloudManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + clusterDomain: + type: string + clusterSpecList: + items: + description: |- + ClusterSpecItem is the mongodb multi-cluster spec that is specific to a + particular Kubernetes cluster, this maps to the statefulset created in each cluster + properties: + clusterName: + description: |- + ClusterName is name of the cluster where the MongoDB Statefulset will be scheduled, the + name should have a one on one mapping with the service-account created in the central cluster + to talk to the workload clusters. + type: string + externalAccess: + description: ExternalAccessConfiguration provides external + access configuration for Multi-Cluster. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default + (NodePort) Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be + added to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica + Set + type: integer + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB + resources only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + service: + description: this is an optional service, it will get the + name "-service" in case not provided + type: string + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + connectivity: + properties: + replicaSetHorizons: + description: |- + ReplicaSetHorizons holds list of maps of horizons to be configured in each of MongoDB processes. + Horizons map horizon names to the node addresses for each process in the replicaset, e.g.: + [ + { + "internal": "my-rs-0.my-internal-domain.com:31843", + "external": "my-rs-0.my-external-domain.com:21467" + }, + { + "internal": "my-rs-1.my-internal-domain.com:31843", + "external": "my-rs-1.my-external-domain.com:21467" + }, + ... + ] + The key of each item in the map is an arbitrary, user-chosen string that + represents the name of the horizon. The value of the item is the host and, + optionally, the port that this mongod node will be connected to from. + items: + additionalProperties: + type: string + type: object + type: array + type: object + credentials: + description: Name of the Secret holding credentials information + type: string + externalAccess: + description: ExternalAccessConfiguration provides external access + configuration. + properties: + externalDomain: + description: An external domain that is used for exposing + MongoDB to the outside world. + type: string + externalService: + description: Provides a way to override the default (NodePort) + Service + properties: + annotations: + additionalProperties: + type: string + description: A map of annotations that shall be added + to the externally available Service. + type: object + spec: + description: A wrapper for the Service spec object. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + featureCompatibilityVersion: + type: string + memberConfig: + description: MemberConfig allows to specify votes, priorities + and tags for each of the mongodb process. + items: + properties: + priority: + type: string + tags: + additionalProperties: + type: string + type: object + votes: + type: integer + type: object + type: array + members: + description: Amount of members for this MongoDB Replica Set + maximum: 50 + minimum: 3 + type: integer + monitoringAgent: + description: |- + Specify configuration like startup flags just for the MonitoringAgent. + These take precedence over + the flags set in AutomationAgent + properties: + startupOptions: + additionalProperties: + type: string + description: |- + StartupParameters can be used to configure the startup parameters with which the agent starts. That also contains + log rotation settings as defined here: + type: object + required: + - startupOptions + type: object + opsManager: + properties: + configMapRef: + properties: + name: + type: string + type: object + type: object + passwordSecretKeyRef: + description: |- + PasswordSecretKeyRef contains a reference to the secret which contains the password + for the mongodb-ops-manager SCRAM-SHA user + properties: + key: + type: string + name: + type: string + required: + - name + type: object + podSpec: + properties: + persistence: + description: Note, that this field is used by MongoDB resources + only, let's keep it here for simplicity + properties: + multiple: + properties: + data: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + journal: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + logs: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + single: + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + type: object + podTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + prometheus: + description: Enables Prometheus integration on the AppDB. + properties: + metricsPath: + description: Indicates path to the metrics endpoint. + pattern: ^\/[a-z0-9]+$ + type: string + passwordSecretRef: + description: Name of a Secret containing a HTTP Basic Auth + Password. + properties: + key: + description: Key is the key in the secret storing this + password. Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + port: + description: Port where metrics endpoint will bind to. Defaults + to 9216. + type: integer + tlsSecretKeyRef: + description: |- + Name of a Secret (type kubernetes.io/tls) holding the certificates to use in the + Prometheus endpoint. + properties: + key: + description: Key is the key in the secret storing this + password. Defaults to "password" + type: string + name: + description: Name is the name of the secret storing this + user's password + type: string + required: + - name + type: object + username: + description: HTTP Basic Auth Username for metrics endpoint. + type: string + required: + - passwordSecretRef + - username + type: object + security: + properties: + authentication: + description: |- + Authentication holds various authentication related settings that affect + this MongoDB resource. + properties: + agents: + description: Agents contains authentication configuration + properties for the agents + properties: + automationLdapGroupDN: + type: string + automationPasswordSecretRef: + description: SecretKeySelector selects a key of a + Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + automationUserName: + type: string + clientCertificateSecretRef: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + description: Mode is the desired Authentication mode + that the agents will use + type: string + required: + - mode + type: object + enabled: + type: boolean + ignoreUnknownUsers: + description: IgnoreUnknownUsers maps to the inverse of + auth.authoritativeSet + type: boolean + internalCluster: + type: string + ldap: + description: LDAP Configuration + properties: + authzQueryTemplate: + type: string + bindQueryPasswordSecretRef: + properties: + name: + type: string + required: + - name + type: object + bindQueryUser: + type: string + caConfigMapRef: + description: Allows to point at a ConfigMap/key with + a CA file to mount on the Pod + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + servers: + items: + type: string + type: array + timeoutMS: + type: integer + transportSecurity: + enum: + - tls + - none + type: string + userCacheInvalidationInterval: + type: integer + userToDNMapping: + type: string + validateLDAPServerConfig: + type: boolean + type: object + modes: + items: + enum: + - X509 + - SCRAM + - SCRAM-SHA-1 + - MONGODB-CR + - SCRAM-SHA-256 + - LDAP + type: string + type: array + requireClientTLSAuthentication: + description: Clients should present valid TLS certificates + type: boolean + required: + - enabled + type: object + certsSecretPrefix: + type: string + roles: + items: + properties: + authenticationRestrictions: + items: + properties: + clientSource: + items: + type: string + type: array + serverAddress: + items: + type: string + type: array + type: object + type: array + db: + type: string + privileges: + items: + properties: + actions: + items: + type: string + type: array + resource: + properties: + cluster: + type: boolean + collection: + type: string + db: + type: string + type: object + required: + - actions + - resource + type: object + type: array + role: + type: string + roles: + items: + properties: + db: + type: string + role: + type: string + required: + - db + - role + type: object + type: array + required: + - db + - role + type: object + type: array + tls: + properties: + additionalCertificateDomains: + items: + type: string + type: array + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used to validate the certificates created already. + type: string + enabled: + description: |- + DEPRECATED please enable TLS by setting `security.certsSecretPrefix` or `security.tls.secretRef.prefix`. + Enables TLS for this resource. This will make the operator try to mount a + Secret with a defined name (-cert). + This is only used when enabling TLS on a MongoDB resource, and not on the + AppDB, where TLS is configured by setting `secretRef.Name`. + type: boolean + type: object + type: object + service: + description: this is an optional service, it will get the name + "-svc" in case not provided + type: string + topology: + enum: + - SingleCluster + - MultiCluster + type: string + type: + enum: + - Standalone + - ReplicaSet + - ShardedCluster + type: string + version: + pattern: ^[0-9]+.[0-9]+.[0-9]+(-.+)?$|^$ + type: string + required: + - version + type: object + backup: + description: Backup + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + blockStores: + items: + description: |- + DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores + Optionally references the user if the Mongodb is configured with authentication + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + required: + - mongodbResourceRef + - name + type: object + type: array + enabled: + description: Enabled indicates if Backups will be enabled for + this Ops Manager. + type: boolean + encryption: + description: Encryption settings + properties: + kmip: + description: Kmip corresponds to the KMIP configuration assigned + to the Ops Manager Project's configuration. + properties: + server: + description: KMIP Server configuration + properties: + ca: + description: |- + CA corresponds to a ConfigMap containing an entry for the CA certificate (ca.pem) + used for KMIP authentication + type: string + url: + description: |- + KMIP Server url in the following format: hostname:port + Valid examples are: + 10.10.10.3:5696 + my-kmip-server.mycorp.com:5696 + kmip-svc.svc.cluster.local:5696 + pattern: '[^\:]+:[0-9]{0,5}' + type: string + required: + - ca + - url + type: object + required: + - server + type: object + type: object + externalServiceEnabled: + type: boolean + fileSystemStores: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + headDB: + description: HeadDB specifies configuration options for the HeadDB + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + jvmParameters: + items: + type: string + type: array + logging: + properties: + LogBackAccessRef: + description: LogBackAccessRef points at a ConfigMap/key with + the logback access configuration file to mount on the Pod + properties: + name: + type: string + type: object + LogBackRef: + description: LogBackRef points at a ConfigMap/key with the + logback configuration file to mount on the Pod + properties: + name: + type: string + type: object + type: object + members: + description: Members indicate the number of backup daemon pods + to create. + minimum: 1 + type: integer + opLogStores: + description: OplogStoreConfigs describes the list of oplog store + configs used for backup + items: + description: |- + DataStoreConfig is the description of the config used to reference to database. Reused by Oplog and Block stores + Optionally references the user if the Mongodb is configured with authentication + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + required: + - mongodbResourceRef + - name + type: object + type: array + queryableBackupSecretRef: + description: |- + QueryableBackupSecretRef references the secret which contains the pem file which is used + for queryable backup. This will be mounted into the Ops Manager pod. + properties: + name: + type: string + required: + - name + type: object + s3OpLogStores: + description: S3OplogStoreConfigs describes the list of s3 oplog + store configs used for backup. + items: + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + customCertificate: + description: |- + Set this to "true" to use the appDBCa as a CA to access S3. + Deprecated: This has been replaced by CustomCertificateSecretRefs, + In the future all custom certificates, which includes the appDBCa + for s3Config should be configured in CustomCertificateSecretRefs instead. + type: boolean + customCertificateSecretRefs: + description: |- + CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + that apply to the associated S3 bucket. + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + irsaEnabled: + description: |- + This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + type: boolean + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + pathStyleAccessEnabled: + type: boolean + s3BucketEndpoint: + type: string + s3BucketName: + type: string + s3RegionOverride: + type: string + s3SecretRef: + description: |- + S3SecretRef is the secret that contains the AWS credentials used to access S3 + It is optional because the credentials can be provided via AWS IRSA + properties: + name: + type: string + required: + - name + type: object + required: + - name + - pathStyleAccessEnabled + - s3BucketEndpoint + - s3BucketName + type: object + type: array + s3Stores: + items: + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + customCertificate: + description: |- + Set this to "true" to use the appDBCa as a CA to access S3. + Deprecated: This has been replaced by CustomCertificateSecretRefs, + In the future all custom certificates, which includes the appDBCa + for s3Config should be configured in CustomCertificateSecretRefs instead. + type: boolean + customCertificateSecretRefs: + description: |- + CustomCertificateSecretRefs is a list of valid Certificate Authority certificate secrets + that apply to the associated S3 bucket. + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + irsaEnabled: + description: |- + This is only set to "true" when a user is running in EKS and is using AWS IRSA to configure + S3 snapshot store. For more details refer this: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ + type: boolean + mongodbResourceRef: + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + mongodbUserRef: + properties: + name: + type: string + required: + - name + type: object + name: + type: string + pathStyleAccessEnabled: + type: boolean + s3BucketEndpoint: + type: string + s3BucketName: + type: string + s3RegionOverride: + type: string + s3SecretRef: + description: |- + S3SecretRef is the secret that contains the AWS credentials used to access S3 + It is optional because the credentials can be provided via AWS IRSA + properties: + name: + type: string + required: + - name + type: object + required: + - name + - pathStyleAccessEnabled + - s3BucketEndpoint + - s3BucketName + type: object + type: array + statefulSet: + description: |- + StatefulSetConfiguration holds the optional custom StatefulSet + that should be merged into the operator created one. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - enabled + type: object + clusterDomain: + format: hostname + type: string + clusterName: + description: |- + Deprecated: This has been replaced by the ClusterDomain which should be + used instead + format: hostname + type: string + clusterSpecList: + items: + description: ClusterSpecOMItem defines members cluster details for + Ops Manager multi-cluster deployment. + properties: + backup: + description: |- + Backup contains settings to override from top-level `spec.backup` for this member cluster. + If the value is not set here, then the value is taken from `spec.backup`. + properties: + assignmentLabels: + description: Assignment Labels set in the Ops Manager + items: + type: string + type: array + headDB: + description: HeadDB specifies configuration options for + the HeadDB + properties: + labelSelector: + type: object + x-kubernetes-preserve-unknown-fields: true + storage: + type: string + storageClass: + type: string + type: object + jvmParameters: + items: + type: string + type: array + members: + description: Members indicate the number of backup daemon + pods to create. + minimum: 0 + type: integer + statefulSet: + description: StatefulSetConfiguration specified optional + overrides for backup datemon statefulset. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper + around Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + type: object + clusterDomain: + description: Cluster domain to override the default *.svc.cluster.local + if the default cluster domain has been changed on a cluster + level. + format: hostname + type: string + clusterName: + description: |- + ClusterName is name of the cluster where the Ops Manager Statefulset will be scheduled. + The operator is using ClusterName to find API credentials in `mongodb-enterprise-operator-member-list` config map to use for this member cluster. + If the credentials are not found, then the member cluster is considered unreachable and ignored in the reconcile process. + type: string + configuration: + additionalProperties: + type: string + description: |- + The configuration properties passed to Ops Manager and Backup Daemon in this cluster. + If specified (not empty) then this field overrides `spec.configuration` field entirely. + If not specified, then `spec.configuration` field is used for the Ops Manager and Backup Daemon instances in this cluster. + type: object + externalConnectivity: + description: |- + MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + accessing Ops Manager instances in this member cluster from outside the Kubernetes cluster. + If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be + directly passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this + Service when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to + this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, + this applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + jvmParameters: + description: |- + JVM parameters to pass to Ops Manager and Backup Daemon instances in this member cluster. + If specified (not empty) then this field overrides `spec.jvmParameters` field entirely. + If not specified, then `spec.jvmParameters` field is used for the Ops Manager and Backup Daemon instances in this cluster. + items: + type: string + type: array + members: + description: Number of Ops Manager instances in this member + cluster. + type: integer + statefulSet: + description: |- + Configure custom StatefulSet configuration to override in Ops Manager's statefulset in this member cluster. + If specified (even if provided empty) then this field overrides `spec.externalConnectivity` field entirely. + If not specified, then `spec.externalConnectivity` field is used for the Ops Manager and Backup Daemon instances in this cluster. + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around + Labels and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + required: + - members + type: object + type: array + configuration: + additionalProperties: + type: string + description: The configuration properties passed to Ops Manager/Backup + Daemon + type: object + externalConnectivity: + description: |- + MongoDBOpsManagerExternalConnectivity if sets allows for the creation of a Service for + accessing this Ops Manager resource from outside the Kubernetes cluster. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be directly + passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this Service + when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, this + applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + internalConnectivity: + description: |- + InternalConnectivity if set allows for overriding the settings of the default service + used for internal connectivity to the OpsManager servers. + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a list of annotations to be directly + passed to the Service object. + type: object + clusterIP: + description: ClusterIP IP that will be assigned to this Service + when creating a ClusterIP type Service + type: string + externalTrafficPolicy: + description: |- + ExternalTrafficPolicy mechanism to preserve the client source IP. + Only supported on GCE and Google Kubernetes Engine. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP IP that will be assigned to this LoadBalancer. + type: string + port: + description: Port in which this `Service` will listen to, this + applies to `NodePort`. + format: int32 + type: integer + type: + description: Type of the `Service` to be created. + enum: + - LoadBalancer + - NodePort + - ClusterIP + type: string + required: + - type + type: object + jvmParameters: + description: Custom JVM parameters passed to the Ops Manager JVM + items: + type: string + type: array + logging: + properties: + LogBackAccessRef: + description: LogBackAccessRef points at a ConfigMap/key with the + logback access configuration file to mount on the Pod + properties: + name: + type: string + type: object + LogBackRef: + description: LogBackRef points at a ConfigMap/key with the logback + configuration file to mount on the Pod + properties: + name: + type: string + type: object + type: object + opsManagerURL: + description: |- + OpsManagerURL specified the URL with which the operator and AppDB monitoring agent should access Ops Manager instance (or instances). + When not set, the operator is using FQDN of Ops Manager's headless service `{name}-svc.{namespace}.svc.cluster.local` to connect to the instance. If that URL cannot be used, then URL in this field should be provided for the operator to connect to Ops Manager instances. + type: string + replicas: + minimum: 1 + type: integer + security: + description: Configure HTTPS. + properties: + certsSecretPrefix: + type: string + tls: + properties: + ca: + type: string + secretRef: + properties: + name: + type: string + required: + - name + type: object + type: object + type: object + statefulSet: + description: Configure custom StatefulSet configuration + properties: + metadata: + description: StatefulSetMetadataWrapper is a wrapper around Labels + and Annotations + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + topology: + description: |- + Topology sets the desired cluster topology of Ops Manager deployment. + It defaults (and if not set) to SingleCluster. If MultiCluster specified, + then clusterSpecList field is mandatory and at least one member cluster has to be specified. + enum: + - SingleCluster + - MultiCluster + type: string + version: + type: string + required: + - applicationDatabase + - version + type: object + status: + properties: + applicationDatabase: + properties: + backup: + properties: + statusName: + type: string + required: + - statusName + type: object + clusterStatusList: + items: + properties: + clusterName: + type: string + members: + type: integer + type: object + type: array + configServerCount: + type: integer + featureCompatibilityVersion: + type: string + lastTransition: + type: string + link: + type: string + members: + type: integer + message: + type: string + mongodsPerShardCount: + type: integer + mongosCount: + type: integer + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + shardCount: + type: integer + sizeStatusInClusters: + description: MongodbShardedSizeStatusInClusters describes the + number and sizes of replica sets members deployed across member + clusters + properties: + configServerMongodsInClusters: + additionalProperties: + type: integer + type: object + mongosCountInClusters: + additionalProperties: + type: integer + type: object + shardMongodsInClusters: + additionalProperties: + type: integer + type: object + shardOverridesInClusters: + additionalProperties: + additionalProperties: + type: integer + type: object + type: object + type: object + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + - version + type: object + backup: + properties: + clusterStatusList: + items: + properties: + clusterName: + type: string + replicas: + type: integer + type: object + type: array + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + type: object + opsManager: + properties: + clusterStatusList: + items: + properties: + clusterName: + type: string + replicas: + type: integer + type: object + type: array + lastTransition: + type: string + message: + type: string + observedGeneration: + format: int64 + type: integer + phase: + type: string + pvc: + items: + properties: + phase: + type: string + statefulsetName: + type: string + required: + - phase + - statefulsetName + type: object + type: array + replicas: + type: integer + resourcesNotReady: + items: + description: ResourceNotReady describes the dependent resource + which is not ready yet + properties: + errors: + items: + properties: + message: + type: string + reason: + type: string + type: object + type: array + kind: + description: ResourceKind specifies a kind of a Kubernetes + resource. Used in status of a Custom Resource + type: string + message: + type: string + name: + type: string + required: + - kind + - name + type: object + type: array + url: + type: string + version: + type: string + warnings: + items: + type: string + type: array + required: + - phase + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/public/docs/assets/image--000.png b/public/docs/assets/image--000.png new file mode 100644 index 000000000..7c2ee0d6e Binary files /dev/null and b/public/docs/assets/image--000.png differ diff --git a/public/docs/assets/image--002.png b/public/docs/assets/image--002.png new file mode 100644 index 000000000..cbebc06e7 Binary files /dev/null and b/public/docs/assets/image--002.png differ diff --git a/public/docs/assets/image--004.png b/public/docs/assets/image--004.png new file mode 100644 index 000000000..96934769b Binary files /dev/null and b/public/docs/assets/image--004.png differ diff --git a/public/docs/assets/image--008.png b/public/docs/assets/image--008.png new file mode 100644 index 000000000..ef776fd77 Binary files /dev/null and b/public/docs/assets/image--008.png differ diff --git a/public/docs/assets/image--014.png b/public/docs/assets/image--014.png new file mode 100644 index 000000000..8d0c75e7a Binary files /dev/null and b/public/docs/assets/image--014.png differ diff --git a/public/docs/assets/image--030.png b/public/docs/assets/image--030.png new file mode 100644 index 000000000..3321b971e Binary files /dev/null and b/public/docs/assets/image--030.png differ diff --git a/public/docs/assets/image--032.png b/public/docs/assets/image--032.png new file mode 100644 index 000000000..510c89a9e Binary files /dev/null and b/public/docs/assets/image--032.png differ diff --git a/public/docs/assets/image--034.png b/public/docs/assets/image--034.png new file mode 100644 index 000000000..e21b43232 Binary files /dev/null and b/public/docs/assets/image--034.png differ diff --git a/public/docs/openshift-marketplace.md b/public/docs/openshift-marketplace.md new file mode 100644 index 000000000..ea639c1a2 --- /dev/null +++ b/public/docs/openshift-marketplace.md @@ -0,0 +1,149 @@ +# OpenShift MongoDB Enterprise Kubernetes Operator +## Operator Service Catalog and Marketplace + +This installation document is a guide for deploying MongoDB Enterprise Kubernetes Operator, Ops Manager and first MongoDB DataBase using OpenShift Operator catalog or Marketplace. + +## Configuring required components + +Step 1: Create a namespace to install MongoDB + +``` +oc create ns mongodb +``` + +Step 2: Install the operator in the cluster in the namespace created above + +![Installed Operators](assets/image--000.png) + +Step 3: Wait for the Operator to be deployed. + +![Operator Installed](assets/image--002.png) + +Step 4: Deploy MongoDB Ops Manager. + +Ops Manager is an Enterprise Control Plane for all your MongoDB Clusters. It is a extensive application and may seem complicated. Please visit [Documentation](https://docs.mongodb.com/kubernetes-operator/stable/om-resources/) to plan and configure production deployments. + +*Only a single Ops Manager deployment is required for all MongoDB clusters in your organization. This step could be skipped if Ops Manager is already deployed. Alternatively [Cloud Manager](https://cloud.mongodb.com) - hosted Ops Manager could be used instead.* + +![Screenshot](assets/image--004.png) + + +To deploy a very simple Ops Manager configuration two steps are required. +1. Create Admin Credential Secret +```bash +create secret generic ops-manager-admin-secret \ +--from-literal=Username="jane.doe@example.com" \ +--from-literal=Password="Passw0rd." \ +--from-literal=FirstName="Jane" \ +--from-literal=LastName="Doe" -n mongodb +``` +2. Deploy Ops Manager instance with CRD +![Screenshot](assets/image--008.png) + +With sample yaml CRD definition + +```yaml +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager + namespace: mongodb +spec: + # the version of Ops Manager to use + version: 4.4.1 + + # the name of the secret containing admin user credentials. + adminCredentials: ops-manager-admin-secret + + externalConnectivity: + type: LoadBalancer + + # the Replica Set backing Ops Manager. + # appDB has the SCRAM-SHA authentication mode always enabled + applicationDatabase: + members: 3 +``` + +Change the `adminCredentials` property to link to the name of the secret created previously. In this example it is ops-manager-admin. + +`Click create.` + +>For more detailed installation visit our blog post: https://www.mongodb.com/blog/post/running-mongodb-ops-manager-in-kubernetes + +Step 7: Verify MongoDB Ops Manager is successfully deployed. Verify Ops Manager resource and ensure that ops-manager resource reached Running state : +`oc describe om ops-manager` + + +>NOTE: Wait for the secret ops-manager-admin-key to be created. It contains Global Admin Programmatic API that will be required in the subsequent steps. We recommend to create new Programmatic API Key scoped to a single Ops Manager Organization https://docs.opsmanager.mongodb.com/rapid/tutorial/manage-programmatic-api-keys/#mms-prog-api-key + + +Please note OpsManager URL exposed by LoadBalancer before moving to the next Section + +## Deploy MongoDB + +In order to create MongoDB Cluster three Kubernetes resources need to be deployed. https://docs.mongodb.com/kubernetes-operator/stable/mdb-resources/ + +1. Kubernetes ConfigMap that contain settings for Operator to connect to Ops Manager + ```bash + os create configmap \ + --from-literal="baseUrl=" \ + --from-literal="projectName=" \ #Optional + --from-literal="orgId=" + ``` + >OpsManagerURL is an Ops Manager url including port (default 8080) noted in Step 7. + +>Documentation: https://docs.mongodb.com/kubernetes-operator/stable/tutorial/create-project-using-configmap/#create-k8s-project + + +2. Kubernetes Secret containing Programmatic API Key to Operator to connect to Ops Manager. +> ops-manager-admin-key secret could be used instead for none production deployments. + +``` +oc -n \ + create secret generic \ + --from-literal="user=" \ + --from-literal="publicApiKey=" + ``` + +For instructions on how to create Ops Manager Organization and Programmatic API Key please refer to documentation: https://docs.mongodb.com/kubernetes-operator/stable/tutorial/create-operator-credentials/#create-k8s-credentials + +3. Deploy Ops Manager +![Deploy MongoDB](assets/image--030.png) + +Click on the first tile to create the MongoDB Deployment Instance + +![Deploy MongoDB](assets/image--032.png) + +* Choose a name for MongoDB cluster 'metadata.name` +* Substitute the values `spec.OpsManager` with a reference to the config map `` +* Substitute the values `spec.credentials` with secret name ``. + +`Click Create. ` + +>For comprehensive Documentation, please visit https://docs.mongodb.com/kubernetes-operator/stable/mdb-resources/ + +### Verify MongoDB cluster is operational + +Verify Status of MongoDB Resource reached ``Running`` state +>Optionally monitor state of pods, sts and services linked to MongoDB CRD +![Deploy MongoDB](assets/image--034.png) + +>Note: MongoDB Enterprise Operator logs are the best source to start troubleshooting any issues with deployments + +### Connect to MongoDB Cluster + +MongoDB Enterprise Operator create Kubernetes Service For each MongoDB deployed using default port 27017. + +` ..svc.` + +MongoDB Connection String could be built using SRV record + +` mongodb+srv://..svc.` + +***In order to connect to MongoDB from outside of OpenShift cluster an ingress route needs to be created manually. Operator does not create ingress or external services.*** + +***MongoDB ReplicaSet External connectivity requires Split Horizon Configuration: [Connect to a MongoDB Database Resource from Outside Kubernetes](https://docs.mongodb.com/kubernetes-operator/stable/tutorial/connect-from-outside-k8s/)*** + +>EXAMPLE +To connect to a sharded cluster resource named shardedcluster, you might use the following connection string:
``mongo --host shardedcluster-mongos-0.shardedcluster-svc.mongodb.svc.cluster.local --port 27017`` + diff --git a/public/docs/upgrading-to-ops-manager-5.md b/public/docs/upgrading-to-ops-manager-5.md new file mode 100644 index 000000000..0c2839ab3 --- /dev/null +++ b/public/docs/upgrading-to-ops-manager-5.md @@ -0,0 +1,125 @@ +# Upgrading to Ops Manager 5.0.0 + +In Ops Manager 5.0.0 the old-style _Personal API Keys_ have been deprecated. In +order to upgrade to Ops Manager 5.0.0 a new set of _Programmatic API Keys_ will +have to be generated, before the upgrade process. + +In this document we explain how to generate a _Programmatic API Key_ using +[mongocli](https://docs.mongodb.com/mongocli/stable/). + +## Obtain your current credentials + +The credentials used by the Operator are stored in a `Secret`. This secret will +be in the same `Namespace` as the Operator and the name will have the following +structure: + +``` +--admin-key +``` + +We can fetch the current credentials in order to use them later with the +following code snippet, make sure you set `om_resource_name` and `namespace` to +the correct values: + +```sh +om_resource_name="ops-manager-resource-name" +namespace="ops-manager-resource-namespace" +public_key=$(kubectl get "secret/${namespace}-${resource_name}-admin-key" -o jsonpath='{.data.user}' | base64 -d) +private_key=$(kubectl get "secret/${namespace}-${resource_name}-admin-key" -o jsonpath='{.data.publicApiKey}' | base64 -d) +``` + +## Using `mongocli` + +We will use [mongocli](https://docs.mongodb.com/mongocli/stable/) to create the +different resources we need: a _whitelist_ and a _programmatic API key_. + +Make sure you visit `mongocli` and install it before proceeding. + +### Configuring `mongocli` + +To configure `mongocli` to be able to talk to Ops Manager: + +``` +mongocli config --service ops-manager +``` + +`mongocli` will ask for your _Ops Manager URL_, _Public API Key_ and _Private +API Key_. Use the ones you fetched in the previous command, they have been +stored in the `public_key` and `private_key` variables for you. + +### Create a Global Access List + +We first need to start creating a [Global Access +List](https://docs.mongodb.com/mongocli/stable/command/mongocli-iam-globalAccessLists-create/#std-label-mongocli-iam-globalAccessLists-create) + +``` +mongocli iam globalAccessLists create --cidr "" --desc "Our first range of allowed IPs" +``` + +You can add as many as you need for your organization. For instance, to allow +access from all the Kubernetes private network: + +``` +mongocli iam globalAccessLists create --cidr "10.0.0.0/8" --desc "Allow access from internal network." +``` + +- Please note: some clusters might use a different network configuration. + Consult you Kubernetes provider or administrator to find out the correct + configuration for your Kubernetes cluster. + +### Create a Programmatic API Key + +A new [Programmatic API +Key](https://docs.mongodb.com/mongocli/stable/command/mongocli-iam-globalApiKeys-create/#std-label-mongocli-iam-globalApiKeys-create) +needs to be created, we will also use `mongocli` for this: + +``` +mongocli iam globalApiKeys create --role GLOBAL_OWNER --desc "New API Key for the Kubernetes Operator" +``` + +The output of this command will be similar to: + +```json +{ + "id": "60ed976ec409d34da670bffe", + "desc": "New programmatic API key for the operator", + "roles": [ + { + "roleName": "GLOBAL_OWNER" + } + ], + "privateKey": "1980bd92-f81a-41e1-b302-a2308fcc450a", + "publicKey": "dhjjpfgf" +} +``` + +Make sure you write down the `privateKey` and `publicKey`. It is not possible to +recover the `privateKey` part at a later stage. + +## Configure the Operator to use the new credentials + +Edit the _API Key_ `Secret` in place with `kubectl edit` or apply the following +yaml segment: + +```sh +cat <--admin-key + namespace: +stringData: + publicApiKey: "" + user: "" +EOF +``` + +- Please note that the returned `publicKey` corresponds to the `user` entry and + the `privateKey` corresponds to the `publicApiKey`. + +# Proceed with the Upgrade + +After the API Key has been changed to a new _Programmatic API Key_ it will be +possible to upgrade the `MongoDBOpsManager` resource to latest version: 5.0.0. diff --git a/public/grafana/sample_dashboard.json b/public/grafana/sample_dashboard.json new file mode 100644 index 000000000..85be3cc78 --- /dev/null +++ b/public/grafana/sample_dashboard.json @@ -0,0 +1,1364 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "iteration": 1650460419048, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "type": "row" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "maxDataPoints": null, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Last \\(not null\\)$/", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "exemplar": true, + "expr": "mongodb_uptimeMillis", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Uptime (minutes)", + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "seriesToRows", + "reducers": [ + "lastNotNull" + ] + } + } + ], + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 5, + "maxDataPoints": null, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "7.5.2", + "repeat": null, + "targets": [ + { + "exemplar": true, + "expr": "mongodb_connections_available", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Available", + "refId": "A" + }, + { + "exemplar": true, + "expr": "mongodb_connections_active", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Active", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "seriesToRows", + "reducers": [ + "lastNotNull" + ] + } + } + ], + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "maxDataPoints": null, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{pod=~\"$Cluster.*\", container=~\"mongodb.*\"})\n", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Working", + "refId": "A" + }, + { + "exemplar": true, + "expr": ":node_memory_MemAvailable_bytes:sum", + "hide": false, + "interval": "", + "legendFormat": "Available", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mongo Pods Memory Usage in GB / Total Available on Cluster", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "seriesToRows", + "reducers": [] + } + } + ], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "maxDataPoints": null, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"$Cluster.*\", image!~\"sha.*\", container=~\"mongo.*\"}[5m]))", + "hide": false, + "interval": "", + "legendFormat": "Usage", + "refId": "Used" + }, + { + "exemplar": true, + "expr": "cluster:node_cpu:sum_rate5m", + "hide": false, + "interval": "", + "legendFormat": "Available", + "refId": "Available" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mongo Pods CPU Usage / Total Available on Cluster", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "seriesToRows", + "reducers": [] + } + } + ], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "exemplar": true, + "expr": "max(mongodb_catalogStats_collections{pod=~\"$Cluster.*\"})", + "instant": false, + "interval": "", + "legendFormat": "Collections", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(mongodb_catalogStats_capped{pod=~\"$Cluster.*\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Capped Collections", + "refId": "B" + }, + { + "exemplar": true, + "expr": "max(mongodb_catalogStats_timeseries{pod=~\"$Cluster.*\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Timeseries", + "refId": "C" + }, + { + "exemplar": true, + "expr": "max(mongodb_catalogStats_views{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Views", + "refId": "D" + } + ], + "title": "Catalog Stats", + "type": "bargauge" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 9, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltipOptions": { + "mode": "single" + } + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "exemplar": true, + "expr": "sum(mongodb_globalLock_activeClients_total{pod=~\"$Cluster.*\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(mongodb_globalLock_activeClients_readers{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Readers", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(mongodb_globalLock_activeClients_writers{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Writers", + "refId": "C" + } + ], + "title": "Global Locks", + "transformations": [], + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "mongodb_metrics_cursor_open_total{pod=~\"$Cluster.*\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{pod}} Open", + "refId": "A" + }, + { + "exemplar": true, + "expr": "mongodb_metrics_cursor_open_noTimeout{pod=~\"$Cluster.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pod}} Open No Timeout", + "refId": "B" + }, + { + "exemplar": true, + "expr": "mongodb_metrics_cursor_timed_out{pod=~\"$Cluster.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pod}} Timed Out", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cursors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "mongodb_metrics_document_inserted{pod=~\"$Cluster.*\"}", + "interval": "", + "legendFormat": "{{pod}} Inserted", + "refId": "A" + }, + { + "exemplar": true, + "expr": "mongodb_metrics_document_returned{pod=~\"$Cluster.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pod}} Returned", + "refId": "B" + }, + { + "exemplar": true, + "expr": "mongodb_metrics_document_deleted{pod=~\"$Cluster.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pod}} Deleted", + "refId": "C" + }, + { + "exemplar": true, + "expr": "mongodb_metrics_document_updated{pod=~\"$Cluster.*\"}", + "hide": false, + "interval": "", + "legendFormat": "{{pod}} Updated", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Documents", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "exemplar": true, + "expr": "sum(mongodb_metrics_repl_network_bytes{pod=~\"$Cluster.*\"})", + "instant": false, + "interval": "", + "legendFormat": "Total Usage", + "refId": "A" + } + ], + "title": "Replication Network Usage", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + } + ], + "type": "stat" + }, + { + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "exemplar": true, + "expr": "sum(mongodb_metrics_repl_network_ops{pod=~\"$Cluster.*\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Replication Operations", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(mongodb_network_bytesIn{pod=~\"$Cluster.*\"})", + "interval": "", + "legendFormat": "Bytes In", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(mongodb_network_bytesOut{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Bytes Out", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_reads_latency{pod=~\"$Cluster.*\"})", + "interval": "", + "legendFormat": "Reads", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_commands_latency{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Commands", + "refId": "C" + }, + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_transactions_latency{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Transactions", + "refId": "D" + }, + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_writes_latency{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Writes", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Latencies", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_reads_ops{pod=~\"$Cluster.*\"})", + "interval": "", + "legendFormat": "Reads", + "refId": "A" + }, + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_commands_ops{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Commands", + "refId": "C" + }, + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_transactions_ops{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Transactions", + "refId": "D" + }, + { + "exemplar": true, + "expr": "max(mongodb_opLatencies_writes_ops{pod=~\"$Cluster.*\"})", + "hide": false, + "interval": "", + "legendFormat": "Writes", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Ops", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "replica-set-with-prom", + "value": "replica-set-with-prom" + }, + "datasource": null, + "definition": "label_values(mongodb_connections_available, cl_name)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(mongodb_connections_available, cl_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "MongoDB Dashboard Copy", + "uid": "_y8XBgynz", + "version": 22 +} \ No newline at end of file diff --git a/public/mongodb-enterprise-multi-cluster.yaml b/public/mongodb-enterprise-multi-cluster.yaml new file mode 100644 index 000000000..ad43e7de0 --- /dev/null +++ b/public/mongodb-enterprise-multi-cluster.yaml @@ -0,0 +1,354 @@ +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster-mongodb-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster-mongodb-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-multi-cluster-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-operator-multi-cluster +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-database-pods + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-ops-manager + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-appdb +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-sa.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb-enterprise-operator-multi-cluster + namespace: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator-multi-cluster + app.kubernetes.io/instance: mongodb-enterprise-operator-multi-cluster + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator-multi-cluster + app.kubernetes.io/instance: mongodb-enterprise-operator-multi-cluster + spec: + serviceAccountName: mongodb-enterprise-operator-multi-cluster + securityContext: + runAsNonRoot: true + runAsUser: 2000 + containers: + - name: mongodb-enterprise-operator-multi-cluster + image: "quay.io/mongodb/mongodb-enterprise-operator-ubi:1.32.0" + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + - -watch-resource=mongodbmulticluster + command: + - /usr/local/bin/mongodb-enterprise-operator + volumeMounts: + - mountPath: /etc/config/kubeconfig + name: kube-config-volume + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: prod + - name: MDB_DEFAULT_ARCHITECTURE + value: non-static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1h" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/mongodb-enterprise-database-ubi + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-database-ubi + - name: INIT_DATABASE_VERSION + value: 1.32.0 + - name: DATABASE_VERSION + value: 1.32.0 + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi + - name: INIT_OPS_MANAGER_VERSION + value: 1.32.0 + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-appdb-ubi + - name: INIT_APPDB_VERSION + value: 1.32.0 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent-ubi" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: ubi8 + - name: PERFORM_FAILOVER + value: 'true' + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "1" + volumes: + - name: kube-config-volume + secret: + defaultMode: 420 + secretName: mongodb-enterprise-operator-multi-cluster-kubeconfig diff --git a/public/mongodb-enterprise-openshift.yaml b/public/mongodb-enterprise-openshift.yaml new file mode 100644 index 000000000..8a0186e50 --- /dev/null +++ b/public/mongodb-enterprise-openshift.yaml @@ -0,0 +1,566 @@ +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-operator +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-database-pods + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-ops-manager + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-appdb +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-sa.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator + app.kubernetes.io/instance: mongodb-enterprise-operator + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator + app.kubernetes.io/instance: mongodb-enterprise-operator + spec: + serviceAccountName: mongodb-enterprise-operator + containers: + - name: mongodb-enterprise-operator + image: "quay.io/mongodb/mongodb-enterprise-operator-ubi:1.32.0" + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + command: + - /usr/local/bin/mongodb-enterprise-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: prod + - name: MDB_DEFAULT_ARCHITECTURE + value: non-static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MANAGED_SECURITY_CONTEXT + value: 'true' + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1h" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/mongodb-enterprise-database-ubi + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-database-ubi + - name: INIT_DATABASE_VERSION + value: 1.32.0 + - name: DATABASE_VERSION + value: 1.32.0 + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi + - name: INIT_OPS_MANAGER_VERSION + value: 1.32.0 + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-appdb-ubi + - name: INIT_APPDB_VERSION + value: 1.32.0 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent-ubi" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: ubi8 + - name: PERFORM_FAILOVER + value: 'true' + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "1" + - name: RELATED_IMAGE_MONGODB_ENTERPRISE_DATABASE_IMAGE_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-database-ubi:1.32.0" + - name: RELATED_IMAGE_INIT_DATABASE_IMAGE_REPOSITORY_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-init-database-ubi:1.32.0" + - name: RELATED_IMAGE_INIT_OPS_MANAGER_IMAGE_REPOSITORY_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi:1.32.0" + - name: RELATED_IMAGE_INIT_APPDB_IMAGE_REPOSITORY_1_32_0 + value: "quay.io/mongodb/mongodb-enterprise-init-appdb-ubi:1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_13_8702_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.13.8702-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_107_0_15_8741_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:107.0.15.8741-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_2_8729_1 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_4_8770_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.4.8770-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_108_0_6_8796_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.6.8796-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_33_7866_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.33.7866-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_34_7888_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.34.7888-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_12_0_35_7911_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:12.0.35.7911-1_1.32.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1_1_30_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1_1.30.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1_1_31_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1_1.31.0" + - name: RELATED_IMAGE_AGENT_IMAGE_13_32_0_9397_1_1_32_0 + value: "quay.io/mongodb/mongodb-agent-ubi:13.32.0.9397-1_1.32.0" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_25 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.25" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_26 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.26" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_6_0_27 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:6.0.27" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_7_0_13 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:7.0.13" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_7_0_14 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:7.0.14" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_7_0_15 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:7.0.15" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_8_0_4 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:8.0.4" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_8_0_5 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:8.0.5" + - name: RELATED_IMAGE_OPS_MANAGER_IMAGE_REPOSITORY_8_0_6 + value: "quay.io/mongodb/mongodb-enterprise-ops-manager-ubi:8.0.6" + # since the official server images end with a different suffix we can re-use the same $mongodbImageEnv + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_1_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.1-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_2_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.2-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_3_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.3-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_4_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.4-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_5_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.5-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_6_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.6-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_7_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.7-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_8_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.8-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_9_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.9-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_10_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.10-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_11_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.11-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_12_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.12-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_13_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.13-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_14_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.14-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_15_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.15-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_16_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.16-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_17_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.17-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_18_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.18-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_19_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.19-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_20_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.20-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_21_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:4.4.21-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_1_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.1-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_2_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.2-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_3_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.3-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_4_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.4-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_5_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.5-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_6_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.6-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_7_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.7-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_8_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.8-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_9_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.9-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_10_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.10-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_11_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.11-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_12_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.12-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_13_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.13-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_14_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.14-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_15_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.15-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_16_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.16-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_17_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.17-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_18_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:5.0.18-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_1_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.1-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_2_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.2-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_3_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.3-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_4_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.4-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_6_0_5_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:6.0.5-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_8_0_0_ubi8 + value: "quay.io/mongodb/mongodb-enterprise-server:8.0.0-ubi8" + - name: RELATED_IMAGE_MONGODB_IMAGE_8_0_0_ubi9 + value: "quay.io/mongodb/mongodb-enterprise-server:8.0.0-ubi9" + # mongodbLegacyAppDb will be deleted in 1.23 release + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_11_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.11-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_2_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.2-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_24_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.24-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_6_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.6-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_2_8_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.2.8-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_0_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.0-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_11_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.11-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_4_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.4-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_4_4_21_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.21-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_1_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.1-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_5_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.5-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_6_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.6-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_7_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.7-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_14_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.14-ent" + - name: RELATED_IMAGE_MONGODB_IMAGE_5_0_18_ent + value: "quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:5.0.18-ent" diff --git a/public/mongodb-enterprise.yaml b/public/mongodb-enterprise.yaml new file mode 100644 index 000000000..1673c4d6c --- /dev/null +++ b/public/mongodb-enterprise.yaml @@ -0,0 +1,345 @@ +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook +rules: + - apiGroups: + - "admissionregistration.k8s.io" + resources: + - validatingwebhookconfigurations + verbs: + - get + - create + - update + - delete + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# Additional ClusterRole for clusterVersionDetection +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-cluster-telemetry +rules: + # Non-resource URL permissions + - nonResourceURLs: + - "/version" + verbs: + - get + # Cluster-scoped resource permissions + - apiGroups: + - '' + resources: + - namespaces + resourceNames: + - kube-system + verbs: + - get + - apiGroups: + - '' + resources: + - nodes + verbs: + - list +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-webhook-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-mongodb-webhook +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +# ClusterRoleBinding for clusterVersionDetection +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator-mongodb-cluster-telemetry-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-cluster-telemetry +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - '' + resources: + - secrets + - configmaps + verbs: + - get + - list + - create + - update + - delete + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - watch + - delete + - update + - apiGroups: + - '' + resources: + - pods + verbs: + - get + - list + - watch + - delete + - deletecollection + - apiGroups: + - mongodb.com + verbs: + - '*' + resources: + - mongodb + - mongodb/finalizers + - mongodbusers + - mongodbusers/finalizers + - opsmanagers + - opsmanagers/finalizers + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodb/status + - mongodbusers/status + - opsmanagers/status + - mongodbmulticluster/status + + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - delete + - list + - watch + - patch + - update +--- +# Source: enterprise-operator/templates/operator-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-operator +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-database-pods + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-ops-manager + namespace: mongodb +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - apiGroups: + - '' + resources: + - pods + verbs: + - patch + - delete + - get +--- +# Source: enterprise-operator/templates/database-roles.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mongodb-enterprise-appdb + namespace: mongodb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-appdb +subjects: + - kind: ServiceAccount + name: mongodb-enterprise-appdb + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator-sa.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +--- +# Source: enterprise-operator/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb-enterprise-operator + namespace: mongodb +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator + app.kubernetes.io/instance: mongodb-enterprise-operator + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/name: mongodb-enterprise-operator + app.kubernetes.io/instance: mongodb-enterprise-operator + spec: + serviceAccountName: mongodb-enterprise-operator + securityContext: + runAsNonRoot: true + runAsUser: 2000 + containers: + - name: mongodb-enterprise-operator + image: "quay.io/mongodb/mongodb-enterprise-operator-ubi:1.32.0" + imagePullPolicy: Always + args: + - -watch-resource=mongodb + - -watch-resource=opsmanagers + - -watch-resource=mongodbusers + command: + - /usr/local/bin/mongodb-enterprise-operator + resources: + limits: + cpu: 1100m + memory: 1Gi + requests: + cpu: 500m + memory: 200Mi + env: + - name: OPERATOR_ENV + value: prod + - name: MDB_DEFAULT_ARCHITECTURE + value: non-static + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY + value: "1h" + - name: MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY + value: "168h" + - name: CLUSTER_CLIENT_TIMEOUT + value: "10" + - name: IMAGE_PULL_POLICY + value: Always + # Database + - name: MONGODB_ENTERPRISE_DATABASE_IMAGE + value: quay.io/mongodb/mongodb-enterprise-database-ubi + - name: INIT_DATABASE_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-database-ubi + - name: INIT_DATABASE_VERSION + value: 1.32.0 + - name: DATABASE_VERSION + value: 1.32.0 + # Ops Manager + - name: OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-ops-manager-ubi + - name: INIT_OPS_MANAGER_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-ops-manager-ubi + - name: INIT_OPS_MANAGER_VERSION + value: 1.32.0 + # AppDB + - name: INIT_APPDB_IMAGE_REPOSITORY + value: quay.io/mongodb/mongodb-enterprise-init-appdb-ubi + - name: INIT_APPDB_VERSION + value: 1.32.0 + - name: OPS_MANAGER_IMAGE_PULL_POLICY + value: Always + - name: AGENT_IMAGE + value: "quay.io/mongodb/mongodb-agent-ubi:108.0.2.8729-1" + - name: MDB_AGENT_IMAGE_REPOSITORY + value: "quay.io/mongodb/mongodb-agent-ubi" + - name: MONGODB_IMAGE + value: mongodb-enterprise-server + - name: MONGODB_REPO_URL + value: quay.io/mongodb + - name: MDB_IMAGE_TYPE + value: ubi8 + - name: PERFORM_FAILOVER + value: 'true' + - name: MDB_MAX_CONCURRENT_RECONCILES + value: "1" diff --git a/public/multi_cluster_verify/sample-service.yaml b/public/multi_cluster_verify/sample-service.yaml new file mode 100644 index 000000000..559840c60 --- /dev/null +++ b/public/multi_cluster_verify/sample-service.yaml @@ -0,0 +1,85 @@ +apiVersion: v1 +kind: Service +metadata: + name: helloworld1 + labels: + app: helloworld1 + service: helloworld1 +spec: + ports: + - port: 5000 + name: http + selector: + app: helloworld +--- +apiVersion: v1 +kind: Service +metadata: + name: helloworld2 + labels: + app: helloworld2 + service: helloworld2 +spec: + ports: + - port: 5000 + name: http + selector: + app: helloworld +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helloworld-v1 + labels: + app: helloworld + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: helloworld + version: v1 + template: + metadata: + labels: + app: helloworld + version: v1 + spec: + containers: + - name: helloworld + image: docker.io/istio/examples-helloworld-v1 + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent #Always + ports: + - containerPort: 5000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helloworld-v2 + labels: + app: helloworld + version: v2 +spec: + replicas: 1 + selector: + matchLabels: + app: helloworld + version: v2 + template: + metadata: + labels: + app: helloworld + version: v2 + spec: + containers: + - name: helloworld + image: docker.io/istio/examples-helloworld-v2 + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent #Always + ports: + - containerPort: 5000 diff --git a/public/opa_examples/README.md b/public/opa_examples/README.md new file mode 100644 index 000000000..354bd3e78 --- /dev/null +++ b/public/opa_examples/README.md @@ -0,0 +1,54 @@ + +# Open Policy Agent Samples + +This is a library of sample policies for [OPA Gatekeeper](https://open-policy-agent.github.io/gatekeeper/website/docs/) . You can edit and apply any of the policies or use them as a springboard to create your own. Each policy is enclosed in its own directory in the form of a policy_name.yaml file ([Constraint Template](https://open-policy-agent.github.io/gatekeeper/website/docs/constrainttemplates)) and constraints.yaml (Constraint). + +**Instructions for use** + +You will need OPA Gatekeeper installed on your Kubernetes cluster. Follow the instructions [here](https://open-policy-agent.github.io/gatekeeper/website/docs/install). + + cd + kubectl apply -f .yaml + kubectl apply -f constraints.yaml + +**Verifying installed Constraint Templates and Constraints** + + kubectl get constrainttemplates + kubectl get constraints + +**Deleting Constraints and Constraint Templates** + + kubectl delete contraint + kubectl delete constrainttemplate + +# Library Folders + +This section explains the purpose of the policies contained in each folder. It is listed according to the folder names. + +## debugging + +This folder contains policies that blocks all MongoDB and MongoDBOpsManager resources. It can be used to log all the review objects on the admission controller and you can use the output to craft your own policies. This is explained [here](https://open-policy-agent.github.io/gatekeeper/website/docs/debug). + +## mongodb_allow_replicaset + +This folder contains policies that only allows MongoDB replicasets to be deployed + +## mongodb_allowed_versions + +This folder contains policies that only allow specific MongoDB versions to be deployed + +## mongodb_strict_tls + +This folder contains policies that only allows strict TLS mode for MongoDB deployments + +## ops_manager_allowed_versions + +This folder contains policies that only allows specific Ops Manager versions to be deployed + +## ops_manager_replica_members + +This folder contains policies that locks the appDB members and the Ops Manager replicas to a certain number + +## ops_manager_wizardless + +This folder contains policies that only allows wizardless installation of Ops Manager \ No newline at end of file diff --git a/public/opa_examples/debugging/constraint_template.yaml b/public/opa_examples/debugging/constraint_template.yaml new file mode 100644 index 000000000..9894fb8ef --- /dev/null +++ b/public/opa_examples/debugging/constraint_template.yaml @@ -0,0 +1,17 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: k8sdenyall +spec: + crd: + spec: + names: + kind: K8sDenyAll + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package k8sdenyall + + violation[{"msg": msg}] { + msg := sprintf("REVIEW OBJECT: %v", [input.review]) + } diff --git a/public/opa_examples/debugging/constraints.yaml b/public/opa_examples/debugging/constraints.yaml new file mode 100644 index 000000000..4a87203aa --- /dev/null +++ b/public/opa_examples/debugging/constraints.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sDenyAll +metadata: + name: deny-all-namespaces +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDB"] + - apiGroups: ["mongodb.com"] + kinds: ["MongoDBOpsManager"] diff --git a/public/opa_examples/mongodb_allow_replicaset/constraints.yaml b/public/opa_examples/mongodb_allow_replicaset/constraints.yaml new file mode 100644 index 000000000..c453302c2 --- /dev/null +++ b/public/opa_examples/mongodb_allow_replicaset/constraints.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: MongoDBAllowReplicaset +metadata: + name: mongodb-allow-replicaset-only +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDB"] \ No newline at end of file diff --git a/public/opa_examples/mongodb_allow_replicaset/mongodb_allow_replicaset.yaml b/public/opa_examples/mongodb_allow_replicaset/mongodb_allow_replicaset.yaml new file mode 100644 index 000000000..a5549e1c7 --- /dev/null +++ b/public/opa_examples/mongodb_allow_replicaset/mongodb_allow_replicaset.yaml @@ -0,0 +1,25 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: mongodballowreplicaset + annotations: + description: >- + Allows only replica set deployment of MongoDB + + The type setting for MongoDB should be replicaset +spec: + crd: + spec: + names: + kind: MongoDBAllowReplicaset + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package mongodballowreplicaset + + violation[{"msg": msg}] { + deployment_type = object.get(input.review.object.spec, "type", "none") + not deployment_type == "replicaset" + msg := sprintf("Only replicaset deployment of MongoDB allowed, requested %v", [deployment_type]) + } + diff --git a/public/opa_examples/mongodb_allowed_versions/constraints.yaml b/public/opa_examples/mongodb_allowed_versions/constraints.yaml new file mode 100644 index 000000000..f5df64f07 --- /dev/null +++ b/public/opa_examples/mongodb_allowed_versions/constraints.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: MongoDBAllowedVersions +metadata: + name: mongodb-allowed-versions-only +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDB"] diff --git a/public/opa_examples/mongodb_allowed_versions/mongodb_allowed_versions.yaml b/public/opa_examples/mongodb_allowed_versions/mongodb_allowed_versions.yaml new file mode 100644 index 000000000..6a34a0eba --- /dev/null +++ b/public/opa_examples/mongodb_allowed_versions/mongodb_allowed_versions.yaml @@ -0,0 +1,28 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: mongodballowedversions + annotations: + description: >- + Requires MongoDB deployment to be within the allowed versions + + The setting version should be within the pinned allowed values +spec: + crd: + spec: + names: + kind: MongoDBAllowedVersions + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package mongodballowedversions + + allowed_versions = ["4.5.0", "5.0.0"] + + violation[{"msg": msg}] { + version = object.get(input.review.object.spec, "version", "none") + not q[version] + msg := sprintf("MongoDB deployment needs to be one of the allowed versions: ", [allowed_versions]) + } + + q[version] { version := allowed_versions[_] } diff --git a/public/opa_examples/mongodb_strict_tls/constraints.yaml b/public/opa_examples/mongodb_strict_tls/constraints.yaml new file mode 100644 index 000000000..17b6ea2b7 --- /dev/null +++ b/public/opa_examples/mongodb_strict_tls/constraints.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: MongoDBStrictTLS +metadata: + name: mongodb-strict-tls-only +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDB"] diff --git a/public/opa_examples/mongodb_strict_tls/mongodb_strict_tls.yaml b/public/opa_examples/mongodb_strict_tls/mongodb_strict_tls.yaml new file mode 100644 index 000000000..e02c08a60 --- /dev/null +++ b/public/opa_examples/mongodb_strict_tls/mongodb_strict_tls.yaml @@ -0,0 +1,36 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: mongodbstricttls + annotations: + description: >- + Requires MongoDB deployment to be in strict TLS mode + + The setting ssl mode needs to be requireSSL and tls enabled should be true +spec: + crd: + spec: + names: + kind: MongoDBStrictTLS + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package mongodbstricttls + + default check_tls_strict = true + + tls := object.get(input.review.object.spec.security.tls, "enabled", false) + + tls_mode := object.get(input.review.object.spec.additionalMongodConfig.net.ssl, "mode", "none") + + check_tls_strict = false { + not tls + } + check_tls_strict = false { + tls_mode != "requireSSL" + } + + violation[{"msg": msg}] { + not check_tls_strict + msg := sprintf("MongoDB deployment needs to be TLS and mode should be requireSSL, TLS enabled set to %v and mode set to %v", [tls, tls_mode]) + } diff --git a/public/opa_examples/ops_manager_allowed_versions/constraints.yaml b/public/opa_examples/ops_manager_allowed_versions/constraints.yaml new file mode 100644 index 000000000..5c38bcecd --- /dev/null +++ b/public/opa_examples/ops_manager_allowed_versions/constraints.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: OpsManagerAllowedVersions +metadata: + name: ops-manager-allowed-versions-only +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDBOpsManager"] diff --git a/public/opa_examples/ops_manager_allowed_versions/ops_manager_allowed_versions.yaml b/public/opa_examples/ops_manager_allowed_versions/ops_manager_allowed_versions.yaml new file mode 100644 index 000000000..67e9bb9b5 --- /dev/null +++ b/public/opa_examples/ops_manager_allowed_versions/ops_manager_allowed_versions.yaml @@ -0,0 +1,28 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: opsmanagerallowedversions + annotations: + description: >- + Requires Ops Manager to be within the allowed versions + + The setting version should be within the pinned allowed values +spec: + crd: + spec: + names: + kind: OpsManagerAllowedVersions + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package opsmanagerallowedversions + + allowed_versions = ["4.4.5", "5.0.0"] + + violation[{"msg": msg}] { + version = object.get(input.review.object.spec, "version", "none") + not q[version] + msg := sprintf("Ops Manager needs to be one of the allowed versions: ", [allowed_versions]) + } + + q[version] { version := allowed_versions[_] } diff --git a/public/opa_examples/ops_manager_replica_members/constraints.yaml b/public/opa_examples/ops_manager_replica_members/constraints.yaml new file mode 100644 index 000000000..d9c01088a --- /dev/null +++ b/public/opa_examples/ops_manager_replica_members/constraints.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: OpsManagerReplicaMembers +metadata: + name: ops-manager-replicamembers-only +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDBOpsManager"] diff --git a/public/opa_examples/ops_manager_replica_members/ops_manager_replica_members.yaml b/public/opa_examples/ops_manager_replica_members/ops_manager_replica_members.yaml new file mode 100644 index 000000000..ebc950759 --- /dev/null +++ b/public/opa_examples/ops_manager_replica_members/ops_manager_replica_members.yaml @@ -0,0 +1,37 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: opsmanagerreplicamembers + annotations: + description: >- + Requires Ops Manager install to be 1 replica and 3 members + + The setting applicationDatabase.members should be 3 and replicas should be 0 +spec: + crd: + spec: + names: + kind: OpsManagerReplicaMembers + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package opsmanagerreplicamembers + + default ops_conditions = true + + replicas := object.get(input.review.object.spec, "replicas", 0) + + dbmembers := object.get(input.review.object.spec.applicationDatabase, "members", 0) + + violation[{"msg": msg}] { + not ops_conditions + msg := sprintf("Ops Manager needs to have 1 replica and 3 members, current config is %v replica and %v members.", [replicas, dbmembers]) + } + + ops_conditions = false { + replicas != 1 + } + + ops_conditions = false { + dbmembers != 3 + } diff --git a/public/opa_examples/ops_manager_wizardless/constraints.yaml b/public/opa_examples/ops_manager_wizardless/constraints.yaml new file mode 100644 index 000000000..dd5b8fe6e --- /dev/null +++ b/public/opa_examples/ops_manager_wizardless/constraints.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: OpsManagerWizardless +metadata: + name: ops-manager-wizardless-only +spec: + match: + kinds: + - apiGroups: ["mongodb.com"] + kinds: ["MongoDBOpsManager"] diff --git a/public/opa_examples/ops_manager_wizardless/ops_manager_wizardless_template.yaml b/public/opa_examples/ops_manager_wizardless/ops_manager_wizardless_template.yaml new file mode 100644 index 000000000..077283812 --- /dev/null +++ b/public/opa_examples/ops_manager_wizardless/ops_manager_wizardless_template.yaml @@ -0,0 +1,24 @@ +apiVersion: templates.gatekeeper.sh/v1beta1 +kind: ConstraintTemplate +metadata: + name: opsmanagerwizardless + annotations: + description: >- + Requires Ops Manager install to be wizardless + + The setting mms.ignoreInitiaUiSetup needs to be true +spec: + crd: + spec: + names: + kind: OpsManagerWizardless + targets: + - target: admission.k8s.gatekeeper.sh + rego: | + package opsmanagerwizardless + + violation[{"msg": msg}] { + value := object.get(input.review.object.spec.configuration, "mms.ignoreInitialUiSetup", "false") + not value == "true" + msg := sprintf("Wizard based setup of Ops Manager is not allowed. mms.ignoreInitialUiSetup needs to be true, currently is %v", [value]) + } diff --git a/public/samples/appdb_multicluster/ops-manager-multi-cluster-appdb.yaml b/public/samples/appdb_multicluster/ops-manager-multi-cluster-appdb.yaml new file mode 100644 index 000000000..60c49590d --- /dev/null +++ b/public/samples/appdb_multicluster/ops-manager-multi-cluster-appdb.yaml @@ -0,0 +1,44 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: +spec: + replicas: 1 + version: + adminCredentials: # Should match metadata.name + # in the Kubernetes secret + # for the admin user + + externalConnectivity: + type: LoadBalancer + security: + certsSecretPrefix: # Required. Text to prefix + # the name of the secret that contains + # Ops Manager's TLS certificate. + tls: + ca: "om-http-cert-ca" # Optional. Name of the ConfigMap file + # containing the certificate authority that + # signs the certificates used by the Ops + # Manager custom resource. + + applicationDatabase: + topology: MultiCluster # if you want to deploy AppDB deployment accross multiple clusters + clusterSpecList: + # distribution of AppDB nodes across multiple Kubernetes clusters + - clusterName: cluster1.example.com + members: 4 + - clusterName: cluster2.example.com + members: 3 + - clusterName: cluster3.example.com + members: 2 + version: "4.4.0-ubi8" + security: + certsSecretPrefix: # Required. Text to prefix to the + # name of the secret that contains the Application + # Database's TLS certificate. Name the secret + # --db-cert. + tls: + ca: "appdb-ca" # Optional. Name of the ConfigMap file + # containing the certicate authority that + # signs the certificates used by the + # application database. diff --git a/public/samples/mongodb/affinity/replica-set-affinity.yaml b/public/samples/mongodb/affinity/replica-set-affinity.yaml new file mode 100644 index 000000000..1191ca39f --- /dev/null +++ b/public/samples/mongodb/affinity/replica-set-affinity.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.2.1-ent + service: my-service + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: ReplicaSet + + persistent: true + podSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + memory: 512M + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + topologyKey: nodeId diff --git a/public/samples/mongodb/affinity/sharded-cluster-affinity.yaml b/public/samples/mongodb/affinity/sharded-cluster-affinity.yaml new file mode 100644 index 000000000..ac109cf9c --- /dev/null +++ b/public/samples/mongodb/affinity/sharded-cluster-affinity.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.2.1-ent + service: my-service + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: ShardedCluster + + persistent: true + configSrvPodSpec: + podTemplate: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + topologyKey: nodeId + mongosPodSpec: + podTemplate: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + topologyKey: nodeId + shardPodSpec: + podTemplate: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + topologyKey: nodeId diff --git a/public/samples/mongodb/affinity/standalone-affinity.yaml b/public/samples/mongodb/affinity/standalone-affinity.yaml new file mode 100644 index 000000000..0e7993a22 --- /dev/null +++ b/public/samples/mongodb/affinity/standalone-affinity.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone +spec: + version: 4.2.1-ent + service: my-service + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: Standalone + + persistent: true + podSpec: + podTemplate: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + topologyKey: nodeId \ No newline at end of file diff --git a/public/samples/mongodb/agent-startup-options/replica-set-agent-startup-options.yaml b/public/samples/mongodb/agent-startup-options/replica-set-agent-startup-options.yaml new file mode 100644 index 000000000..8e651a77c --- /dev/null +++ b/public/samples/mongodb/agent-startup-options/replica-set-agent-startup-options.yaml @@ -0,0 +1,21 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-agent-parameters +spec: + members: 3 + version: 4.4.0-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + # optional. Allows to pass custom flags that will be used + # when launching the mongodb agent. All values must be strings + # The full list of available settings is at: + # https://docs.opsmanager.mongodb.com/current/reference/mongodb-agent-settings/ + agent: + startupOptions: + maxLogFiles: "30" + dialTimeoutSeconds: "40" diff --git a/public/samples/mongodb/agent-startup-options/sharded-cluster-agent-startup-options.yaml b/public/samples/mongodb/agent-startup-options/sharded-cluster-agent-startup-options.yaml new file mode 100644 index 000000000..a32a9c04f --- /dev/null +++ b/public/samples/mongodb/agent-startup-options/sharded-cluster-agent-startup-options.yaml @@ -0,0 +1,45 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster-options +spec: + version: 4.4.0-ent + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 1 + + mongos: + # optional. Allows to pass custom flags that will be used + # when launching the mongodb agent for mongos processes. + # All values must be string + # The full list of available settings is at: + # https://docs.opsmanager.mongodb.com/current/reference/mongodb-agent-settings/ + agent: + startupOptions: + maxLogFiles: "30" + + configSrv: + # optional. Allows to pass custom flags that will be used + # when launching the mongodb agent for Config Server mongod processes. + # All values must be string + # The full list of available settings is at: + # https://docs.opsmanager.mongodb.com/current/reference/mongodb-agent-settings/ + agent: + startupOptions: + dialTimeoutSeconds: "40" + shard: + # optional. Allows to pass custom flags that will be used + # when launching the mongodb agent for Shards mongod processes. + # All values must be string + # The full list of available settings is at: + # https://docs.opsmanager.mongodb.com/current/reference/mongodb-agent-settings/ + agent: + startupOptions: + serverSelectionTimeoutSeconds: "20" diff --git a/public/samples/mongodb/agent-startup-options/standalone-agent-startup-options.yaml b/public/samples/mongodb/agent-startup-options/standalone-agent-startup-options.yaml new file mode 100644 index 000000000..55ae9a0e5 --- /dev/null +++ b/public/samples/mongodb/agent-startup-options/standalone-agent-startup-options.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone +spec: + version: 4.4.0-ent + service: my-service + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: Standalone + + persistent: true + # optional. Allows to pass custom flags that will be used + # when launching the mongodb agent. All values must be strings + # The full list of available settings is at: + # https://docs.opsmanager.mongodb.com/current/reference/mongodb-agent-settings/ + agent: + startupOptions: + maxLogFiles: "30" + dialTimeoutSeconds: "40" diff --git a/public/samples/mongodb/authentication/ldap/replica-set/replica-set-ldap-user.yaml b/public/samples/mongodb/authentication/ldap/replica-set/replica-set-ldap-user.yaml new file mode 100644 index 000000000..0864ffe58 --- /dev/null +++ b/public/samples/mongodb/authentication/ldap/replica-set/replica-set-ldap-user.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-ldap-user +spec: + username: my-ldap-user + db: $external + mongodbResourceRef: + name: my-ldap-enabled-replica-set # The name of the MongoDB resource this user will be added to + roles: + - db: admin + name: clusterAdmin + - db: admin + name: userAdminAnyDatabase + - db: admin + name: readWrite diff --git a/public/samples/mongodb/authentication/ldap/replica-set/replica-set-ldap.yaml b/public/samples/mongodb/authentication/ldap/replica-set/replica-set-ldap.yaml new file mode 100644 index 000000000..46c518449 --- /dev/null +++ b/public/samples/mongodb/authentication/ldap/replica-set/replica-set-ldap.yaml @@ -0,0 +1,67 @@ +# Creates a MongoDB Replica Set with LDAP Authentication Enabled. +# LDAP is an Enterprise-only feature. + +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-ldap-enabled-replica-set +spec: + type: ReplicaSet + members: 3 + version: 4.0.4-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + # Enabled LDAP Authentication Mode + modes: ["LDAP"] + + # LDAP related configuration + ldap: + # Specify the hostname:port combination of one or + # more LDAP servers + servers: + - "" + - "" + + # Set to "tls" to use LDAP over TLS. Leave blank if + # LDAP server does not accept TLS. + transportSecurity: "tls" + + # ConfigMap containing a CA certificate that validates + # the LDAP server's TLS certificate. + caConfigMapRef: + name: "" + key: "" + + # Specify the LDAP Distinguished Name to which + # MongoDB binds when connecting to the LDAP server + bindQueryUser: "cn=admin,dc=example,dc=org" + + # Specify the password with which MongoDB binds + # when connecting to an LDAP server. This is a + # reference to a Secret Kubernetes Object containing + # one "password" key. + bindQueryPasswordSecretRef: + name: "" + + # Select True to validate the LDAP server configuration or False to skip validation. + validateLDAPServerConfig: false + + # LDAP-formatted query URL template executed by MongoDB to obtain the LDAP groups that the user belongs to + authzQueryTemplate: "{USER}?memberOf?base" + + # Maps the username provided to mongod or mongos for authentication to an LDAP Distinguished Name (DN). + userToDNMapping: '[{match: "CN=mms-automation-agent,(.+),L=NY,ST=NY,C=US", substitution: "uid=mms-automation-agent,{0},dc=example,dc=org"}, {match: "(.+)", substitution:"uid={0},ou=groups,dc=example,dc=org"}]' + + # Specify how long an authentication request should wait before timing out. In milliseconds. + timeoutMS: 10000 + + # Specify how long MongoDB waits to flush the LDAP user cache. In seconds. + userCacheInvalidationInterval: 30 diff --git a/public/samples/mongodb/authentication/ldap/sharded-cluster/sharded-cluster-ldap-user.yaml b/public/samples/mongodb/authentication/ldap/sharded-cluster/sharded-cluster-ldap-user.yaml new file mode 100644 index 000000000..252c93e7c --- /dev/null +++ b/public/samples/mongodb/authentication/ldap/sharded-cluster/sharded-cluster-ldap-user.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-ldap-user +spec: + username: my-ldap-user + db: $external + mongodbResourceRef: + name: my-ldap-enabled-sharded-cluster # The name of the MongoDB resource this user will be added to + roles: + - db: admin + name: clusterAdmin + - db: admin + name: userAdminAnyDatabase + - db: admin + name: readWrite diff --git a/public/samples/mongodb/authentication/ldap/sharded-cluster/sharded-cluster-ldap.yaml b/public/samples/mongodb/authentication/ldap/sharded-cluster/sharded-cluster-ldap.yaml new file mode 100644 index 000000000..c1fe491ba --- /dev/null +++ b/public/samples/mongodb/authentication/ldap/sharded-cluster/sharded-cluster-ldap.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-ldap-enabled-sharded-cluster +spec: + type: ShardedCluster + + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + + version: 4.0.4-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + + # Enabled LDAP Authentication Mode + modes: ["LDAP"] + + # LDAP related configuration + ldap: + # Specify the hostname:port combination of one or + # more LDAP servers + servers: + - "" + - "" + + # Set to "tls" to use LDAP over TLS. Leave blank if + # LDAP server does not accept TLS. + transportSecurity: "tls" + + # ConfigMap containing a CA certificate that validates + # the LDAP server's TLS certificate. + caConfigMapRef: + name: "" + key: "" + + # Specify the LDAP Distinguished Name to which + # MongoDB binds when connecting to the LDAP server + bindQueryUser: "cn=admin,dc=example,dc=org" + + # Specify the password with which MongoDB binds + # when connecting to an LDAP server. This is a + # reference to a Secret Kubernetes Object containing + # one "password" key. + bindQueryPasswordSecretRef: + name: "" + diff --git a/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-password.yaml b/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-password.yaml new file mode 100644 index 000000000..6fa081096 --- /dev/null +++ b/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-password.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-scram-secret +type: Opaque +stringData: + password: my-replica-set-password diff --git a/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-sha.yaml b/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-sha.yaml new file mode 100644 index 000000000..11653baa8 --- /dev/null +++ b/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-sha.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-scram-enabled-replica-set +spec: + type: ReplicaSet + members: 3 + + # Using a version >= 4.0 will enable SCRAM-SHA-256 authentication + # setting a version < 4.0 will enable SCRAM-SHA-1/MONGODB-CR authentication + version: 4.0.4-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + modes: ["SCRAM"] # Valid authentication modes are "SCRAM', "SCRAM-SHA-1", "MONGODB-CR", "X509" and "LDAP" + + # Optional field - ignoreUnknownUsers + # A value of true means that any users not configured via the Operator or the Ops Manager or Cloud Manager UI + # will not be altered in any way + + # If you need to manage MongoDB users directly via the mongods, set this value to true + ignoreUnknownUsers: true # default value false + diff --git a/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-user.yaml b/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-user.yaml new file mode 100644 index 000000000..2a6db8589 --- /dev/null +++ b/public/samples/mongodb/authentication/scram/replica-set/replica-set-scram-user.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-scram-user +spec: + passwordSecretKeyRef: + name: my-scram-secret # the name of the secret that stores this user's password + key: password # the key in the secret that stores the password + username: my-scram-user + db: admin + mongodbResourceRef: + name: my-scram-enabled-replica-set # The name of the MongoDB resource this user will be added to + roles: + - db: admin + name: clusterAdmin + - db: admin + name: readWriteAnyDatabase + - db: admin + name: dbAdminAnyDatabase diff --git a/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-password.yaml b/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-password.yaml new file mode 100644 index 000000000..fb1b353af --- /dev/null +++ b/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-password.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-scram-secret +type: Opaque +stringData: + password: my-sharded-cluster-password diff --git a/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-sha.yaml b/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-sha.yaml new file mode 100644 index 000000000..f950e0808 --- /dev/null +++ b/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-sha.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-scram-enabled-sharded-cluster +spec: + type: ShardedCluster + + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + + # Using a version >= 4.0 will enable SCRAM-SHA-256 authentication + # setting a version < 4.0 will enable SCRAM-SHA-1/MONGODB-CR authentication + version: 4.0.4-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + modes: ["SCRAM"] # Valid authentication modes are "SCRAM', "SCRAM-SHA-1", "MONGODB-CR", "X509" and "LDAP" + + # Optional field - ignoreUnknownUsers + # A value of true means that any users not configured via the Operator or the Ops Manager or Cloud Manager UI + # will not be altered in any way + + # If you need to manage MongoDB users directly via the mongods, set this value to true + ignoreUnknownUsers: true # default value false + diff --git a/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-user.yaml b/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-user.yaml new file mode 100644 index 000000000..a62c3f10d --- /dev/null +++ b/public/samples/mongodb/authentication/scram/sharded-cluster/sharded-cluster-scram-user.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-scram-user +spec: + passwordSecretKeyRef: + name: my-scram-secret # the name of the secret that stores this user's password + key: password # the key in the secret that stores the password + username: my-scram-user + db: admin + mongodbResourceRef: + name: my-scram-enabled-sharded-cluster # The name of the MongoDB resource this user will be added to + roles: + - db: admin + name: clusterAdmin + - db: admin + name: userAdminAnyDatabase + diff --git a/public/samples/mongodb/authentication/scram/standalone/standalone-scram-password.yaml b/public/samples/mongodb/authentication/scram/standalone/standalone-scram-password.yaml new file mode 100644 index 000000000..92101f7f5 --- /dev/null +++ b/public/samples/mongodb/authentication/scram/standalone/standalone-scram-password.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-scram-secret +type: Opaque +stringData: + password: my-standalone-password diff --git a/public/samples/mongodb/authentication/scram/standalone/standalone-scram-sha.yaml b/public/samples/mongodb/authentication/scram/standalone/standalone-scram-sha.yaml new file mode 100644 index 000000000..dd418652d --- /dev/null +++ b/public/samples/mongodb/authentication/scram/standalone/standalone-scram-sha.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-scram-enabled-standalone +spec: + type: Standalone + + # Using a version >= 4.0 will enable SCRAM-SHA-256 authentication + version: 4.4.0-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + authentication: + enabled: true + modes: ["SCRAM"] # Valid authentication modes are "SCRAM' and "X509" + + # Optional field - ignoreUnknownUsers + # A value of true means that any users not configured via the Operator or the Ops Manager or Cloud Manager UI + # will not be altered in any way + + # If you need to manage MongoDB users directly via the mongods, set this value to true + ignoreUnknownUsers: true # default value false + diff --git a/public/samples/mongodb/authentication/scram/standalone/standalone-scram-user.yaml b/public/samples/mongodb/authentication/scram/standalone/standalone-scram-user.yaml new file mode 100644 index 000000000..02948d75c --- /dev/null +++ b/public/samples/mongodb/authentication/scram/standalone/standalone-scram-user.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-scram-user +spec: + passwordSecretKeyRef: + name: my-scram-secret # the name of the secret that stores this user's password + key: password # the key in the secret that stores the password + username: my-scram-user + db: admin + mongodbResourceRef: + name: my-scram-enabled-standalone # The name of the MongoDB resource this user will be added to + roles: + - db: admin + name: readWrite + - db: admin + name: userAdminAnyDatabase diff --git a/public/samples/mongodb/authentication/x509/replica-set/replica-set-x509.yaml b/public/samples/mongodb/authentication/x509/replica-set/replica-set-x509.yaml new file mode 100644 index 000000000..45af6b3f2 --- /dev/null +++ b/public/samples/mongodb/authentication/x509/replica-set/replica-set-x509.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + type: ReplicaSet + + members: 3 + version: 4.0.4-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + # look into `replica-set-persistent-volumes.yaml` for an example on how to use + # Kubernetes Persistent Volumes in your MDB deployment. + persistent: false + + # This will create a TLS & x509 enabled Replica Set, which means that all the traffic + # between members of the Replica Set and clients, will be encrypted using TLS + # certificates. These certificates will be generated on the fly by the operator + # using the Kubernetes CA. + # + # More information about setting up x509 client authentication in Ops Manager: + # + # https://docs.opsmanager.mongodb.com/current/tutorial/enable-x509-authentication-for-group + # + # Please refer to Kubernetes TLS Documentation on how to approve these certs: + # + # https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + # + security: + tls: + enabled: true + authentication: + enabled: true + modes: ["X509"] diff --git a/public/samples/mongodb/authentication/x509/replica-set/user.yaml b/public/samples/mongodb/authentication/x509/replica-set/user.yaml new file mode 100644 index 000000000..b63967107 --- /dev/null +++ b/public/samples/mongodb/authentication/x509/replica-set/user.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-replica-set-x509-user +spec: + username: CN=my-replica-set-x509-user,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US + db: $external + mongodbResourceRef: + name: my-replica-set + roles: + - db: admin + name: dbOwner diff --git a/public/samples/mongodb/authentication/x509/sharded-cluster/sharded-cluster-x509.yaml b/public/samples/mongodb/authentication/x509/sharded-cluster/sharded-cluster-x509.yaml new file mode 100644 index 000000000..fbee157dd --- /dev/null +++ b/public/samples/mongodb/authentication/x509/sharded-cluster/sharded-cluster-x509.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-x509-enabled-sc +spec: + type: ShardedCluster + + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + + version: 4.0.6-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + + # This will create a TLS & x509 enabled Sharded Cluster, which means that all the traffic + # between members of the Shards and clients, will be encrypted using TLS + # certificates. These certificates will be generated on the fly by the operator + # using the Kubernetes CA. + # + # More information about setting up x509 client authentication in Ops Manager: + # + # https://docs.opsmanager.mongodb.com/current/tutorial/enable-x509-authentication-for-group + # + # Please refer to Kubernetes TLS Documentation on how to approve these certs: + # + # https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + # + security: + authentication: + enabled: true + modes: ["X509"] + internalCluster: "X509" + tls: + enabled: true diff --git a/public/samples/mongodb/authentication/x509/sharded-cluster/user.yaml b/public/samples/mongodb/authentication/x509/sharded-cluster/user.yaml new file mode 100644 index 000000000..67005b429 --- /dev/null +++ b/public/samples/mongodb/authentication/x509/sharded-cluster/user.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBUser +metadata: + name: my-sharded-cluster-x509-user +spec: + username: CN=my-sharded-cluster-x509-user,OU=cloud,O=MongoDB,L=New York,ST=New York,C=US + db: $external + mongodbResourceRef: + name: my-replica-set + roles: + - db: admin + name: dbOwner diff --git a/public/samples/mongodb/backup/replica-set-backup-disabled.yaml b/public/samples/mongodb/backup/replica-set-backup-disabled.yaml new file mode 100644 index 000000000..9d6043c24 --- /dev/null +++ b/public/samples/mongodb/backup/replica-set-backup-disabled.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-backup-disabled +spec: + members: 3 + version: 4.4.0-ent + type: ReplicaSet + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + backup: + mode: disabled diff --git a/public/samples/mongodb/backup/replica-set-backup.yaml b/public/samples/mongodb/backup/replica-set-backup.yaml new file mode 100644 index 000000000..17dded8c9 --- /dev/null +++ b/public/samples/mongodb/backup/replica-set-backup.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-backup +spec: + members: 3 + version: 4.4.0-ent + type: ReplicaSet + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + backup: + mode: enabled diff --git a/public/samples/mongodb/external-connectivity/replica-set-external.yaml b/public/samples/mongodb/external-connectivity/replica-set-external.yaml new file mode 100644 index 000000000..0b31af10c --- /dev/null +++ b/public/samples/mongodb/external-connectivity/replica-set-external.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-externally-connectible-rs +spec: + type: ReplicaSet + + members: 3 + version: 4.2.1-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + security: + tls: + # TLS must be enabled to allow external connectivity + enabled: true + + connectivity: + # replicaSetHorizons consists of a list of maps where each map represents a node within + # the replica set and maps names of DNS horizons to externally connectable DNS names. + # In the following example, this would allow a client to make a replica set connection + # from outside the replica set using a connection string like + # mongodb://mdb0-test-website.com:1337,mdb1-test-website.com:1338,mdb2-test-website.com:1339. + # The length of the replicaSetHorizons list must be equal to the number of the members in the + # replica set and each member should have all of the same DNS horizon names specified. + replicaSetHorizons: + - "test-horizon-1": "mdb0-test-website.com:1337" + "test-horizon-2": "mdb0-test-internal-website.com:2337" + - "test-horizon-1": "mdb1-test-website.com:1338" + "test-horizon-2": "mdb1-test-internal-website.com:2338" + - "test-horizon-1": "mdb2-test-website.com:1339" + "test-horizon-2": "mdb2-test-internal-website.com:2339" diff --git a/public/samples/mongodb/minimal/replica-set.yaml b/public/samples/mongodb/minimal/replica-set.yaml new file mode 100644 index 000000000..849fb2514 --- /dev/null +++ b/public/samples/mongodb/minimal/replica-set.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.4.0-ent + type: ReplicaSet + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + + podSpec: + # 'podTemplate' allows to set custom fields in PodTemplateSpec. + # (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podtemplatespec-v1-core) + # for the Database StatefulSet. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 700M + requests: + cpu: "1" + memory: 500M diff --git a/public/samples/mongodb/minimal/sharded-cluster.yaml b/public/samples/mongodb/minimal/sharded-cluster.yaml new file mode 100644 index 000000000..05571f3b0 --- /dev/null +++ b/public/samples/mongodb/minimal/sharded-cluster.yaml @@ -0,0 +1,68 @@ +# +# This is a minimal config. To see all the options available, refer to the +# "extended" directory +# +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.4.0-ent + type: ShardedCluster + + # Before you create this object, you'll need to create a project ConfigMap and a + # credentials Secret. For instructions on how to do this, please refer to our + # documentation, here: + # https://docs.opsmanager.mongodb.com/current/tutorial/install-k8s-operator + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + # This flag allows the creation of pods without persistent volumes. This is for + # testing only, and must not be used in production. 'false' will disable + # Persistent Volume Claims. The default is 'true' + persistent: false + + configSrvPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 700M + requests: + cpu: "1" + memory: 500M + shardPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 700M + requests: + cpu: "1" + memory: 500M + + mongosPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "1" + memory: 200M + requests: + cpu: "0.5" + memory: 100M diff --git a/public/samples/mongodb/minimal/standalone.yaml b/public/samples/mongodb/minimal/standalone.yaml new file mode 100644 index 000000000..85e04ba57 --- /dev/null +++ b/public/samples/mongodb/minimal/standalone.yaml @@ -0,0 +1,40 @@ +# +# This is a minimal config. To see all the options available, refer to the +# "extended" directory +# +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone +spec: + version: 4.4.0-ent + type: Standalone + # Before you create this object, you'll need to create a project ConfigMap and a + # credentials Secret. For instructions on how to do this, please refer to our + # documentation, here: + # https://docs.opsmanager.mongodb.com/current/tutorial/install-k8s-operator + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + # This flag allows the creation of pods without persistent volumes. This is for + # testing only, and must not be used in production. 'false' will disable + # Persistent Volume Claims. The default is 'true' + persistent: false + + podSpec: + # 'podTemplate' allows to set custom fields in PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podtemplatespec-v1-core) + # for the Database StatefulSet. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 700M + requests: + cpu: "1" + memory: 500M diff --git a/public/samples/mongodb/mongodb-options/replica-set-mongod-options.yaml b/public/samples/mongodb/mongodb-options/replica-set-mongod-options.yaml new file mode 100644 index 000000000..b001177df --- /dev/null +++ b/public/samples/mongodb/mongodb-options/replica-set-mongod-options.yaml @@ -0,0 +1,19 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-options +spec: + members: 3 + version: 4.2.8-ent + type: ReplicaSet + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + # optional. Allows to pass custom MongoDB process configuration + additionalMongodConfig: + systemLog: + logAppend: true + systemLog.verbosity: 4 + operationProfiling.mode: slowOp diff --git a/public/samples/mongodb/mongodb-options/sharded-cluster-mongod-options.yaml b/public/samples/mongodb/mongodb-options/sharded-cluster-mongod-options.yaml new file mode 100644 index 000000000..fa5d40287 --- /dev/null +++ b/public/samples/mongodb/mongodb-options/sharded-cluster-mongod-options.yaml @@ -0,0 +1,33 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster-options +spec: + version: 4.2.8-ent + type: ShardedCluster + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 1 + mongos: + # optional. Allows to pass custom configuration for mongos processes + additionalMongodConfig: + systemLog: + logAppend: true + verbosity: 4 + configSrv: + # optional. Allows to pass custom configuration for Config Server mongod processes + additionalMongodConfig: + operationProfiling: + mode: slowOp + shard: + additionalMongodConfig: + # optional. Allows to pass custom configuration for Shards mongod processes + storage: + journal: + commitIntervalMs: 50 diff --git a/public/samples/mongodb/persistent-volumes/replica-set-persistent-volumes.yaml b/public/samples/mongodb/persistent-volumes/replica-set-persistent-volumes.yaml new file mode 100644 index 000000000..31866b23c --- /dev/null +++ b/public/samples/mongodb/persistent-volumes/replica-set-persistent-volumes.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.2.1-ent + service: my-service + + # Indicates featureCompatibilityVersion. This attribute will make the data + # format to persist in a particular version, maybe older, allowing for + # future downgrades if necessary. + featureCompatibilityVersion: "4.0" + + # Please Note: The default Kubernetes cluster domain is `cluster.local`. + # If your cluster has been configured with another domain, you can specify it + # with the `clusterDomain` attribute. + # clusterDomain: mycompany.net + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: ReplicaSet + + # log level affects the level of logging for the agent. Use DEBUG cautiously + # as log file size may grow very quickly. + logLevel: WARN + + persistent: true + + podSpec: + # `podTemplate.spec.containers[].resources` should be specified otherwise, WiredTiger + # cache won't be calculated properly by MongoDB daemon. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.25" + memory: 512M + + # "multiple" persistence allows to mount different directories to different + # Persistent Volumes + persistence: + multiple: + data: + storage: 10Gi + journal: + storage: 1Gi + labelSelector: + matchLabels: + app: "my-app" + logs: + storage: 500M + storageClass: standard diff --git a/public/samples/mongodb/persistent-volumes/sharded-cluster-persistent-volumes.yaml b/public/samples/mongodb/persistent-volumes/sharded-cluster-persistent-volumes.yaml new file mode 100644 index 000000000..898c0f865 --- /dev/null +++ b/public/samples/mongodb/persistent-volumes/sharded-cluster-persistent-volumes.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.2.1-ent + service: my-service + + # Indicates featureCompatibilityVersion. This attribute will make the data + # format to persist in a particular version, maybe older, allowing for + # future downgrades if necessary. + featureCompatibilityVersion: "4.0" + + # Please Note: The default Kubernetes cluster domain is `cluster.local`. + # If your cluster has been configured with another domain, you can specify it + # with the `clusterDomain` attribute. + # clusterDomain: mycompany.net + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: ShardedCluster + + # if "persistence" element is omitted then Operator uses the default size (5G) for + # mounting single Persistent Volume for config server. + persistent: true + + configSrvPodSpec: + # `podTemplate.spec.containers[].resources` should be specified, otherwise WiredTiger + # cache won't be calculated properly by MongoDB daemon. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.8" + memory: 512M + + mongosPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + + shardPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + memory: 3G + + persistence: + multiple: + # if the child of "multiple" is omitted then the default size will be used. + # 16G for "data", 1G for "journal", 3Gb for "logs". + data: + storage: 20G + logs: + storage: 4G + storageClass: standard diff --git a/public/samples/mongodb/persistent-volumes/standalone-persistent-volumes.yaml b/public/samples/mongodb/persistent-volumes/standalone-persistent-volumes.yaml new file mode 100644 index 000000000..6e69ba1ab --- /dev/null +++ b/public/samples/mongodb/persistent-volumes/standalone-persistent-volumes.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone +spec: + version: 4.2.1-ent + service: my-service + + # Indicates featureCompatibilityVersion. This attribute will make the data + # format to persist in a particular version, maybe older, allowing for + # future downgrades if necessary. + featureCompatibilityVersion: "4.0" + + # Please Note: The default Kubernetes cluster domain is `cluster.local`. + # If your cluster has been configured with another domain, you can specify it + # with the `clusterDomain` attribute. + # clusterDomain: mycompany.net + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: Standalone + + persistent: true + + + podSpec: + # `podTemplate.spec.containers[].resources` should be specified, otherwise WiredTiger + # cache won't be calculated properly by MongoDB daemon. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.25" + memory: 512M + + # "single" persistence allows to mount different directories to single + # Persistent Volume. + persistence: + single: + storage: 12G + storageClass: standard + labelSelector: + matchExpressions: + - {key: environment, operator: In, values: [dev]} diff --git a/public/samples/mongodb/pod-template/initcontainer-sysctl_config.yaml b/public/samples/mongodb/pod-template/initcontainer-sysctl_config.yaml new file mode 100644 index 000000000..80f40acb8 --- /dev/null +++ b/public/samples/mongodb/pod-template/initcontainer-sysctl_config.yaml @@ -0,0 +1,25 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set + namespace: mongodb +spec: + members: 3 + version: 4.2.2 + type: ReplicaSet + + cloudManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + podSpec: + podTemplate: + spec: + initContainers: + - name: "apply-sysctl-test" + image: "busybox:latest" + securityContext: + privileged: true + command: ["sysctl", "-w", "net.ipv4.tcp_keepalive_time=120"] diff --git a/public/samples/mongodb/pod-template/replica-set-pod-template.yaml b/public/samples/mongodb/pod-template/replica-set-pod-template.yaml new file mode 100644 index 000000000..ef27625cf --- /dev/null +++ b/public/samples/mongodb/pod-template/replica-set-pod-template.yaml @@ -0,0 +1,42 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set-pod-template +spec: + members: 3 + version: 4.2.11-ent + type: ReplicaSet + + opsManager: + configMapRef: + name: my-project + + credentials: my-credentials + + podSpec: + # `podTemplate` allows to set custom fields in PodTemplateSpec for the + # Database Pods. + # For more information see: + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podtemplatespec-v1-core + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.8" + memory: 1G + + # Another container will be added to each pod as a sidecar. + - name: standalone-sidecar + image: busybox + command: ["sleep"] + args: [ "infinity" ] + resources: + limits: + cpu: "1" + memory: 512M + requests: + cpu: 500m diff --git a/public/samples/mongodb/pod-template/sharded-cluster-pod-template.yaml b/public/samples/mongodb/pod-template/sharded-cluster-pod-template.yaml new file mode 100644 index 000000000..1bfd1f8dc --- /dev/null +++ b/public/samples/mongodb/pod-template/sharded-cluster-pod-template.yaml @@ -0,0 +1,72 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster-pod-template +spec: + type: ShardedCluster + version: 4.0.14-ent + + shardCount: 1 + mongodsPerShardCount: 1 + mongosCount: 1 + configServerCount: 1 + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + configSrvPodSpec: + podAntiAffinityTopologyKey: nodeId + podTemplate: + spec: + terminationGracePeriodSeconds: 120 + + # `podTemplate.spec.containers[].resources` should be specified, otherwise WiredTiger + # cache won't be calculated properly by MongoDB daemon. + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + # Note, that this topology key will overwrite the antiAffinity + # topologyKey set by the Operator from + # 'spec.configSrvPodSpec.podAntiAffinityTopologyKey'. + topologyKey: "failure-domain.beta.kubernetes.io/zone" + weight: 30 + + mongosPodSpec: + podTemplate: + spec: + restartPolicy: Never + serviceAccountName: the-custom-user + + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + + shardPodSpec: + podTemplate: + metadata: + annotations: + key1: value1 + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + tolerations: + - key: "key" + operator: "Exists" + effect: "NoSchedule" diff --git a/public/samples/mongodb/pod-template/standalone-pod-template.yaml b/public/samples/mongodb/pod-template/standalone-pod-template.yaml new file mode 100644 index 000000000..684d98845 --- /dev/null +++ b/public/samples/mongodb/pod-template/standalone-pod-template.yaml @@ -0,0 +1,17 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone-pod-template +spec: + version: 4.2.11-ent + type: Standalone + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + podSpec: + podTemplate: + spec: + hostAliases: + - ip: "1.2.3.4" + hostnames: ["hostname"] diff --git a/public/samples/mongodb/podspec/replica-set-podspec.yaml b/public/samples/mongodb/podspec/replica-set-podspec.yaml new file mode 100644 index 000000000..e775be650 --- /dev/null +++ b/public/samples/mongodb/podspec/replica-set-podspec.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 4.0.0-ent + service: my-service + + # Indicates featureCompatibilityVersion. This attribute will make the data + # format to persist in a particular version, maybe older, allowing for + # future downgrades if necessary. + featureCompatibilityVersion: "4.0" + + # Please Note: The default Kubernetes cluster domain is `cluster.local`. + # If your cluster has been configured with another domain, you can specify it + # with the `clusterDomain` attribute. + # clusterDomain: mycompany.net + + opsManager: + configMapRef: + name: my-project + + credentials: my-credentials + type: ReplicaSet + + # log level affects the level of logging for the agent. Use DEBUG cautiously + # as log file size may grow very quickly. + logLevel: WARN + + persistent: true + podSpec: + # `podTemplate.spec.containers[].resources` should be specified otherwise, WiredTiger + # cache won't be calculated properly by MongoDB daemon. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.25" + memory: 512M + + # "multiple" persistence allows to mount different directories to different + # Persistent Volumes. + persistence: + multiple: + data: + storage: 10Gi + journal: + storage: 1Gi + labelSelector: + matchLabels: + app: "my-app" + logs: + storage: 500M + storageClass: standard + + # For podAffinity and nodeAffinity see Kubernetes Docs + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + podAntiAffinityTopologyKey: nodeId + + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 diff --git a/public/samples/mongodb/podspec/sharded-cluster-podspec.yaml b/public/samples/mongodb/podspec/sharded-cluster-podspec.yaml new file mode 100644 index 000000000..0f032490b --- /dev/null +++ b/public/samples/mongodb/podspec/sharded-cluster-podspec.yaml @@ -0,0 +1,103 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 4.0.0-ent + service: my-service + + # Indicates featureCompatibilityVersion. This attribute will make the data + # format to persist in a particular version, maybe older, allowing for + # future downgrades if necessary. + featureCompatibilityVersion: "4.0" + + # Please Note: The default Kubernetes cluster domain is `cluster.local`. + # If your cluster has been configured with another domain, you can specify it + # with the `clusterDomain` attribute. + # clusterDomain: mycompany.net + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: ShardedCluster + + persistent: true + configSrvPodSpec: + # `podTemplate.spec.containers[].resources` should be specified otherwise, WiredTiger + # cache won't be calculated properly by MongoDB daemon. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.8" + memory: 1G + + # If "persistence" element is omitted then Operator uses the default size + # (5G) for mounting single Persistent Volume for config server. + + # For podAffinity and nodeAffinity see Kubernetes Docs + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + podAntiAffinityTopologyKey: kubernetes.io/hostname + + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + + mongosPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.8" + memory: 1G + + podAntiAffinityTopologyKey: rackId + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: another-node-label-key + operator: In + values: + - another-node-label-value + + shardPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "0.6" + memory: 3G + + persistence: + multiple: + # if the child of "multiple" is omitted then the default size will be used. + # 16G for "data", 1G for "journal", 3Gb for "logs" + data: + storage: 20G + logs: + storage: 4G + storageClass: standard + + podAntiAffinityTopologyKey: kubernetes.io/hostname diff --git a/public/samples/mongodb/podspec/standalone-podspec.yaml b/public/samples/mongodb/podspec/standalone-podspec.yaml new file mode 100644 index 000000000..c9b4254c2 --- /dev/null +++ b/public/samples/mongodb/podspec/standalone-podspec.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-standalone +spec: + version: 4.0.0-ent + service: my-service + + # Indicates featureCompatibilityVersion. This attribute will make the data + # format to persist in a particular version, maybe older, allowing for + # future downgrades if necessary. + featureCompatibilityVersion: "4.0" + + # Please Note: The default Kubernetes cluster domain is `cluster.local`. + # If your cluster has been configured with another domain, you can specify it + # with the `clusterDomain` attribute. + # clusterDomain: mycompany.net + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: Standalone + + persistent: true + podSpec: + # `podTemplate.spec.containers[].resources` should be specified otherwise, WiredTiger + # cache won't be calculated properly by MongoDB daemon. + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + # For more information about Pod and container resource management, see: + # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: "0.8" + memory: 1G + + # "single" persistence allows to mount different directories to single + # Persistent Volume. + persistence: + single: + storage: 12G + storageClass: standard + labelSelector: + matchExpressions: + - {key: environment, operator: In, values: [dev]} + + # For podAffinity and nodeAffinity see Kubernetes Docs + # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: security + operator: In + values: + - S1 + topologyKey: failure-domain.beta.kubernetes.io/zone + + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/e2e-az-name + operator: In + values: + - e2e-az1 + - e2e-az2 diff --git a/public/samples/mongodb/project.yaml b/public/samples/mongodb/project.yaml new file mode 100644 index 000000000..190673739 --- /dev/null +++ b/public/samples/mongodb/project.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-project +data: + projectName: My Ops/Cloud Manager Project + baseUrl: http://my-ops-cloud-manager-url + + # Optional parameters + + # If orgId is omitted a new organization will be created, with the same name as the Project. + orgId: my-org-id diff --git a/public/samples/mongodb/prometheus/replica-set.yaml b/public/samples/mongodb/prometheus/replica-set.yaml new file mode 100644 index 000000000..3184e2398 --- /dev/null +++ b/public/samples/mongodb/prometheus/replica-set.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-replica-set +spec: + members: 3 + version: 5.0.6-ent + + opsManager: + configMapRef: + name: my-project + + credentials: my-credentials + type: ReplicaSet + + persistent: true + + prometheus: + passwordSecretRef: + # SecretRef to a Secret with a 'password' entry on it. + name: prometheus-password + + # change this value to your Prometheus username + username: + + # Enables HTTPS on the prometheus scrapping endpoint + # This should be a reference to a Secret type kuberentes.io/tls + # tlsSecretKeyRef: + # name: + + # Port for Prometheus, default is 9216 + # port: 9216 + # + # Metrics path for Prometheus, default is /metrics + # metricsPath: '/metrics' + +--- +apiVersion: v1 +kind: Secret +metadata: + name: prometheus-password +type: Opaque +stringData: + password: diff --git a/public/samples/mongodb/prometheus/sharded-cluster.yaml b/public/samples/mongodb/prometheus/sharded-cluster.yaml new file mode 100644 index 000000000..c3fd7c737 --- /dev/null +++ b/public/samples/mongodb/prometheus/sharded-cluster.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster +spec: + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + version: 5.0.6-ent + type: ShardedCluster + + # Before you create this object, you'll need to create a project ConfigMap and a + # credentials Secret. For instructions on how to do this, please refer to our + # documentation, here: + # https://docs.opsmanager.mongodb.com/current/tutorial/install-k8s-operator + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + # This flag allows the creation of pods without persistent volumes. This is for + # testing only, and must not be used in production. 'false' will disable + # Persistent Volume Claims. The default is 'true' + persistent: false + + prometheus: + passwordSecretRef: + # SecretRef to a Secret with a 'password' entry on it. + name: prometheus-password + + # change this value to your Prometheus username + username: + + # Enables HTTPS on the prometheus scrapping endpoint + # This should be a reference to a Secret type kuberentes.io/tls + # tlsSecretKeyRef: + # name: + + # Port for Prometheus, default is 9216 + # port: 9216 + # + # Metrics path for Prometheus, default is /metrics + # metricsPath: '/metrics' + + configSrvPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 700M + requests: + cpu: "1" + memory: 500M + shardPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "2" + memory: 700M + requests: + cpu: "1" + memory: 500M + + mongosPodSpec: + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: "1" + memory: 200M + requests: + cpu: "0.5" + memory: 100M diff --git a/public/samples/mongodb/tls/replica-set/replica-set-tls.yaml b/public/samples/mongodb/tls/replica-set/replica-set-tls.yaml new file mode 100644 index 000000000..ce6c0fb00 --- /dev/null +++ b/public/samples/mongodb/tls/replica-set/replica-set-tls.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-tls-enabled-rs +spec: + type: ReplicaSet + + members: 3 + version: 4.0.4-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + # look into `replica-set-persistent-volumes.yaml` for an example on how to use + # Kubernetes Persistent Volumes in your MDB deployment. + persistent: false + + # This will create a TLS enabled Replica Set, which means that all the traffic + # between members of the Replica Set and clients, will be encrypted using TLS + # certificates. + security: + # The operator will look for a secret name mdb-my-tls-enabled-rs-cert + certsSecretPrefix: mdb + tls: + ca: custom-ca + enabled: true + + # The default TLS mode is 'requireTLS' but it can be customized using the + # the `additionalMongodConfig` structure. Please find more information here: + # https://docs.mongodb.com/manual/reference/configuration-options/#net.ssl.mode + additionalMongodConfig: + net: + ssl: + mode: "preferSSL" diff --git a/public/samples/mongodb/tls/sharded-cluster/sharded-cluster-tls.yaml b/public/samples/mongodb/tls/sharded-cluster/sharded-cluster-tls.yaml new file mode 100644 index 000000000..1e092a177 --- /dev/null +++ b/public/samples/mongodb/tls/sharded-cluster/sharded-cluster-tls.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-sharded-cluster +spec: + type: ShardedCluster + + shardCount: 2 + mongodsPerShardCount: 3 + mongosCount: 2 + configServerCount: 3 + + version: 4.0.6-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + + persistent: false + + # This will create a TLS enabled Sharded Cluster, which means that + # all the traffic between Shards and clients will be encrypted using + # TLS certificates. These certificates will be generated on the fly + # by the operator using the Kubernetes CA. + security: + # The operator will look for secrets with the following names: + # mdb-my-sharded-cluster-mongos-cert + # mdb-my-sharded-cluster-config-cert + # mdb-my-sharded-cluster--cert + # Where x is all numbers between 0 and the number of shards (excluded) + certsSecretPrefix: mdb + tls: + ca: custom-ca + enabled: true diff --git a/public/samples/mongodb/tls/standalone/standalone-tls.yaml b/public/samples/mongodb/tls/standalone/standalone-tls.yaml new file mode 100644 index 000000000..5fab6a702 --- /dev/null +++ b/public/samples/mongodb/tls/standalone/standalone-tls.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: my-tls-standalone +spec: + version: 4.0.14-ent + + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + type: Standalone + + persistent: true + + # This will create a TLS enabled Standalone which means that the + # traffic will be encrypted using TLS certificates. These + # certificates will be generated on the fly by the operatror using + # the Kubernetes CA. + # Please refer to Kubernetes TLS Documentation on how to approve these certs: + # + # https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + # + security: + tls: + enabled: true diff --git a/public/samples/mongodb_multicluster/replica-set-configure-storage.yaml b/public/samples/mongodb_multicluster/replica-set-configure-storage.yaml new file mode 100644 index 000000000..d7adf8b57 --- /dev/null +++ b/public/samples/mongodb_multicluster/replica-set-configure-storage.yaml @@ -0,0 +1,70 @@ +# provide statefulset override per cluster +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: cluster1.mongokubernetes.com + members: 2 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar1 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + # to override the default storage size of the "data" PV + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 1Gi + - clusterName: cluster2.mongokubernetes.com + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar2 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 1Gi + - clusterName: cluster3.mongokubernetes.com + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar3 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 1Gi diff --git a/public/samples/mongodb_multicluster/replica-set-sts-override.yaml b/public/samples/mongodb_multicluster/replica-set-sts-override.yaml new file mode 100644 index 000000000..e4c572bd7 --- /dev/null +++ b/public/samples/mongodb_multicluster/replica-set-sts-override.yaml @@ -0,0 +1,68 @@ +# provide statefulset override per cluster +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + persistent: false + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + - clusterName: cluster1.mongokubernetes.com + members: 2 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar1 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + # to override the default storage class for the pv + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "gp2" + - clusterName: cluster2.mongokubernetes.com + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar2 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "gp2" + - clusterName: cluster3.mongokubernetes.com + members: 1 + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar3 + image: busybox + command: ["sleep"] + args: [ "infinity" ] + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "gp2" diff --git a/public/samples/mongodb_multicluster/replica-set.yaml b/public/samples/mongodb_multicluster/replica-set.yaml new file mode 100644 index 000000000..23c567f21 --- /dev/null +++ b/public/samples/mongodb_multicluster/replica-set.yaml @@ -0,0 +1,23 @@ +# sample mongodb-multi replicaset yaml +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + persistent: false + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + # cluster names where you want to deploy the replicaset + - clusterName: cluster1.mongokubernetes.com + members: 2 + - clusterName: cluster2.mongokubernetes.com + members: 1 + - clusterName: cluster3.mongokubernetes.com + members: 2 diff --git a/public/samples/multi-cluster-cli-gitops/README.md b/public/samples/multi-cluster-cli-gitops/README.md new file mode 100644 index 000000000..1b4bd6729 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/README.md @@ -0,0 +1,22 @@ +# Multi-Cluster CLI GitOps Samples + +This is an example of using the `multi-cluster-cli` in a [GitOps](https://www.weave.works/technologies/gitops/) operating model to perform a recovery of the dataplane +in a multi-cluster deployment scenario. For more details on managing multi-cluster resources with the kubernetes operator see [the official documentation](https://www.mongodb.com/docs/kubernetes-operator/master/multi-cluster/). The example is applicable for an [ArgoCD](https://argo-cd.readthedocs.io/) configuration. + +## ArgoCD configuration +The files in the [argocd](./argocd) contain an [AppProject](./argocd/project.yaml) and an [Application](./argocd/application.yaml) linked to it which allows the synchronization of `MongoDBMulti` resources from a Git repo. + +## Multi-Cluster CLI Job setup +To enable the manual disaster recovery using the CLI, this sample provides a [Job](./resources/job.yaml) which runs the recovery subcommand as a [PreSync hook](https://argo-cd.readthedocs.io/en/stable/user-guide/resource_hooks/). This ensures that the multicluster environment is configured before the application of the modified [`MongoDBMulti`](./resources/replica-set.yaml) resource. The `Job` mounts the same `kubeconfig` that the operator is using to connect to the clusters defined in your architecture. + +## RBAC Settings for the Central and Member clusters +The RBAC settings for the operator are typically creating using the CLI. In cases, where it is not possible, you can adjust and apply the YAML files from the [rbac](./resources/rbac) directory. + +### Build the multi-cluster CLI image +You can build a minimal image containing the CLI executable using the `Dockerfile` [provided in this repo](./../../tools/multicluster/Dockerfile). +``` shell +git clone https://github.com/mongodb/mongodb-enterprise-kubernetes +cd mongodb-enterprise-kubernetes/tools/multicluster +docker build . -t "your-registry/multi-cluster-cli:latest" +docker push "your-registry/multi-cluster-cli:latest" +``` diff --git a/public/samples/multi-cluster-cli-gitops/argocd/application.yaml b/public/samples/multi-cluster-cli-gitops/argocd/application.yaml new file mode 100644 index 000000000..870170ec2 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/argocd/application.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: multi-cluster-replica-set + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io + labels: + name: database +spec: + project: my-project + source: + repoURL: https://github.com/mongodb/mongodb-enterprise-kubernetes + targetRevision: "fix/ubi-8-repo-names" + path: samples/multi-cluster-cli-gitops + destination: + server: https://central.mongokubernetes.com + namespace: mongodb + syncPolicy: + automated: + prune: true + syncOptions: + - CreateNamespace=true diff --git a/public/samples/multi-cluster-cli-gitops/argocd/project.yaml b/public/samples/multi-cluster-cli-gitops/argocd/project.yaml new file mode 100644 index 000000000..d9d917827 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/argocd/project.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: my-project + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: Example Project + sourceRepos: + - '*' + destinations: + - namespace: mongodb + server: https://central.mongokubernetes.com + clusterResourceWhitelist: + # Allow MongoDBMulti resources to be synced + - group: '' + kind: MongoDBMultiCluster + # Allow Jobs to be created (used for sync hooks in this example) + - group: '' + kind: Job + - group: '' + kind: Namespace diff --git a/public/samples/multi-cluster-cli-gitops/resources/job.yaml b/public/samples/multi-cluster-cli-gitops/resources/job.yaml new file mode 100644 index 000000000..4d476cc9b --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/resources/job.yaml @@ -0,0 +1,36 @@ +# Sample PreSync job to perform the manual dataplane recovery before a replica set sync +--- +apiVersion: batch/v1 +kind: Job +metadata: + generateName: multicluster-cli-recover- + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded +spec: + template: + spec: + containers: + - name: multicluster-cli + image: your-registry/multi-cluster-cli + env: + - name: KUBECONFIG + value: /etc/config/kubeconfig + args: + - "-central-cluster=central.mongokubernetes.com" + - "-member-clusters=cluster1.mongokubernetes.com,cluster2.mongokubernetes.com,cluster4.mongokubernetes.com" + - "-member-cluster-namespace=mongodb" + - "-central-cluster-namespace=mongodb" + - "-operator-name=mongodb-enterprise-operator-multi-cluster" + - "-source-cluster=cluster1.mongokubernetes.com" + volumeMounts: + - mountPath: /etc/config/kubeconfig + name: kube-config-volume + restartPolicy: Never + volumes: + - name: kube-config-volume + secret: + defaultMode: 420 + secretName: mongodb-enterprise-operator-multi-cluster-kubeconfig + + backoffLimit: 2 diff --git a/public/samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_central_cluster.yaml b/public/samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_central_cluster.yaml new file mode 100644 index 000000000..c50b0dbc1 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_central_cluster.yaml @@ -0,0 +1,99 @@ +# Central Cluster, cluster-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-cluster-role +rules: +- apiGroups: + - mongodb.com + resources: + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodbmulticluster/status + - mongodbusers + - mongodbusers/status + - opsmanagers + - opsmanagers/finalizers + - opsmanagers/status + - mongodb + - mongodb/finalizers + - mongodb/status + verbs: + - '*' +- apiGroups: + - "" + resources: + - secrets + - configmaps + - services + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch + +--- +# Central Cluster, cluster-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-cluster-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-multi-cluster-role +subjects: +- kind: ServiceAccount + name: test-service-account + namespace: central-namespace + +--- +# Central Cluster, cluster-scoped resources +apiVersion: v1 +kind: ServiceAccount +imagePullSecrets: +- name: image-registries-secret +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: test-service-account + namespace: central-namespace + +--- diff --git a/public/samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_member_cluster.yaml b/public/samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_member_cluster.yaml new file mode 100644 index 000000000..481c08885 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_member_cluster.yaml @@ -0,0 +1,113 @@ +# Member Cluster, cluster-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-cluster-role +rules: +- apiGroups: + - "" + resources: + - secrets + - configmaps + - services + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch + +--- +# Member Cluster, cluster-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-cluster-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mongodb-enterprise-operator-multi-cluster-role +subjects: +- kind: ServiceAccount + name: test-service-account + namespace: member-namespace + +--- +# Member Cluster, cluster-scoped resources +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-appdb + namespace: member-namespace + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-database-pods + namespace: member-namespace + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-ops-manager + namespace: member-namespace + +--- +apiVersion: v1 +kind: ServiceAccount +imagePullSecrets: +- name: image-registries-secret +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: test-service-account + namespace: member-namespace + +--- diff --git a/public/samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_central_cluster.yaml b/public/samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_central_cluster.yaml new file mode 100644 index 000000000..315bb019b --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_central_cluster.yaml @@ -0,0 +1,94 @@ +# Central Cluster, namespace-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-role + namespace: central-namespace +rules: +- apiGroups: + - mongodb.com + resources: + - mongodbmulticluster + - mongodbmulticluster/finalizers + - mongodbmulticluster/status + - mongodbusers + - mongodbusers/status + - opsmanagers + - opsmanagers/finalizers + - opsmanagers/status + - mongodb + - mongodb/finalizers + - mongodb/status + verbs: + - '*' +- apiGroups: + - "" + resources: + - secrets + - configmaps + - services + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- +# Central Cluster, namespace-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-role-binding + namespace: central-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-operator-multi-role +subjects: +- kind: ServiceAccount + name: test-service-account + namespace: central-namespace + +--- +# Central Cluster, namespace-scoped resources +apiVersion: v1 +kind: ServiceAccount +imagePullSecrets: +- name: image-registries-secret +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: test-service-account + namespace: central-namespace + +--- diff --git a/public/samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_member_cluster.yaml b/public/samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_member_cluster.yaml new file mode 100644 index 000000000..04f4a4980 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_member_cluster.yaml @@ -0,0 +1,150 @@ +# Member Cluster, namespace-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-appdb + namespace: member-namespace +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - patch + - delete + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-role + namespace: member-namespace +rules: +- apiGroups: + - "" + resources: + - secrets + - configmaps + - services + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - create + - update + - delete + - watch + - deletecollection +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- +# Member Cluster, namespace-scoped resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-appdb + namespace: member-namespace +roleRef: + apiGroup: "" + kind: Role + name: mongodb-enterprise-appdb +subjects: +- kind: ServiceAccount + name: mongodb-enterprise-appdb + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-operator-multi-role-binding + namespace: member-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: mongodb-enterprise-operator-multi-role +subjects: +- kind: ServiceAccount + name: test-service-account + namespace: member-namespace + +--- +# Member Cluster, namespace-scoped resources +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-appdb + namespace: member-namespace + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-database-pods + namespace: member-namespace + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: mongodb-enterprise-ops-manager + namespace: member-namespace + +--- +apiVersion: v1 +kind: ServiceAccount +imagePullSecrets: +- name: image-registries-secret +metadata: + creationTimestamp: null + labels: + multi-cluster: "true" + name: test-service-account + namespace: member-namespace + +--- diff --git a/public/samples/multi-cluster-cli-gitops/resources/replica-set.yaml b/public/samples/multi-cluster-cli-gitops/resources/replica-set.yaml new file mode 100644 index 000000000..a27486078 --- /dev/null +++ b/public/samples/multi-cluster-cli-gitops/resources/replica-set.yaml @@ -0,0 +1,23 @@ +# sample mongodb-multi replicaset yaml +--- +apiVersion: mongodb.com/v1 +kind: MongoDBMultiCluster +metadata: + name: multi-replica-set +spec: + version: 4.4.0-ent + type: ReplicaSet + persistent: false + duplicateServiceObjects: false + credentials: my-credentials + opsManager: + configMapRef: + name: my-project + clusterSpecList: + # cluster names where you want to deploy the replicaset + - clusterName: cluster1.mongokubernetes.com + members: 2 + - clusterName: cluster2.mongokubernetes.com + members: 1 + - clusterName: cluster4.mongokubernetes.com + members: 2 diff --git a/public/samples/ops-manager/ops-manager-appdb-agent-startup-parameters.yaml b/public/samples/ops-manager/ops-manager-appdb-agent-startup-parameters.yaml new file mode 100644 index 000000000..4631f31de --- /dev/null +++ b/public/samples/ops-manager/ops-manager-appdb-agent-startup-parameters.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + # the number of Ops Manager instances to run. Set to value bigger + # than 1 to get high availability and upgrades without downtime + replicas: 3 + + # the version of Ops Manager distro to use + version: 5.0.5 + + # optional. Specify the custom cluster domain of the Kubernetes cluster if it's different from the default one ('cluster.local'). + # This affects the urls generated by the Operator. + # This field is also used for Application Database url + clusterDomain: mycompany.net + + # the name of the secret containing admin user credentials. + # Either remove the secret or change the password using Ops Manager UI after the Ops Manager + # resource is created! + adminCredentials: ops-manager-admin-secret + + # the application database backing Ops Manager. Replica Set is the only supported type + # Application database has the SCRAM-SHA authentication mode always enabled + applicationDatabase: + version: "4.4.11-ent" + members: 3 + # optional. Allows to pass custom flags that will be used + # when launching the mongodb agent. All values must be strings + agent: + startupOptions: + serverSelectionTimeoutSeconds: "20" diff --git a/public/samples/ops-manager/ops-manager-appdb-custom-images.yaml b/public/samples/ops-manager/ops-manager-appdb-custom-images.yaml new file mode 100644 index 000000000..04733a561 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-appdb-custom-images.yaml @@ -0,0 +1,24 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + version: 5.0.5 + replicas: 3 + adminCredentials: ops-manager-admin-secret + backup: + enabled: false + applicationDatabase: + # The version specified must match the one in the image provided in the `mongod` field + version: 4.4.11-ent + members: 3 + podSpec: + podTemplate: + spec: + containers: + - name: mongodb-agent + image: 'quay.io/mongodb/mongodb-agent:10.29.0.6830-1' + - name: mongod + image: 'quay.io/mongodb/mongodb-enterprise-appdb-database:4.4.11-ent' + - name: mongodb-agent-monitoring + image: 'quay.io/mongodb/mongodb-agent:10.29.0.6830-1' diff --git a/public/samples/ops-manager/ops-manager-backup.yaml b/public/samples/ops-manager/ops-manager-backup.yaml new file mode 100644 index 000000000..c9558fd21 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-backup.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-backup +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + + # optional. Enabled by default + # Allows to configure backup in Ops Manager + backup: + enabled: true + # optional. Defaults to 1 if not set. + # Configures the number of backup daemons to create + members: 2 + + # optional. Configured by default if backup is enabled. + # Configures Head db storage parameters + headDB: + # optional. Default storage is 30G + storage: 50G + # optional + labelSelector: + matchLabels: + app: "my-app" + # Configures the list of Oplog Store Configs + opLogStores: + - name: oplog1 + # reference to MongoDB Custom Resource. The Operator watches changes in it and updates Oplog configuration + # in Ops Manager + mongodbResourceRef: + name: om-mongodb-oplog + # optional. Specify if Oplog database has SCRAM-SHA authentication enabled + mongodbUserRef: + name: admin-user + + # Configures the list of S3 Oplog Store Configs + s3OpLogStores: + - name: my-s3-oplog-store + # the name of the secret which contains aws credentials + s3SecretRef: + name: my-aws-creds + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: my-s3-oplog-store-bucket-name + pathStyleAccessEnabled: true + + # Configures the list of S3 Snapshot Configs. Application database is used as a database for S3 metadata + # by default + # Note, that either S3 Snapshot or Blockstore config needs to be specified to backup MongoDB deployments + s3Stores: + - name: s3store1 + # the name of the secret which contains aws credentials + s3SecretRef: + name: my-aws-creds + s3BucketEndpoint: s3.us-east-1.amazonaws.com + s3BucketName: my-bucket-name + pathStyleAccessEnabled: true + # Configures the list of Blockstore Configs + blockStores: + - name: blockStore1 + # reference to MongoDB Custom Resource. The Operator watches changes in it and updates Blockstore configuration + # in Ops Manager + mongodbResourceRef: + name: my-mongodb-blockstore + + # The secret referenced by this field contains the certificates used to enable Queryable Backups https://docs.opsmanager.mongodb.com/current/tutorial/query-backup/ + queryableBackupSecretRef: + name: queryable-backup-pem-secret + + applicationDatabase: + members: 3 + version: 4.4.11-ent diff --git a/public/samples/ops-manager/ops-manager-disable-appdb-process.yaml b/public/samples/ops-manager/ops-manager-disable-appdb-process.yaml new file mode 100644 index 000000000..4cbcf1783 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-disable-appdb-process.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + replicas: 3 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + configuration: + mms.fromEmailAddr: "admin@example.com" + + applicationDatabase: + members: 3 + version: 4.4.11-ent + automationConfig: + processes: + # this will disable the second AppDB process to allow for manual backups to be taken. + - name: ops-manager-db-1 + disabled: true diff --git a/public/samples/ops-manager/ops-manager-external.yaml b/public/samples/ops-manager/ops-manager-external.yaml new file mode 100644 index 000000000..90cd204b9 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-external.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-external +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + + configuration: + # set this property to allow Ops Manager to manage deployments outside of + # Kubernetes cluster. This must be equal to an externally connectible DNS + mms.centralUrl: http:// + + # optional. Disabled by default. Creates an additional service to make Ops Manager reachable from + # outside of the Kubernetes cluster. + externalConnectivity: + # LoadBalancer|NodePort + type: LoadBalancer + # optional. Corresponds to NodePort port + port: 30100 + # optional + loadBalancerIP: 123.456.789 + # optional + externalTrafficPolicy: Local + # optional + # For more information, read https://kubernetes.io/docs/concepts/services-networking/service/ about + # what kind of annotations you can use, based on your Cloud provider. + annotations: + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012 + + applicationDatabase: + members: 3 + version: 4.4.11-ent diff --git a/public/samples/ops-manager/ops-manager-ignore-ui-setup.yaml b/public/samples/ops-manager/ops-manager-ignore-ui-setup.yaml new file mode 100644 index 000000000..908f4f442 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-ignore-ui-setup.yaml @@ -0,0 +1,27 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-ignore-ui +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + + configuration: + # passing mms.ignoreInitialUiSetup=true allows to avoid the setup wizard in Ops Manager. Note, that + # this requires to set some mandatory configuration properties, see + # https://docs.opsmanager.mongodb.com/current/reference/configuration/index.html#mms.ignoreInitialUiSetup + mms.ignoreInitialUiSetup: "true" + automation.versions.source: mongodb + mms.adminEmailAddr: support@example.com + mms.fromEmailAddr: support@example.com + mms.replyToEmailAddr: support@example.com + mms.mail.hostname: email-smtp.us-east-1.amazonaws.com + mms.mail.port: "465" + mms.mail.ssl: "true" + mms.mail.transport: smtp + mms.minimumTLSVersion: TLSv1.2 + + applicationDatabase: + version: "4.4.11-ent" + members: 3 diff --git a/public/samples/ops-manager/ops-manager-local-mode.yaml b/public/samples/ops-manager/ops-manager-local-mode.yaml new file mode 100644 index 000000000..b8d93f33c --- /dev/null +++ b/public/samples/ops-manager/ops-manager-local-mode.yaml @@ -0,0 +1,40 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-localmode +spec: + replicas: 2 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + configuration: + # this enables local mode in Ops Manager + automation.versions.source: local + + statefulSet: + spec: + # the Persistent Volume Claim will be created for each Ops Manager Pod + volumeClaimTemplates: + - metadata: + name: mongodb-versions + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20G + template: + spec: + containers: + - name: mongodb-ops-manager + volumeMounts: + - name: mongodb-versions + # this is the directory in each Pod where all MongoDB + # archives must be put + mountPath: /mongodb-ops-manager/mongodb-releases + + + backup: + enabled: false + + applicationDatabase: + version: "4.4.11-ent" + members: 3 diff --git a/public/samples/ops-manager/ops-manager-non-root.yaml b/public/samples/ops-manager/ops-manager-non-root.yaml new file mode 100644 index 000000000..1ba0d4f28 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-non-root.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + replicas: 1 + version: 5.0.5 + + adminCredentials: ops-manager-admin-secret + + applicationDatabase: + members: 3 + version: 4.4.11-ent + + + # The statefulSet entry will modify the way the StatefulSet holding the + # Ops Manager and Backup Daemon Pods will be created. In this case we can + # specify a non-default SecurityContext. + statefulSet: + spec: + template: + spec: + securityContext: + fsGroup: 5000 + runAsUser: 5000 diff --git a/public/samples/ops-manager/ops-manager-pod-spec.yaml b/public/samples/ops-manager/ops-manager-pod-spec.yaml new file mode 100644 index 000000000..eb4c9d64b --- /dev/null +++ b/public/samples/ops-manager/ops-manager-pod-spec.yaml @@ -0,0 +1,73 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-pod-spec +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + configuration: + mms.testUtil.enabled: "true" + backup: + enabled: true + statefulSet: + spec: + template: + spec: + hostAliases: + - ip: "1.2.3.4" + hostnames: ["hostname"] + containers: + - name: mongodb-backup-daemon + resources: + requests: + cpu: '0.50' + memory: 4500M + + statefulSet: + spec: + template: + metadata: + annotations: + key1: value1 + spec: + volumes: + - name: mongod-binaries-volume + # replace this with a real persistent volume + emptyDir: {} + containers: + - name: mongodb-ops-manager + volumeMounts: + - mountPath: /mongodb-ops-manager/mongodb-releases/ + name: mongod-binaries-volume + resources: + limits: + cpu: '0.70' + memory: 6G + tolerations: + - key: "key" + operator: "Exists" + effect: "NoSchedule" + + applicationDatabase: + members: 3 + version: 4.4.11-ent + podSpec: + cpu: '0.25' + memory: 350M + persistence: + single: + storage: 1G + podTemplate: + spec: + # This container will be added to each pod as a sidecar + containers: + - name: appdb-sidecar + image: busybox + command: ["sleep"] + args: [ "infinity" ] + resources: + limits: + cpu: "1" + requests: + cpu: 500m diff --git a/public/samples/ops-manager/ops-manager-remote-mode.yaml b/public/samples/ops-manager/ops-manager-remote-mode.yaml new file mode 100644 index 000000000..80c5fa253 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-remote-mode.yaml @@ -0,0 +1,150 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-remotemode +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + configuration: + # Change this url to point to the nginx server deployed below + automation.versions.download.baseUrl: http://nginx-svc..svc.cluster.local:80 + # Ops Manager versions prior 4.4.11 require this flag to be set to "false" to make MongoDB downloads work correctly + # for "remote" mode + automation.versions.download.baseUrl.allowOnlyAvailableBuilds: "false" + automation.versions.source: remote + backup: + enabled: false + + applicationDatabase: + version: "4.4.11-ent" + members: 3 +--- +# The nginx deployment allows to deploy the web server that will serve mongodb binaries to the MongoDBOpsManager resource +# The example below provides the binaries for 4.4.0 mongodb (community and enterprise) for ubuntu and rhel (necessary if +# the cluster is Openshift) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - image: nginx:1.14.2 + imagePullPolicy: IfNotPresent + name: nginx + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /mongodb-ops-manager/mongodb-releases/linux + name: mongodb-versions + - name: nginx-conf + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + initContainers: + - name: setting-up-rhel-mongodb-4-4 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel80-4.4.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/linux/mongodb-linux-x86_64-rhel80-4.4.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases/linux + - name: setting-up-ubuntu-mongodb-4-4 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1804-4.4.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/linux/mongodb-linux-x86_64-ubuntu1804-4.4.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases/linux + + - name: setting-up-rhel-mongodb-4-4-ent + image: curlimages/curl:latest + command: + - curl + - -L + - https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-rhel80-4.4.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/linux/mongodb-linux-x86_64-enterprise-rhel80-4.4.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases/linux + - name: setting-up-ubuntu-mongodb-4-4-ent + image: curlimages/curl:latest + command: + - curl + - -L + - https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu1804-4.4.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/linux/mongodb-linux-x86_64-enterprise-ubuntu1804-4.4.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases/linux + - name: setting-up-rhel-mongodb-8-0-0 + image: curlimages/curl:latest + command: + - curl + - -L + - https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-rhel8-8.0.0.tgz + - -o + - /mongodb-ops-manager/mongodb-releases/mongodb-linux-x86_64-rhel8-8.0.0.tgz + volumeMounts: + - name: mongodb-versions + mountPath: /mongodb-ops-manager/mongodb-releases + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - name: mongodb-versions + emptyDir: {} + - configMap: + name: nginx-conf + name: nginx-conf + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf +data: + nginx.conf: | + events {} + http { + server { + server_name localhost; + listen 80; + location /linux/ { + alias /mongodb-ops-manager/mongodb-releases/linux/; + } + } + } + +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-svc + labels: + app: nginx +spec: + ports: + - port: 80 + protocol: TCP + selector: + app: nginx diff --git a/public/samples/ops-manager/ops-manager-scram.yaml b/public/samples/ops-manager/ops-manager-scram.yaml new file mode 100644 index 000000000..b7c00ac99 --- /dev/null +++ b/public/samples/ops-manager/ops-manager-scram.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-scram +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + + # the application database backing Ops Manager. Replica Set is the only supported type + # Application database has the SCRAM-SHA authentication mode always enabled + applicationDatabase: + version: "4.4.11-ent" + members: 3 + podSpec: + cpu: '0.25' + # optional. Specifies the secret which contains the password used to connect to the database + # If not specified - the Operator will generate a random password + passwordSecretKeyRef: + name: my-password-secret + # optional, default is "password" + key: password-key diff --git a/public/samples/ops-manager/ops-manager-tls.yaml b/public/samples/ops-manager/ops-manager-tls.yaml new file mode 100644 index 000000000..517349b6a --- /dev/null +++ b/public/samples/ops-manager/ops-manager-tls.yaml @@ -0,0 +1,38 @@ +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager-tls +spec: + replicas: 1 + version: 5.0.5 + adminCredentials: ops-manager-admin-secret + + configuration: + # Local/hybrid mode is necessary when Ops Manager has TLS with custom CA enabled as Agents running in MongoDB + # containers cannot download from the Internet in this case and need to download directly from Ops Manager + automation.versions.source: local + + security: + # enables TLS for Ops Manager allowing it to serve traffic over HTTPS + tls: + # secret containing the TLS certificate signed by known or custom CA. The secret must have a key "server.pem" + # and value of .pem file containing private key and TLS certificate + secretRef: certs-for-ops-manager + + backup: + enabled: false + + applicationDatabase: + members: 3 + version: 4.4.11-ent + security: + # enables TLS mode for application database Replica Set + tls: + # optional, ConfigMap containing custom CA certificate + # Will be used by Ops Manager to establish secure connection to application database + ca: issuer-ca-config-map + # the prefix of the secret containing the TLS certificate signed by known or custom CA. The secret must have a key "server.pem" + # and value of .pem file containing private key and TLS certificate + # the name of this secret must be appdb-ops-manager-tls-db-cert + secretRef: + prefix: appdb diff --git a/public/samples/ops-manager/ops-manager.yaml b/public/samples/ops-manager/ops-manager.yaml new file mode 100644 index 000000000..d959ef0cd --- /dev/null +++ b/public/samples/ops-manager/ops-manager.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + # the number of Ops Manager instances to run. Set to value bigger + # than 1 to get high availability and upgrades without downtime + replicas: 3 + + # the version of Ops Manager distro to use + version: 5.0.5 + + # optional. Specify the custom cluster domain of the Kubernetes cluster if it's different from the default one ('cluster.local'). + # This affects the urls generated by the Operator. + # This field is also used for Application Database url + # clusterDomain: mycompany.net + + # the name of the secret containing admin user credentials. + # Either remove the secret or change the password using Ops Manager UI after the Ops Manager + # resource is created! + adminCredentials: ops-manager-admin-secret + + # optional. The Ops Manager configuration. All the values must be of type string + configuration: + mms.fromEmailAddr: "admin@example.com" + + # the application database backing Ops Manager. Replica Set is the only supported type + # Application database has the SCRAM-SHA authentication mode always enabled + applicationDatabase: + members: 3 + version: 4.4.11-ent + # optional. Allows to pass custom MongoDB process configuration + additionalMongodConfig: + operationProfiling: + mode: slowOp diff --git a/public/samples/sharded_multicluster/example-sharded-cluster-deployment.yaml b/public/samples/sharded_multicluster/example-sharded-cluster-deployment.yaml new file mode 100644 index 000000000..2aac5bb52 --- /dev/null +++ b/public/samples/sharded_multicluster/example-sharded-cluster-deployment.yaml @@ -0,0 +1,110 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sc +spec: + topology: MultiCluster + type: ShardedCluster + # this deployment will have 3 shards + shardCount: 3 + # you cannot specify mongodsPerShardCount, configServerCount and mongosCount + # in MultiCluster topology + version: 8.0.3 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + + shardPodSpec: # applies to all shards on all clusters + persistence: + single: + # all pods for all shards on all clusters will use that storage size in their + # PersistentVolumeClaim unless overridden in spec.shard.clusterSpecList or + # spec.shardOverrides. + storage: 10G + + configSrvPodSpec: # applies to all config server nodes in all clusters + persistence: + multiple: + data: + storage: 2G + journal: + storage: 1G + logs: + storage: 1G + + # consider this section as a default configuration for ALL shards + shard: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + # each shard will have only one mongod process deployed in this cluster + members: 1 + memberConfig: + - votes: 1 + priority: "20" # we increase the priority to have primary in this cluster + - clusterName: kind-e2e-cluster-2 + # one member in this cluster, no votes and priority defined means it'll get + # the default values votes=1, priority="1" + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 1 # one member in this cluster + + shardOverrides: # here you specify customizations for specific shards + # here you specify to which shard names the following configuration will + # apply + - shardNames: + - sc-0 + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + # all fields here are optional + # shard "sc-0" will have two members instead of one, which was defined as the + # default for all shards in spec.shard.clusterSpecList[0].members + members: 2 + memberConfig: + - votes: 1 + # shard "sc-0" should not have primary in this cluster like every other shard + priority: "1" + - votes: 1 + priority: "1" + - clusterName: kind-e2e-cluster-2 + members: 2 # shard "sc-0" will have two members instead of one + memberConfig: + - votes: 1 + # both processes of shard "sc-0" in this cluster will have the same + # likelihood to become a primary member + priority: "20" + - votes: 1 + priority: "20" + # We need to specify the list of all clusters on which this shard will be + # deployed. + - clusterName: kind-e2e-cluster-3 + # If the clusterName element is omitted here, it will be considered as an + # override for this shard, so that the operator shouldn't deploy any member + # to it. + # No fields are mandatory in here, though. In case a field is not set, it's + # not overridden and the default value is taken from a top level spec.shard + # settings. + + configSrv: + # the same configuration fields are available as in + # spec.shard.clusterSpecList. + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 1 + + mongos: + # the same configuration fields are available as in + # spec.shard.clusterSpecList apart from storage and replica-set related + # fields. + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 1 + - clusterName: kind-e2e-cluster-2 + members: 1 + - clusterName: kind-e2e-cluster-3 + members: 1 diff --git a/public/samples/sharded_multicluster/pod_template_config_servers.yaml b/public/samples/sharded_multicluster/pod_template_config_servers.yaml new file mode 100644 index 000000000..3c1b0a511 --- /dev/null +++ b/public/samples/sharded_multicluster/pod_template_config_servers.yaml @@ -0,0 +1,80 @@ +# This file is a minimal example of how to define global custom pod templates and persistence settings +# and how to override them in clusterSpecList for Config Servers +# It is similar to how we define them for shards, except that there is no shardOverrides +# Note that mongos settings work in the same way +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: pod-template-config-servers + namespace: mongodb-test +spec: + shardCount: 3 + topology: MultiCluster + type: ShardedCluster + version: 8.0.3 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + + # doc-region-start: configSrv + configSrvPodSpec: # applicable to all members in all clusters + persistence: + single: + storage: "5G" + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 0.5 + memory: 1.0G + limits: + cpu: 1.0 + memory: 2.0G + configSrv: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + # The below statefulset override is applicable only to pods in kind-e2e-cluster-1 + # Specs will be merged, the "request" field defined above will still be applied to containers in this cluster + # However, limits will be replaced with below values, because clusterSpecList.statefulSet.spec.template has a + # higher priority than configSrvPodSpec.podTemplate + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: 1.0 + memory: 2.5G + # In clusterSpecList.podSpec, only persistence field must be used, the podTemplate field is ignored. + podSpec: # In kind-e2e-cluster-1, we replace the persistence settings defined in configSrvPodSpec + persistence: + multiple: + journal: + storage: "6G" + data: + storage: "7G" + logs: + storage: "6G" + - clusterName: kind-e2e-cluster-2 + members: 1 + # doc-highlight-end: configSrv + mongos: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + + shard: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 diff --git a/public/samples/sharded_multicluster/pod_template_shards_0.yaml b/public/samples/sharded_multicluster/pod_template_shards_0.yaml new file mode 100644 index 000000000..7f5dd441d --- /dev/null +++ b/public/samples/sharded_multicluster/pod_template_shards_0.yaml @@ -0,0 +1,80 @@ +# This file is a minimal example of how to define global custom pod templates +# and persistence settings and how to override them in clusterSpecList +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: pod-template-shards-0 + namespace: mongodb-test +spec: + shardCount: 3 + topology: MultiCluster + type: ShardedCluster + version: 8.0.3 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + mongos: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + configSrv: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + + shardPodSpec: # applicable to all shards in all clusters + persistence: + single: + storage: "5G" + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 0.5 + memory: 1.0G + limits: + cpu: 1.0 + memory: 2.0G + shard: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + # The below statefulset override is applicable only to pods in kind-e2e-cluster-1 + # Specs will be merged, the "request" field defined above will still be + # applied to containers in this cluster. + # However, limits will be replaced with below values, because + # clusterSpecList.statefulSet.spec.template has a + # higher priority than shardPodSpec.podTemplate + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: 1.0 + memory: 2.5G + # In clusterSpecList.podSpec, only persistence field must be used, the + # podTemplate field is ignored. + # In kind-e2e-cluster-1, we replace the persistence settings defined in + # shardPodSpec + podSpec: + persistence: + multiple: + journal: + storage: "6G" + data: + storage: "7G" + logs: + storage: "6G" + - clusterName: kind-e2e-cluster-2 + members: 1 diff --git a/public/samples/sharded_multicluster/pod_template_shards_1.yaml b/public/samples/sharded_multicluster/pod_template_shards_1.yaml new file mode 100644 index 000000000..51ad52d3d --- /dev/null +++ b/public/samples/sharded_multicluster/pod_template_shards_1.yaml @@ -0,0 +1,109 @@ +# This file is a minimal example of how to define custom pod templates and +# persistence settings at the cluster level, and in shard overrides. +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: pod-template-shards-1 + namespace: mongodb-test +spec: + shardCount: 3 + topology: MultiCluster + type: ShardedCluster + version: 8.0.3 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + mongos: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + configSrv: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + + shard: + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + # Statefulset and PodSPec settings below apply to members of all shards in kind-e2e-cluster-1 + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + limits: + cpu: 1.0 + memory: 2.0G + # In clusterSpecList.podSpec, only persistence field must be used, the + # podTemplate field is ignored. + # In kind-e2e-cluster-1, we define custom persistence settings + podSpec: + persistence: + multiple: + journal: + storage: "5G" + data: + storage: "5G" + logs: + storage: "5G" + - clusterName: kind-e2e-cluster-2 + members: 1 + + shardOverrides: + - shardNames: [ "pod-template-shards-1-2" ] + # This override will apply to shard of index 2 + # Statefulset settings defined at this level (shardOverrides.statefulSet) + # apply to members of shard 2 in ALL clusters. + # This field has higher priority than shard.clusterSpecList.statefulSet, but + # lower than shardOverrides.clusterSpecList.statefulSet + # It has a merge policy, which means that the limits defined above for the + # mongodb-enterprise-database container field still apply to all members in + # that shard, except if overridden. + statefulSet: + spec: + template: + spec: + containers: + - name: sidecar-shard-2 + image: busybox + command: [ "sleep" ] + args: [ "infinity" ] + clusterSpecList: + - clusterName: kind-e2e-cluster-1 + members: 2 + - clusterName: kind-e2e-cluster-2 + members: 1 + # The below statefulset override is applicable only to members of shard 2, in cluster 1 + # Specs will be merged, the "limits" field defined above will still be applied + # to containers in this cluster together with the requests field below. + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: # We add a requests field in shard 2, cluster 1 + cpu: 0.5 + memory: 1.0G + + podSpec: + # In shardOverrides.clusterSpecList.podSpec, only persistence field must be + # used, the podTemplate field is ignored. + persistence: # we assign additional disk resources in shard 2, cluster 1 + multiple: + journal: + storage: "6G" + data: + storage: "6G" + logs: + storage: "6G" diff --git a/public/samples/sharded_multicluster/shardSpecificPodSpec_migration.yaml b/public/samples/sharded_multicluster/shardSpecificPodSpec_migration.yaml new file mode 100644 index 000000000..f9f79fecc --- /dev/null +++ b/public/samples/sharded_multicluster/shardSpecificPodSpec_migration.yaml @@ -0,0 +1,84 @@ +# This file is an example of how to migrate from the old deprecated +# ShardSpecificPodSpec field to the new shardOverrides fields +# for single cluster deployments. +# The settings specified in shardOverrides are the exact equivalent to the +# ones in shardSpecificPodSpec, showing how to replicate them +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: shardspecificpodspec-migration + namespace: mongodb-test +spec: + # There are 4 shards in this cluster, but the shardSpecificPodSpec field + # doesn't need to have on entry per shard, it can have less + shardCount: 4 + mongodsPerShardCount: 2 + mongosCount: 1 + configServerCount: 3 + topology: SingleCluster + type: ShardedCluster + version: 8.0.3 + opsManager: + configMapRef: + name: my-project + credentials: my-credentials + persistent: true + + shardPodSpec: + # default persistence configuration for all shards in all clusters + persistence: + single: + storage: "5G" + shardSpecificPodSpec: # deprecated way of overriding shards (array) + - persistence: # shard of index 0 + single: + storage: "6G" + # Specify resources settings to enterprise database container in shard 0 + podTemplate: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 0.5 + memory: 1G + limits: + cpu: 1.0 + memory: 2.0G + - persistence: # shard of index 1 + single: + storage: "7G" + - persistence: # shard of index 2 + single: + storage: "7G" + + # The below shardOverrides replicate the same shards configuration as the one + # specified above in shardSpecificPodSpec + shardOverrides: + - shardNames: [ "shardspecificpodspec-migration-0" ] # overriding shard #0 + podSpec: + persistence: + single: + storage: "6G" + statefulSet: + spec: + template: + spec: + containers: + - name: mongodb-enterprise-database + resources: + requests: + cpu: 0.5 + memory: 1G + limits: + cpu: 1.0 + memory: 2.0G + + # The ShardSpecificPodSpec field above has the same configuration for shards + # 1 and 2. It is possible to specify both shard names in the override and not + # duplicate that configuration + - shardNames: [ "shardspecificpodspec-migration-1", "shardspecificpodspec-migration-2" ] + podSpec: + persistence: + single: + storage: "7G" diff --git a/public/samples/single-sharded-overrides.yaml b/public/samples/single-sharded-overrides.yaml new file mode 100644 index 000000000..caea2128b --- /dev/null +++ b/public/samples/single-sharded-overrides.yaml @@ -0,0 +1,31 @@ +apiVersion: mongodb.com/v1 +kind: MongoDB +metadata: + name: sh-single-overrides +spec: + shardCount: 2 + mongodsPerShardCount: 1 + mongosCount: 1 + configServerCount: 1 + version: "7.0.15-ent" + type: ShardedCluster + configSrvPodSpec: + persistence: + single: + storage: 0.5G + shardPodSpec: + persistence: + single: + storage: 1G + shardOverrides: + - shardNames: [sh-single-overrides-0] + members: 3 + - shardNames: [sh-single-overrides-1] + podSpec: + persistence: + single: + storage: 2Gi + opsManager: + configMapRef: + name: my-project + credentials: my-credentials \ No newline at end of file diff --git a/public/support/certificate_rotation.sh b/public/support/certificate_rotation.sh new file mode 100755 index 000000000..36da403be --- /dev/null +++ b/public/support/certificate_rotation.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# +# certificate_rotation.sh +# +# WARNING: This script is provided as a guide-only and it is not meant +# to be used in production environment. +# +# Use this script as a guide on how to rotate TLS certificates on a +# MongoDB Resource. During this process there will be no downtime on +# the Mdb resource. +# + +# +# shellcheck disable=SC2119 +# shellcheck disable=SC2039 +# + +usage() { + local script_name + script_name=$(basename "${0}") + echo "Usage:" + echo "${script_name} " +} + +if [ -z "${2}" ]; then + usage + exit 1 +fi + +namespace="${1}" +mdb_resource_name="${2}" +mdb_resource_type=$(kubectl -n "${namespace}" get "mdb/${mdb_resource_name}" -o jsonpath='{.spec.type}') +mdb_resource_members=$(kubectl -n "${namespace}" get "mdb/${mdb_resource_name}" -o jsonpath='{.spec.members}') +mdb_resource_members=$(("${mdb_resource_members}" - 1)) + +if [[ ${mdb_resource_type} != "ReplicaSet" ]]; then + echo "Only Replica Set TLS certificates are supported as of now." + exit 1 +fi + +echo "Removing existing CSRs if they still exist." +for i in $(seq 0 ${mdb_resource_members}); do + kubectl delete "csr/${mdb_resource_name}-${i}.${namespace}" || true +done + +echo "Removing the 'Secret' object holding the current certificates and private keys." +kubectl -n "${namespace}" delete "secret/${mdb_resource_name}-cert" + +timestamp=$(date --rfc-3339=ns) +echo "Triggering a reconciliation for the Operator to notice the missing certs." +kubectl -n "${namespace}" patch "mdb/${mdb_resource_name}" --type='json' \ + -p='[{"op": "add", "path": "/metadata/annotations/timestamp", "value": "'"${timestamp}"'"}]' + +echo "Wait until the operator recreates the CSRs." +while true; do + all_created=0 + for i in $(seq 0 "${mdb_resource_members}"); do + if ! kubectl get "csr/${mdb_resource_name}-${i}.${namespace}" -o name > /dev/null ; then + all_created=1 + fi + done + if [[ ${all_created} != 0 ]]; then + sleep 10 + else + break + fi +done + +echo "CSRs have been generated. Approving certificates." +for i in $(seq 0 ${mdb_resource_members}); do + kubectl certificate approve "${mdb_resource_name}-${i}.${namespace}" +done + +echo "A this point, the operator should take the new certificates and generate the Secret." +while ! kubectl -n "${namespace}" get "secret/${mdb_resource_name}-cert" &> /dev/null; do + printf "." + sleep 10 +done + +echo "Secret with certificates has been created, proceeding with a rolling restart of the Mdb resource" +kubectl -n "${namespace}" rollout restart sts "${mdb_resource_name}" + +echo "The Mdb resource is being restarted now, it should take a few minutes to reach Running state again." diff --git a/public/support/mdb_operator_diagnostic_data.sh b/public/support/mdb_operator_diagnostic_data.sh new file mode 100755 index 000000000..c6412ae5d --- /dev/null +++ b/public/support/mdb_operator_diagnostic_data.sh @@ -0,0 +1,340 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# +# mdb_operator_diagnostic_data.sh +# +# Use this script to gather data about your MongoDB Enterprise Kubernetes Operator +# and the MongoDB Resources deployed with it. +# + +usage() { + local script_name + script_name=$(basename "${0}") + echo "------------------------------------------------------------------------------" + echo "Usage:" + echo "${script_name} [] [] [--om] [--private]" + echo "------------------------------------------------------------------------------" + echo "#Scenario 01: Collecting MongoDB Logs (Operator in same namespace as MongoDB):" + echo "------------------------------------------------------------------------------" + echo "Example: Operator_Namespace: mongodb, Deployment_Namespace: mongodb, Deployment_Name: myreplicaset" + echo "Usage: ${script_name} mongodb myreplicaset" + echo "For OpenShift: ${script_name} mongodb myreplicaset mongodb enterprise-operator" + echo "------------------------------------------------------------------------------" + echo "#Scenario 02: Collecting MongoDB Logs (Operator in different namespace as MongoDB):" + echo "------------------------------------------------------------------------------" + echo "Example: Operator_Namespace: mdboperator, Deployment_Namespace: mongodb, Deployment_Name: myreplicaset" + echo "Usage: ${script_name} mongodb myreplicaset mdboperator" + echo "For OpenShift: ${script_name} mongodb myreplicaset mdboperator enterprise-operator" + echo "------------------------------------------------------------------------------" + echo "#Scenario 03: Collecting Ops Manager Logs (Operator in same namespace as Ops Manager):" + echo "------------------------------------------------------------------------------" + echo "Example: Operator_Namespace: mongodb, Deployment_Namespace: mongodb, Deployment_Name: ops-manager" + echo "Usage: ${script_name} mongodb ops-manager --om" + echo "For OpenShift: ${script_name} mongodb ops-manager mongodb enterprise-operator --om" + echo "------------------------------------------------------------------------------" + echo "#Scenario 04: Collecting Ops Manager Logs (Operator in different namespace as Ops Manager):" + echo "Example: Operator_Namespace: mdboperator, Deployment_Namespace: mongodb, Deployment_Name: ops-manager" + echo "Usage: ${script_name} mongodb ops-manager mdboperator --om" + echo "For OpenShift: ${script_name} mongodb ops-manager mdboperator enterprise-operator --om" + echo "------------------------------------------------------------------------------" + echo "#Scenario 05: Collecting MongoDB Logs but for multi-cluster (Operator is in a different namespace as MongoDB). Note: Member Cluster and Central Cluster is now given via an environment Flag:" + echo 'Example: MEMBER_CLUSTERS:"kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3", CENTRAL_CLUSTER:"kind-e2e-operator", Operator_Namespace: mdboperator, Deployment_Namespace: mongodb, Deployment_Name: multi-replica-set' + echo "Usage: MEMBER_CLUSTERS='kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3' CENTRAL_CLUSTER='kind-e2e-operator' ${script_name} mongodb multi-replica-set mdboperator enterprise-operator" + echo "For OpenShift: MEMBER_CLUSTERS='kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3' CENTRAL_CLUSTER='kind-e2e-operator' ${script_name} mongodb multi-replica-set mdboperator enterprise-operator" + echo "------------------------------------------------------------------------------" +} + +contains() { + local e match=$1 + shift + for e; do [[ "${e}" == "${match}" ]] && return 0; done + return 1 +} + +if [ $# -lt 2 ]; then + usage >&2 + exit 1 +fi + +namespace="${1}" +mdb_resource="${2}" + +collect_om=0 +contains "--om" "$@" && collect_om=1 + +if [ ${collect_om} == 1 ]; then + if [[ $3 == "--om" ]]; then + operator_namespace="${1}" + operator_name="mongodb-enterprise-operator" + om_resource_name="${2}" + elif [[ $4 == "--om" ]]; then + operator_namespace="${3}" + operator_name="mongodb-enterprise-operator" + om_resource_name="${2}" + elif [[ $5 == "--om" ]]; then + operator_namespace="${3}" + operator_name="$4" + om_resource_name="${2}" + fi +else + operator_name="${4:-mongodb-enterprise-operator}" + operator_namespace="${3:-$1}" +fi + +dump_all() { + local central=${1} + if [ "${central}" == "member" ]; then + if [ "${collect_om}" == 0 ]; then + pod_name=$(kubectl get pods -l controller -n "${namespace}" --no-headers=true | awk '{print $1}' | head -n 1) + database_container_pretty_name=$(kubectl -n "${namespace}" exec -it "${pod_name}" -- sh -c "cat /etc/*release" | grep "PRETTY_NAME" | cut -d'=' -f 2) + echo "+ Database is running on: ${database_container_pretty_name}" + fi + + statefulset_filename="statefulset.yaml" + echo "+ Saving StatefulSet state to ${statefulset_filename}" + kubectl -n "${namespace}" -l controller get "sts" -o yaml >"${log_dir}/${statefulset_filename}" + + echo "+ Deployment Pods" + kubectl -n "${namespace}" get pods | grep -E "^${mdb_resource}-+" + + echo "+ Saving Pods state to ${mdb_resource}-N.logs" + pods_in_namespace=$(kubectl get pods --namespace "${namespace}" --selector=controller=mongodb-enterprise-operator --no-headers -o custom-columns=":metadata.name") + + mdb_container_name="mongodb-enterprise-database" + for pod in ${pods_in_namespace}; do + if ! kubectl -n "${namespace}" get pod "${pod}" --no-headers -o custom-columns=":spec.containers[*].name" | grep -E "^${mdb_container_name}$" > /dev/null; then + continue + fi + kubectl -n "${namespace}" logs "${pod}" -c "${mdb_container_name}" --tail 2000 >"${log_dir}/${pod}.log" + kubectl -n "${namespace}" get event --field-selector "involvedObject.name=${pod}" >"${log_dir}/${pod}_events.log" + done + + echo "+ Persistent Volumes" + kubectl -n "${namespace}" get pv + + echo "+ Persistent Volume Claims" + kubectl -n "${namespace}" get pvc + + pv_filename="persistent_volumes.yaml" + echo "+ Saving Persistent Volumes state to ${pv_filename}" + kubectl -n "${namespace}" get pv -o yaml >"${log_dir}/${pv_filename}" + + pvc_filename="persistent_volume_claims.yaml" + echo "+ Saving Persistent Volumes Claims state to ${pvc_filename}" + kubectl -n "${namespace}" get pvc -o yaml >"${log_dir}/${pvc_filename}" + + services_filename="services.yaml" + echo "+ Services" + kubectl -n "${namespace}" get services + + echo "+ Saving Services state to ${services_filename}" + kubectl -n "${namespace}" get services -o yaml >"${log_dir}/${services_filename}" + + echo "+ Saving Events for the Namespace" + kubectl -n "${namespace}" get events >"${log_dir}/events.log" + + else + echo "++ MongoDB Resource Running Environment" + + if [ -z "${CENTRAL_CLUSTER}" ]; then + crd_filename="crd_mdb.yaml" + echo "+ Saving MDB Customer Resource Definition into ${crd_filename}" + kubectl -n "${operator_namespace}" get crd/mongodb.mongodb.com -o yaml >"${log_dir}/${crd_filename}" + mdb_resource_name="mdb/${mdb_resource}" + resource_filename="mdb_object_${mdb_resource}.yaml" + else + crd_filename="crd_mdbmc.yaml" + echo "+ Saving MDBMC Customer Resource Definition into ${crd_filename}" + echo kubectl -n "${operator_namespace}" get crd/mongodbmulticluster.mongodb.com -o yaml >"${log_dir}/${crd_filename}" + mdb_resource_name="mdbmc/${mdb_resource}" + resource_filename="mdbmc_object_${mdb_resource}.yaml" + fi + + project_filename="project.yaml" + project_name=$(kubectl -n "${namespace}" get "${mdb_resource_name}" -o jsonpath='{.spec.opsManager.configMapRef.name}') + credentials_name=$(kubectl -n "${namespace}" get "${mdb_resource_name}" -o jsonpath='{.spec.credentials}') + + echo "+ MongoDB Resource Status" + kubectl -n "${namespace}" get "${mdb_resource_name}" -o yaml >"${log_dir}/${resource_filename}" + + echo "+ Saving Project YAML file to ${project_filename}" + kubectl -n "${namespace}" get "configmap/${project_name}" -o yaml >"${log_dir}/${project_filename}" + credentials_user=$(kubectl -n "${namespace}" get "secret/${credentials_name}" -o jsonpath='{.data.user}' | base64 --decode) + echo "+ User configured is (credentials.user): ${credentials_user}" + + echo "= To get the Secret Public API Key use: kubectl -n ${namespace} get secret/${credentials_name} -o jsonpath='{.data.publicApiKey}' | base64 --decode)" + + echo "+ Certificates (no private keys are captured)" + csr_filename="csr.text" + kubectl get csr | grep "${namespace}" || true + echo "+ Saving Certificate state into ${csr_filename}" + kubectl describe "$(kubectl get csr -o name | grep "${namespace}")" >"${log_dir}/${csr_filename}" || true + + echo "++ MongoDBUser Resource Status" + mdbusers_filename="mdbu.yaml" + kubectl -n "${namespace}" get mdbu + echo "+ Saving MongoDBUsers to ${mdbusers_filename}" + kubectl -n "${namespace}" get mdbu >"${log_dir}/${mdbusers_filename}" + + crdu_filename="crd_mdbu.yaml" + echo "+ Saving MongoDBUser Customer Resource Definition into ${crdu_filename}" + kubectl -n "${namespace}" get crd/mongodbusers.mongodb.com -o yaml >"${log_dir}/${crdu_filename}" + fi + +} + +MEMBER_CLUSTERS=${MEMBER_CLUSTERS:-} +CENTRAL_CLUSTER=${CENTRAL_CLUSTER:-} + +current_date="$(date +%Y-%m-%d_%H_%M)" + +private_mode=1 +contains "--private" "$@" && private_mode=0 + +log_dir="logs_${current_date}" +mkdir -p "${log_dir}" &>/dev/null + +if [ -n "${CENTRAL_CLUSTER}" ]; then + if [ -z "${MEMBER_CLUSTERS}" ]; then + echo "CENTRAL_CLUSTER is set but no MEMBER_CLUSTERS" + exit 1 + else + echo "starting with the CENTRAL_CLUSTER!" + kubectl config use-context "${CENTRAL_CLUSTER}" + fi +fi + +if [ -n "${MEMBER_CLUSTERS}" ]; then + if [ -z "${CENTRAL_CLUSTER}" ]; then + echo "MEMBER_CLUSTERS is set but no CENTRAL_CLUSTER" + exit 1 + fi +fi + +if ! kubectl get "namespace/${namespace}" &>/dev/null; then + echo "Error fetching namespace. Make sure name ${namespace} for Namespace is correct." + exit 1 +fi + +if [ ${collect_om} == 0 ]; then + if [ -z "${CENTRAL_CLUSTER}" ]; then + if ! kubectl -n "${namespace}" get "mdb/${mdb_resource}" &>/dev/null; then + echo "Error fetching the MongoDB resource. Make sure the '${namespace}/${mdb_resource}' is correct." + exit 1 + fi + else + if ! kubectl -n "${namespace}" get "mdbmc/${mdb_resource}" &>/dev/null; then + echo "Error fetching the MongoDB MultiCluster resource. Make sure the '${namespace}/${mdb_resource}' is correct." + exit 1 + fi + fi +fi + +echo "++ Versions" +mdb_operator_pod=$(kubectl -n "${operator_namespace}" get pods -l "app.kubernetes.io/component=controller" -o name | cut -d'/' -f 2) +echo "${operator_namespace}" +echo "+ Operator Pod: pod/${mdb_operator_pod}" + +if ! kubectl -n "${operator_namespace}" get "pod/${mdb_operator_pod}" &>/dev/null; then + echo "Error fetching the MongoDB Operator Deployment. Make sure the pod/${mdb_operator_pod} exist and it is running." + exit 1 +fi + +if ! kubectl -n "${namespace}" get om -o wide &>/dev/null; then + echo "Error fetching the MongoDB OpsManager Resource." +fi + +if [ ${private_mode} == 0 ]; then + echo "+ Running on private mode. Make sure you don't share the results of this run outside your organization." +fi + +mdb_operator_filename="operator.yaml" +echo "+ Saving Operator Deployment into ${mdb_operator_filename}" +kubectl -n "${operator_namespace}" get "pod/${mdb_operator_pod}" -o yaml >"${log_dir}/${mdb_operator_filename}" + +echo "+ Kubernetes Version Reported by kubectl" +kubectl version + +if type oc &>/dev/null; then + echo "+ Kubernetes Version Reported by oc" + oc version +fi + +operator_logs_filename="${operator_name}_${current_date}.logs" +echo "+ Saving Operator logs to file ${operator_logs_filename}" +kubectl -n "${operator_namespace}" logs "pod/${mdb_operator_pod}" --tail 2000 >"${log_dir}/${operator_logs_filename}" + +operator_container_pretty_name=$(kubectl -n "${operator_namespace}" exec -it "${mdb_operator_pod}" -- sh -c "cat /etc/*release" | grep "PRETTY_NAME" | cut -d'=' -f 2) +echo "+ Operator is running on: ${operator_container_pretty_name}" + +echo "++ Kubernetes Cluster Ecosystem" +echo "+ Kubectl Cluster Information" +kubectl cluster-info + +if [ ${private_mode} == 0 ]; then + kubectl_cluster_info_filename="kubectl_cluster_info_${current_date}.logs" + echo "+ Saving Cluster Info to file ${kubectl_cluster_info_filename} (this might take a few minutes)" + kubectl cluster-info dump | gzip >"${log_dir}/${kubectl_cluster_info_filename}.gz" +else + echo "= Skipping Kubectl cluster information dump, use --private to enable." +fi + +kubectl_sc_dump_filename="kubectl_storage_class_${current_date}.yaml" +kubectl get storageclass -o yaml >"${log_dir}/${kubectl_sc_dump_filename}" + +nodes_filename="nodes.yaml" +echo "+ Nodes" +kubectl get nodes + +echo "+ Saving Nodes full state to ${nodes_filename}" +kubectl get nodes -o yaml >"${log_dir}/${nodes_filename}" + +if [ ${collect_om} == 0 ]; then + if [ -n "${CENTRAL_CLUSTER}" ]; then + for member_cluster in ${MEMBER_CLUSTERS}; do + echo "Dumping diagnostics for context ${member_cluster}" + kubectl config use-context "${member_cluster}" + dump_all "member" + done + echo "Dumping diagnostics for context ${CENTRAL_CLUSTER}" + kubectl config use-context "${CENTRAL_CLUSTER}" + dump_all "central" + else + dump_all "member" + dump_all "central" + fi +fi + +if [ ${collect_om} == 1 ]; then + ops_manager_filename="ops_manager.yaml" + echo "+ Saving OpsManager Status" + kubectl -n "${namespace}" get om -o wide + echo "+ Saving OpsManager Status to ${ops_manager_filename}" + kubectl -n "${namespace}" get om -o yaml >"${log_dir}/${ops_manager_filename}" + echo "+ Saving Pods state to ${om_resource_name}-N.logs" + pods_in_namespace=$(kubectl -n "${namespace}" get pods -o name -l "app=${om_resource_name}-svc" | cut -d'/' -f 2) + for pod in ${pods_in_namespace}; do + kubectl -n "${namespace}" logs "${pod}" --tail 2000 >"${log_dir}/${pod}.log" + echo "Collecting Events: ${pod}" + kubectl -n "${namespace}" get event --field-selector "involvedObject.name=${pod}" >"${log_dir}/${pod}_events.log" + done + echo "+ Saving AppDB Pods state to ${om_resource_name}-db-N-.logs" + pods_in_namespace=$(kubectl -n "${namespace}" get pods -o name -l "app=${om_resource_name}-db-svc" | cut -d'/' -f 2) + for pod in ${pods_in_namespace}; do + kubectl -n "${namespace}" logs "${pod}" -c "mongod" --tail 2000 >"${log_dir}/${pod}-mongod.log" + kubectl -n "${namespace}" logs "${pod}" -c "mongodb-agent" --tail 2000 >"${log_dir}/${pod}-mongodb-agent.log" + kubectl -n "${namespace}" logs "${pod}" -c "mongodb-agent-monitoring" --tail 2000 >"${log_dir}/${pod}-mongodb-agent-monitoring.log" + echo "Collecting Events: ${pod}" + kubectl -n "${namespace}" get event --field-selector "involvedObject.name=${pod}" >"${log_dir}/${pod}_events.log" + done +fi + +echo "++ Compressing files" +compressed_logs_filename="${namespace}__${mdb_resource}__${current_date}.tar.gz" +tar -czf "${compressed_logs_filename}" -C "${log_dir}" . + +echo "- All logs have been captured and compressed into the file ${compressed_logs_filename}." +echo "- If support is needed, please attach this file to an email to provide you with a better support experience." +echo "- If there are additional logs that your organization is capturing, they should be made available in case of a support request." diff --git a/public/tools/multicluster/.gitignore b/public/tools/multicluster/.gitignore new file mode 100644 index 000000000..bf5b02441 --- /dev/null +++ b/public/tools/multicluster/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +.idea +.vscode +*.log +tmp/ +dist/ diff --git a/public/tools/multicluster/.goreleaser.yaml b/public/tools/multicluster/.goreleaser.yaml new file mode 100644 index 000000000..b5b8c04bd --- /dev/null +++ b/public/tools/multicluster/.goreleaser.yaml @@ -0,0 +1,52 @@ +project_name: kubectl-mongodb + +before: + hooks: + - go mod tidy + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + hooks: + # This will notarize Apple binaries and replace goreleaser bins with the notarized ones + post: + - cmd: ./kubectl_mac_notarize.sh + output: true + - cmd: ./sign.sh {{ .Path }} + env: + - GRS_USERNAME={{ .Env.GRS_USERNAME }} + - GRS_PASSWORD={{ .Env.GRS_PASSWORD }} + - PKCS11_URI={{ .Env.PKCS11_URI }} + - ARTIFACTORY_URL={{ .Env.ARTIFACTORY_URL }} + - SIGNING_IMAGE_URI={{ .Env.SIGNING_IMAGE_URI }} + - ARTIFACTORY_USERNAME=mongodb-enterprise-kubernetes-operator + - ARTIFACTORY_PASSWORD={{ .Env.ARTIFACTORY_PASSWORD }} + - cmd: ./verify.sh {{ .Path }} && echo "VERIFIED OK" + +archives: + - format: tar.gz + name_template: "kubectl-mongodb_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + files: + # Include signature files in each archive along with the binary, strip_parent avoid nested folders + - src: "./dist/kubectl-mongodb_{{ .Os }}_{{ .Arch }}*{{ .Amd64 }}/kubectl-mongodb.sig" + strip_parent: true +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + skip: true + +release: + prerelease: auto + draft: true + name_template: "MongoDB Enterprise Kubernetes Operator {{ .Version }}" + +git: + tag_sort: -version:creatordate diff --git a/public/tools/multicluster/Dockerfile b/public/tools/multicluster/Dockerfile new file mode 100644 index 000000000..86fa3a2a0 --- /dev/null +++ b/public/tools/multicluster/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.24 as builder +WORKDIR /go/src +ADD . . + +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /go/bin/mongodb-multicluster + +FROM scratch +COPY --from=builder /go/bin/mongodb-multicluster /go/bin/mongodb-multicluster + +ENTRYPOINT [ "/go/bin/mongodb-multicluster" ] diff --git a/public/tools/multicluster/LICENSE b/public/tools/multicluster/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/public/tools/multicluster/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/public/tools/multicluster/cmd/debug.go b/public/tools/multicluster/cmd/debug.go new file mode 100644 index 000000000..b0e48e916 --- /dev/null +++ b/public/tools/multicluster/cmd/debug.go @@ -0,0 +1,138 @@ +package cmd + +import ( + "fmt" + "os" + "strings" + + "k8s.io/client-go/tools/clientcmd" + + "github.com/10gen/ops-manager-kubernetes/multi/pkg/common" + "github.com/10gen/ops-manager-kubernetes/multi/pkg/debug" + "github.com/spf13/cobra" +) + +type Flags struct { + common.Flags + Anonymize bool + UseOwnerRef bool +} + +func (f *Flags) ParseDebugFlags() error { + if len(common.MemberClusters) > 0 { + f.MemberClusters = strings.Split(common.MemberClusters, ",") + } + + configFilePath := common.LoadKubeConfigFilePath() + kubeconfig, err := clientcmd.LoadFromFile(configFilePath) + if err != nil { + return fmt.Errorf("error loading kubeconfig file '%s': %s", configFilePath, err) + } + if len(f.CentralCluster) == 0 { + f.CentralCluster = kubeconfig.CurrentContext + f.CentralClusterNamespace = kubeconfig.Contexts[kubeconfig.CurrentContext].Namespace + } + + return nil +} + +var debugFlags = &Flags{} + +func init() { + rootCmd.AddCommand(debugCmd) + + debugCmd.Flags().StringVar(&common.MemberClusters, "member-clusters", "", "Comma separated list of member clusters. [optional]") + debugCmd.Flags().StringVar(&debugFlags.CentralCluster, "central-cluster", "", "The central cluster the operator will be deployed in. [optional]") + debugCmd.Flags().StringVar(&debugFlags.MemberClusterNamespace, "member-cluster-namespace", "", "The namespace the member cluster resources will be deployed to. [optional]") + debugCmd.Flags().StringVar(&debugFlags.CentralClusterNamespace, "central-cluster-namespace", "", "The namespace the Operator will be deployed to. [optional]") + debugCmd.Flags().StringVar(&common.MemberClustersApiServers, "member-clusters-api-servers", "", "Comma separated list of api servers addresses. [optional, default will take addresses from KUBECONFIG env var]") + debugCmd.Flags().BoolVar(&debugFlags.Anonymize, "anonymize", true, "True if anonymization should be turned on") + debugCmd.Flags().BoolVar(&debugFlags.UseOwnerRef, "ownerRef", false, "True if the collection should be made with owner references (consider turning it on after CLOUDP-176772 is fixed)") +} + +var debugCmd = &cobra.Command{ + Use: "debug", + Short: "Downloads all resources required for debugging and stores them into the disk", + Long: `'debug' downloads all resources required for debugging and stores them into the disk. + +Example: + +kubectl-mongodb debug +kubectl-mongodb debug setup --central-cluster="operator-cluster" --member-clusters="cluster-1,cluster-2,cluster-3" --member-cluster-namespace=mongodb --central-cluster-namespace=mongodb + +`, + Run: func(cmd *cobra.Command, args []string) { + err := debugFlags.ParseDebugFlags() + if err != nil { + fmt.Printf("error parsing flags: %s\n", err) + os.Exit(1) + } + clientMap, err := common.CreateClientMap(debugFlags.MemberClusters, debugFlags.CentralCluster, common.LoadKubeConfigFilePath(), common.GetKubernetesClient) + if err != nil { + fmt.Printf("failed to create clientset map: %s", err) + os.Exit(1) + } + + var collectors []debug.Collector + collectors = append(collectors, &debug.StatefulSetCollector{}) + collectors = append(collectors, &debug.ConfigMapCollector{}) + collectors = append(collectors, &debug.SecretCollector{}) + collectors = append(collectors, &debug.ServiceAccountCollector{}) + collectors = append(collectors, &debug.RolesCollector{}) + collectors = append(collectors, &debug.RolesBindingsCollector{}) + collectors = append(collectors, &debug.MongoDBCollector{}) + collectors = append(collectors, &debug.MongoDBMultiClusterCollector{}) + collectors = append(collectors, &debug.MongoDBUserCollector{}) + collectors = append(collectors, &debug.OpsManagerCollector{}) + collectors = append(collectors, &debug.MongoDBCommunityCollector{}) + collectors = append(collectors, &debug.EventsCollector{}) + collectors = append(collectors, &debug.LogsCollector{}) + collectors = append(collectors, &debug.AgentHealthFileCollector{}) + + var anonymizer debug.Anonymizer + if debugFlags.Anonymize { + anonymizer = &debug.SensitiveDataAnonymizer{} + } else { + anonymizer = &debug.NoOpAnonymizer{} + } + + var filter debug.Filter + + if debugFlags.UseOwnerRef { + filter = &debug.WithOwningReference{} + } else { + filter = &debug.AcceptAllFilter{} + } + + var collectionResults []debug.CollectionResult + + collectionResults = append(collectionResults, debug.Collect(cmd.Context(), clientMap[debugFlags.CentralCluster], debugFlags.CentralCluster, debugFlags.CentralClusterNamespace, filter, collectors, anonymizer)) + + if len(debugFlags.MemberClusters) > 0 { + for i := range debugFlags.MemberClusters { + collectionResults = append(collectionResults, debug.Collect(cmd.Context(), clientMap[debugFlags.MemberClusters[i]], debugFlags.MemberClusters[i], debugFlags.MemberClusterNamespace, filter, collectors, anonymizer)) + } + } + + fmt.Printf("==== Report ====\n\n") + fmt.Printf("Anonymisation: %v\n", debugFlags.Anonymize) + fmt.Printf("Following owner refs: %v\n", debugFlags.UseOwnerRef) + fmt.Printf("Collected data from %d clusters\n", len(collectionResults)) + fmt.Printf("\n\n==== Collected Data ====\n\n") + + storeDirectory, err := debug.DebugDirectory() + if err != nil { + fmt.Printf("failed to obtain directory for collecting the results: %v", err) + os.Exit(1) + } + + if len(collectionResults) > 0 { + directoryName, compressedFileName, err := debug.WriteToFile(storeDirectory, collectionResults...) + if err != nil { + panic(err) + } + fmt.Printf("Debug data file (compressed): %v\n", compressedFileName) + fmt.Printf("Debug data directory: %v\n", directoryName) + } + }, +} diff --git a/public/tools/multicluster/cmd/multicluster.go b/public/tools/multicluster/cmd/multicluster.go new file mode 100644 index 000000000..c04f955af --- /dev/null +++ b/public/tools/multicluster/cmd/multicluster.go @@ -0,0 +1,17 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +// multiclusterCmd represents the multicluster command +var multiclusterCmd = &cobra.Command{ + Use: "multicluster", + Short: "Manage MongoDB multicluster environments on k8s", + Long: `'multicluster' is the toplevel command for managing +multicluster environments that hold MongoDB resources.`, +} + +func init() { + rootCmd.AddCommand(multiclusterCmd) +} diff --git a/public/tools/multicluster/cmd/recover.go b/public/tools/multicluster/cmd/recover.go new file mode 100644 index 000000000..69ec40290 --- /dev/null +++ b/public/tools/multicluster/cmd/recover.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "fmt" + "os" + "strings" + + "github.com/10gen/ops-manager-kubernetes/multi/pkg/common" + + "github.com/spf13/cobra" + "golang.org/x/xerrors" + "k8s.io/client-go/tools/clientcmd" +) + +func init() { + multiclusterCmd.AddCommand(recoverCmd) + + recoverCmd.Flags().StringVar(&common.MemberClusters, "member-clusters", "", "Comma separated list of member clusters. [required]") + recoverCmd.Flags().StringVar(&RecoverFlags.ServiceAccount, "service-account", "mongodb-enterprise-operator-multi-cluster", "Name of the service account which should be used for the Operator to communicate with the member clusters. [optional, default: mongodb-enterprise-operator-multi-cluster]") + recoverCmd.Flags().StringVar(&RecoverFlags.CentralCluster, "central-cluster", "", "The central cluster the operator will be deployed in. [required]") + recoverCmd.Flags().StringVar(&RecoverFlags.MemberClusterNamespace, "member-cluster-namespace", "", "The namespace the member cluster resources will be deployed to. [required]") + recoverCmd.Flags().StringVar(&RecoverFlags.CentralClusterNamespace, "central-cluster-namespace", "", "The namespace the Operator will be deployed to. [required]") + recoverCmd.Flags().BoolVar(&RecoverFlags.Cleanup, "cleanup", false, "Delete all previously created resources except for namespaces. [optional default: false]") + recoverCmd.Flags().BoolVar(&RecoverFlags.ClusterScoped, "cluster-scoped", false, "Create ClusterRole and ClusterRoleBindings for member clusters. [optional default: false]") + recoverCmd.Flags().StringVar(&RecoverFlags.OperatorName, "operator-name", common.DefaultOperatorName, "Name used to identify the deployment of the operator. [optional, default: mongodb-enterprise-operator]") + recoverCmd.Flags().BoolVar(&RecoverFlags.InstallDatabaseRoles, "install-database-roles", false, "Install the ServiceAccounts and Roles required for running database workloads in the member clusters. [optional default: false]") + recoverCmd.Flags().StringVar(&RecoverFlags.SourceCluster, "source-cluster", "", "The source cluster for recovery. This has to be one of the healthy member cluster that is the source of truth for new cluster configuration. [required]") + recoverCmd.Flags().BoolVar(&RecoverFlags.CreateServiceAccountSecrets, "create-service-account-secrets", true, "Create service account token secrets. [optional default: true]") + recoverCmd.Flags().StringVar(&common.MemberClustersApiServers, "member-clusters-api-servers", "", "Comma separated list of api servers addresses. [optional, default will take addresses from KUBECONFIG env var]") +} + +// recoverCmd represents the recover command +var recoverCmd = &cobra.Command{ + Use: "recover", + Short: "Recover the multicluster environment for MongoDB resources after a dataplane failure", + Long: `'recover' re-configures a failed multicluster environment to a enable the shuffling of dataplane +resources to a new healthy topology. + +Example: + +kubectl-mongodb multicluster recover --central-cluster="operator-cluster" --member-clusters="cluster-1,cluster-3,cluster-4" --member-cluster-namespace="mongodb-fresh" --central-cluster-namespace="mongodb" --operator-name=mongodb-enterprise-operator-multi-cluster --source-cluster="cluster-1" + +`, + Run: func(cmd *cobra.Command, args []string) { + if err := parseRecoverFlags(args); err != nil { + fmt.Printf("error parsing flags: %s\n", err) + os.Exit(1) + } + + clientMap, err := common.CreateClientMap(RecoverFlags.MemberClusters, RecoverFlags.CentralCluster, common.LoadKubeConfigFilePath(), common.GetKubernetesClient) + if err != nil { + fmt.Printf("failed to create clientset map: %s", err) + os.Exit(1) + } + + if err := common.EnsureMultiClusterResources(cmd.Context(), RecoverFlags, clientMap); err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := common.ReplaceClusterMembersConfigMap(cmd.Context(), clientMap[RecoverFlags.CentralCluster], RecoverFlags); err != nil { + fmt.Println(err) + os.Exit(1) + } + }, +} + +var RecoverFlags = common.Flags{} + +func parseRecoverFlags(args []string) error { + if common.AnyAreEmpty(common.MemberClusters, RecoverFlags.ServiceAccount, RecoverFlags.CentralCluster, RecoverFlags.MemberClusterNamespace, RecoverFlags.CentralClusterNamespace, RecoverFlags.SourceCluster) { + return xerrors.Errorf("non empty values are required for [service-account, member-clusters, central-cluster, member-cluster-namespace, central-cluster-namespace, source-cluster]") + } + + RecoverFlags.MemberClusters = strings.Split(common.MemberClusters, ",") + if !common.Contains(RecoverFlags.MemberClusters, RecoverFlags.SourceCluster) { + return xerrors.Errorf("source-cluster has to be one of the healthy member clusters: %s", common.MemberClusters) + } + + if strings.TrimSpace(common.MemberClustersApiServers) != "" { + RecoverFlags.MemberClusterApiServerUrls = strings.Split(common.MemberClustersApiServers, ",") + if len(RecoverFlags.MemberClusterApiServerUrls) != len(RecoverFlags.MemberClusters) { + return xerrors.Errorf("expected %d addresses in member-clusters-api-servers parameter but got %d", len(RecoverFlags.MemberClusters), len(RecoverFlags.MemberClusterApiServerUrls)) + } + } + + configFilePath := common.LoadKubeConfigFilePath() + kubeconfig, err := clientcmd.LoadFromFile(configFilePath) + if err != nil { + return xerrors.Errorf("error loading kubeconfig file '%s': %w", configFilePath, err) + } + if len(RecoverFlags.MemberClusterApiServerUrls) == 0 { + if RecoverFlags.MemberClusterApiServerUrls, err = common.GetMemberClusterApiServerUrls(kubeconfig, RecoverFlags.MemberClusters); err != nil { + return err + } + } + return nil +} diff --git a/public/tools/multicluster/cmd/root.go b/public/tools/multicluster/cmd/root.go new file mode 100644 index 000000000..17c99aac9 --- /dev/null +++ b/public/tools/multicluster/cmd/root.go @@ -0,0 +1,59 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "os/signal" + "runtime/debug" + "syscall" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "kubectl-mongodb", + Short: "Manage and configure MongoDB resources on k8s", + Long: `This application is a tool to simplify maintenance tasks +of MongoDB resources in your kubernetes cluster. + `, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute(ctx context.Context) { + ctx, cancel := context.WithCancel(ctx) + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-signalChan + cancel() + }() + buildInfo, ok := debug.ReadBuildInfo() + if ok { + rootCmd.Long += getBuildInfoString(buildInfo) + } + err := rootCmd.ExecuteContext(ctx) + if err != nil { + os.Exit(1) + } +} + +func getBuildInfoString(buildInfo *debug.BuildInfo) string { + var vcsHash string + var vcsTime string + for _, setting := range buildInfo.Settings { + if setting.Key == "vcs.revision" { + vcsHash = setting.Value + } + if setting.Key == "vcs.time" { + vcsTime = setting.Value + } + } + + buildInfoStr := fmt.Sprintf("\nBuild: %s, %s", vcsHash, vcsTime) + return buildInfoStr +} diff --git a/public/tools/multicluster/cmd/setup.go b/public/tools/multicluster/cmd/setup.go new file mode 100644 index 000000000..9db08a67e --- /dev/null +++ b/public/tools/multicluster/cmd/setup.go @@ -0,0 +1,100 @@ +package cmd + +import ( + "fmt" + "os" + "runtime/debug" + "strings" + + "github.com/10gen/ops-manager-kubernetes/multi/pkg/common" + + "github.com/spf13/cobra" + "golang.org/x/xerrors" + "k8s.io/client-go/tools/clientcmd" +) + +func init() { + multiclusterCmd.AddCommand(setupCmd) + + setupCmd.Flags().StringVar(&common.MemberClusters, "member-clusters", "", "Comma separated list of member clusters. [required]") + setupCmd.Flags().StringVar(&setupFlags.ServiceAccount, "service-account", "mongodb-enterprise-operator-multi-cluster", "Name of the service account which should be used for the Operator to communicate with the member clusters. [optional, default: mongodb-enterprise-operator-multi-cluster]") + setupCmd.Flags().StringVar(&setupFlags.CentralCluster, "central-cluster", "", "The central cluster the operator will be deployed in. [required]") + setupCmd.Flags().StringVar(&setupFlags.MemberClusterNamespace, "member-cluster-namespace", "", "The namespace the member cluster resources will be deployed to. [required]") + setupCmd.Flags().StringVar(&setupFlags.CentralClusterNamespace, "central-cluster-namespace", "", "The namespace the Operator will be deployed to. [required]") + setupCmd.Flags().BoolVar(&setupFlags.Cleanup, "cleanup", false, "Delete all previously created resources except for namespaces. [optional default: false]") + setupCmd.Flags().BoolVar(&setupFlags.ClusterScoped, "cluster-scoped", false, "Create ClusterRole and ClusterRoleBindings for member clusters. [optional default: false]") + setupCmd.Flags().BoolVar(&setupFlags.CreateTelemetryClusterRoles, "create-telemetry-roles", true, "Create ClusterRole and ClusterRoleBindings for member clusters for telemetry. [optional default: true]") + setupCmd.Flags().BoolVar(&setupFlags.InstallDatabaseRoles, "install-database-roles", false, "Install the ServiceAccounts and Roles required for running database workloads in the member clusters. [optional default: false]") + setupCmd.Flags().BoolVar(&setupFlags.CreateServiceAccountSecrets, "create-service-account-secrets", true, "Create service account token secrets. [optional default: true]") + setupCmd.Flags().StringVar(&setupFlags.ImagePullSecrets, "image-pull-secrets", "", "Name of the secret for imagePullSecrets to set in created service accounts") + setupCmd.Flags().StringVar(&common.MemberClustersApiServers, "member-clusters-api-servers", "", "Comma separated list of api servers addresses. [optional, default will take addresses from KUBECONFIG env var]") +} + +// setupCmd represents the setup command +var setupCmd = &cobra.Command{ + Use: "setup", + Short: "Setup the multicluster environment for MongoDB resources", + Long: `'setup' configures the central and member clusters in preparation for a MongoDBMultiCluster deployment. + +Example: + +kubectl-mongodb multicluster setup --central-cluster="operator-cluster" --member-clusters="cluster-1,cluster-2,cluster-3" --member-cluster-namespace=mongodb --central-cluster-namespace=mongodb --create-service-account-secrets --install-database-roles + +`, + Run: func(cmd *cobra.Command, _ []string) { + if err := parseSetupFlags(); err != nil { + fmt.Printf("error parsing flags: %s\n", err) + os.Exit(1) + } + + buildInfo, ok := debug.ReadBuildInfo() + if ok { + fmt.Println(getBuildInfoString(buildInfo)) + } + + clientMap, err := common.CreateClientMap(setupFlags.MemberClusters, setupFlags.CentralCluster, common.LoadKubeConfigFilePath(), common.GetKubernetesClient) + if err != nil { + fmt.Printf("failed to create clientset map: %s", err) + os.Exit(1) + } + + if err := common.EnsureMultiClusterResources(cmd.Context(), setupFlags, clientMap); err != nil { + fmt.Println(err) + os.Exit(1) + } + + if err := common.ReplaceClusterMembersConfigMap(cmd.Context(), clientMap[setupFlags.CentralCluster], setupFlags); err != nil { + fmt.Println(err) + os.Exit(1) + } + }, +} + +var setupFlags = common.Flags{} + +func parseSetupFlags() error { + if common.AnyAreEmpty(common.MemberClusters, setupFlags.ServiceAccount, setupFlags.CentralCluster, setupFlags.MemberClusterNamespace, setupFlags.CentralClusterNamespace) { + return xerrors.Errorf("non empty values are required for [service-account, member-clusters, central-cluster, member-cluster-namespace, central-cluster-namespace]") + } + + setupFlags.MemberClusters = strings.Split(common.MemberClusters, ",") + + if strings.TrimSpace(common.MemberClustersApiServers) != "" { + setupFlags.MemberClusterApiServerUrls = strings.Split(common.MemberClustersApiServers, ",") + if len(setupFlags.MemberClusterApiServerUrls) != len(setupFlags.MemberClusters) { + return xerrors.Errorf("expected %d addresses in member-clusters-api-servers parameter but got %d", len(setupFlags.MemberClusters), len(setupFlags.MemberClusterApiServerUrls)) + } + } + + configFilePath := common.LoadKubeConfigFilePath() + kubeconfig, err := clientcmd.LoadFromFile(configFilePath) + if err != nil { + return xerrors.Errorf("error loading kubeconfig file '%s': %w", configFilePath, err) + } + if len(setupFlags.MemberClusterApiServerUrls) == 0 { + if setupFlags.MemberClusterApiServerUrls, err = common.GetMemberClusterApiServerUrls(kubeconfig, setupFlags.MemberClusters); err != nil { + return err + } + } + return nil +} diff --git a/public/tools/multicluster/go.mod b/public/tools/multicluster/go.mod new file mode 100644 index 000000000..5d13d6fab --- /dev/null +++ b/public/tools/multicluster/go.mod @@ -0,0 +1,62 @@ +module github.com/10gen/ops-manager-kubernetes/multi + +go 1.24.0 + +require ( + github.com/ghodss/yaml v1.0.0 + github.com/spf13/cobra v1.6.1 + github.com/stretchr/testify v1.8.4 + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 + k8s.io/api v0.29.11 + k8s.io/apimachinery v0.29.11 + k8s.io/client-go v0.29.11 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 +) + +// force pin, until we update the direct dependencies +require google.golang.org/protobuf v1.33.0 // indirect + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/net v0.36.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/public/tools/multicluster/go.sum b/public/tools/multicluster/go.sum new file mode 100644 index 000000000..b481c3cd5 --- /dev/null +++ b/public/tools/multicluster/go.sum @@ -0,0 +1,175 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.11 h1:6FwDo33f1WX5Yu0RQTX9YAd3wth8Ik0B4SXQKsoQfbk= +k8s.io/api v0.29.11/go.mod h1:3TDAW1OpFbz/Yx5r0W06b6eiAfHEwtH61VYDzpTU4Ng= +k8s.io/apimachinery v0.29.11 h1:55+6ue9advpA7T0sX2ZJDHCLKuiFfrAAR/39VQN9KEQ= +k8s.io/apimachinery v0.29.11/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= +k8s.io/client-go v0.29.11 h1:mBX7Ub0uqpLMwWz3J/AGS/xKOZsjr349qZ1vxVoL1l8= +k8s.io/client-go v0.29.11/go.mod h1:WOEoi/eLg2YEg3/yEd7YK3CNScYkM8AEScQadxUnaTE= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/public/tools/multicluster/install_istio_separate_network.sh b/public/tools/multicluster/install_istio_separate_network.sh new file mode 100755 index 000000000..5277d5698 --- /dev/null +++ b/public/tools/multicluster/install_istio_separate_network.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +set -eux + +# define here or provide the cluster names externally +export CTX_CLUSTER1=${CTX_CLUSTER1} +export CTX_CLUSTER2=${CTX_CLUSTER2} +export CTX_CLUSTER3=${CTX_CLUSTER3} +export ISTIO_VERSION=${ISTIO_VERSION} + +# download Istio under the path +curl -L https://istio.io/downloadIstio | sh - + +# checks if external IP has been assigned to a service object, in our case we are interested in east-west gateway +function_check_external_ip_assigned() { + while : ; do + ip=$(kubectl --context="$1" get svc istio-eastwestgateway -n istio-system --output jsonpath='{.status.loadBalancer.ingress[0].ip}') + if [ -n "$ip" ] + then + echo "external ip assigned $ip" + break + else + echo "waiting for external ip to be assigned" + fi +done +} + +cd istio-${ISTIO_VERSION} +mkdir -p certs +pushd certs + +# create root trust for the clusters +make -f ../tools/certs/Makefile.selfsigned.mk root-ca +make -f ../tools/certs/Makefile.selfsigned.mk ${CTX_CLUSTER1}-cacerts +make -f ../tools/certs/Makefile.selfsigned.mk ${CTX_CLUSTER2}-cacerts +make -f ../tools/certs/Makefile.selfsigned.mk ${CTX_CLUSTER3}-cacerts + +kubectl --context="${CTX_CLUSTER1}" create ns istio-system +kubectl --context="${CTX_CLUSTER1}" create secret generic cacerts -n istio-system \ + --from-file=${CTX_CLUSTER1}/ca-cert.pem \ + --from-file=${CTX_CLUSTER1}/ca-key.pem \ + --from-file=${CTX_CLUSTER1}/root-cert.pem \ + --from-file=${CTX_CLUSTER1}/cert-chain.pem + +kubectl --context="${CTX_CLUSTER2}" create ns istio-system +kubectl --context="${CTX_CLUSTER2}" create secret generic cacerts -n istio-system \ + --from-file=${CTX_CLUSTER2}/ca-cert.pem \ + --from-file=${CTX_CLUSTER2}/ca-key.pem \ + --from-file=${CTX_CLUSTER2}/root-cert.pem \ + --from-file=${CTX_CLUSTER2}/cert-chain.pem + +kubectl --context="${CTX_CLUSTER3}" create ns istio-system +kubectl --context="${CTX_CLUSTER3}" create secret generic cacerts -n istio-system \ + --from-file=${CTX_CLUSTER3}/ca-cert.pem \ + --from-file=${CTX_CLUSTER3}/ca-key.pem \ + --from-file=${CTX_CLUSTER3}/root-cert.pem \ + --from-file=${CTX_CLUSTER3}/cert-chain.pem +popd + +# label namespace in cluster1 +kubectl --context="${CTX_CLUSTER1}" get namespace istio-system && \ + kubectl --context="${CTX_CLUSTER1}" label namespace istio-system topology.istio.io/network=network1 + +cat < cluster1.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster1 + network: network1 +EOF +bin/istioctl install --context="${CTX_CLUSTER1}" -f cluster1.yaml +samples/multicluster/gen-eastwest-gateway.sh \ + --mesh mesh1 --cluster cluster1 --network network1 | \ + bin/istioctl --context="${CTX_CLUSTER1}" install -y -f - + + +# check if external IP is assigned to east-west gateway in cluster1 +function_check_external_ip_assigned "${CTX_CLUSTER1}" + + +# expose services in cluster1 +kubectl --context="${CTX_CLUSTER1}" apply -n istio-system -f \ + samples/multicluster/expose-services.yaml + + +kubectl --context="${CTX_CLUSTER2}" get namespace istio-system && \ + kubectl --context="${CTX_CLUSTER2}" label namespace istio-system topology.istio.io/network=network2 + + +cat < cluster2.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster2 + network: network2 +EOF + +bin/istioctl install --context="${CTX_CLUSTER2}" -f cluster2.yaml + +samples/multicluster/gen-eastwest-gateway.sh \ + --mesh mesh1 --cluster cluster2 --network network2 | \ + bin/istioctl --context="${CTX_CLUSTER2}" install -y -f - + +# check if external IP is assigned to east-west gateway in cluster2 +function_check_external_ip_assigned "${CTX_CLUSTER2}" + +kubectl --context="${CTX_CLUSTER2}" apply -n istio-system -f \ + samples/multicluster/expose-services.yaml + +# cluster3 +kubectl --context="${CTX_CLUSTER3}" get namespace istio-system && \ + kubectl --context="${CTX_CLUSTER3}" label namespace istio-system topology.istio.io/network=network3 + +cat < cluster3.yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + values: + global: + meshID: mesh1 + multiCluster: + clusterName: cluster3 + network: network3 +EOF + +bin/istioctl install --context="${CTX_CLUSTER3}" -f cluster3.yaml + +samples/multicluster/gen-eastwest-gateway.sh \ + --mesh mesh1 --cluster cluster3 --network network3 | \ + bin/istioctl --context="${CTX_CLUSTER3}" install -y -f - + + +# check if external IP is assigned to east-west gateway in cluster3 +function_check_external_ip_assigned "${CTX_CLUSTER3}" + +kubectl --context="${CTX_CLUSTER3}" apply -n istio-system -f \ + samples/multicluster/expose-services.yaml + + +# enable endpoint discovery +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER1}" \ + -n istio-system \ + --name=cluster1 | \ + kubectl apply -f - --context="${CTX_CLUSTER2}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER1}" \ + -n istio-system \ + --name=cluster1 | \ + kubectl apply -f - --context="${CTX_CLUSTER3}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + -n istio-system \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER2}" \ + -n istio-system \ + --name=cluster2 | \ + kubectl apply -f - --context="${CTX_CLUSTER3}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER3}" \ + -n istio-system \ + --name=cluster3 | \ + kubectl apply -f - --context="${CTX_CLUSTER1}" + +bin/istioctl x create-remote-secret \ + --context="${CTX_CLUSTER3}" \ + -n istio-system \ + --name=cluster3 | \ + kubectl apply -f - --context="${CTX_CLUSTER2}" + + # cleanup: delete the istio repo at the end +cd .. +rm -r istio-${ISTIO_VERSION} +rm -f cluster1.yaml cluster2.yaml cluster3.yaml diff --git a/public/tools/multicluster/kubectl_mac_notarize.sh b/public/tools/multicluster/kubectl_mac_notarize.sh new file mode 100755 index 000000000..06a69e640 --- /dev/null +++ b/public/tools/multicluster/kubectl_mac_notarize.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2022 MongoDB Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -Eeou pipefail + +# Notarize generated binaries with Apple and replace the original binary with the notarized one +# This depends on binaries being generated in a goreleaser manner and gon being set up. +# goreleaser should already take care of calling this script as a hook. + +if [[ -f "./dist/kubectl-mongodb_darwin_amd64_v1/kubectl-mongodb" && -f "./dist/kubectl-mongodb_darwin_arm64/kubectl-mongodb" && ! -f "./dist/kubectl-mongodb_macos_signed.zip" ]]; then + echo "notarizing macOs binaries" + zip -r ./dist/kubectl-mongodb_amd64_arm64_bin.zip ./dist/kubectl-mongodb_darwin_amd64_v1/kubectl-mongodb ./dist/kubectl-mongodb_darwin_arm64/kubectl-mongodb # The Notarization Service takes an archive as input + "${workdir:-.}"/linux_amd64/macnotary \ + -f ./dist/kubectl-mongodb_amd64_arm64_bin.zip \ + -m notarizeAndSign -u https://dev.macos-notary.build.10gen.cc/api \ + -b com.mongodb.mongodb-kubectl-mongodb \ + -o ./dist/kubectl-mongodb_macos_signed.zip + + echo "replacing original files" + unzip -oj ./dist/kubectl-mongodb_macos_signed.zip dist/kubectl-mongodb_darwin_amd64_v1/kubectl-mongodb -d ./dist/kubectl-mongodb_darwin_amd64_v1/ + unzip -oj ./dist/kubectl-mongodb_macos_signed.zip dist/kubectl-mongodb_darwin_arm64/kubectl-mongodb -d ./dist/kubectl-mongodb_darwin_arm64/ +fi diff --git a/public/tools/multicluster/licenses.csv b/public/tools/multicluster/licenses.csv new file mode 100644 index 000000000..adb7a5331 --- /dev/null +++ b/public/tools/multicluster/licenses.csv @@ -0,0 +1,42 @@ + +github.com/davecgh/go-spew/spew,v1.1.1,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC +github.com/emicklei/go-restful/v3,v3.11.0,https://github.com/emicklei/go-restful/blob/v3.11.0/LICENSE,MIT +github.com/ghodss/yaml,v1.0.0,https://github.com/ghodss/yaml/blob/v1.0.0/LICENSE,MIT +github.com/go-logr/logr,v1.4.1,https://github.com/go-logr/logr/blob/v1.4.1/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,v0.19.6,https://github.com/go-openapi/jsonpointer/blob/v0.19.6/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,v0.20.2,https://github.com/go-openapi/jsonreference/blob/v0.20.2/LICENSE,Apache-2.0 +github.com/go-openapi/swag,v0.22.3,https://github.com/go-openapi/swag/blob/v0.22.3/LICENSE,Apache-2.0 +github.com/gogo/protobuf,v1.3.2,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause +github.com/golang/protobuf,v1.5.4,https://github.com/golang/protobuf/blob/v1.5.4/LICENSE,BSD-3-Clause +github.com/google/gnostic-models,v0.6.8,https://github.com/google/gnostic-models/blob/v0.6.8/LICENSE,Apache-2.0 +github.com/google/gofuzz,v1.2.0,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 +github.com/google/uuid,v1.3.0,https://github.com/google/uuid/blob/v1.3.0/LICENSE,BSD-3-Clause +github.com/gorilla/websocket,v1.5.0,https://github.com/gorilla/websocket/blob/v1.5.0/LICENSE,BSD-2-Clause +github.com/imdario/mergo,v0.3.6,https://github.com/imdario/mergo/blob/v0.3.6/LICENSE,BSD-3-Clause +github.com/josharian/intern,v1.0.0,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT +github.com/json-iterator/go,v1.1.12,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT +github.com/mailru/easyjson,v0.7.7,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT +github.com/moby/spdystream,v0.2.0,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 +github.com/modern-go/concurrent,v0.0.0-20180306012644-bacd9c7ef1dd,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 +github.com/modern-go/reflect2,v1.0.2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 +github.com/munnerz/goautoneg,v0.0.0-20191010083416-a7dc8b61c822,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause +github.com/mxk/go-flowrate/flowrate,v0.0.0-20140419014527-cca7078d478f,https://github.com/mxk/go-flowrate/blob/cca7078d478f/LICENSE,BSD-3-Clause +github.com/spf13/cobra,v1.6.1,https://github.com/spf13/cobra/blob/v1.6.1/LICENSE.txt,Apache-2.0 +github.com/spf13/pflag,v1.0.5,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause +google.golang.org/protobuf,v1.33.0,https://github.com/protocolbuffers/protobuf-go/blob/v1.33.0/LICENSE,BSD-3-Clause +gopkg.in/inf.v0,v0.9.1,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/yaml.v2,v2.4.0,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 +gopkg.in/yaml.v3,v3.0.1,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT +k8s.io/api,v0.29.11,https://github.com/kubernetes/api/blob/v0.29.11/LICENSE,Apache-2.0 +k8s.io/apimachinery/pkg,v0.29.11,https://github.com/kubernetes/apimachinery/blob/v0.29.11/LICENSE,Apache-2.0 +k8s.io/apimachinery/third_party/forked/golang,v0.29.11,https://github.com/kubernetes/apimachinery/blob/v0.29.11/third_party/forked/golang/LICENSE,BSD-3-Clause +k8s.io/client-go,v0.29.11,https://github.com/kubernetes/client-go/blob/v0.29.11/LICENSE,Apache-2.0 +k8s.io/klog/v2,v2.120.1,https://github.com/kubernetes/klog/blob/v2.120.1/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg,v0.0.0-20231010175941-2dd684a91f00,https://github.com/kubernetes/kube-openapi/blob/2dd684a91f00/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json,v0.0.0-20231010175941-2dd684a91f00,https://github.com/kubernetes/kube-openapi/blob/2dd684a91f00/pkg/internal/third_party/go-json-experiment/json/LICENSE,BSD-3-Clause +k8s.io/kube-openapi/pkg/validation/spec,v0.0.0-20231010175941-2dd684a91f00,https://github.com/kubernetes/kube-openapi/blob/2dd684a91f00/pkg/validation/spec/LICENSE,Apache-2.0 +k8s.io/utils,v0.0.0-20240502163921-fe8a2dddb1d0,https://github.com/kubernetes/utils/blob/fe8a2dddb1d0/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang/net,v0.0.0-20240502163921-fe8a2dddb1d0,https://github.com/kubernetes/utils/blob/fe8a2dddb1d0/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/json,v0.0.0-20221116044647-bc3834ca7abd,https://github.com/kubernetes-sigs/json/blob/bc3834ca7abd/LICENSE,Apache-2.0 +sigs.k8s.io/structured-merge-diff/v4,v4.4.1,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.4.1/LICENSE,Apache-2.0 +sigs.k8s.io/yaml,v1.3.0,https://github.com/kubernetes-sigs/yaml/blob/v1.3.0/LICENSE,MIT diff --git a/public/tools/multicluster/main.go b/public/tools/multicluster/main.go new file mode 100644 index 000000000..3fe6b9832 --- /dev/null +++ b/public/tools/multicluster/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "context" + + "github.com/10gen/ops-manager-kubernetes/multi/cmd" +) + +func main() { + ctx := context.Background() + cmd.Execute(ctx) +} diff --git a/public/tools/multicluster/pkg/common/common.go b/public/tools/multicluster/pkg/common/common.go new file mode 100644 index 000000000..740a81733 --- /dev/null +++ b/public/tools/multicluster/pkg/common/common.go @@ -0,0 +1,1097 @@ +package common + +import ( + "context" + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/ghodss/yaml" + "golang.org/x/xerrors" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" +) + +type clusterType string + +// This tool handles the creation of ServiceAccounts and roles across multiple clusters. +// Service Accounts, Roles and RoleBindings are created in all the member clusters and the central cluster. +// The Service Account token secrets from the member clusters are merged into a KubeConfig file which is then +// created in the central cluster. + +var ( + MemberClusters string + MemberClustersApiServers string +) + +var ( + PollingInterval = time.Millisecond * 100 + PollingTimeout = time.Second * 5 +) + +const ( + clusterTypeCentral clusterType = "CENTRAL" + clusterTypeMember clusterType = "MEMBER" +) + +// Flags holds all the fields provided by the user. +type Flags struct { + MemberClusters []string + MemberClusterApiServerUrls []string + ServiceAccount string + CentralCluster string + MemberClusterNamespace string + CentralClusterNamespace string + Cleanup bool + ClusterScoped bool + InstallDatabaseRoles bool + CreateTelemetryClusterRoles bool + OperatorName string + SourceCluster string + CreateServiceAccountSecrets bool + ImagePullSecrets string +} + +const ( + KubeConfigSecretName = "mongodb-enterprise-operator-multi-cluster-kubeconfig" + KubeConfigSecretKey = "kubeconfig" + AppdbServiceAccount = "mongodb-enterprise-appdb" + DatabasePodsServiceAccount = "mongodb-enterprise-database-pods" + OpsManagerServiceAccount = "mongodb-enterprise-ops-manager" + AppdbRole = "mongodb-enterprise-appdb" + AppdbRoleBinding = "mongodb-enterprise-appdb" + DefaultOperatorName = "mongodb-enterprise-operator" + DefaultOperatorConfigMapName = DefaultOperatorName + "-member-list" +) + +// KubeConfigFile represents the contents of a KubeConfig file. +type KubeConfigFile struct { + ApiVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Clusters []KubeConfigClusterItem `json:"clusters"` + Contexts []KubeConfigContextItem `json:"contexts"` + Users []KubeConfigUserItem `json:"users"` +} + +type KubeConfigClusterItem struct { + Name string `json:"name"` + Cluster KubeConfigCluster `json:"cluster"` +} + +type KubeConfigCluster struct { + CertificateAuthorityData []byte `json:"certificate-authority-data"` + Server string `json:"server"` +} + +type KubeConfigContextItem struct { + Name string `json:"name"` + Context KubeConfigContext `json:"context"` +} + +type KubeConfigContext struct { + Cluster string `json:"cluster"` + Namespace string `json:"namespace"` + User string `json:"user"` +} + +type KubeConfigUserItem struct { + Name string `json:"name"` + User KubeConfigUser `json:"user"` +} + +type KubeConfigUser struct { + Token string `json:"token"` +} + +// multiClusterLabels the labels that will be applied to every resource created by this tool. +func multiClusterLabels() map[string]string { + return map[string]string{ + "multi-cluster": "true", + } +} + +// performCleanup cleans up all of the resources that were created by this script in the past. +func performCleanup(ctx context.Context, clientMap map[string]KubeClient, flags Flags) error { + for _, cluster := range flags.MemberClusters { + c := clientMap[cluster] + if err := cleanupClusterResources(ctx, c, cluster, flags.MemberClusterNamespace); err != nil { + return xerrors.Errorf("failed cleaning up cluster %s namespace %s: %w", cluster, flags.MemberClusterNamespace, err) + } + } + c := clientMap[flags.CentralCluster] + if err := cleanupClusterResources(ctx, c, flags.CentralCluster, flags.CentralClusterNamespace); err != nil { + return xerrors.Errorf("failed cleaning up cluster %s namespace %s: %w", flags.CentralCluster, flags.CentralClusterNamespace, err) + } + return nil +} + +// cleanupClusterResources cleans up all the resources created by this tool in a given namespace. +func cleanupClusterResources(ctx context.Context, clientset KubeClient, clusterName, namespace string) error { + listOpts := metav1.ListOptions{ + LabelSelector: "multi-cluster=true", + } + + // clean up secrets + secretList, err := clientset.CoreV1().Secrets(namespace).List(ctx, listOpts) + if err != nil { + return err + } + + if secretList != nil { + for _, s := range secretList.Items { + fmt.Printf("Deleting Secret: %s in cluster %s\n", s.Name, clusterName) + if err := clientset.CoreV1().Secrets(namespace).Delete(ctx, s.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + } + + // clean up service accounts + serviceAccountList, err := clientset.CoreV1().ServiceAccounts(namespace).List(ctx, listOpts) + if err != nil { + return err + } + + if serviceAccountList != nil { + for _, sa := range serviceAccountList.Items { + fmt.Printf("Deleting ServiceAccount: %s in cluster %s\n", sa.Name, clusterName) + if err := clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, sa.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + } + + // clean up roles + roleList, err := clientset.RbacV1().Roles(namespace).List(ctx, listOpts) + if err != nil { + return err + } + + for _, r := range roleList.Items { + fmt.Printf("Deleting Role: %s in cluster %s\n", r.Name, clusterName) + if err := clientset.RbacV1().Roles(namespace).Delete(ctx, r.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + + // clean up roles + roles, err := clientset.RbacV1().Roles(namespace).List(ctx, listOpts) + if err != nil { + return err + } + + if roles != nil { + for _, r := range roles.Items { + fmt.Printf("Deleting Role: %s in cluster %s\n", r.Name, clusterName) + if err := clientset.RbacV1().Roles(namespace).Delete(ctx, r.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + } + + // clean up role bindings + roleBindings, err := clientset.RbacV1().RoleBindings(namespace).List(ctx, listOpts) + if !errors.IsNotFound(err) && err != nil { + return err + } + + if roleBindings != nil { + for _, crb := range roleBindings.Items { + fmt.Printf("Deleting RoleBinding: %s in cluster %s\n", crb.Name, clusterName) + if err := clientset.RbacV1().RoleBindings(namespace).Delete(ctx, crb.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + } + + // clean up cluster role bindings + clusterRoleBindings, err := clientset.RbacV1().ClusterRoleBindings().List(ctx, listOpts) + if !errors.IsNotFound(err) && err != nil { + return err + } + + if clusterRoleBindings != nil { + for _, crb := range clusterRoleBindings.Items { + fmt.Printf("Deleting ClusterRoleBinding: %s in cluster %s\n", crb.Name, clusterName) + if err := clientset.RbacV1().ClusterRoleBindings().Delete(ctx, crb.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + } + + // clean up cluster roles + clusterRoles, err := clientset.RbacV1().ClusterRoles().List(ctx, listOpts) + if !errors.IsNotFound(err) && err != nil { + return err + } + + if clusterRoles != nil { + for _, cr := range clusterRoles.Items { + fmt.Printf("Deleting ClusterRole: %s in cluster %s\n", cr.Name, clusterName) + if err := clientset.RbacV1().ClusterRoles().Delete(ctx, cr.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + } + + return nil +} + +// ensureNamespace creates the namespace with the given clientset. +func ensureNamespace(ctx context.Context, clientSet KubeClient, nsName string) error { + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + Labels: multiClusterLabels(), + }, + } + _, err := clientSet.CoreV1().Namespaces().Create(ctx, &ns, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("failed to create namespace %s: %w", ns.Name, err) + } + + return nil +} + +// ensureAllClusterNamespacesExist makes sure the namespace we will be creating exists in all clusters. +func ensureAllClusterNamespacesExist(ctx context.Context, clientSets map[string]KubeClient, f Flags) error { + for _, clusterName := range f.MemberClusters { + if err := ensureNamespace(ctx, clientSets[clusterName], f.MemberClusterNamespace); err != nil { + return xerrors.Errorf("failed to ensure namespace %s in member cluster %s: %w", f.MemberClusterNamespace, clusterName, err) + } + if f.CentralClusterNamespace != f.MemberClusterNamespace { + if err := ensureNamespace(ctx, clientSets[clusterName], f.CentralClusterNamespace); err != nil { + return xerrors.Errorf("failed to ensure namespace %s in member cluster %s: %w", f.CentralClusterNamespace, clusterName, err) + } + } + } + if err := ensureNamespace(ctx, clientSets[f.CentralCluster], f.CentralClusterNamespace); err != nil { + return xerrors.Errorf("failed to ensure namespace %s in central cluster %s: %w", f.CentralClusterNamespace, f.CentralCluster, err) + } + return nil +} + +// EnsureMultiClusterResources copies the ServiceAccount Secret tokens from the specified +// member clusters, merges them into a KubeConfig file and creates a Secret in the central cluster +// with the contents. +func EnsureMultiClusterResources(ctx context.Context, flags Flags, clientMap map[string]KubeClient) error { + if flags.Cleanup { + if err := performCleanup(ctx, clientMap, flags); err != nil { + return xerrors.Errorf("failed performing Cleanup of resources: %w", err) + } + } + + if err := ensureAllClusterNamespacesExist(ctx, clientMap, flags); err != nil { + return xerrors.Errorf("failed ensuring namespaces: %w", err) + } + fmt.Println("Ensured namespaces exist in all clusters.") + + if err := createOperatorServiceAccountsAndRoles(ctx, clientMap, flags); err != nil { + return xerrors.Errorf("failed creating service accounts and roles in all clusters: %w", err) + } + fmt.Println("Ensured ServiceAccounts and Roles.") + + secrets, err := getAllMemberClusterServiceAccountSecretTokens(ctx, clientMap, flags) + if err != nil { + return xerrors.Errorf("failed to get service account secret tokens: %w", err) + } + + if len(secrets) != len(flags.MemberClusters) { + return xerrors.Errorf("required %d serviceaccount tokens but found only %d\n", len(flags.MemberClusters), len(secrets)) + } + + kubeConfig, err := createKubeConfigFromServiceAccountTokens(secrets, flags) + if err != nil { + return xerrors.Errorf("failed to create kube config from service account tokens: %w", err) + } + + kubeConfigBytes, err := yaml.Marshal(kubeConfig) + if err != nil { + return xerrors.Errorf("failed to marshal kubeconfig: %w", err) + } + + centralClusterClient := clientMap[flags.CentralCluster] + if err != nil { + return xerrors.Errorf("failed to get central cluster clientset: %w", err) + } + + if err := createKubeConfigSecret(ctx, centralClusterClient, kubeConfigBytes, flags); err != nil { + return xerrors.Errorf("failed creating KubeConfig secret: %w", err) + } + + if flags.SourceCluster != "" { + if err := setupDatabaseRoles(ctx, clientMap, flags); err != nil { + return xerrors.Errorf("failed setting up database roles: %w", err) + } + fmt.Println("Ensured database Roles in member clusters.") + } else if flags.InstallDatabaseRoles { + if err := installDatabaseRoles(ctx, clientMap, flags); err != nil { + return xerrors.Errorf("failed installing database roles: %w", err) + } + fmt.Println("Ensured database Roles in member clusters.") + } + + return nil +} + +// createKubeConfigSecret creates the secret containing the KubeConfig file made from the various +// service account tokens in the member clusters. +func createKubeConfigSecret(ctx context.Context, centralClusterClient kubernetes.Interface, kubeConfigBytes []byte, flags Flags) error { + kubeConfigSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: KubeConfigSecretName, + Namespace: flags.CentralClusterNamespace, + Labels: multiClusterLabels(), + }, + Data: map[string][]byte{ + KubeConfigSecretKey: kubeConfigBytes, + }, + } + + fmt.Printf("Creating KubeConfig secret %s/%s in cluster %s\n", flags.CentralClusterNamespace, kubeConfigSecret.Name, flags.CentralCluster) + _, err := centralClusterClient.CoreV1().Secrets(flags.CentralClusterNamespace).Create(ctx, &kubeConfigSecret, metav1.CreateOptions{}) + + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("failed creating secret: %w", err) + } + + if errors.IsAlreadyExists(err) { + _, err = centralClusterClient.CoreV1().Secrets(flags.CentralClusterNamespace).Update(ctx, &kubeConfigSecret, metav1.UpdateOptions{}) + if err != nil { + return xerrors.Errorf("failed updating existing secret: %w", err) + } + } + + return nil +} + +func getCentralRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + Verbs: []string{"*"}, + Resources: []string{ + "mongodbmulticluster", "mongodbmulticluster/finalizers", "mongodbmulticluster/status", + "mongodbusers", "mongodbusers/status", + "opsmanagers", "opsmanagers/finalizers", "opsmanagers/status", + "mongodb", "mongodb/finalizers", "mongodb/status", + }, + APIGroups: []string{"mongodb.com"}, + }, + } +} + +func buildCentralEntityRole(namespace string) rbacv1.Role { + rules := append(getCentralRules(), getMemberRules()...) + return rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mongodb-enterprise-operator-multi-role", + Namespace: namespace, + Labels: multiClusterLabels(), + }, + Rules: rules, + } +} + +func buildCentralEntityClusterRole() rbacv1.ClusterRole { + rules := append(getCentralRules(), getMemberRules()...) + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"list", "watch"}, + Resources: []string{"namespaces"}, + APIGroups: []string{""}, + }) + + return rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mongodb-enterprise-operator-multi-cluster-role", + Labels: multiClusterLabels(), + }, + Rules: rules, + } +} + +func getMemberRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "create", "update", "delete", "watch", "deletecollection"}, + Resources: []string{"secrets", "configmaps", "services"}, + APIGroups: []string{""}, + }, + { + Verbs: []string{"get", "list", "create", "update", "delete", "watch", "deletecollection"}, + Resources: []string{"statefulsets"}, + APIGroups: []string{"apps"}, + }, + { + Verbs: []string{"get", "list", "create", "update", "watch", "patch"}, + Resources: []string{"persistentvolumeclaims"}, + APIGroups: []string{""}, + }, + { + Verbs: []string{"get", "list", "watch", "delete", "deletecollection"}, + Resources: []string{"pods"}, + APIGroups: []string{""}, + }, + } +} + +func buildMemberEntityRole(namespace string) rbacv1.Role { + return rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mongodb-enterprise-operator-multi-role", + Namespace: namespace, + Labels: multiClusterLabels(), + }, + Rules: getMemberRules(), + } +} + +func buildMemberEntityClusterRole() rbacv1.ClusterRole { + rules := append(getMemberRules(), rbacv1.PolicyRule{ + Verbs: []string{"list", "watch"}, + Resources: []string{"namespaces"}, + APIGroups: []string{""}, + }) + + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"get"}, + Resources: []string{"nodes"}, + APIGroups: []string{""}, + }) + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"get"}, + Resources: []string{"namespaces"}, + APIGroups: []string{""}, + ResourceNames: []string{"kube-system"}, + }) + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"get"}, + NonResourceURLs: []string{"/version"}, + }) + + return rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mongodb-enterprise-operator-multi-cluster-role", + Labels: multiClusterLabels(), + }, + Rules: rules, + } +} + +func buildClusterRoleTelemetry() rbacv1.ClusterRole { + var rules []rbacv1.PolicyRule + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"list"}, + Resources: []string{"nodes"}, + APIGroups: []string{""}, + }) + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"get"}, + Resources: []string{"namespaces"}, + APIGroups: []string{""}, + ResourceNames: []string{"kube-system"}, + }) + rules = append(rules, rbacv1.PolicyRule{ + Verbs: []string{"get"}, + NonResourceURLs: []string{"/version"}, + }) + + return rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mongodb-enterprise-operator-multi-cluster-role-telemetry", + Labels: multiClusterLabels(), + }, + Rules: rules, + } +} + +// buildRoleBinding creates the RoleBinding which binds the Role to the given ServiceAccount. +func buildRoleBinding(role rbacv1.Role, serviceAccount string, serviceAccountNamespace string) rbacv1.RoleBinding { + return rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mongodb-enterprise-operator-multi-role-binding", + Labels: multiClusterLabels(), + Namespace: role.Namespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccount, + Namespace: serviceAccountNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: role.Name, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +// buildClusterRoleBinding creates the ClusterRoleBinding which binds the ClusterRole to the given ServiceAccount. +func buildClusterRoleBinding(clusterRole rbacv1.ClusterRole, serviceAccountName, serviceAccountNamespace, clusterRoleBindingName string) rbacv1.ClusterRoleBinding { + return rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleBindingName, + Labels: multiClusterLabels(), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccountName, + Namespace: serviceAccountNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: clusterRole.Name, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +// createRoles creates the ServiceAccount and Roles, RoleBindings, ClusterRoles and ClusterRoleBindings required. +func createRoles(ctx context.Context, c KubeClient, serviceAccountName, serviceAccountNamespace, namespace string, clusterScoped, telemetryClusterRoles bool, clusterType clusterType) error { + var err error + + if telemetryClusterRoles { + clusterRoleTelemetry := buildClusterRoleTelemetry() + _, err = c.RbacV1().ClusterRoles().Create(ctx, &clusterRoleTelemetry, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating cluster role: %w", err) + } + fmt.Printf("created clusterrole: %s\n", clusterRoleTelemetry.Name) + if err = createClusterRoleBinding(ctx, c, serviceAccountName, serviceAccountNamespace, "mongodb-enterprise-operator-multi-telemetry-cluster-role-binding", clusterRoleTelemetry); err != nil { + return err + } + + } + + if !clusterScoped { + var role rbacv1.Role + if clusterType == clusterTypeCentral { + role = buildCentralEntityRole(namespace) + } else { + role = buildMemberEntityRole(namespace) + } + + _, err = c.RbacV1().Roles(namespace).Create(ctx, &role, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + if errors.IsAlreadyExists(err) { + if _, err := c.RbacV1().Roles(namespace).Update(ctx, &role, metav1.UpdateOptions{}); err != nil { + return xerrors.Errorf("error updating role: %w", err) + } + } else { + return xerrors.Errorf("error creating role: %w", err) + } + } + + roleBinding := buildRoleBinding(role, serviceAccountName, serviceAccountNamespace) + _, err = c.RbacV1().RoleBindings(namespace).Create(ctx, &roleBinding, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + if _, err := c.RbacV1().RoleBindings(namespace).Update(ctx, &roleBinding, metav1.UpdateOptions{}); err != nil { + return xerrors.Errorf("error updating role binding: %w", err) + } + } else { + return xerrors.Errorf("error creating role binding: %w", err) + } + } + + return nil + } + + var clusterRole rbacv1.ClusterRole + if clusterType == clusterTypeCentral { + clusterRole = buildCentralEntityClusterRole() + } else { + clusterRole = buildMemberEntityClusterRole() + } + + _, err = c.RbacV1().ClusterRoles().Create(ctx, &clusterRole, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating cluster role: %w", err) + } + fmt.Printf("created clusterrole: %s\n", clusterRole.Name) + + if err = createClusterRoleBinding(ctx, c, serviceAccountName, serviceAccountNamespace, "mongodb-enterprise-operator-multi-cluster-role-binding", clusterRole); err != nil { + return err + } + return nil +} + +func createClusterRoleBinding(ctx context.Context, c KubeClient, serviceAccountName string, serviceAccountNamespace string, clusterRoleBindingName string, clusterRole rbacv1.ClusterRole) error { + clusterRoleBinding := buildClusterRoleBinding(clusterRole, serviceAccountName, serviceAccountNamespace, clusterRoleBindingName) + _, err := c.RbacV1().ClusterRoleBindings().Create(ctx, &clusterRoleBinding, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating cluster role binding: %w", err) + } + fmt.Printf("created clusterrolebinding: %s\n", clusterRoleBinding.Name) + return nil +} + +// createOperatorServiceAccountsAndRoles creates the required ServiceAccounts in all member clusters. +func createOperatorServiceAccountsAndRoles(ctx context.Context, clientMap map[string]KubeClient, f Flags) error { + fmt.Printf("creating central cluster roles in cluster: %s\n", f.CentralCluster) + centralClusterClient := clientMap[f.CentralCluster] + _, err := createServiceAccount(ctx, centralClusterClient, f.ServiceAccount, f.CentralClusterNamespace, f.ImagePullSecrets) + if err != nil { + return xerrors.Errorf("error creating service account: %w", err) + } + if f.CreateServiceAccountSecrets { + if err := createServiceAccountTokenSecret(ctx, centralClusterClient, f.CentralClusterNamespace, f.ServiceAccount); err != nil { + return err + } + } + + if err := createRoles(ctx, centralClusterClient, f.ServiceAccount, f.CentralClusterNamespace, f.CentralClusterNamespace, f.ClusterScoped, f.CreateTelemetryClusterRoles, clusterTypeCentral); err != nil { + return err + } + + // in case the operator namespace (CentralClusterNamespace) is different from member cluster namespace we need + // to provide roles and role binding to the operator's SA in member namespace + if f.CentralClusterNamespace != f.MemberClusterNamespace { + if err := createRoles(ctx, centralClusterClient, f.ServiceAccount, f.CentralClusterNamespace, f.MemberClusterNamespace, f.ClusterScoped, f.CreateTelemetryClusterRoles, clusterTypeCentral); err != nil { + return err + } + } + + for _, memberCluster := range f.MemberClusters { + if memberCluster == f.CentralCluster { + // we've already done that for central cluster + continue + } + fmt.Printf("creating member roles in cluster: %s\n", memberCluster) + memberClusterClient := clientMap[memberCluster] + _, err := createServiceAccount(ctx, memberClusterClient, f.ServiceAccount, f.CentralClusterNamespace, f.ImagePullSecrets) + if err != nil { + return xerrors.Errorf("error creating service account: %w", err) + } + + if f.CreateServiceAccountSecrets { + if err := createServiceAccountTokenSecret(ctx, memberClusterClient, f.CentralClusterNamespace, f.ServiceAccount); err != nil { + return err + } + } + + if err := createRoles(ctx, memberClusterClient, f.ServiceAccount, f.CentralClusterNamespace, f.MemberClusterNamespace, f.ClusterScoped, f.CreateTelemetryClusterRoles, clusterTypeMember); err != nil { + return err + } + if err := createRoles(ctx, memberClusterClient, f.ServiceAccount, f.CentralClusterNamespace, f.CentralClusterNamespace, f.ClusterScoped, f.CreateTelemetryClusterRoles, clusterTypeMember); err != nil { + return err + } + } + + return nil +} + +func createServiceAccountTokenSecret(ctx context.Context, c kubernetes.Interface, namespace string, serviceAccountName string) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-token-secret", serviceAccountName), + Namespace: namespace, + Annotations: map[string]string{ + "kubernetes.io/service-account.name": serviceAccountName, + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + } + + _, err := c.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("cannot create secret %+v: %w", *secret, err) + } + + return nil +} + +// createKubeConfigFromServiceAccountTokens builds up a KubeConfig from the ServiceAccount tokens provided. +func createKubeConfigFromServiceAccountTokens(serviceAccountTokens map[string]corev1.Secret, flags Flags) (KubeConfigFile, error) { + config := &KubeConfigFile{ + Kind: "Config", + ApiVersion: "v1", + } + + for i, clusterName := range flags.MemberClusters { + tokenSecret := serviceAccountTokens[clusterName] + ca, ok := tokenSecret.Data["ca.crt"] + if !ok { + return KubeConfigFile{}, xerrors.Errorf("key 'ca.crt' missing from token secret %s", tokenSecret.Name) + } + + token, ok := tokenSecret.Data["token"] + if !ok { + return KubeConfigFile{}, xerrors.Errorf("key 'token' missing from token secret %s", tokenSecret.Name) + } + + config.Clusters = append(config.Clusters, KubeConfigClusterItem{ + Name: clusterName, + Cluster: KubeConfigCluster{ + CertificateAuthorityData: ca, + Server: flags.MemberClusterApiServerUrls[i], + }, + }) + + ns := flags.MemberClusterNamespace + if flags.ClusterScoped { + ns = "" + } + + config.Contexts = append(config.Contexts, KubeConfigContextItem{ + Name: clusterName, + Context: KubeConfigContext{ + Cluster: clusterName, + Namespace: ns, + User: clusterName, + }, + }) + + config.Users = append(config.Users, KubeConfigUserItem{ + Name: clusterName, + User: KubeConfigUser{ + Token: string(token), + }, + }) + } + return *config, nil +} + +// getAllMemberClusterServiceAccountSecretTokens returns a slice of secrets that should all be +// copied in the central cluster for the operator to use. +func getAllMemberClusterServiceAccountSecretTokens(ctx context.Context, clientSetMap map[string]KubeClient, flags Flags) (map[string]corev1.Secret, error) { + allSecrets := map[string]corev1.Secret{} + + for _, cluster := range flags.MemberClusters { + c := clientSetMap[cluster] + serviceAccountNamespace := flags.CentralClusterNamespace + sa, err := getServiceAccount(ctx, c, serviceAccountNamespace, flags.ServiceAccount, cluster) + if err != nil { + return nil, xerrors.Errorf("failed getting service account: %w", err) + } + + // Wait for the token secret to be created and populated with service account token data + var tokenSecret *corev1.Secret + if err := wait.PollWithContext(ctx, PollingInterval, PollingTimeout, func(ctx context.Context) (done bool, err error) { + tokenSecret, err = getServiceAccountToken(ctx, c, *sa) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } else { + return true, err + } + } + + if _, ok := tokenSecret.Data["ca.crt"]; !ok { + return false, nil + } + if _, ok := tokenSecret.Data["token"]; !ok { + return false, nil + } + + return true, nil + }); err != nil { + return nil, xerrors.Errorf("failed getting service account token secret: %w", err) + } + + allSecrets[cluster] = *tokenSecret + } + return allSecrets, nil +} + +func getServiceAccount(ctx context.Context, lister kubernetes.Interface, namespace string, name string, memberClusterName string) (*corev1.ServiceAccount, error) { + sa, err := lister.CoreV1().ServiceAccounts(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, xerrors.Errorf("failed to get service account %s/%s in member cluster %s: %w", namespace, name, memberClusterName, err) + } + return sa, nil +} + +// getServiceAccountToken returns the Secret containing the ServiceAccount token +func getServiceAccountToken(ctx context.Context, secretLister KubeClient, sa corev1.ServiceAccount) (*corev1.Secret, error) { + secretList, err := secretLister.CoreV1().Secrets(sa.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, xerrors.Errorf("failed to list secrets in member cluster namespace %s: %w", sa.Namespace, err) + } + for _, secret := range secretList.Items { + // found the associated service account token. + if strings.HasPrefix(secret.Name, fmt.Sprintf("%s-token", sa.Name)) { + return &secret, nil + } + } + return nil, xerrors.Errorf("no service account token found for serviceaccount: %s", sa.Name) +} + +// copySecret copies a Secret from a source cluster to a target cluster +func copySecret(ctx context.Context, src, dst KubeClient, namespace, name string) error { + secret, err := src.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return xerrors.Errorf("failed retrieving secret: %s from source cluster: %w", name, err) + } + _, err = dst.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: secret.Labels, + }, + Data: secret.Data, + }, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return err + } + return nil +} + +func createServiceAccount(ctx context.Context, c KubeClient, serviceAccountName, namespace string, imagePullSecrets string) (corev1.ServiceAccount, error) { + sa := corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: serviceAccountName, + Namespace: namespace, + Labels: multiClusterLabels(), + }, + } + + if imagePullSecrets != "" { + sa.ImagePullSecrets = []corev1.LocalObjectReference{ + {Name: "image-registries-secret"}, + } + } + + _, err := c.CoreV1().ServiceAccounts(sa.Namespace).Create(ctx, &sa, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + _, err = c.CoreV1().ServiceAccounts(sa.Namespace).Update(ctx, &sa, metav1.UpdateOptions{}) + } + if err != nil { + return corev1.ServiceAccount{}, xerrors.Errorf("error creating/updating service account: %w", err) + } + return sa, nil +} + +func createDatabaseRole(ctx context.Context, c KubeClient, roleName, namespace string) error { + role := rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + Namespace: namespace, + Labels: multiClusterLabels(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"patch", "delete", "get"}, + }, + }, + } + roleBinding := rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleName, + Namespace: namespace, + Labels: multiClusterLabels(), + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: roleName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: AppdbServiceAccount, + }, + }, + } + _, err := c.RbacV1().Roles(role.Namespace).Create(ctx, &role, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating role: %w", err) + } + + _, err = c.RbacV1().RoleBindings(roleBinding.Namespace).Create(ctx, &roleBinding, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating role binding: %w", err) + } + return nil +} + +// createDatabaseRoles creates the default ServiceAccounts, Roles and RoleBindings required for running database +// instances in a member cluster. +func createDatabaseRoles(ctx context.Context, client KubeClient, f Flags) error { + if _, err := createServiceAccount(ctx, client, AppdbServiceAccount, f.MemberClusterNamespace, f.ImagePullSecrets); err != nil { + return err + } + if _, err := createServiceAccount(ctx, client, DatabasePodsServiceAccount, f.MemberClusterNamespace, f.ImagePullSecrets); err != nil { + return err + } + if _, err := createServiceAccount(ctx, client, OpsManagerServiceAccount, f.MemberClusterNamespace, f.ImagePullSecrets); err != nil { + return err + } + if err := createDatabaseRole(ctx, client, AppdbRole, f.MemberClusterNamespace); err != nil { + return err + } + return nil +} + +// copyDatabaseRoles copies the ServiceAccounts, Roles and RoleBindings required for running database instances +// in a member cluster. This is used for adding new member clusters by copying over the configuration of a healthy +// source cluster. +func copyDatabaseRoles(ctx context.Context, src, dst KubeClient, namespace string) error { + appdbSA, err := src.CoreV1().ServiceAccounts(namespace).Get(ctx, AppdbServiceAccount, metav1.GetOptions{}) + if err != nil { + return xerrors.Errorf("failed retrieving service account %s from source cluster: %w", AppdbServiceAccount, err) + } + dbpodsSA, err := src.CoreV1().ServiceAccounts(namespace).Get(ctx, DatabasePodsServiceAccount, metav1.GetOptions{}) + if err != nil { + return xerrors.Errorf("failed retrieving service account %s from source cluster: %w", DatabasePodsServiceAccount, err) + } + opsManagerSA, err := src.CoreV1().ServiceAccounts(namespace).Get(ctx, OpsManagerServiceAccount, metav1.GetOptions{}) + if err != nil { + return xerrors.Errorf("failed retrieving service account %s from source cluster: %w", OpsManagerServiceAccount, err) + } + appdbR, err := src.RbacV1().Roles(namespace).Get(ctx, AppdbRole, metav1.GetOptions{}) + if err != nil { + return xerrors.Errorf("failed retrieving role %s from source cluster: %w", AppdbRole, err) + } + appdbRB, err := src.RbacV1().RoleBindings(namespace).Get(ctx, AppdbRoleBinding, metav1.GetOptions{}) + if err != nil { + return xerrors.Errorf("failed retrieving role binding %s from source cluster: %w", AppdbRoleBinding, err) + } + if len(appdbSA.ImagePullSecrets) > 0 { + if err := copySecret(ctx, src, dst, namespace, appdbSA.ImagePullSecrets[0].Name); err != nil { + fmt.Printf("failed creating image pull secret %s: %s\n", appdbSA.ImagePullSecrets[0].Name, err) + } + } + if len(dbpodsSA.ImagePullSecrets) > 0 { + if err := copySecret(ctx, src, dst, namespace, dbpodsSA.ImagePullSecrets[0].Name); err != nil { + fmt.Printf("failed creating image pull secret %s: %s\n", dbpodsSA.ImagePullSecrets[0].Name, err) + } + } + if len(opsManagerSA.ImagePullSecrets) > 0 { + if err := copySecret(ctx, src, dst, namespace, opsManagerSA.ImagePullSecrets[0].Name); err != nil { + fmt.Printf("failed creating image pull secret %s: %s\n", opsManagerSA.ImagePullSecrets[0].Name, err) + } + } + _, err = dst.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: appdbSA.Name, + Labels: appdbSA.Labels, + }, + ImagePullSecrets: appdbSA.DeepCopy().ImagePullSecrets, + }, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating service account: %w", err) + } + _, err = dst.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: dbpodsSA.Name, + Labels: dbpodsSA.Labels, + }, + ImagePullSecrets: dbpodsSA.DeepCopy().ImagePullSecrets, + }, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating service account: %w", err) + } + _, err = dst.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: opsManagerSA.Name, + Labels: opsManagerSA.Labels, + }, + ImagePullSecrets: opsManagerSA.DeepCopy().ImagePullSecrets, + }, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating service account: %w", err) + } + + _, err = dst.RbacV1().Roles(namespace).Create(ctx, &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: appdbR.Name, + Labels: appdbR.Labels, + }, + Rules: appdbR.DeepCopy().Rules, + }, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating role: %w", err) + } + _, err = dst.RbacV1().RoleBindings(namespace).Create(ctx, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: appdbRB.Name, + Labels: appdbRB.Labels, + }, + Subjects: appdbRB.DeepCopy().Subjects, + RoleRef: appdbRB.DeepCopy().RoleRef, + }, metav1.CreateOptions{}) + if !errors.IsAlreadyExists(err) && err != nil { + return xerrors.Errorf("error creating role binding: %w", err) + } + + return nil +} + +func installDatabaseRoles(ctx context.Context, clientSet map[string]KubeClient, f Flags) error { + for _, clusterName := range f.MemberClusters { + if err := createDatabaseRoles(ctx, clientSet[clusterName], f); err != nil { + return err + } + } + + return nil +} + +// setupDatabaseRoles installs the required database roles in the member clusters. +// The CommonFlags passed to the CLI must contain a healthy source member cluster which will be treated as +// the source of truth for all the member clusters. +func setupDatabaseRoles(ctx context.Context, clientSet map[string]KubeClient, f Flags) error { + for _, clusterName := range f.MemberClusters { + if clusterName != f.SourceCluster { + if err := copyDatabaseRoles(ctx, clientSet[f.SourceCluster], clientSet[clusterName], f.MemberClusterNamespace); err != nil { + return err + } + } + } + + return nil +} + +// ReplaceClusterMembersConfigMap creates the configmap used by the operator to know which clusters are members of the multi-cluster setup. +// This will replace the existing configmap. +// NOTE: the configmap is hardcoded to be DefaultOperatorConfigMapName +func ReplaceClusterMembersConfigMap(ctx context.Context, centralClusterClient KubeClient, flags Flags) error { + members := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultOperatorConfigMapName, + Namespace: flags.CentralClusterNamespace, + Labels: multiClusterLabels(), + }, + Data: map[string]string{}, + } + + addToSet(flags.MemberClusters, &members) + + fmt.Printf("Creating Member list Configmap %s/%s in cluster %s\n", flags.CentralClusterNamespace, DefaultOperatorConfigMapName, flags.CentralCluster) + _, err := centralClusterClient.CoreV1().ConfigMaps(flags.CentralClusterNamespace).Create(ctx, &members, metav1.CreateOptions{}) + + if err != nil && !errors.IsAlreadyExists(err) { + return xerrors.Errorf("failed creating secret: %w", err) + } + + if errors.IsAlreadyExists(err) { + if _, err := centralClusterClient.CoreV1().ConfigMaps(flags.CentralClusterNamespace).Update(ctx, &members, metav1.UpdateOptions{}); err != nil { + return xerrors.Errorf("error creating configmap: %w", err) + } + } + + return nil +} + +func addToSet(memberClusters []string, into *corev1.ConfigMap) { + // override or add + for _, memberCluster := range memberClusters { + into.Data[memberCluster] = "" + } +} diff --git a/public/tools/multicluster/pkg/common/common_test.go b/public/tools/multicluster/pkg/common/common_test.go new file mode 100644 index 000000000..4ef2611ec --- /dev/null +++ b/public/tools/multicluster/pkg/common/common_test.go @@ -0,0 +1,921 @@ +package common + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/clientcmd" +) + +const testKubeconfig = `apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: ZHNqaA== + server: https://api.member-cluster-0 + name: member-cluster-0 +- cluster: + certificate-authority-data: ZHNqaA== + server: https://api.member-cluster-1 + name: member-cluster-1 +- cluster: + certificate-authority-data: ZHNqaA== + server: https://api.member-cluster-2 + name: member-cluster-2 +contexts: +- context: + cluster: member-cluster-0 + namespace: citi + user: member-cluster-0 + name: member-cluster-0 +- context: + cluster: member-cluster-1 + namespace: citi + user: member-cluster-1 + name: member-cluster-1 +- context: + cluster: member-cluster-2 + namespace: citi + user: member-cluster-2 + name: member-cluster-2 +current-context: member-cluster-0 +kind: Config +preferences: {} +users: +- name: member-cluster-0 + user: + client-certificate-data: ZHNqaA== + client-key-data: ZHNqaA== +` + +func init() { + // we lower this to not make unit tests fast + PollingInterval = time.Millisecond + PollingTimeout = time.Second * 1 +} + +func testFlags(t *testing.T, cleanup bool) Flags { + memberClusters := []string{"member-cluster-0", "member-cluster-1", "member-cluster-2"} + kubeconfig, err := clientcmd.Load([]byte(testKubeconfig)) + assert.NoError(t, err) + + memberClusterApiServerUrls, err := GetMemberClusterApiServerUrls(kubeconfig, memberClusters) + assert.NoError(t, err) + + return Flags{ + MemberClusterApiServerUrls: memberClusterApiServerUrls, + MemberClusters: memberClusters, + ServiceAccount: "test-service-account", + CentralCluster: "central-cluster", + MemberClusterNamespace: "member-namespace", + CentralClusterNamespace: "central-namespace", + Cleanup: cleanup, + ClusterScoped: false, + CreateTelemetryClusterRoles: true, + OperatorName: "mongodb-enterprise-operator", + CreateServiceAccountSecrets: true, + } +} + +func TestNamespaces_GetsCreated_WhenTheyDoNotExit(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + assert.NoError(t, err) + + assertMemberClusterNamespacesExist(t, ctx, clientMap, flags) + assertCentralClusterNamespacesExist(t, ctx, clientMap, flags) +} + +func TestExistingNamespaces_DoNotCause_AlreadyExistsErrors(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags, namespaceResourceType) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + assert.NoError(t, err) + + assertMemberClusterNamespacesExist(t, ctx, clientMap, flags) + assertCentralClusterNamespacesExist(t, ctx, clientMap, flags) +} + +func TestServiceAccount_GetsCreate_WhenTheyDoNotExit(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertServiceAccountsExist(t, ctx, clientMap, flags) +} + +func TestExistingServiceAccounts_DoNotCause_AlreadyExistsErrors(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags, serviceAccountResourceType) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertServiceAccountsExist(t, ctx, clientMap, flags) +} + +func TestDatabaseRoles_GetCreated(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + flags.ClusterScoped = true + flags.InstallDatabaseRoles = true + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertDatabaseRolesExist(t, ctx, clientMap, flags) +} + +func TestRoles_GetsCreated_WhenTheyDoesNotExit(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertMemberRolesExist(t, ctx, clientMap, flags) +} + +func TestExistingRoles_DoNotCause_AlreadyExistsErrors(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags, roleResourceType) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertMemberRolesExist(t, ctx, clientMap, flags) +} + +func TestClusterRoles_DoNotGetCreated_WhenNotSpecified(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + flags.ClusterScoped = false + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertMemberRolesExist(t, ctx, clientMap, flags) + assertCentralRolesExist(t, ctx, clientMap, flags) +} + +func Test_TelemetryClusterRoles_GetCreated_WhenNotSpecified(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertClusterRoles(t, ctx, clientMap, flags, false, true, clusterTypeCentral) +} + +func TestClusterRoles_GetCreated_WhenSpecified(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + flags.ClusterScoped = true + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertMemberRolesDoNotExist(t, ctx, clientMap, flags) + assertMemberClusterRolesExist(t, ctx, clientMap, flags) +} + +func TestCentralCluster_GetsRegularRoleCreated_WhenClusterScoped_IsSpecified(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + flags.ClusterScoped = true + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + assert.NoError(t, err) +} + +func TestCentralCluster_GetsRegularRoleCreated_WhenNonClusterScoped_IsSpecified(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + flags.ClusterScoped = false + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + require.NoError(t, err) + assertCentralRolesExist(t, ctx, clientMap, flags) +} + +func TestPerformCleanup(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, true) + flags.ClusterScoped = true + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + t.Run("Resources get created with labels", func(t *testing.T) { + assertMemberClusterRolesExist(t, ctx, clientMap, flags) + assertMemberClusterNamespacesExist(t, ctx, clientMap, flags) + assertCentralClusterNamespacesExist(t, ctx, clientMap, flags) + assertServiceAccountsExist(t, ctx, clientMap, flags) + }) + + err = performCleanup(ctx, clientMap, flags) + require.NoError(t, err) + + t.Run("Resources with labels are removed", func(t *testing.T) { + assertMemberRolesDoNotExist(t, ctx, clientMap, flags) + assertMemberClusterRolesDoNotExist(t, ctx, clientMap, flags) + assertCentralRolesDoNotExist(t, ctx, clientMap, flags) + }) + + t.Run("Namespaces are preserved", func(t *testing.T) { + assertMemberClusterNamespacesExist(t, ctx, clientMap, flags) + assertCentralClusterNamespacesExist(t, ctx, clientMap, flags) + }) +} + +func TestCreateKubeConfig_IsComposedOf_ServiceAccountTokens_InAllClusters(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + + err := EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + kubeConfig, err := readKubeConfig(ctx, clientMap[flags.CentralCluster], flags.CentralClusterNamespace) + assert.NoError(t, err) + + assert.Equal(t, "Config", kubeConfig.Kind) + assert.Equal(t, "v1", kubeConfig.ApiVersion) + assert.Len(t, kubeConfig.Contexts, len(flags.MemberClusters)) + assert.Len(t, kubeConfig.Clusters, len(flags.MemberClusters)) + + for i, kubeConfigCluster := range kubeConfig.Clusters { + assert.Equal(t, flags.MemberClusters[i], kubeConfigCluster.Name, "Name of cluster should be set to the member clusters.") + expectedCaBytes, err := readSecretKey(ctx, clientMap[flags.MemberClusters[i]], fmt.Sprintf("%s-token-secret", flags.ServiceAccount), flags.CentralClusterNamespace, "ca.crt") + + assert.NoError(t, err) + assert.Contains(t, string(expectedCaBytes), flags.MemberClusters[i]) + assert.Equal(t, 0, bytes.Compare(expectedCaBytes, kubeConfigCluster.Cluster.CertificateAuthorityData), "CA should be read from Service Account token Secret.") + assert.Equal(t, fmt.Sprintf("https://api.%s", flags.MemberClusters[i]), kubeConfigCluster.Cluster.Server, "Server should be correctly configured based on cluster name.") + } + + for i, user := range kubeConfig.Users { + tokenBytes, err := readSecretKey(ctx, clientMap[flags.MemberClusters[i]], fmt.Sprintf("%s-token-secret", flags.ServiceAccount), flags.CentralClusterNamespace, "token") + assert.NoError(t, err) + assert.Equal(t, flags.MemberClusters[i], user.Name, "User name should be the name of the cluster.") + assert.Equal(t, string(tokenBytes), user.User.Token, "Token from the service account secret should be set.") + } +} + +func TestKubeConfigSecret_IsCreated_InCentralCluster(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + + err := EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + centralClusterClient := clientMap[flags.CentralCluster] + kubeConfigSecret, err := centralClusterClient.CoreV1().Secrets(flags.CentralClusterNamespace).Get(ctx, KubeConfigSecretName, metav1.GetOptions{}) + + assert.NoError(t, err) + assert.NotNil(t, kubeConfigSecret) +} + +func TestKubeConfigSecret_IsNotCreated_InMemberClusters(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + + err := EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + for _, memberCluster := range flags.MemberClusters { + memberClient := clientMap[memberCluster] + kubeConfigSecret, err := memberClient.CoreV1().Secrets(flags.CentralClusterNamespace).Get(ctx, KubeConfigSecretName, metav1.GetOptions{}) + assert.True(t, errors.IsNotFound(err)) + assert.Nil(t, kubeConfigSecret) + } +} + +func TestChangingOneServiceAccountToken_ChangesOnlyThatEntry_InKubeConfig(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + clientMap := getClientResources(ctx, flags) + + err := EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + kubeConfigBefore, err := readKubeConfig(ctx, clientMap[flags.CentralCluster], flags.CentralClusterNamespace) + assert.NoError(t, err) + + firstClusterClient := clientMap[flags.MemberClusters[0]] + + // simulate a service account token changing, re-running the script should leave the other clusters unchanged. + newServiceAccountToken := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-token-secret", flags.ServiceAccount), + Namespace: flags.CentralClusterNamespace, + }, + Data: map[string][]byte{ + "token": []byte("new-token-data"), + "ca.crt": []byte("new-ca-crt"), + }, + } + + _, err = firstClusterClient.CoreV1().Secrets(flags.CentralClusterNamespace).Update(ctx, &newServiceAccountToken, metav1.UpdateOptions{}) + assert.NoError(t, err) + + flags.CreateServiceAccountSecrets = false + err = EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + kubeConfigAfter, err := readKubeConfig(ctx, clientMap[flags.CentralCluster], flags.CentralClusterNamespace) + assert.NoError(t, err) + + assert.NotEqual(t, kubeConfigBefore.Users[0], kubeConfigAfter.Users[0], "Cluster 0 users should have been modified.") + assert.NotEqual(t, kubeConfigBefore.Clusters[0], kubeConfigAfter.Clusters[0], "Cluster 1 clusters should have been modified") + + assert.Equal(t, "new-token-data", kubeConfigAfter.Users[0].User.Token, "first user token should have been updated.") + assert.Equal(t, []byte("new-ca-crt"), kubeConfigAfter.Clusters[0].Cluster.CertificateAuthorityData, "CA for cluster 0 should have been updated.") + + assert.Equal(t, kubeConfigBefore.Users[1], kubeConfigAfter.Users[1], "Cluster 1 users should have remained unchanged") + assert.Equal(t, kubeConfigBefore.Clusters[1], kubeConfigAfter.Clusters[1], "Cluster 1 clusters should have remained unchanged") + + assert.Equal(t, kubeConfigBefore.Users[2], kubeConfigAfter.Users[2], "Cluster 2 users should have remained unchanged") + assert.Equal(t, kubeConfigBefore.Clusters[2], kubeConfigAfter.Clusters[2], "Cluster 2 clusters should have remained unchanged") +} + +func TestGetMemberClusterApiServerUrls(t *testing.T) { + t.Run("Test comma separated string returns correct values", func(t *testing.T) { + kubeconfig, err := clientcmd.Load([]byte(testKubeconfig)) + assert.NoError(t, err) + + apiUrls, err := GetMemberClusterApiServerUrls(kubeconfig, []string{"member-cluster-0", "member-cluster-1", "member-cluster-2"}) + assert.Nil(t, err) + assert.Len(t, apiUrls, 3) + assert.Equal(t, apiUrls[0], "https://api.member-cluster-0") + assert.Equal(t, apiUrls[1], "https://api.member-cluster-1") + assert.Equal(t, apiUrls[2], "https://api.member-cluster-2") + }) + + t.Run("Test missing cluster lookup returns error", func(t *testing.T) { + kubeconfig, err := clientcmd.Load([]byte(testKubeconfig)) + assert.NoError(t, err) + + _, err = GetMemberClusterApiServerUrls(kubeconfig, []string{"member-cluster-0", "member-cluster-1", "member-cluster-missing"}) + assert.Error(t, err) + }) +} + +func TestMemberClusterUris(t *testing.T) { + ctx := context.Background() + t.Run("Uses server values set in CommonFlags", func(t *testing.T) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + flags := testFlags(t, false) + flags.MemberClusterApiServerUrls = []string{"cluster1-url", "cluster2-url", "cluster3-url"} + clientMap := getClientResources(ctx, flags) + + err := EnsureMultiClusterResources(ctx, flags, clientMap) + require.NoError(t, err) + + kubeConfig, err := readKubeConfig(ctx, clientMap[flags.CentralCluster], flags.CentralClusterNamespace) + assert.NoError(t, err) + + for i, c := range kubeConfig.Clusters { + assert.Equal(t, flags.MemberClusterApiServerUrls[i], c.Cluster.Server) + } + + assert.NoError(t, err) + }) +} + +func TestReplaceClusterMembersConfigMap(t *testing.T) { + ctx := context.Background() + flags := testFlags(t, false) + + clientMap := getClientResources(ctx, flags) + client := clientMap[flags.CentralCluster] + + { + flags.MemberClusters = []string{"member-1", "member-2", "member-3", "member-4"} + err := ReplaceClusterMembersConfigMap(ctx, client, flags) + assert.NoError(t, err) + + cm, err := client.CoreV1().ConfigMaps(flags.CentralClusterNamespace).Get(ctx, DefaultOperatorConfigMapName, metav1.GetOptions{}) + assert.NoError(t, err) + + expected := map[string]string{} + for _, cluster := range flags.MemberClusters { + expected[cluster] = "" + } + assert.Equal(t, cm.Data, expected) + } + + { + flags.MemberClusters = []string{"member-1", "member-2"} + err := ReplaceClusterMembersConfigMap(ctx, client, flags) + cm, err := client.CoreV1().ConfigMaps(flags.CentralClusterNamespace).Get(ctx, DefaultOperatorConfigMapName, metav1.GetOptions{}) + assert.NoError(t, err) + + expected := map[string]string{} + for _, cluster := range flags.MemberClusters { + expected[cluster] = "" + } + + assert.Equal(t, cm.Data, expected) + } +} + +// TestPrintingOutRolesServiceAccountsAndRoleBindings is not an ordinary test. It updates the RBAC samples in the +// samples/multi-cluster-cli-gitops/resources/rbac directory. By default, this test is not executed. If you indent to run +// it, please set EXPORT_RBAC_SAMPLES variable to "true". +func TestPrintingOutRolesServiceAccountsAndRoleBindings(t *testing.T) { + ctx := context.Background() + if os.Getenv("EXPORT_RBAC_SAMPLES") != "true" { // nolint:forbidigo + t.Skip("Skipping as EXPORT_RBAC_SAMPLES is false") + } + + flags := testFlags(t, false) + flags.ClusterScoped = true + flags.InstallDatabaseRoles = true + + { + sb := &strings.Builder{} + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + cr, err := clientMap[flags.CentralCluster].RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + crb, err := clientMap[flags.CentralCluster].RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + sa, err := clientMap[flags.CentralCluster].CoreV1().ServiceAccounts(flags.CentralClusterNamespace).List(ctx, metav1.ListOptions{}) + + sb = marshalToYaml(t, sb, "Central Cluster, cluster-scoped resources", "rbac.authorization.k8s.io/v1", "ClusterRole", cr.Items) + sb = marshalToYaml(t, sb, "Central Cluster, cluster-scoped resources", "rbac.authorization.k8s.io/v1", "ClusterRoleBinding", crb.Items) + sb = marshalToYaml(t, sb, "Central Cluster, cluster-scoped resources", "v1", "ServiceAccount", sa.Items) + + _ = os.WriteFile("../../samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_central_cluster.yaml", []byte(sb.String()), os.ModePerm) + } + + { + sb := &strings.Builder{} + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + cr, err := clientMap[flags.MemberClusters[0]].RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + crb, err := clientMap[flags.MemberClusters[0]].RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + sa, err := clientMap[flags.MemberClusters[0]].CoreV1().ServiceAccounts(flags.MemberClusterNamespace).List(ctx, metav1.ListOptions{}) + + sb = marshalToYaml(t, sb, "Member Cluster, cluster-scoped resources", "rbac.authorization.k8s.io/v1", "ClusterRole", cr.Items) + sb = marshalToYaml(t, sb, "Member Cluster, cluster-scoped resources", "rbac.authorization.k8s.io/v1", "ClusterRoleBinding", crb.Items) + sb = marshalToYaml(t, sb, "Member Cluster, cluster-scoped resources", "v1", "ServiceAccount", sa.Items) + + _ = os.WriteFile("../../samples/multi-cluster-cli-gitops/resources/rbac/cluster_scoped_member_cluster.yaml", []byte(sb.String()), os.ModePerm) + } + + { + sb := &strings.Builder{} + flags.ClusterScoped = false + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + r, err := clientMap[flags.CentralCluster].RbacV1().Roles(flags.CentralClusterNamespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + rb, err := clientMap[flags.CentralCluster].RbacV1().RoleBindings(flags.CentralClusterNamespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + sa, err := clientMap[flags.CentralCluster].CoreV1().ServiceAccounts(flags.CentralClusterNamespace).List(ctx, metav1.ListOptions{}) + + sb = marshalToYaml(t, sb, "Central Cluster, namespace-scoped resources", "rbac.authorization.k8s.io/v1", "Role", r.Items) + sb = marshalToYaml(t, sb, "Central Cluster, namespace-scoped resources", "rbac.authorization.k8s.io/v1", "RoleBinding", rb.Items) + sb = marshalToYaml(t, sb, "Central Cluster, namespace-scoped resources", "v1", "ServiceAccount", sa.Items) + + _ = os.WriteFile("../../samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_central_cluster.yaml", []byte(sb.String()), os.ModePerm) + } + + { + sb := &strings.Builder{} + flags.ClusterScoped = false + + clientMap := getClientResources(ctx, flags) + err := EnsureMultiClusterResources(ctx, flags, clientMap) + + r, err := clientMap[flags.MemberClusters[0]].RbacV1().Roles(flags.MemberClusterNamespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + rb, err := clientMap[flags.MemberClusters[0]].RbacV1().RoleBindings(flags.MemberClusterNamespace).List(ctx, metav1.ListOptions{}) + assert.NoError(t, err) + sa, err := clientMap[flags.MemberClusters[0]].CoreV1().ServiceAccounts(flags.MemberClusterNamespace).List(ctx, metav1.ListOptions{}) + + sb = marshalToYaml(t, sb, "Member Cluster, namespace-scoped resources", "rbac.authorization.k8s.io/v1", "Role", r.Items) + sb = marshalToYaml(t, sb, "Member Cluster, namespace-scoped resources", "rbac.authorization.k8s.io/v1", "RoleBinding", rb.Items) + sb = marshalToYaml(t, sb, "Member Cluster, namespace-scoped resources", "v1", "ServiceAccount", sa.Items) + + _ = os.WriteFile("../../samples/multi-cluster-cli-gitops/resources/rbac/namespace_scoped_member_cluster.yaml", []byte(sb.String()), os.ModePerm) + } +} + +func marshalToYaml[T interface{}](t *testing.T, sb *strings.Builder, comment string, apiVersion string, kind string, items []T) *strings.Builder { + sb.WriteString(fmt.Sprintf("# %s\n", comment)) + for _, cr := range items { + sb.WriteString(fmt.Sprintf("apiVersion: %s\n", apiVersion)) + sb.WriteString(fmt.Sprintf("kind: %s\n", kind)) + marshalledBytes, err := yaml.Marshal(cr) + assert.NoError(t, err) + sb.WriteString(string(marshalledBytes)) + sb.WriteString("\n---\n") + } + return sb +} + +func TestConvertToSet(t *testing.T) { + type args struct { + memberClusters []string + cm *corev1.ConfigMap + } + tests := []struct { + name string + args args + expected map[string]string + }{ + { + name: "new members", + args: args{ + memberClusters: []string{"kind-1", "kind-2", "kind-3"}, + cm: &corev1.ConfigMap{Data: map[string]string{}}, + }, + expected: map[string]string{"kind-1": "", "kind-2": "", "kind-3": ""}, + }, + { + name: "one override and one new", + args: args{ + memberClusters: []string{"kind-1", "kind-2", "kind-3"}, + cm: &corev1.ConfigMap{Data: map[string]string{"kind-1": "", "kind-0": ""}}, + }, + expected: map[string]string{"kind-1": "", "kind-2": "", "kind-3": "", "kind-0": ""}, + }, + { + name: "one new ones", + args: args{ + memberClusters: []string{}, + cm: &corev1.ConfigMap{Data: map[string]string{"kind-1": "", "kind-0": ""}}, + }, + expected: map[string]string{"kind-1": "", "kind-0": ""}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + addToSet(tt.args.memberClusters, tt.args.cm) + assert.Equal(t, tt.expected, tt.args.cm.Data) + }) + } +} + +// assertMemberClusterNamespacesExist asserts the Namespace in the member clusters exists. +func assertMemberClusterNamespacesExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + for _, clusterName := range flags.MemberClusters { + client := clientMap[clusterName] + ns, err := client.CoreV1().Namespaces().Get(ctx, flags.MemberClusterNamespace, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, ns) + assert.Equal(t, flags.MemberClusterNamespace, ns.Name) + assert.Equal(t, ns.Labels, multiClusterLabels()) + } +} + +// assertCentralClusterNamespacesExist asserts the Namespace in the central cluster exists. +func assertCentralClusterNamespacesExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + client := clientMap[flags.CentralCluster] + ns, err := client.CoreV1().Namespaces().Get(ctx, flags.CentralClusterNamespace, metav1.GetOptions{}) + require.NoError(t, err) + assert.NotNil(t, ns) + assert.Equal(t, flags.CentralClusterNamespace, ns.Name) + assert.Equal(t, ns.Labels, multiClusterLabels()) +} + +// assertServiceAccountsAreCorrect asserts the ServiceAccounts are created as expected. +func assertServiceAccountsExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + for _, clusterName := range flags.MemberClusters { + client := clientMap[clusterName] + sa, err := client.CoreV1().ServiceAccounts(flags.CentralClusterNamespace).Get(ctx, flags.ServiceAccount, metav1.GetOptions{}) + require.NoError(t, err) + assert.NotNil(t, sa) + assert.Equal(t, flags.ServiceAccount, sa.Name) + assert.Equal(t, sa.Labels, multiClusterLabels()) + } + + client := clientMap[flags.CentralCluster] + sa, err := client.CoreV1().ServiceAccounts(flags.CentralClusterNamespace).Get(ctx, flags.ServiceAccount, metav1.GetOptions{}) + require.NoError(t, err) + assert.NotNil(t, sa) + assert.Equal(t, flags.ServiceAccount, sa.Name) + assert.Equal(t, sa.Labels, multiClusterLabels()) +} + +// assertDatabaseRolesExist asserts the DatabaseRoles are created as expected. +func assertDatabaseRolesExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + for _, clusterName := range flags.MemberClusters { + client := clientMap[clusterName] + + // appDB service account + sa, err := client.CoreV1().ServiceAccounts(flags.MemberClusterNamespace).Get(ctx, AppdbServiceAccount, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, sa) + assert.Equal(t, sa.Labels, multiClusterLabels()) + + // database pods service account + sa, err = client.CoreV1().ServiceAccounts(flags.MemberClusterNamespace).Get(ctx, DatabasePodsServiceAccount, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, sa) + assert.Equal(t, sa.Labels, multiClusterLabels()) + + // ops manager service account + sa, err = client.CoreV1().ServiceAccounts(flags.MemberClusterNamespace).Get(ctx, OpsManagerServiceAccount, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, sa) + assert.Equal(t, sa.Labels, multiClusterLabels()) + + // appdb role + r, err := client.RbacV1().Roles(flags.MemberClusterNamespace).Get(ctx, AppdbRole, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, r) + assert.Equal(t, r.Labels, multiClusterLabels()) + assert.Equal(t, []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"patch", "delete", "get"}, + }, + }, r.Rules) + + // appdb rolebinding + rb, err := client.RbacV1().RoleBindings(flags.MemberClusterNamespace).Get(ctx, AppdbRoleBinding, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, r) + assert.Equal(t, rb.Labels, multiClusterLabels()) + assert.Equal(t, []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: AppdbServiceAccount, + }, + }, rb.Subjects) + assert.Equal(t, rbacv1.RoleRef{ + Kind: "Role", + Name: AppdbRole, + }, rb.RoleRef) + } +} + +// assertMemberClusterRolesExist should be used when member cluster cluster roles should exist. +func assertMemberClusterRolesExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + assertClusterRoles(t, ctx, clientMap, flags, true, true, clusterTypeMember) +} + +// assertMemberClusterRolesDoNotExist should be used when member cluster cluster roles should not exist. +func assertMemberClusterRolesDoNotExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + assertClusterRoles(t, ctx, clientMap, flags, false, false, clusterTypeCentral) +} + +// assertClusterRoles should be used to assert the existence of member-cluster cluster roles. The boolean +// shouldExist should be true for roles existing, and false for cluster roles not existing. +// telemetryShouldExist should be true for roles existing, and false for cluster roles not existing. +func assertClusterRoles(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags, clusterScopeShouldExist bool, telemetryShouldExist bool, clusterType clusterType) { + var expectedClusterRole rbacv1.ClusterRole + if clusterType == clusterTypeCentral { + expectedClusterRole = buildCentralEntityClusterRole() + } else { + expectedClusterRole = buildMemberEntityClusterRole() + } + assertClusterRoleMembers(t, ctx, clientMap, flags, clusterScopeShouldExist, expectedClusterRole) + assertClusterRoleCentral(t, ctx, clientMap, flags, clusterScopeShouldExist, expectedClusterRole) + + expectedClusterRoleTelemetry := buildClusterRoleTelemetry() + assertClusterRoleMembers(t, ctx, clientMap, flags, telemetryShouldExist, expectedClusterRoleTelemetry) + assertClusterRoleCentral(t, ctx, clientMap, flags, telemetryShouldExist, expectedClusterRoleTelemetry) +} + +func assertClusterRoleCentral(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags, shouldExist bool, expectedClusterRole rbacv1.ClusterRole) { + clusterRole, err := clientMap[flags.CentralCluster].RbacV1().ClusterRoles().Get(ctx, expectedClusterRole.Name, metav1.GetOptions{}) + if shouldExist { + assert.Nil(t, err) + assert.NotNil(t, clusterRole) + } else { + assert.Error(t, err) + } +} + +func assertClusterRoleMembers(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags, shouldExist bool, expectedClusterRole rbacv1.ClusterRole) { + for _, clusterName := range flags.MemberClusters { + client := clientMap[clusterName] + role, err := client.RbacV1().ClusterRoles().Get(ctx, expectedClusterRole.Name, metav1.GetOptions{}) + if shouldExist { + assert.NoError(t, err) + assert.NotNil(t, role) + assert.Equal(t, expectedClusterRole, *role) + } else { + assert.Error(t, err) + assert.Nil(t, role) + } + } +} + +// assertMemberRolesExist should be used when member cluster roles should exist. +func assertMemberRolesExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + assertMemberRolesAreCorrect(t, ctx, clientMap, flags, true) +} + +// assertMemberRolesDoNotExist should be used when member cluster roles should not exist. +func assertMemberRolesDoNotExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + assertMemberRolesAreCorrect(t, ctx, clientMap, flags, false) +} + +// assertMemberRolesAreCorrect should be used to assert the existence of member cluster roles. The boolean +// shouldExist should be true for roles existing, and false for roles not existing. +func assertMemberRolesAreCorrect(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags, shouldExist bool) { + expectedRole := buildMemberEntityRole(flags.MemberClusterNamespace) + + for _, clusterName := range flags.MemberClusters { + client := clientMap[clusterName] + role, err := client.RbacV1().Roles(flags.MemberClusterNamespace).Get(ctx, expectedRole.Name, metav1.GetOptions{}) + if shouldExist { + assert.NoError(t, err) + assert.NotNil(t, role) + assert.Equal(t, expectedRole, *role) + } else { + assert.Error(t, err) + assert.Nil(t, role) + } + } +} + +// assertCentralRolesExist should be used when central cluster roles should exist. +func assertCentralRolesExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + assertCentralRolesAreCorrect(t, ctx, clientMap, flags, true) +} + +// assertCentralRolesDoNotExist should be used when central cluster roles should not exist. +func assertCentralRolesDoNotExist(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags) { + assertCentralRolesAreCorrect(t, ctx, clientMap, flags, false) +} + +// assertCentralRolesAreCorrect should be used to assert the existence of central cluster roles. The boolean +// shouldExist should be true for roles existing, and false for roles not existing. +func assertCentralRolesAreCorrect(t *testing.T, ctx context.Context, clientMap map[string]KubeClient, flags Flags, shouldExist bool) { + client := clientMap[flags.CentralCluster] + + // should never have a cluster role + clusterRole := buildCentralEntityClusterRole() + cr, err := client.RbacV1().ClusterRoles().Get(ctx, clusterRole.Name, metav1.GetOptions{}) + + assert.True(t, errors.IsNotFound(err)) + assert.Nil(t, cr) + + expectedRole := buildCentralEntityRole(flags.CentralClusterNamespace) + role, err := client.RbacV1().Roles(flags.CentralClusterNamespace).Get(ctx, expectedRole.Name, metav1.GetOptions{}) + + if shouldExist { + assert.NoError(t, err, "should always create a role for central cluster") + assert.NotNil(t, role) + assert.Equal(t, expectedRole, *role) + } else { + assert.Error(t, err) + assert.Nil(t, role) + } +} + +// resourceType indicates a type of resource that is created during the tests. +type resourceType string + +var ( + serviceAccountResourceType resourceType = "ServiceAccount" + namespaceResourceType resourceType = "Namespace" + roleBindingResourceType resourceType = "RoleBinding" + roleResourceType resourceType = "Role" +) + +// getClientResources returns a map of cluster name to fake.Clientset +func getClientResources(ctx context.Context, flags Flags, resourceTypes ...resourceType) map[string]KubeClient { + clientMap := make(map[string]KubeClient) + + for _, clusterName := range flags.MemberClusters { + if clusterName == flags.CentralCluster { + continue + } + clientMap[clusterName] = NewKubeClientContainer(nil, newFakeClientset(ctx, clusterName, nil), nil) + } + clientMap[flags.CentralCluster] = NewKubeClientContainer(nil, newFakeClientset(ctx, flags.CentralCluster, nil), nil) + + return clientMap +} + +func newFakeClientset(ctx context.Context, clusterName string, resources []runtime.Object) *fake.Clientset { + clientset := fake.NewSimpleClientset(resources...) + informerFactory := informers.NewSharedInformerFactory(clientset, time.Second) + secretInformer := informerFactory.Core().V1().Secrets().Informer() + _, err := secretInformer.AddEventHandler(&cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + s := obj.(*corev1.Secret).DeepCopy() + // simulate populating the service account secret token data into the secret + // it's done automatically by k8s + onSecretCreate(s, clusterName, clientset, ctx) + }, + }) + + informerFactory.Start(ctx.Done()) + informerFactory.WaitForCacheSync(ctx.Done()) + + if err != nil { + panic(fmt.Errorf("%w", err)) + } + + return clientset +} + +func onSecretCreate(s *corev1.Secret, clusterName string, clientset *fake.Clientset, ctx context.Context) { + // simulate populating the service account secret token data into the secret + // it's done automatically by k8s + if s.Type == corev1.SecretTypeServiceAccountToken { + // random delay to ensure the code is polling for the data set by k8s + time.Sleep(time.Millisecond * time.Duration(1+rand.Intn(5))) + if s.Data == nil { + s.Data = map[string][]byte{} + } + s.Data["ca.crt"] = []byte(fmt.Sprintf("ca.crt: %s", clusterName)) + s.Data["token"] = []byte(fmt.Sprintf("token: %s", clusterName)) + if _, err := clientset.CoreV1().Secrets(s.Namespace).Update(ctx, s, metav1.UpdateOptions{}); err != nil { + panic(err) + } + } +} + +// containsResourceType returns true if r is in resourceTypes, otherwise false. +func containsResourceType(resourceTypes []resourceType, r resourceType) bool { + for _, rt := range resourceTypes { + if rt == r { + return true + } + } + return false +} + +// readSecretKey reads a key from a Secret in the given namespace with the given name. +func readSecretKey(ctx context.Context, client KubeClient, secretName, namespace, key string) ([]byte, error) { + tokenSecret, err := client.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return tokenSecret.Data[key], nil +} + +// readKubeConfig reads the KubeConfig file from the secret in the given cluster and namespace. +func readKubeConfig(ctx context.Context, client KubeClient, namespace string) (KubeConfigFile, error) { + kubeConfigSecret, err := client.CoreV1().Secrets(namespace).Get(ctx, KubeConfigSecretName, metav1.GetOptions{}) + if err != nil { + return KubeConfigFile{}, err + } + + kubeConfigBytes := kubeConfigSecret.Data[KubeConfigSecretKey] + result := KubeConfigFile{} + if err := yaml.Unmarshal(kubeConfigBytes, &result); err != nil { + return KubeConfigFile{}, err + } + + return result, nil +} diff --git a/public/tools/multicluster/pkg/common/kubeclientcontainer.go b/public/tools/multicluster/pkg/common/kubeclientcontainer.go new file mode 100644 index 000000000..4547ac399 --- /dev/null +++ b/public/tools/multicluster/pkg/common/kubeclientcontainer.go @@ -0,0 +1,303 @@ +package common + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + admissionregistrationv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1" + admissionregistrationv1alpha1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1" + admissionregistrationv1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" + apiserverinternalv1alpha1 "k8s.io/client-go/kubernetes/typed/apiserverinternal/v1alpha1" + appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" + appsv1beta1 "k8s.io/client-go/kubernetes/typed/apps/v1beta1" + appsv1beta2 "k8s.io/client-go/kubernetes/typed/apps/v1beta2" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authenticationv1alpha1 "k8s.io/client-go/kubernetes/typed/authentication/v1alpha1" + authenticationv1beta1 "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + authorizationv1beta1 "k8s.io/client-go/kubernetes/typed/authorization/v1beta1" + autoscalingv1 "k8s.io/client-go/kubernetes/typed/autoscaling/v1" + autoscalingv2 "k8s.io/client-go/kubernetes/typed/autoscaling/v2" + autoscalingv2beta1 "k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1" + autoscalingv2beta2 "k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + batchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" + certificatesv1 "k8s.io/client-go/kubernetes/typed/certificates/v1" + certificatesv1alpha1 "k8s.io/client-go/kubernetes/typed/certificates/v1alpha1" + certificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" + coordinationv1beta1 "k8s.io/client-go/kubernetes/typed/coordination/v1beta1" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + discoveryv1 "k8s.io/client-go/kubernetes/typed/discovery/v1" + discoveryv1beta1 "k8s.io/client-go/kubernetes/typed/discovery/v1beta1" + eventsv1 "k8s.io/client-go/kubernetes/typed/events/v1" + eventsv1beta1 "k8s.io/client-go/kubernetes/typed/events/v1beta1" + extensionsv1beta1 "k8s.io/client-go/kubernetes/typed/extensions/v1beta1" + v1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1" + flowcontrolv1beta1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1" + flowcontrolv1beta2 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2" + flowcontrolv1beta3 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3" + networkingv1 "k8s.io/client-go/kubernetes/typed/networking/v1" + networkingv1alpha1 "k8s.io/client-go/kubernetes/typed/networking/v1alpha1" + networkingv1beta1 "k8s.io/client-go/kubernetes/typed/networking/v1beta1" + nodev1 "k8s.io/client-go/kubernetes/typed/node/v1" + nodev1alpha1 "k8s.io/client-go/kubernetes/typed/node/v1alpha1" + nodev1beta1 "k8s.io/client-go/kubernetes/typed/node/v1beta1" + policyv1 "k8s.io/client-go/kubernetes/typed/policy/v1" + policyv1beta1 "k8s.io/client-go/kubernetes/typed/policy/v1beta1" + rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" + rbacv1alpha1 "k8s.io/client-go/kubernetes/typed/rbac/v1alpha1" + rbacv1beta1 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" + resourcev1alpha2 "k8s.io/client-go/kubernetes/typed/resource/v1alpha2" + schedulingv1 "k8s.io/client-go/kubernetes/typed/scheduling/v1" + schedulingv1alpha1 "k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1" + schedulingv1beta1 "k8s.io/client-go/kubernetes/typed/scheduling/v1beta1" + storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1" + storagev1alpha1 "k8s.io/client-go/kubernetes/typed/storage/v1alpha1" + storagev1beta1 "k8s.io/client-go/kubernetes/typed/storage/v1beta1" + "k8s.io/client-go/rest" +) + +// KubeClient is wrapper (decorator pattern) over the static and dynamic Kube Clients. +// It provides capabilities of both interfaces along with access to the initial REST configuration. +type KubeClient interface { + kubernetes.Interface + dynamic.Interface + GetRestConfig() *rest.Config +} + +var _ KubeClient = &KubeClientContainer{} + +type KubeClientContainer struct { + staticClient kubernetes.Interface + dynamicClient dynamic.Interface + restConfig *rest.Config +} + +func (k *KubeClientContainer) FlowcontrolV1() v1.FlowcontrolV1Interface { + panic("implement me") +} + +func (k *KubeClientContainer) CertificatesV1alpha1() certificatesv1alpha1.CertificatesV1alpha1Interface { + // TODO implement me + panic("implement me") +} + +func (k *KubeClientContainer) ResourceV1alpha2() resourcev1alpha2.ResourceV1alpha2Interface { + // TODO implement me + panic("implement me") +} + +func (k *KubeClientContainer) AdmissionregistrationV1alpha1() admissionregistrationv1alpha1.AdmissionregistrationV1alpha1Interface { + return k.staticClient.AdmissionregistrationV1alpha1() +} + +func (k *KubeClientContainer) AuthenticationV1alpha1() authenticationv1alpha1.AuthenticationV1alpha1Interface { + return k.staticClient.AuthenticationV1alpha1() +} + +func (k *KubeClientContainer) FlowcontrolV1beta3() flowcontrolv1beta3.FlowcontrolV1beta3Interface { + return k.staticClient.FlowcontrolV1beta3() +} + +func (k *KubeClientContainer) NetworkingV1alpha1() networkingv1alpha1.NetworkingV1alpha1Interface { + return k.staticClient.NetworkingV1alpha1() +} + +func (k *KubeClientContainer) Discovery() discovery.DiscoveryInterface { + return k.staticClient.Discovery() +} + +func (k *KubeClientContainer) AdmissionregistrationV1() admissionregistrationv1.AdmissionregistrationV1Interface { + return k.staticClient.AdmissionregistrationV1() +} + +func (k *KubeClientContainer) AdmissionregistrationV1beta1() admissionregistrationv1beta1.AdmissionregistrationV1beta1Interface { + return k.staticClient.AdmissionregistrationV1beta1() +} + +func (k *KubeClientContainer) InternalV1alpha1() apiserverinternalv1alpha1.InternalV1alpha1Interface { + return k.staticClient.InternalV1alpha1() +} + +func (k *KubeClientContainer) AppsV1() appsv1.AppsV1Interface { + return k.staticClient.AppsV1() +} + +func (k *KubeClientContainer) AppsV1beta1() appsv1beta1.AppsV1beta1Interface { + return k.staticClient.AppsV1beta1() +} + +func (k *KubeClientContainer) AppsV1beta2() appsv1beta2.AppsV1beta2Interface { + return k.staticClient.AppsV1beta2() +} + +func (k *KubeClientContainer) AuthenticationV1() authenticationv1.AuthenticationV1Interface { + return k.staticClient.AuthenticationV1() +} + +func (k *KubeClientContainer) AuthenticationV1beta1() authenticationv1beta1.AuthenticationV1beta1Interface { + return k.staticClient.AuthenticationV1beta1() +} + +func (k *KubeClientContainer) AuthorizationV1() authorizationv1.AuthorizationV1Interface { + return k.staticClient.AuthorizationV1() +} + +func (k *KubeClientContainer) AuthorizationV1beta1() authorizationv1beta1.AuthorizationV1beta1Interface { + return k.staticClient.AuthorizationV1beta1() +} + +func (k *KubeClientContainer) AutoscalingV1() autoscalingv1.AutoscalingV1Interface { + return k.staticClient.AutoscalingV1() +} + +func (k *KubeClientContainer) AutoscalingV2() autoscalingv2.AutoscalingV2Interface { + return k.staticClient.AutoscalingV2() +} + +func (k *KubeClientContainer) AutoscalingV2beta1() autoscalingv2beta1.AutoscalingV2beta1Interface { + return k.staticClient.AutoscalingV2beta1() +} + +func (k *KubeClientContainer) AutoscalingV2beta2() autoscalingv2beta2.AutoscalingV2beta2Interface { + return k.staticClient.AutoscalingV2beta2() +} + +func (k *KubeClientContainer) BatchV1() batchv1.BatchV1Interface { + return k.staticClient.BatchV1() +} + +func (k *KubeClientContainer) BatchV1beta1() batchv1beta1.BatchV1beta1Interface { + // TODO implement me + panic("implement me") +} + +func (k *KubeClientContainer) CertificatesV1() certificatesv1.CertificatesV1Interface { + return k.staticClient.CertificatesV1() +} + +func (k *KubeClientContainer) CertificatesV1beta1() certificatesv1beta1.CertificatesV1beta1Interface { + return k.staticClient.CertificatesV1beta1() +} + +func (k *KubeClientContainer) CoordinationV1beta1() coordinationv1beta1.CoordinationV1beta1Interface { + return k.staticClient.CoordinationV1beta1() +} + +func (k *KubeClientContainer) CoordinationV1() coordinationv1.CoordinationV1Interface { + return k.staticClient.CoordinationV1() +} + +func (k *KubeClientContainer) CoreV1() corev1.CoreV1Interface { + return k.staticClient.CoreV1() +} + +func (k *KubeClientContainer) DiscoveryV1() discoveryv1.DiscoveryV1Interface { + return k.staticClient.DiscoveryV1() +} + +func (k *KubeClientContainer) DiscoveryV1beta1() discoveryv1beta1.DiscoveryV1beta1Interface { + return k.staticClient.DiscoveryV1beta1() +} + +func (k KubeClientContainer) EventsV1() eventsv1.EventsV1Interface { + return k.staticClient.EventsV1() +} + +func (k *KubeClientContainer) EventsV1beta1() eventsv1beta1.EventsV1beta1Interface { + return k.staticClient.EventsV1beta1() +} + +func (k *KubeClientContainer) ExtensionsV1beta1() extensionsv1beta1.ExtensionsV1beta1Interface { + return k.staticClient.ExtensionsV1beta1() +} + +func (k *KubeClientContainer) FlowcontrolV1beta1() flowcontrolv1beta1.FlowcontrolV1beta1Interface { + return k.staticClient.FlowcontrolV1beta1() +} + +func (k *KubeClientContainer) FlowcontrolV1beta2() flowcontrolv1beta2.FlowcontrolV1beta2Interface { + return k.staticClient.FlowcontrolV1beta2() +} + +func (k *KubeClientContainer) NetworkingV1() networkingv1.NetworkingV1Interface { + return k.staticClient.NetworkingV1() +} + +func (k *KubeClientContainer) NetworkingV1beta1() networkingv1beta1.NetworkingV1beta1Interface { + return k.staticClient.NetworkingV1beta1() +} + +func (k *KubeClientContainer) NodeV1() nodev1.NodeV1Interface { + return k.staticClient.NodeV1() +} + +func (k *KubeClientContainer) NodeV1alpha1() nodev1alpha1.NodeV1alpha1Interface { + return k.staticClient.NodeV1alpha1() +} + +func (k *KubeClientContainer) NodeV1beta1() nodev1beta1.NodeV1beta1Interface { + return k.staticClient.NodeV1beta1() +} + +func (k *KubeClientContainer) PolicyV1() policyv1.PolicyV1Interface { + return k.staticClient.PolicyV1() +} + +func (k *KubeClientContainer) PolicyV1beta1() policyv1beta1.PolicyV1beta1Interface { + return k.staticClient.PolicyV1beta1() +} + +func (k *KubeClientContainer) RbacV1() rbacv1.RbacV1Interface { + return k.staticClient.RbacV1() +} + +func (k *KubeClientContainer) RbacV1beta1() rbacv1beta1.RbacV1beta1Interface { + return k.staticClient.RbacV1beta1() +} + +func (k *KubeClientContainer) RbacV1alpha1() rbacv1alpha1.RbacV1alpha1Interface { + return k.staticClient.RbacV1alpha1() +} + +func (k *KubeClientContainer) SchedulingV1alpha1() schedulingv1alpha1.SchedulingV1alpha1Interface { + return k.staticClient.SchedulingV1alpha1() +} + +func (k *KubeClientContainer) SchedulingV1beta1() schedulingv1beta1.SchedulingV1beta1Interface { + return k.staticClient.SchedulingV1beta1() +} + +func (k *KubeClientContainer) SchedulingV1() schedulingv1.SchedulingV1Interface { + return k.staticClient.SchedulingV1() +} + +func (k *KubeClientContainer) StorageV1beta1() storagev1beta1.StorageV1beta1Interface { + return k.staticClient.StorageV1beta1() +} + +func (k *KubeClientContainer) StorageV1() storagev1.StorageV1Interface { + return k.staticClient.StorageV1() +} + +func (k *KubeClientContainer) StorageV1alpha1() storagev1alpha1.StorageV1alpha1Interface { + return k.staticClient.StorageV1alpha1() +} + +func (k *KubeClientContainer) Resource(resource schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { + return k.dynamicClient.Resource(resource) +} + +func (k *KubeClientContainer) GetRestConfig() *rest.Config { + return k.restConfig +} + +func NewKubeClientContainer(restConfig *rest.Config, staticClient kubernetes.Interface, dynamicClient dynamic.Interface) *KubeClientContainer { + return &KubeClientContainer{ + staticClient: staticClient, + dynamicClient: dynamicClient, + restConfig: restConfig, + } +} diff --git a/public/tools/multicluster/pkg/common/kubeconfig.go b/public/tools/multicluster/pkg/common/kubeconfig.go new file mode 100644 index 000000000..a353b6e0c --- /dev/null +++ b/public/tools/multicluster/pkg/common/kubeconfig.go @@ -0,0 +1,83 @@ +package common + +import ( + "os" + "path/filepath" + + "golang.org/x/xerrors" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/homedir" +) + +const ( + kubeConfigEnv = "KUBECONFIG" +) + +// LoadKubeConfigFilePath returns the path of the local KubeConfig file. +func LoadKubeConfigFilePath() string { + env := os.Getenv(kubeConfigEnv) // nolint:forbidigo + if env != "" { + return env + } + return filepath.Join(homedir.HomeDir(), ".kube", "config") +} + +// GetMemberClusterApiServerUrls returns the slice of member cluster api urls that should be used. +func GetMemberClusterApiServerUrls(kubeconfig *clientcmdapi.Config, clusterNames []string) ([]string, error) { + var urls []string + for _, name := range clusterNames { + if cluster := kubeconfig.Clusters[name]; cluster != nil { + urls = append(urls, cluster.Server) + } else { + return nil, xerrors.Errorf("cluster '%s' not found in kubeconfig", name) + } + } + return urls, nil +} + +// CreateClientMap crates a map of all MultiClusterClient for every member cluster, and the operator cluster. +func CreateClientMap(memberClusters []string, operatorCluster, kubeConfigPath string, getClient func(clusterName string, kubeConfigPath string) (KubeClient, error)) (map[string]KubeClient, error) { + clientMap := map[string]KubeClient{} + for _, c := range memberClusters { + clientset, err := getClient(c, kubeConfigPath) + if err != nil { + return nil, xerrors.Errorf("failed to create clientset map: %w", err) + } + clientMap[c] = clientset + } + + clientset, err := getClient(operatorCluster, kubeConfigPath) + if err != nil { + return nil, xerrors.Errorf("failed to create clientset map: %w", err) + } + clientMap[operatorCluster] = clientset + return clientMap, nil +} + +// GetKubernetesClient returns a kubernetes.Clientset using the given context from the +// specified KubeConfig filepath. +func GetKubernetesClient(context, kubeConfigPath string) (KubeClient, error) { + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeConfigPath}, + &clientcmd.ConfigOverrides{ + CurrentContext: context, + }).ClientConfig() + if err != nil { + return nil, xerrors.Errorf("failed to create client config: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, xerrors.Errorf("failed to create kubernetes clientset: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, xerrors.Errorf("failed to create dynamic kubernetes clientset: %w", err) + } + + return NewKubeClientContainer(config, clientset, dynamicClient), nil +} diff --git a/public/tools/multicluster/pkg/common/utils.go b/public/tools/multicluster/pkg/common/utils.go new file mode 100644 index 000000000..77980e187 --- /dev/null +++ b/public/tools/multicluster/pkg/common/utils.go @@ -0,0 +1,21 @@ +package common + +// Contains checks if a string is present in the provided slice. +func Contains(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + return false +} + +// AnyAreEmpty returns true if any of the given strings have the zero value. +func AnyAreEmpty(values ...string) bool { + for _, v := range values { + if v == "" { + return true + } + } + return false +} diff --git a/public/tools/multicluster/pkg/debug/anonymize.go b/public/tools/multicluster/pkg/debug/anonymize.go new file mode 100644 index 000000000..f1f2005bd --- /dev/null +++ b/public/tools/multicluster/pkg/debug/anonymize.go @@ -0,0 +1,30 @@ +package debug + +import v1 "k8s.io/api/core/v1" + +const ( + MASKED_TEXT = "***MASKED***" +) + +type Anonymizer interface { + AnonymizeSecret(secret *v1.Secret) *v1.Secret +} + +var _ Anonymizer = &NoOpAnonymizer{} + +type NoOpAnonymizer struct{} + +func (n *NoOpAnonymizer) AnonymizeSecret(secret *v1.Secret) *v1.Secret { + return secret +} + +var _ Anonymizer = &SensitiveDataAnonymizer{} + +type SensitiveDataAnonymizer struct{} + +func (n *SensitiveDataAnonymizer) AnonymizeSecret(secret *v1.Secret) *v1.Secret { + for key := range secret.Data { + secret.Data[key] = []byte(MASKED_TEXT) + } + return secret +} diff --git a/public/tools/multicluster/pkg/debug/anonymize_test.go b/public/tools/multicluster/pkg/debug/anonymize_test.go new file mode 100644 index 000000000..c4b8e78ed --- /dev/null +++ b/public/tools/multicluster/pkg/debug/anonymize_test.go @@ -0,0 +1,40 @@ +package debug + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" +) + +func TestNoOpAnonymizer_AnonymizeSecret(t *testing.T) { + // given + text := "test" + anonymizer := NoOpAnonymizer{} + + // when + result := anonymizer.AnonymizeSecret(&v1.Secret{ + Data: map[string][]byte{ + text: []byte(text), + }, + }) + + // then + assert.Equal(t, text, string(result.Data[text])) +} + +func TestSensitiveDataAnonymizer_AnonymizeSecret(t *testing.T) { + // given + text := "test" + anonymizer := SensitiveDataAnonymizer{} + + // when + result := anonymizer.AnonymizeSecret(&v1.Secret{ + Data: map[string][]byte{ + text: []byte(text), + }, + }) + + // then + assert.Equal(t, MASKED_TEXT, string(result.Data[text])) +} diff --git a/public/tools/multicluster/pkg/debug/collectors.go b/public/tools/multicluster/pkg/debug/collectors.go new file mode 100644 index 000000000..8428bb8b5 --- /dev/null +++ b/public/tools/multicluster/pkg/debug/collectors.go @@ -0,0 +1,377 @@ +package debug + +import ( + "bufio" + "bytes" + "context" + "fmt" + "strings" + + "k8s.io/utils/ptr" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + "github.com/10gen/ops-manager-kubernetes/multi/pkg/common" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + // TODO: Report a bug on inconsistent naming (plural vs singular). + MongoDBCommunityGVR = schema.GroupVersionResource{Group: "mongodbcommunity.mongodb.com", Version: "v1", Resource: "mongodbcommunity"} + MongoDBGVR = schema.GroupVersionResource{Group: "mongodb.com", Version: "v1", Resource: "mongodb"} + MongoDBMultiClusterGVR = schema.GroupVersionResource{Group: "mongodb.com", Version: "v1", Resource: "mongodbmulticlusters"} + MongoDBUsersGVR = schema.GroupVersionResource{Group: "mongodb.com", Version: "v1", Resource: "mongodbusers"} + OpsManagerSchemeGVR = schema.GroupVersionResource{Group: "mongodb.com", Version: "v1", Resource: "opsmanagers"} +) + +const ( + redColor = "\033[31m" + resetColor = "\033[0m" +) + +type Filter interface { + Accept(object runtime.Object) bool +} + +var _ Filter = &AcceptAllFilter{} + +type AcceptAllFilter struct{} + +func (a *AcceptAllFilter) Accept(_ runtime.Object) bool { + return true +} + +var _ Filter = &WithOwningReference{} + +type WithOwningReference struct{} + +func (a *WithOwningReference) Accept(object runtime.Object) bool { + typeAccessor, err := meta.Accessor(object) + if err != nil { + return true + } + + for _, or := range typeAccessor.GetOwnerReferences() { + if strings.Contains(strings.ToLower(or.Kind), "mongo") { + return true + } + } + return false +} + +type RawFile struct { + Name string + ContainerName string + content []byte +} + +type Collector interface { + Collect(context.Context, common.KubeClient, string, Filter, Anonymizer) ([]runtime.Object, []RawFile, error) +} + +var _ Collector = &StatefulSetCollector{} + +type StatefulSetCollector struct{} + +func (s *StatefulSetCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, _ Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.AppsV1().StatefulSets(namespace).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &ConfigMapCollector{} + +type ConfigMapCollector struct{} + +func (s *ConfigMapCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, _ Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.CoreV1().ConfigMaps(namespace).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &SecretCollector{} + +type SecretCollector struct{} + +func (s *SecretCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + var ret []runtime.Object + secrets, err := kubeClient.CoreV1().Secrets(namespace).List(ctx, v1.ListOptions{}) + if err != nil { + return nil, nil, err + } + for i := range secrets.Items { + item := secrets.Items[i] + if filter.Accept(&item) { + ret = append(ret, anonymizer.AnonymizeSecret(&item)) + } + } + return ret, nil, nil +} + +var _ Collector = &ServiceAccountCollector{} + +type ServiceAccountCollector struct{} + +func (s *ServiceAccountCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.CoreV1().ServiceAccounts(namespace).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &RolesCollector{} + +type RolesCollector struct{} + +func (s *RolesCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.RbacV1().Roles(namespace).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &RolesBindingsCollector{} + +type RolesBindingsCollector struct{} + +func (s *RolesBindingsCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.RbacV1().RoleBindings(namespace).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &MongoDBCollector{} + +type MongoDBCollector struct{} + +func (s *MongoDBCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.Resource(MongoDBGVR).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &MongoDBMultiClusterCollector{} + +type MongoDBMultiClusterCollector struct{} + +func (s *MongoDBMultiClusterCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.Resource(MongoDBMultiClusterGVR).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &MongoDBUserCollector{} + +type MongoDBUserCollector struct{} + +func (s *MongoDBUserCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.Resource(MongoDBUsersGVR).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &OpsManagerCollector{} + +type OpsManagerCollector struct{} + +func (s *OpsManagerCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.Resource(OpsManagerSchemeGVR).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &MongoDBCommunityCollector{} + +type MongoDBCommunityCollector struct{} + +func (s *MongoDBCommunityCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.Resource(MongoDBCommunityGVR).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &EventsCollector{} + +type EventsCollector struct{} + +func (s *EventsCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + return genericCollect(ctx, kubeClient, namespace, filter, func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) { + return kubeClient.EventsV1().Events(namespace).List(ctx, v1.ListOptions{}) + }) +} + +var _ Collector = &LogsCollector{} + +type LogsCollector struct{} + +func (s *LogsCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{}) + if err != nil { + return nil, nil, err + } + var logsToCollect []RawFile + for podIdx := range pods.Items { + for containerIdx := range pods.Items[podIdx].Spec.Containers { + logsToCollect = append(logsToCollect, RawFile{ + Name: pods.Items[podIdx].Name, + ContainerName: pods.Items[podIdx].Spec.Containers[containerIdx].Name, + }) + } + } + for i := range logsToCollect { + podName := logsToCollect[i].Name + PodLogsConnection := kubeClient.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{ + Follow: false, + TailLines: ptr.To(int64(100)), + Container: logsToCollect[i].ContainerName, + }) + LogStream, err := PodLogsConnection.Stream(ctx) + if err != nil { + fmt.Printf(redColor+"[%T] error from %s/%s, ignoring: %s\n"+resetColor, s, namespace, podName, err) + continue + } + reader := bufio.NewScanner(LogStream) + var line string + for reader.Scan() { + line = fmt.Sprintf("%s\n", reader.Text()) + bytes := []byte(line) + logsToCollect[i].content = append(logsToCollect[i].content, bytes...) + } + LogStream.Close() + } + return nil, logsToCollect, nil +} + +var _ Collector = &AgentHealthFileCollector{} + +type AgentHealthFileCollector struct{} + +func (s *AgentHealthFileCollector) Collect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, anonymizer Anonymizer) ([]runtime.Object, []RawFile, error) { + type AgentHealthFileToCollect struct { + podName string + RawFile rest.ContentConfig + agentFileName string + containerName string + } + + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, v1.ListOptions{}) + if err != nil { + return nil, nil, err + } + var logsToCollect []AgentHealthFileToCollect + var collectedHealthFiles []RawFile + for i, pod := range pods.Items { + add := AgentHealthFileToCollect{ + podName: pods.Items[i].Name, + } + found := false + for _, c := range pod.Spec.Containers { + for _, e := range c.Env { + if "AGENT_STATUS_FILEPATH" == e.Name { + add.agentFileName = e.Value + found = true + break + } + } + if found { + add.containerName = c.Name + break + } + } + + if found { + logsToCollect = append(logsToCollect, add) + } + } + for _, l := range logsToCollect { + add := RawFile{ + Name: l.podName + "-agent-health", + } + content, err := getFileContent(kubeClient.GetRestConfig(), kubeClient, namespace, l.podName, l.containerName, l.agentFileName) + if err == nil { + add.content = content + collectedHealthFiles = append(collectedHealthFiles, add) + } + } + return nil, collectedHealthFiles, nil +} + +// Inspired by https://gist.github.com/kyroy/8453a0c4e075e91809db9749e0adcff2 +func getFileContent(config *rest.Config, clientset common.KubeClient, namespace, podName, containerName, path string) ([]byte, error) { + u := clientset.CoreV1().RESTClient().Post(). + Namespace(namespace). + Name(podName). + Resource("pods"). + SubResource("exec"). + Param("command", "/bin/cat"). + Param("command", path). + Param("container", containerName). + Param("stderr", "true"). + Param("stdout", "true").URL() + + buf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + exec, err := remotecommand.NewSPDYExecutor(config, "POST", u) + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: buf, + Stderr: errBuf, + }) + if err != nil { + return nil, fmt.Errorf("%w Failed obtaining file %s from %v/%v", err, path, namespace, podName) + } + + return buf.Bytes(), nil +} + +type genericLister func(ctx context.Context, kubeClient common.KubeClient, namespace string) (runtime.Object, error) + +func genericCollect(ctx context.Context, kubeClient common.KubeClient, namespace string, filter Filter, lister genericLister) ([]runtime.Object, []RawFile, error) { + var ret []runtime.Object + listAsObject, err := lister(ctx, kubeClient, namespace) + if err != nil { + return nil, nil, err + } + list, err := meta.ExtractList(listAsObject) + if err != nil { + return nil, nil, err + } + for i := range list { + item := list[i] + if filter.Accept(item) { + ret = append(ret, item) + } + } + return ret, nil, nil +} + +type CollectionResult struct { + kubeResources []runtime.Object + rawObjects []RawFile + errors []error + namespace string + context string +} + +func Collect(ctx context.Context, kubeClient common.KubeClient, context string, namespace string, filter Filter, collectors []Collector, anonymizer Anonymizer) CollectionResult { + result := CollectionResult{} + result.context = context + result.namespace = namespace + + for _, collector := range collectors { + collectedKubeObjects, collectedRawObjects, err := collector.Collect(ctx, kubeClient, namespace, filter, anonymizer) + errorString := "" + if err != nil { + errorString = fmt.Sprintf(redColor+" error: %s"+resetColor, err) + } + fmt.Printf("[%T] collected %d kubeObjects, %d rawObjects%s\n", collector, len(collectedKubeObjects), len(collectedRawObjects), errorString) + result.kubeResources = append(result.kubeResources, collectedKubeObjects...) + result.rawObjects = append(result.rawObjects, collectedRawObjects...) + if err != nil { + result.errors = append(result.errors, err) + } + } + return result +} diff --git a/public/tools/multicluster/pkg/debug/collectors_test.go b/public/tools/multicluster/pkg/debug/collectors_test.go new file mode 100644 index 000000000..d7df86c1f --- /dev/null +++ b/public/tools/multicluster/pkg/debug/collectors_test.go @@ -0,0 +1,173 @@ +package debug + +import ( + "context" + "testing" + + "github.com/10gen/ops-manager-kubernetes/multi/pkg/common" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/apps/v1" + v12 "k8s.io/api/core/v1" + v13 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + fake2 "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" +) + +func TestCollectors(t *testing.T) { + ctx := context.Background() + // given + collectors := []Collector{ + &MongoDBCommunityCollector{}, + &MongoDBCollector{}, + &MongoDBMultiClusterCollector{}, + &MongoDBUserCollector{}, + &OpsManagerCollector{}, + &StatefulSetCollector{}, + &SecretCollector{}, + &ConfigMapCollector{}, + &RolesCollector{}, + &ServiceAccountCollector{}, + &RolesBindingsCollector{}, + &ServiceAccountCollector{}, + } + filter := &AcceptAllFilter{} + anonymizer := &NoOpAnonymizer{} + namespace := "test" + testObjectNames := "test" + + kubeClient := kubeClientWithTestingResources(ctx, namespace, testObjectNames) + + // when + for _, collector := range collectors { + kubeObjects, rawObjects, err := collector.Collect(ctx, kubeClient, namespace, filter, anonymizer) + + // then + assert.NoError(t, err) + assert.Equal(t, 1, len(kubeObjects)) + assert.Equal(t, 0, len(rawObjects)) + } +} + +func kubeClientWithTestingResources(ctx context.Context, namespace, testObjectNames string) *common.KubeClientContainer { + resources := []runtime.Object{ + &v12.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + &v1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + &v12.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + &v12.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + &v13.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + &v13.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + &v12.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: testObjectNames, + Namespace: namespace, + }, + }, + } + + // Unfortunately most of the Kind and Resource parts are guessing and making fake.NewSimpleDynamicClientWithCustomListKinds + // happy. Sadly, it uses naming conventions (with List suffix) and tries to guess the plural names - mostly incorrectly. + + scheme := runtime.NewScheme() + MongoDBCommunityGVK := schema.GroupVersionKind{ + Group: MongoDBCommunityGVR.Group, + Version: MongoDBCommunityGVR.Version, + Kind: "MongoDBCommunity", + } + MongoDBGVK := schema.GroupVersionKind{ + Group: MongoDBGVR.Group, + Version: MongoDBGVR.Version, + Kind: "MongoDB", + } + MongoDBUserGVK := schema.GroupVersionKind{ + Group: MongoDBGVR.Group, + Version: MongoDBGVR.Version, + Kind: "MongoDBUser", + } + MongoDBMultiGVK := schema.GroupVersionKind{ + Group: MongoDBGVR.Group, + Version: MongoDBGVR.Version, + Kind: "MongoDBMulti", + } + OpsManagerGVK := schema.GroupVersionKind{ + Group: OpsManagerSchemeGVR.Group, + Version: OpsManagerSchemeGVR.Version, + Kind: "OpsManager", + } + + scheme.AddKnownTypeWithName(MongoDBCommunityGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(MongoDBGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(MongoDBMultiGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(MongoDBUserGVK, &unstructured.Unstructured{}) + + MongoDBCommunityResource := unstructured.Unstructured{} + MongoDBCommunityResource.SetGroupVersionKind(MongoDBCommunityGVK) + MongoDBCommunityResource.SetName(testObjectNames) + + MongoDBResource := unstructured.Unstructured{} + MongoDBResource.SetGroupVersionKind(MongoDBGVK) + MongoDBResource.SetName(testObjectNames) + + MongoDBUserResource := unstructured.Unstructured{} + MongoDBUserResource.SetGroupVersionKind(MongoDBUserGVK) + MongoDBUserResource.SetName(testObjectNames) + + MongoDBMultiClusterResource := unstructured.Unstructured{} + MongoDBMultiClusterResource.SetGroupVersionKind(MongoDBMultiGVK) + MongoDBMultiClusterResource.SetName(testObjectNames) + + OpsManagerResource := unstructured.Unstructured{} + OpsManagerResource.SetGroupVersionKind(OpsManagerGVK) + OpsManagerResource.SetName(testObjectNames) + + dynamicLists := map[schema.GroupVersionResource]string{ + MongoDBCommunityGVR: "MongoDBCommunityList", + MongoDBGVR: "MongoDBList", + MongoDBUsersGVR: "MongoDBUserList", + MongoDBMultiClusterGVR: "MongoDBMultiClusterList", + OpsManagerSchemeGVR: "OpsManagerList", + } + dynamicFake := fake2.NewSimpleDynamicClientWithCustomListKinds(scheme, dynamicLists) + + dynamicFake.Resource(MongoDBMultiClusterGVR).Create(ctx, &MongoDBMultiClusterResource, metav1.CreateOptions{}) + dynamicFake.Resource(MongoDBCommunityGVR).Create(ctx, &MongoDBCommunityResource, metav1.CreateOptions{}) + dynamicFake.Resource(MongoDBGVR).Create(ctx, &MongoDBResource, metav1.CreateOptions{}) + dynamicFake.Resource(MongoDBUsersGVR).Create(ctx, &MongoDBUserResource, metav1.CreateOptions{}) + dynamicFake.Resource(OpsManagerSchemeGVR).Create(ctx, &OpsManagerResource, metav1.CreateOptions{}) + + kubeClient := common.NewKubeClientContainer(nil, fake.NewSimpleClientset(resources...), dynamicFake) + return kubeClient +} diff --git a/public/tools/multicluster/pkg/debug/writer.go b/public/tools/multicluster/pkg/debug/writer.go new file mode 100644 index 000000000..7471bfcd6 --- /dev/null +++ b/public/tools/multicluster/pkg/debug/writer.go @@ -0,0 +1,133 @@ +package debug + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ghodss/yaml" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + DefaultWritePath = ".mongodb/debug" +) + +func WriteToFile(path string, collectionResults ...CollectionResult) (string, string, error) { + err := os.MkdirAll(path, os.ModePerm) + if err != nil { + return "", "", err + } + for _, collectionResult := range collectionResults { + for _, obj := range collectionResult.kubeResources { + data, err := yaml.Marshal(obj) + if err != nil { + return "", "", err + } + meta, err := meta.Accessor(obj) + if err != nil { + return "", "", err + } + kubeType, err := getType(obj) + if err != nil { + return "", "", err + } + fileName := fmt.Sprintf("%s/%s-%s-%s-%s.yaml", path, cleanContext(collectionResult.context), collectionResult.namespace, kubeType, meta.GetName()) + err = os.WriteFile(fileName, data, os.ModePerm) + if err != nil { + return "", "", err + } + } + for _, obj := range collectionResult.rawObjects { + fileName := fmt.Sprintf("%s/%s-%s-%s-%s-%s.txt", path, cleanContext(collectionResult.context), collectionResult.namespace, "txt", obj.ContainerName, obj.Name) + err = os.WriteFile(fileName, obj.content, os.ModePerm) + if err != nil { + return "", "", err + } + } + } + compressedFile, err := compressDirectory(path) + if err != nil { + return "", "", err + } + return path, compressedFile, err +} + +// Inspired by https://stackoverflow.com/questions/37869793/how-do-i-zip-a-directory-containing-sub-directories-or-files-in-golang/63233911#63233911 +func compressDirectory(path string) (string, error) { + fileName := path + ".zip" + file, err := os.Create(fileName) + if err != nil { + return "", err + } + defer file.Close() + + w := zip.NewWriter(file) + defer w.Close() + + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + // Ensure that `path` is not absolute; it should not start with "/". + // This snippet happens to work because I don't use + // absolute paths, but ensure your real-world code + // transforms path into a zip-root relative path. + f, err := w.Create(path) + if err != nil { + return err + } + + _, err = io.Copy(f, file) + if err != nil { + return err + } + + return nil + } + err = filepath.Walk(path, walker) + if err != nil { + return "", err + } + return fileName, nil +} + +func DebugDirectory() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + time, err := time.Now().UTC().MarshalText() + if err != nil { + return "", err + } + return fmt.Sprintf("%s/%s/%s", home, DefaultWritePath, time), nil +} + +// This is a workaround for https://github.com/kubernetes/kubernetes/pull/63972 +func getType(obj runtime.Object) (string, error) { + v, err := conversion.EnforcePtr(obj) + if err != nil { + return "", err + } + return v.Type().String(), nil +} + +func cleanContext(context string) string { + return strings.Replace(context, "/", "-", -1) +} diff --git a/public/tools/multicluster/pkg/debug/writer_test.go b/public/tools/multicluster/pkg/debug/writer_test.go new file mode 100644 index 000000000..47217c364 --- /dev/null +++ b/public/tools/multicluster/pkg/debug/writer_test.go @@ -0,0 +1,93 @@ +package debug + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestWriteToFile(t *testing.T) { + // setup + uniqueTempDir, err := os.MkdirTemp(os.TempDir(), "*-TestWriteToFile") + assert.NoError(t, err) + defer os.RemoveAll(uniqueTempDir) + + // given + testNamespace := "testNamespace" + testContext := "testContext" + testError := fmt.Errorf("test") + testSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: testNamespace, + }, + Data: map[string][]byte{ + "test": []byte("test"), + }, + } + testFile := RawFile{ + Name: "testFile", + content: []byte("test"), + ContainerName: "testContainer", + } + collectionResult := CollectionResult{ + kubeResources: []runtime.Object{testSecret}, + rawObjects: []RawFile{testFile}, + errors: []error{testError}, + namespace: testNamespace, + context: testContext, + } + outputFiles := []string{"testContext-testNamespace-txt-testContainer-testFile.txt", "testContext-testNamespace-v1.Secret-test-secret.yaml"} + + // when + path, compressedFile, err := WriteToFile(uniqueTempDir, collectionResult) + defer os.RemoveAll(path) // This is fine as in case of an empty path, this does nothing + defer os.RemoveAll(compressedFile) + + // then + assert.NoError(t, err) + assert.NotNil(t, path) + assert.NotNil(t, compressedFile) + + files, err := os.ReadDir(uniqueTempDir) + assert.NoError(t, err) + assert.Equal(t, len(outputFiles), len(files)) + for _, outputFile := range outputFiles { + found := false + for _, file := range files { + if strings.Contains(file.Name(), outputFile) { + found = true + break + } + } + assert.Truef(t, found, "File %s not found", outputFile) + } + _, err = os.Stat(compressedFile) + assert.NoError(t, err) +} + +func TestCleanContext(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: "kind-cluster-1", + expected: "kind-cluster-1", + }, + { + input: "api-project-openshiftapps-com:6443/admin-random-v1", + expected: "api-project-openshiftapps-com:6443-admin-random-v1", + }, + } + + for _, tc := range tests { + assert.Equal(t, tc.expected, cleanContext(tc.input)) + } +} diff --git a/public/tools/multicluster/sign.sh b/public/tools/multicluster/sign.sh new file mode 100755 index 000000000..ee787720d --- /dev/null +++ b/public/tools/multicluster/sign.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Sign a binary using garasign credentials +# goreleaser takes care of calling this script as a hook. + +ARTIFACT=$1 +SIGNATURE="${ARTIFACT}.sig" + +TMPDIR=${TMPDIR:-/tmp} +SIGNING_ENVFILE="${TMPDIR}/signing-envfile" + +GRS_USERNAME=${GRS_USERNAME} +GRS_PASSWORD=${GRS_PASSWORD} +PKCS11_URI=${PKCS11_URI} +ARTIFACTORY_URL=${ARTIFACTORY_URL} +SIGNING_IMAGE_URI=${SIGNING_IMAGE_URI} +ARTIFACTORY_PASSWORD=${ARTIFACTORY_PASSWORD} +ARTIFACTORY_USERNAME=${ARTIFACTORY_USERNAME} + +echo "Signing artifact ${ARTIFACT} and saving signature to ${SIGNATURE}" + +{ + echo "GRS_CONFIG_USER1_USERNAME=${GRS_USERNAME}"; + echo "GRS_CONFIG_USER1_PASSWORD=${GRS_PASSWORD}"; + echo "PKCS11_URI=${PKCS11_URI}"; +} > "${SIGNING_ENVFILE}" + +echo "Logging in artifactory.corp" +echo ${ARTIFACTORY_PASSWORD} | docker login --password-stdin --username ${ARTIFACTORY_USERNAME} ${ARTIFACTORY_URL} + +echo "Signing artifact" +echo "Envfile is ${SIGNING_ENVFILE}" +docker run \ + --env-file="${SIGNING_ENVFILE}" \ + --rm \ + -v $(pwd):$(pwd) \ + -w $(pwd) \ + ${SIGNING_IMAGE_URI} \ + cosign sign-blob --key "${PKCS11_URI}" --output-signature ${SIGNATURE} ${ARTIFACT} --yes diff --git a/public/tools/multicluster/verify.sh b/public/tools/multicluster/verify.sh new file mode 100755 index 000000000..f65752c1f --- /dev/null +++ b/public/tools/multicluster/verify.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Verify the signature of a binary with the operator's public key +# goreleaser takes care of calling this script as a hook. + +ARTIFACT=$1 +SIGNATURE="${ARTIFACT}.sig" + +HOSTED_SIGN_PUBKEY="https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem" # to complete +TMPDIR=${TMPDIR:-/tmp} +KEY_FILE="${TMPDIR}/host-public.key" +SIGNING_IMAGE_URI=${SIGNING_IMAGE_URI} + +curl -o ${KEY_FILE} "${HOSTED_SIGN_PUBKEY}" +echo "Verifying signature ${SIGNATURE} of artifact ${ARTIFACT}" +echo "Keyfile is ${KEY_FILE}" + +# When working locally, the following command can be used instead of Docker +# cosign verify-blob --key ${KEY_FILE} --signature ${SIGNATURE} ${ARTIFACT} + +docker run \ + --rm \ + -v $(pwd):$(pwd) \ + -v ${KEY_FILE}:${KEY_FILE} \ + -w $(pwd) \ + ${SIGNING_IMAGE_URI} \ + cosign verify-blob --key ${KEY_FILE} --signature ${SIGNATURE} ${ARTIFACT} + +# Without below line, Evergreen fails at archiving with "open dist/kubectl-[...]/kubectl-mongodb.sig: permission denied +sudo chmod 666 ${SIGNATURE} diff --git a/public/vault_policies/appdb-policy.hcl b/public/vault_policies/appdb-policy.hcl new file mode 100644 index 000000000..de4c056ae --- /dev/null +++ b/public/vault_policies/appdb-policy.hcl @@ -0,0 +1,6 @@ +path "secret/data/mongodbenterprise/appdb/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/mongodbenterprise/appdb/*" { + capabilities = ["list"] +} diff --git a/public/vault_policies/database-policy.hcl b/public/vault_policies/database-policy.hcl new file mode 100644 index 000000000..1ab8798bd --- /dev/null +++ b/public/vault_policies/database-policy.hcl @@ -0,0 +1,6 @@ +path "secret/data/mongodbenterprise/database/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/mongodbenterprise/database/*" { + capabilities = ["list"] +} diff --git a/public/vault_policies/operator-policy.hcl b/public/vault_policies/operator-policy.hcl new file mode 100644 index 000000000..ed5eb39f9 --- /dev/null +++ b/public/vault_policies/operator-policy.hcl @@ -0,0 +1,6 @@ +path "secret/data/mongodbenterprise/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} +path "secret/metadata/mongodbenterprise/*" { + capabilities = ["list", "read"] +} diff --git a/public/vault_policies/opsmanager-policy.hcl b/public/vault_policies/opsmanager-policy.hcl new file mode 100644 index 000000000..9a181dfda --- /dev/null +++ b/public/vault_policies/opsmanager-policy.hcl @@ -0,0 +1,6 @@ +path "secret/data/mongodbenterprise/opsmanager/*" { + capabilities = ["read", "list"] +} +path "secret/metadata/mongodbenterprise/opsmanager/*" { + capabilities = ["list"] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..b7d3a4329 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.black] +line-length = 120 +target-version = ['py39'] +include = '\.pyi?$' + +[tool.isort] +profile = "black" diff --git a/release.json b/release.json index 88983b5b3..433cf60c9 100644 --- a/release.json +++ b/release.json @@ -1,8 +1,344 @@ { - "golang-builder-image": "golang:1.24", - "operator": "0.12.0", - "version-upgrade-hook": "1.0.9", - "readiness-probe": "1.0.22", - "agent": "108.0.2.8729-1", - "agent-tools-version": "100.10.0" + "mongodbToolsBundle": { + "ubi": "mongodb-database-tools-rhel88-x86_64-100.11.0.tgz" + }, + "mongodbOperator": "1.32.0", + "initDatabaseVersion": "1.32.0", + "initOpsManagerVersion": "1.32.0", + "initAppDbVersion": "1.32.0", + "databaseImageVersion": "1.32.0", + "agentVersion": "108.0.2.8729-1", + "openshift": { + "minimumSupportedVersion": "4.6" + }, + "supportedImages": { + "mongodb-kubernetes-readinessprobe": { + "ssdlc_name": "MongoDB Community Operator Readiness Probe", + "versions": [ + "1.0.22" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-kubernetes-operator-version-upgrade-post-start-hook": { + "ssdlc_name": "MongoDB Community Operator Version Upgrade Hook", + "versions": [ + "1.0.9" + ], + "variants": [ + "ubi" + ] + }, + "ops-manager": { + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator Ops Manager", + "versions": [ + "6.0.25", + "6.0.26", + "6.0.27", + "7.0.13", + "7.0.14", + "7.0.15", + "8.0.4", + "8.0.5", + "8.0.6" + ], + "variants": [ + "ubi" + ] + }, + "operator": { + "Description": "We support 3 last versions, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator", + "versions": [ + "1.24.0", + "1.23.0", + "1.22.0", + "1.21.0", + "1.20.1", + "1.20.0", + "1.19.1", + "1.19.0", + "1.18.0", + "1.17.2", + "1.17.1", + "1.17.0", + "1.25.0", + "1.26.0", + "1.27.0", + "1.28.0", + "1.29.0", + "1.30.0", + "1.31.0", + "1.32.0" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-kubernetes-operator": { + "Description": "Community Operator daily rebuilds", + "ssdlc_name": "MongoDB Community Operator", + "versions": [ + "0.12.0", + "0.11.0", + "0.10.0", + "0.9.0", + "0.8.3", + "0.8.2", + "0.8.1", + "0.8.0", + "0.7.9", + "0.7.8", + "0.7.7", + "0.7.6" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-agent": { + "Description": "Agents corresponding to OpsManager 5.x and 6.x series", + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator MongoDB Agent", + "Description for specific versions": { + "11.0.5.6963-1": "An upgraded version for OM 5.0 we use for Operator-only deployments", + "12.0.28.7763-1": "OM 6 basic version" + }, + "versions": [ + "108.0.2.8729-1" + ], + "opsManagerMapping": { + "Description": "These are the agents from which we start supporting static containers.", + "cloud_manager": "13.32.0.9397-1", + "cloud_manager_tools": "100.11.0", + "ops_manager": { + "6.0.25": { + "agent_version": "12.0.33.7866-1", + "tools_version": "100.10.0" + }, + "6.0.26": { + "agent_version": "12.0.34.7888-1", + "tools_version": "100.10.0" + }, + "6.0.27": { + "agent_version": "12.0.35.7911-1", + "tools_version": "100.10.0" + }, + "7.0.13": { + "agent_version": "107.0.13.8702-1", + "tools_version": "100.10.0" + }, + "7.0.14": { + "agent_version": "107.0.13.8702-1", + "tools_version": "100.10.0" + }, + "7.0.15": { + "agent_version": "107.0.15.8741-1", + "tools_version": "100.11.0" + }, + "8.0.4": { + "agent_version": "108.0.4.8770-1", + "tools_version": "100.10.0" + }, + "8.0.5": { + "agent_version": "108.0.4.8770-1", + "tools_version": "100.11.0" + }, + "8.0.6": { + "agent_version": "108.0.6.8796-1", + "tools_version": "100.11.0" + } + } + }, + "variants": [ + "ubi" + ] + }, + "init-ops-manager": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator Init Ops Manager", + "versions": [ + "1.24.0", + "1.23.0", + "1.0.7", + "1.0.8", + "1.0.9", + "1.0.10", + "1.0.11", + "1.0.12", + "1.25.0", + "1.26.0", + "1.27.0", + "1.28.0", + "1.29.0", + "1.30.0", + "1.31.0", + "1.32.0" + ], + "variants": [ + "ubi" + ] + }, + "init-database": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator Init Database", + "versions": [ + "1.24.0", + "1.23.0", + "1.0.9", + "1.0.10", + "1.0.11", + "1.0.12", + "1.0.13", + "1.0.14", + "1.0.15", + "1.0.16", + "1.0.17", + "1.0.18", + "1.0.19", + "1.25.0", + "1.26.0", + "1.27.0", + "1.28.0", + "1.29.0", + "1.30.0", + "1.31.0", + "1.32.0" + ], + "variants": [ + "ubi" + ] + }, + "init-appdb": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator Init AppDB", + "versions": [ + "1.24.0", + "1.23.0", + "1.0.9", + "1.0.10", + "1.0.11", + "1.0.12", + "1.0.13", + "1.0.14", + "1.0.15", + "1.0.16", + "1.0.17", + "1.0.18", + "1.25.0", + "1.26.0", + "1.27.0", + "1.28.0", + "1.29.0", + "1.30.0", + "1.31.0", + "1.32.0" + ], + "variants": [ + "ubi" + ] + }, + "database": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Kubernetes Operator Database", + "versions": [ + "1.24.0", + "1.23.0", + "2.0.2", + "1.25.0", + "1.26.0", + "1.27.0", + "1.28.0", + "1.29.0", + "1.30.0", + "1.31.0", + "1.32.0" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-enterprise-server": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Server", + "versions": [ + "4.4.0-ubi8", + "4.4.1-ubi8", + "4.4.2-ubi8", + "4.4.3-ubi8", + "4.4.4-ubi8", + "4.4.5-ubi8", + "4.4.6-ubi8", + "4.4.7-ubi8", + "4.4.8-ubi8", + "4.4.9-ubi8", + "4.4.10-ubi8", + "4.4.11-ubi8", + "4.4.12-ubi8", + "4.4.13-ubi8", + "4.4.14-ubi8", + "4.4.15-ubi8", + "4.4.16-ubi8", + "4.4.17-ubi8", + "4.4.18-ubi8", + "4.4.19-ubi8", + "4.4.20-ubi8", + "4.4.21-ubi8", + "5.0.0-ubi8", + "5.0.1-ubi8", + "5.0.2-ubi8", + "5.0.3-ubi8", + "5.0.4-ubi8", + "5.0.5-ubi8", + "5.0.6-ubi8", + "5.0.7-ubi8", + "5.0.8-ubi8", + "5.0.9-ubi8", + "5.0.10-ubi8", + "5.0.11-ubi8", + "5.0.12-ubi8", + "5.0.13-ubi8", + "5.0.14-ubi8", + "5.0.15-ubi8", + "5.0.16-ubi8", + "5.0.17-ubi8", + "5.0.18-ubi8", + "6.0.0-ubi8", + "6.0.1-ubi8", + "6.0.2-ubi8", + "6.0.3-ubi8", + "6.0.4-ubi8", + "6.0.5-ubi8", + "8.0.0-ubi8", + "8.0.0-ubi9" + ], + "variants": [ + "ubi" + ] + }, + "appdb-database": { + "Description": "4.2 onwards, see https://www.mongodb.com/support-policy/lifecycles", + "ssdlc_name": "[Deprecated] MongoDB Enterprise Kubernetes Operator AppDB", + "versions": [ + "4.2.11-ent", + "4.2.2-ent", + "4.2.24-ent", + "4.2.6-ent", + "4.2.8-ent", + "4.4.0-ent", + "4.4.11-ent", + "4.4.4-ent", + "4.4.21-ent", + "5.0.1-ent", + "5.0.5-ent", + "5.0.6-ent", + "5.0.7-ent", + "5.0.14-ent", + "5.0.18-ent" + ], + "variants": [ + "ubi" + ] + } + } } diff --git a/requirements.txt b/requirements.txt index e264e2e09..d6045af9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,48 @@ -git+https://github.com/mongodb/sonar@bc7bf7732851425421f3cfe2a19cf50b0460e633 -github-action-templates==0.0.4 +requests==2.32.3 +click==8.0.4 docker==7.1.0 -kubernetes==26.1.0 -jinja2==3.1.4 +Jinja2==3.1.6 +ruamel.yaml==0.18.6 +dnspython>=2.6.1 MarkupSafe==2.0.1 -PyYAML==6.0.1 -black==24.3.0 -mypy==0.961 -tqdm==v4.66.3 -boto3==1.16.21 -pymongo==4.6.3 -dnspython==2.6.1 -requests==2.32.3 -ruamel.yaml==0.17.9 semver==2.13.0 -rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability -setuptools==78.0.1 # not directly required, pinned by Snyk to avoid a vulnerability -certifi>=2022.12.7 # not directly required, pinned by Snyk to avoid a vulnerability -urllib3<2 # not directly required, pinned by Snyk to avoid a vulnerability +chardet==3.0.4 +jsonpatch==1.33 +kubernetes==17.17.0 +pymongo==4.6.3 +pytest==7.4.3 +pytest-asyncio==0.14.0 +PyYAML==6.0.2 +urllib3==1.26.19 +cryptography==44.0.1 +python-dateutil==2.9.0 +python-ldap==3.4.4 +GitPython==3.1.43 +setuptools>=71.0.3 # not directly required, pinned by Snyk to avoid a vulnerability +opentelemetry-api +opentelemetry-sdk +pytest-opentelemetry +jedi +rope +black==24.3.0 +flake8 +isort==5.12.0 +shrub.py==3.6.0 +pytest-mock==3.14.0 +wrapt==1.16.0 +botocore==1.35.29 +boto3==1.35.29 + +# from kubeobject +freezegun==1.5.1 +python-box==7.3.2 +autopep8==1.5.7 +flake8-isort==4.0.0 +mypy==1.14.1 +types-freezegun==0.1.4 +types-PyYAML==6.0.12.20250402 +types-pytz==2021.1.0 +types-python-dateutil==0.1.4 +pipupgrade==1.12.0 +pytest-cov==6.0.0 +pytest-socket==0.7.0 diff --git a/scripts/code_snippets/code_snippets_cleanup.sh b/scripts/code_snippets/code_snippets_cleanup.sh new file mode 100755 index 000000000..5c4736bb4 --- /dev/null +++ b/scripts/code_snippets/code_snippets_cleanup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eou pipefail + +find public/architectures -name "test.sh" -exec sh -c ' + source scripts/code_snippets/sample_test_runner.sh + + pushd "$(dirname $1)" + run_cleanup "test.sh" + run_cleanup "teardown.sh" + rm -rf istio* + rm -rf certs + rm -rf secrets + + popd + ' sh {} \; diff --git a/scripts/code_snippets/gke_multi_cluster_no_mesh_test.sh b/scripts/code_snippets/gke_multi_cluster_no_mesh_test.sh new file mode 100755 index 000000000..97245a8c2 --- /dev/null +++ b/scripts/code_snippets/gke_multi_cluster_no_mesh_test.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eou pipefail +source scripts/dev/set_env_context.sh + +function cleanup() { + if [ "${code_snippets_teardown:-true}" = true ]; then + echo "Deleting clusters and resources" + ./public/architectures/ops-manager-mc-no-mesh/teardown.sh & + ./public/architectures/setup-multi-cluster/setup-externaldns/teardown.sh & + wait + + ./public/architectures/setup-multi-cluster/setup-gke/teardown.sh + elif [ "${code_snippets_reset:-false}" = true ]; then + echo "Deleting resources, keeping the clusters" + ./public/architectures/ops-manager-mc-no-mesh/teardown.sh & + ./public/architectures/mongodb-sharded-mc-no-mesh/teardown.sh & + ./public/architectures/mongodb-replicaset-mc-no-mesh/teardown.sh & + ./public/architectures/setup-multi-cluster/setup-externaldns/teardown.sh & + wait + + ./public/architectures/setup-multi-cluster/setup-operator/teardown.sh + else + echo "Not deleting anything" + fi +} +trap cleanup EXIT + +source public/architectures/setup-multi-cluster/setup-gke/env_variables.sh +./public/architectures/setup-multi-cluster/setup-gke/test.sh + +source public/architectures/setup-multi-cluster/setup-operator/env_variables.sh +./public/architectures/setup-multi-cluster/setup-operator/test.sh + +./public/architectures/setup-multi-cluster/setup-cert-manager/test.sh + +source public/architectures/setup-multi-cluster/setup-externaldns/env_variables.sh +./public/architectures/setup-multi-cluster/setup-externaldns/test.sh + +source public/architectures/ops-manager-mc-no-mesh/env_variables.sh +./public/architectures/ops-manager-mc-no-mesh/test.sh + +source public/architectures/mongodb-replicaset-mc-no-mesh/env_variables.sh +./public/architectures/mongodb-replicaset-mc-no-mesh/test.sh + +source public/architectures/mongodb-sharded-mc-no-mesh/env_variables.sh +./public/architectures/mongodb-sharded-mc-no-mesh/test.sh diff --git a/scripts/code_snippets/gke_multi_cluster_test.sh b/scripts/code_snippets/gke_multi_cluster_test.sh new file mode 100755 index 000000000..850a68e9c --- /dev/null +++ b/scripts/code_snippets/gke_multi_cluster_test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -eou pipefail +source scripts/dev/set_env_context.sh + +function cleanup() { + if [ "${code_snippets_teardown:-true}" = true ]; then + echo "Deleting clusters" + ./public/architectures/setup-multi-cluster/setup-gke/teardown.sh + elif [ "${code_snippets_reset:-false}" = true ]; then + echo "Deleting resources, keeping the clusters" + ./public/architectures/ops-manager-multi-cluster/teardown.sh & + ./public/architectures/mongodb-sharded-multi-cluster/teardown.sh & + ./public/architectures/mongodb-replicaset-multi-cluster/teardown.sh & + wait + + ./public/architectures/setup-multi-cluster/setup-operator/teardown.sh + else + echo "Not deleting anything" + fi +} +trap cleanup EXIT + +source public/architectures/setup-multi-cluster/setup-gke/env_variables.sh +./public/architectures/setup-multi-cluster/setup-gke/test.sh + +source public/architectures/setup-multi-cluster/setup-operator/env_variables.sh +./public/architectures/setup-multi-cluster/setup-operator/test.sh + +./public/architectures/setup-multi-cluster/setup-istio/test.sh + +./public/architectures/setup-multi-cluster/verify-connectivity/test.sh + +./public/architectures/setup-multi-cluster/setup-cert-manager/test.sh + +source public/architectures/ops-manager-multi-cluster/env_variables.sh +./public/architectures/ops-manager-multi-cluster/test.sh + +source public/architectures/mongodb-replicaset-multi-cluster/env_variables.sh +./public/architectures/mongodb-replicaset-multi-cluster/test.sh + +source public/architectures/mongodb-sharded-multi-cluster/env_variables.sh +./public/architectures/mongodb-sharded-multi-cluster/test.sh diff --git a/scripts/code_snippets/sample_commit_output.sh b/scripts/code_snippets/sample_commit_output.sh new file mode 100755 index 000000000..dc5daae69 --- /dev/null +++ b/scripts/code_snippets/sample_commit_output.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail +source scripts/dev/set_env_context.sh + +if [ "${COMMIT_OUTPUT:-false}" = true ]; then + echo "Pushing output files" + branch="meko-snippets-update-$(date "+%Y%m%d%H%M%S")" + git checkout -b "${branch}" + git reset + git add public/architectures/**/*.out + git commit -m "Update code snippets outputs" + git remote set-url origin https://x-access-token:"${GH_TOKEN}"@github.com/10gen/ops-manager-kubernetes.git + git push origin "${branch}" +else + echo "Not pushing output files" +fi diff --git a/scripts/code_snippets/sample_test_runner.sh b/scripts/code_snippets/sample_test_runner.sh new file mode 100755 index 000000000..bc579b732 --- /dev/null +++ b/scripts/code_snippets/sample_test_runner.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +set -eou pipefail + +log_file="$(basename "$0").run.log" +snippets_src_dir="code_snippets" +snippets_run_dir=".generated" + +DEBUG=${DEBUG:-"false"} + +function snippets_list() { + src_dir=$1 + # shellcheck disable=SC2012 + ls -1 "${src_dir}" | sort -t '_' -k1,1n -k2,2 +} + +function run_cleanup() { + script_file=$1 + rm -rf "${snippets_run_dir}" 2>/dev/null || true + rm -rf "log" 2>/dev/null || true + git restore --staged --worktree rm -rf "output" 2>/dev/null || true + rm -rf "${script_file}.run.log" 2>/dev/null || true +} + +function prepare_snippets() { + echo "Generating code snippets in ${snippets_run_dir}..." + + touch "${log_file}" + mkdir log 2>/dev/null || true + mkdir output 2>/dev/null || true + + rm -rf "${snippets_run_dir}" 2>/dev/null || true + mkdir "${snippets_run_dir}" 2>/dev/null || true + + file_list=$(snippets_list "${snippets_src_dir}") + while IFS= read -r file_name; do + file_path="${snippets_run_dir}/${file_name}" + ( + echo "# This file is generated automatically from ${file_path}" + echo "# DO NOT EDIT" + echo "function ${file_name%.sh}() {" + cat "${snippets_src_dir}/${file_name}" + echo "}" + ) > "${file_path}" + done <<< "${file_list}" +} + +function run() { + # shellcheck disable=SC1090 + source "${snippets_run_dir}/$1" + cmd=${1%.sh} + + if grep -q "^${cmd}$" "${log_file}"; then + echo "Skipping ${cmd} as it is already executed." + return 0 + fi + + echo "$(date +"%Y-%m-%d %H:%M:%S") Executing ${cmd}" + + stdout_file="log/${cmd}.stdout.log" + stderr_file="log/${cmd}.stderr.log" + set +e + (set -e; set -x; "${cmd}" >"${stdout_file}" 2>"${stderr_file}") + ret=$? + set -e + if [[ ${ret} == 0 ]]; then + echo "${cmd}" >> "${log_file}" + else + echo "Error running: ${cmd}" + fi + + if [[ ${DEBUG} == "true" || ${ret} != 0 ]]; then + cat "${stdout_file}" + cat "${stderr_file}" + fi + + return ${ret} +} + +function run_for_output() { + # shellcheck disable=SC1090 + source "${snippets_run_dir}/$1" + cmd=${1%.sh} + + if grep -q "^${cmd}$" "${log_file}"; then + echo "Skipping ${cmd} as it is already executed." + return 0 + fi + + echo "$(date +"%Y-%m-%d %H:%M:%S") Executing ${cmd}" + stdout_file="log/${cmd}.stdout.log" + stderr_file="log/${cmd}.stderr.log" + set +e + (set -e; set -x; "${cmd}" >"${stdout_file}" 2>"${stderr_file}") + ret=$? + set -e + if [[ ${ret} == 0 ]]; then + tee "output/${cmd}.out" < "${stdout_file}" + else + echo "Error running: ${cmd}" + fi + + if [[ ${ret} == 0 ]]; then + echo "${cmd}" >> "${log_file}" + fi + + if [[ ${DEBUG} == "true" || ${ret} != 0 ]]; then + cat "${stdout_file}" + cat "${stderr_file}" + fi + + return ${ret} +} diff --git a/scripts/dev/cleanup-evg-docker.sh b/scripts/dev/cleanup-evg-docker.sh new file mode 100644 index 000000000..09d13c08e --- /dev/null +++ b/scripts/dev/cleanup-evg-docker.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Note: running directly docker system prune and related commands won't be enough since +# images are accumulated via containerd which is running in docker. So you need to jump into docker and cleanup +# via crictl + + +# Get all container IDs +container_ids=$(docker ps -q) + +# Iterate through each container ID +for container_id in ${container_ids}; do + echo "Cleaning up container ${container_id}" + # Use docker exec to run crictl rmi --prune inside the container + docker exec "${container_id}" crictl rmi --prune +done + +echo "Cleanup complete!" diff --git a/scripts/dev/configure_docker_auth.sh b/scripts/dev/configure_docker_auth.sh new file mode 100755 index 000000000..d1465bcb6 --- /dev/null +++ b/scripts/dev/configure_docker_auth.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/checks +source scripts/funcs/printing +source scripts/funcs/kubernetes + +check_docker_daemon_is_running() { + if [[ "$(uname -s)" != "Linux" ]]; then + echo "Skipping docker daemon check when not running in Linux" + return 0 + fi + + if systemctl is-active --quiet docker; then + echo "Docker is already running." + else + echo "Docker is not running. Starting Docker..." + # Start the Docker daemon + sudo systemctl start docker + for _ in {1..15}; do + if systemctl is-active --quiet docker; then + echo "Docker started successfully." + return 0 + fi + echo "Waiting for Docker to start..." + sleep 3 + done + fi +} + +remove_element() { + config_option="${1}" + tmpfile=$(mktemp) + jq 'del(.'"${config_option}"')' ~/.docker/config.json >"${tmpfile}" + cp "${tmpfile}" ~/.docker/config.json + rm "${tmpfile}" +} + +# This is the script which performs docker authentication to different registries that we use (so far ECR and RedHat) +# As the result of this login the ~/.docker/config.json will have all the 'auth' information necessary to work with docker registries + +check_docker_daemon_is_running + +if [[ -f ~/.docker/config.json ]]; then + if [[ "${RUNNING_IN_EVG:-""}" == "true" ]]; then + # when running locally we don't need to docker login all the time - we can do it once in 11 hours (ECR tokens expire each 12 hours) + if [[ -n "$(find ~/.docker/config.json -mmin -360 -type f)" ]] && + grep "268558157000" -q ~/.docker/config.json; then + echo "Docker credentials are up to date - not performing the new login!" + exit + fi + fi + + title "Performing docker login to ECR registries" + + # There could be some leftovers on Evergreen + if grep -q "credsStore" ~/.docker/config.json; then + remove_element "credsStore" + fi + if grep -q "credHelpers" ~/.docker/config.json; then + remove_element "credHelpers" + fi +fi + + +echo "$(aws --version)}" + +aws ecr get-login-password --region "us-east-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.us-east-1.amazonaws.com + +# by default docker tries to store credentials in an external storage (e.g. OS keychain) - not in the config.json +# We need to store it as base64 string in config.json instead so we need to remove the "credsStore" element +if grep -q "credsStore" ~/.docker/config.json; then + remove_element "credsStore" + + # login again to store the credentials into the config.json + aws ecr get-login-password --region "us-east-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.us-east-1.amazonaws.com +fi + +aws ecr get-login-password --region "eu-west-1" | docker login --username AWS --password-stdin 268558157000.dkr.ecr.eu-west-1.amazonaws.com + + +create_image_registries_secret diff --git a/scripts/dev/configure_operator.sh b/scripts/dev/configure_operator.sh new file mode 100755 index 000000000..c648b5489 --- /dev/null +++ b/scripts/dev/configure_operator.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + + +# TODO replace in favour of 'evergreen/e2e/configure_operator' +source scripts/dev/set_env_context.sh +source scripts/funcs/kubernetes +source scripts/funcs/printing + +ensure_namespace "${NAMESPACE}" + +export OM_BASE_URL=${OM_HOST} + +title "Configuring config map and secret for the Operator" + +if [[ -z ${OM_HOST} ]]; then + echo "OM_HOST env variable not provided - the default project ConfigMap won't be created!" + echo "You may need to spawn new Ops Manager - call 'make om'/'make om-evg' or add parameters to " + echo "'~/.operator-dev/om' or '~/.operator-dev/contexts/' manually" + echo "(Ignore this if you are working with MongoDBOpsManager custom resource)" +else + config_map_name="my-project" + kubectl delete configmap ${config_map_name} -n "${NAMESPACE}" 2>/dev/null || true + kubectl create configmap ${config_map_name} --from-literal orgId="${OM_ORGID-}" --from-literal "projectName=${NAMESPACE}" --from-literal "baseUrl=${OM_HOST}" -n "${NAMESPACE}" +fi + +if [[ -z ${OM_USER} ]] || [[ -z ${OM_API_KEY} ]]; then + echo "OM_USER and/or OM_API_KEY env variables are not provided - the default credentials Secret won't be created!" + echo "You may need to spawn new Ops Manager - call 'make om'/'make om-evg' or add parameters to " + echo "'~/.operator-dev/om' or '~/.operator-dev/contexts/' manually" + echo "(Ignore this if you are working with MongoDBOpsManager custom resource)" +else + secret_name="my-credentials" + kubectl delete secret ${secret_name} -n "${NAMESPACE}" 2>/dev/null || true + kubectl create secret generic ${secret_name} --from-literal=user="${OM_USER}" --from-literal=publicApiKey="${OM_API_KEY}" -n "${NAMESPACE}" +fi + +# this is the secret for OpsManager CR +om_admin_secret="ops-manager-admin-secret" +kubectl delete secret ${om_admin_secret} -n "${NAMESPACE}" 2>/dev/null || true +kubectl create secret generic ${om_admin_secret} --from-literal=Username="jane.doe@example.com" --from-literal=Password="Passw0rd." --from-literal=FirstName="Jane" --from-literal=LastName="Doe" -n "${NAMESPACE}" + +title "All necessary ConfigMaps and Secrets for the Operator are configured" + diff --git a/scripts/dev/contexts/build_om60_images b/scripts/dev/contexts/build_om60_images new file mode 100644 index 000000000..b82d46d55 --- /dev/null +++ b/scripts/dev/contexts/build_om60_images @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60_image" diff --git a/scripts/dev/contexts/build_om70_images b/scripts/dev/contexts/build_om70_images new file mode 100644 index 000000000..dfd4c775f --- /dev/null +++ b/scripts/dev/contexts/build_om70_images @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70_image" diff --git a/scripts/dev/contexts/build_om80_images b/scripts/dev/contexts/build_om80_images new file mode 100644 index 000000000..a5896ad77 --- /dev/null +++ b/scripts/dev/contexts/build_om80_images @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om80_image" diff --git a/scripts/dev/contexts/e2e_custom_domain_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_custom_domain_mdb_kind_ubi_cloudqa new file mode 100644 index 000000000..c1c23ad1b --- /dev/null +++ b/scripts/dev/contexts/e2e_custom_domain_mdb_kind_ubi_cloudqa @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +# shellcheck disable=SC1091 +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" + +export CLUSTER_DOMAIN="testdomain.local" + +export CUSTOM_MDB_VERSION=6.0.5 diff --git a/scripts/dev/contexts/e2e_kind_olm_ubi b/scripts/dev/contexts/e2e_kind_olm_ubi new file mode 100644 index 000000000..0eb01c411 --- /dev/null +++ b/scripts/dev/contexts/e2e_kind_olm_ubi @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=kind diff --git a/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa new file mode 100644 index 000000000..03384c26c --- /dev/null +++ b/scripts/dev/contexts/e2e_mdb_kind_ubi_cloudqa @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=6.0.5 +export CUSTOM_MDB_PREV_VERSION=5.0.7 diff --git a/scripts/dev/contexts/e2e_mdb_openshift_ubi_cloudqa b/scripts/dev/contexts/e2e_mdb_openshift_ubi_cloudqa new file mode 100644 index 000000000..43b68cc08 --- /dev/null +++ b/scripts/dev/contexts/e2e_mdb_openshift_ubi_cloudqa @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export KUBE_ENVIRONMENT_NAME=openshift_4 +export CLUSTER_TYPE="openshift" +export ecr_registry_needs_auth=ecr-registry +export MANAGED_SECURITY_CONTEXT="true" +export ALWAYS_REMOVE_TESTING_NAMESPACE="true" +export ops_manager_version="cloud_qa" +export MDB_OPERATOR_TELEMETRY_INSTALL_CLUSTER_ROLE_INSTALLATION="false" + +export CUSTOM_MDB_VERSION=6.0.16 + +# shellcheck disable=SC2154 +export OPENSHIFT_URL="${openshift_url}" +# shellcheck disable=SC2154 +export OPENSHIFT_TOKEN="${openshift_token}" diff --git a/scripts/dev/contexts/e2e_multi_cluster_2_clusters b/scripts/dev/contexts/e2e_multi_cluster_2_clusters new file mode 100644 index 000000000..a06efaed9 --- /dev/null +++ b/scripts/dev/contexts/e2e_multi_cluster_2_clusters @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-cluster-1" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2" +export CENTRAL_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" + diff --git a/scripts/dev/contexts/e2e_multi_cluster_kind b/scripts/dev/contexts/e2e_multi_cluster_kind new file mode 100644 index 000000000..340d79d5f --- /dev/null +++ b/scripts/dev/contexts/e2e_multi_cluster_kind @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-operator" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-operator +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" diff --git a/scripts/dev/contexts/e2e_multi_cluster_om_appdb b/scripts/dev/contexts/e2e_multi_cluster_om_appdb new file mode 100644 index 000000000..67c93db11 --- /dev/null +++ b/scripts/dev/contexts/e2e_multi_cluster_om_appdb @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-cluster-1" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-cluster-1 +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="${CUSTOM_OM_VERSION}" diff --git a/scripts/dev/contexts/e2e_multi_cluster_om_operator_not_in_mesh b/scripts/dev/contexts/e2e_multi_cluster_om_operator_not_in_mesh new file mode 100644 index 000000000..12674510d --- /dev/null +++ b/scripts/dev/contexts/e2e_multi_cluster_om_operator_not_in_mesh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-operator" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-operator +export test_pod_cluster=kind-e2e-cluster-1 +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export ops_manager_version="${CUSTOM_OM_VERSION}" diff --git a/scripts/dev/contexts/e2e_om60_kind_ubi b/scripts/dev/contexts/e2e_om60_kind_ubi new file mode 100644 index 000000000..0eb01c411 --- /dev/null +++ b/scripts/dev/contexts/e2e_om60_kind_ubi @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=kind diff --git a/scripts/dev/contexts/e2e_om70_kind_ubi b/scripts/dev/contexts/e2e_om70_kind_ubi new file mode 100644 index 000000000..e28b398bd --- /dev/null +++ b/scripts/dev/contexts/e2e_om70_kind_ubi @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +export KUBE_ENVIRONMENT_NAME=kind + +# this is the only variant to fully deactivate telemetry. +# this is mostly used to verify that deactivation works on the helm chart level +export MDB_OPERATOR_TELEMETRY_ENABLED=false diff --git a/scripts/dev/contexts/e2e_om80_kind_ubi b/scripts/dev/contexts/e2e_om80_kind_ubi new file mode 100644 index 000000000..453a91ae8 --- /dev/null +++ b/scripts/dev/contexts/e2e_om80_kind_ubi @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om80" + +export KUBE_ENVIRONMENT_NAME=kind diff --git a/scripts/dev/contexts/e2e_openshift_static_mdb_ubi_cloudqa b/scripts/dev/contexts/e2e_openshift_static_mdb_ubi_cloudqa new file mode 100644 index 000000000..c498b01b3 --- /dev/null +++ b/scripts/dev/contexts/e2e_openshift_static_mdb_ubi_cloudqa @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export KUBE_ENVIRONMENT_NAME=openshift_4 +export CLUSTER_TYPE="openshift" +export ecr_registry_needs_auth=ecr-registry +export MANAGED_SECURITY_CONTEXT="true" +export ALWAYS_REMOVE_TESTING_NAMESPACE="true" +export ops_manager_version="cloud_qa" +export MDB_OPERATOR_TELEMETRY_INSTALL_CLUSTER_ROLE_INSTALLATION="false" + +export CUSTOM_MDB_VERSION=6.0.16 + +# shellcheck disable=SC2154 +export OPENSHIFT_URL="${openshift_url}" +# shellcheck disable=SC2154 +export OPENSHIFT_TOKEN="${openshift_token}" +export MDB_DEFAULT_ARCHITECTURE=static diff --git a/scripts/dev/contexts/e2e_operator_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_operator_kind_ubi_cloudqa new file mode 100644 index 000000000..dbe8d0666 --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_kind_ubi_cloudqa @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +export ops_manager_version="cloud_qa" diff --git a/scripts/dev/contexts/e2e_operator_no_webhook_roles_cloudqa b/scripts/dev/contexts/e2e_operator_no_webhook_roles_cloudqa new file mode 100644 index 000000000..8a023dc3d --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_no_webhook_roles_cloudqa @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export ops_manager_version="cloud_qa" + +export MDB_HELM_OPERATOR_WEBHOOK_INSTALL_CLUSTER_ROLE="false" diff --git a/scripts/dev/contexts/e2e_operator_perf b/scripts/dev/contexts/e2e_operator_perf new file mode 100644 index 000000000..67424423f --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_perf @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export MDB_MAX_CONCURRENT_RECONCILES=10 +export MDB_DEFAULT_ARCHITECTURE=static +export KUBE_ENVIRONMENT_NAME=performance diff --git a/scripts/dev/contexts/e2e_operator_perf_large b/scripts/dev/contexts/e2e_operator_perf_large new file mode 100644 index 000000000..67424423f --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_perf_large @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export MDB_MAX_CONCURRENT_RECONCILES=10 +export MDB_DEFAULT_ARCHITECTURE=static +export KUBE_ENVIRONMENT_NAME=performance diff --git a/scripts/dev/contexts/e2e_operator_perf_one_thread b/scripts/dev/contexts/e2e_operator_perf_one_thread new file mode 100644 index 000000000..693a94cc9 --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_perf_one_thread @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export MDB_MAX_CONCURRENT_RECONCILES=1 +export MDB_DEFAULT_ARCHITECTURE=static +export KUBE_ENVIRONMENT_NAME=performance + diff --git a/scripts/dev/contexts/e2e_operator_perf_one_thread_large b/scripts/dev/contexts/e2e_operator_perf_one_thread_large new file mode 100644 index 000000000..693a94cc9 --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_perf_one_thread_large @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export MDB_MAX_CONCURRENT_RECONCILES=1 +export MDB_DEFAULT_ARCHITECTURE=static +export KUBE_ENVIRONMENT_NAME=performance + diff --git a/scripts/dev/contexts/e2e_operator_perf_thirty b/scripts/dev/contexts/e2e_operator_perf_thirty new file mode 100644 index 000000000..ee8cbc7a0 --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_perf_thirty @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export MDB_MAX_CONCURRENT_RECONCILES=30 +export MDB_DEFAULT_ARCHITECTURE=static +export KUBE_ENVIRONMENT_NAME=performance diff --git a/scripts/dev/contexts/e2e_operator_perf_thirty_large b/scripts/dev/contexts/e2e_operator_perf_thirty_large new file mode 100644 index 000000000..ee8cbc7a0 --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_perf_thirty_large @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export MDB_MAX_CONCURRENT_RECONCILES=30 +export MDB_DEFAULT_ARCHITECTURE=static +export KUBE_ENVIRONMENT_NAME=performance diff --git a/scripts/dev/contexts/e2e_operator_race_ubi_with_telemetry b/scripts/dev/contexts/e2e_operator_race_ubi_with_telemetry new file mode 100644 index 000000000..5974bc795 --- /dev/null +++ b/scripts/dev/contexts/e2e_operator_race_ubi_with_telemetry @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-operator" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-operator +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" +export BUILD_WITH_RACE_DETECTION=true +export MDB_OPERATOR_TELEMETRY_SEND_ENABLED=true +export MDB_OPERATOR_TELEMETRY_SEND_BASEURL="https://cloud-dev.mongodb.com/" +export MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY="10m" + +# there is no python 3.13 in ubuntu1804-xlarge +export PYTHON_VERSION=3.12 diff --git a/scripts/dev/contexts/e2e_smoke b/scripts/dev/contexts/e2e_smoke new file mode 100644 index 000000000..42c565545 --- /dev/null +++ b/scripts/dev/contexts/e2e_smoke @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export DATABASE_REGISTRY="${QUAY_REGISTRY}" +export APPDB_REGISTRY="${QUAY_REGISTRY}" +export INIT_OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" +export OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" +export OPERATOR_REGISTRY="${QUAY_REGISTRY}" +export INIT_IMAGES_REGISTRY="${QUAY_REGISTRY}" +export INIT_APPDB_REGISTRY="${QUAY_REGISTRY}" +export INIT_DATABASE_REGISTRY="${QUAY_REGISTRY}" +# Since we're sourcing this as an initial step, the jq might not be there. That's why we need bash magic here. +OPERATOR_VERSION="$(grep -o '"mongodbOperator": "[^"]*' release.json | grep -o '[^"]*$')" +export OPERATOR_VERSION +INIT_DATABASE_VERSION="$(grep -o '"initDatabaseVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export INIT_DATABASE_VERSION +INIT_APPDB_VERSION="$(grep -o '"initAppDbVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export INIT_APPDB_VERSION +INIT_OPS_MANAGER_VERSION="$(grep -o '"initOpsManagerVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export INIT_OPS_MANAGER_VERSION +DATABASE_VERSION="$(grep -o '"databaseImageVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export DATABASE_VERSION diff --git a/scripts/dev/contexts/e2e_static_custom_domain_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_static_custom_domain_mdb_kind_ubi_cloudqa new file mode 100644 index 000000000..81b9d8427 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_custom_domain_mdb_kind_ubi_cloudqa @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +# shellcheck disable=SC1091 +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" +export MDB_DEFAULT_ARCHITECTURE=static +export CUSTOM_MDB_VERSION=6.0.16 + +export CLUSTER_DOMAIN="testdomain.local" diff --git a/scripts/dev/contexts/e2e_static_kind_olm_ubi b/scripts/dev/contexts/e2e_static_kind_olm_ubi new file mode 100644 index 000000000..e9f965a2d --- /dev/null +++ b/scripts/dev/contexts/e2e_static_kind_olm_ubi @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +export KUBE_ENVIRONMENT_NAME=kind +export CUSTOM_MDB_PREV_VERSION=4.4.20 +export MDB_DEFAULT_ARCHITECTURE=static diff --git a/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa new file mode 100644 index 000000000..869396d1e --- /dev/null +++ b/scripts/dev/contexts/e2e_static_mdb_kind_ubi_cloudqa @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" +export MDB_DEFAULT_ARCHITECTURE=static + +# This is required to be able to rebuild the om image and use that image which has been rebuild +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_PREV_VERSION=6.0.16 +export CUSTOM_MDB_VERSION=7.0.5 diff --git a/scripts/dev/contexts/e2e_static_multi_cluster_2_clusters b/scripts/dev/contexts/e2e_static_multi_cluster_2_clusters new file mode 100644 index 000000000..c86e8655e --- /dev/null +++ b/scripts/dev/contexts/e2e_static_multi_cluster_2_clusters @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-cluster-1" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2" +export CENTRAL_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" + +export MDB_DEFAULT_ARCHITECTURE=static +export CUSTOM_MDB_VERSION=6.0.5 diff --git a/scripts/dev/contexts/e2e_static_multi_cluster_kind b/scripts/dev/contexts/e2e_static_multi_cluster_kind new file mode 100644 index 000000000..a19bb1726 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_multi_cluster_kind @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-operator" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-operator +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" +export MDB_DEFAULT_ARCHITECTURE=static + +# For upgrade/downgrade tests we need to override this for static containers since we don't have 5.0.x versions +# of MDB binaries for ubi9 +export CUSTOM_MDB_PREV_VERSION=6.0.5 diff --git a/scripts/dev/contexts/e2e_static_multi_cluster_om_appdb b/scripts/dev/contexts/e2e_static_multi_cluster_om_appdb new file mode 100644 index 000000000..3a03c047a --- /dev/null +++ b/scripts/dev/contexts/e2e_static_multi_cluster_om_appdb @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +# The earliest version supporting Static Containers for OM +# We started supporting Static Containers in 6.0.21, except for OM +# This CUSTOM version is required for upgrade tests to work +# shellcheck disable=SC2034 +export CUSTOM_OM_PREV_VERSION=6.0.27 + +# TODO Remove this once the startup script for OM can handle skipping preflight checks. +# As it stands now, OM 7.0.13 will fail the preflight checks in a disaster recovery scenario. +# https://jira.mongodb.org/browse/CLOUDP-297377 +export CUSTOM_OM_VERSION=7.0.12 + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-cluster-1" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="${CUSTOM_OM_VERSION}" +export MDB_DEFAULT_ARCHITECTURE=static +export CUSTOM_APPDB_VERSION="7.0.2-ubi9-20241112T085558Z" + +# clear cloud-qa settings +export OM_ORGID="" diff --git a/scripts/dev/contexts/e2e_static_om60_kind_ubi b/scripts/dev/contexts/e2e_static_om60_kind_ubi new file mode 100644 index 000000000..eaf45fe14 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_om60_kind_ubi @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=kind +export MDB_DEFAULT_ARCHITECTURE=static + +# The earliest version supporting Static Containers for OM +# We started supporting Static Containers in 6.0.21, except for OM +# This CUSTOM version is required for upgrade tests to work +# Default value for this variable in om60 context file is 6.0.0 +# shellcheck disable=SC2034 +export CUSTOM_OM_PREV_VERSION=6.0.27 + +export CUSTOM_MDB_VERSION=6.0.16 +# We can't use a 5.0.x version for this static variant because there's no UBI9 image for the 5.0.x series +export CUSTOM_MDB_PREV_VERSION=6.0.5 +export CUSTOM_APPDB_VERSION="6.0.5-ubi9-20241112T090442Z" diff --git a/scripts/dev/contexts/e2e_static_om70_kind_ubi b/scripts/dev/contexts/e2e_static_om70_kind_ubi new file mode 100644 index 000000000..11115f31d --- /dev/null +++ b/scripts/dev/contexts/e2e_static_om70_kind_ubi @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70" + +export KUBE_ENVIRONMENT_NAME=kind +export MDB_DEFAULT_ARCHITECTURE=static +export CUSTOM_MDB_VERSION=7.0.5 +export CUSTOM_MDB_PREV_VERSION=6.0.16 +export CUSTOM_APPDB_VERSION="7.0.2-ubi9-20241112T085558Z" diff --git a/scripts/dev/contexts/e2e_static_om80_kind_ubi b/scripts/dev/contexts/e2e_static_om80_kind_ubi new file mode 100644 index 000000000..dde8dede2 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_om80_kind_ubi @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om80" + +export KUBE_ENVIRONMENT_NAME=kind +export MDB_DEFAULT_ARCHITECTURE=static +export CUSTOM_APPDB_VERSION="8.0.0-ubi9-20241112T091101Z" diff --git a/scripts/dev/contexts/e2e_static_operator_kind_ubi_cloudqa b/scripts/dev/contexts/e2e_static_operator_kind_ubi_cloudqa new file mode 100644 index 000000000..b493e00ef --- /dev/null +++ b/scripts/dev/contexts/e2e_static_operator_kind_ubi_cloudqa @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export ops_manager_version="cloud_qa" +export MDB_DEFAULT_ARCHITECTURE=static + +export CUSTOM_MDB_VERSION=6.0.16 +# We can't use a 5.0.x version for this static variant because there's no UBI9 image for the 5.0.x series +export CUSTOM_MDB_PREV_VERSION=6.0.5 +export CUSTOM_APPDB_VERSION="6.0.5-ubi9-20241112T090442Z" diff --git a/scripts/dev/contexts/e2e_static_smoke b/scripts/dev/contexts/e2e_static_smoke new file mode 100644 index 000000000..4e7b94e69 --- /dev/null +++ b/scripts/dev/contexts/e2e_static_smoke @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export DATABASE_REGISTRY="${QUAY_REGISTRY}" +export APPDB_REGISTRY="${QUAY_REGISTRY}" +export INIT_OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" +export OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" +export OPERATOR_REGISTRY="${QUAY_REGISTRY}" +export INIT_IMAGES_REGISTRY="${QUAY_REGISTRY}" +export INIT_APPDB_REGISTRY="${QUAY_REGISTRY}" +export INIT_DATABASE_REGISTRY="${QUAY_REGISTRY}" +# Since we're sourcing this as an initial step, the jq might not be there. That's why we need bash magic here. +OPERATOR_VERSION="$(grep -o '"mongodbOperator": "[^"]*' release.json | grep -o '[^"]*$')" +export OPERATOR_VERSION +INIT_DATABASE_VERSION="$(grep -o '"initDatabaseVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export INIT_DATABASE_VERSION +INIT_APPDB_VERSION="$(grep -o '"initAppDbVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export INIT_APPDB_VERSION +INIT_OPS_MANAGER_VERSION="$(grep -o '"initOpsManagerVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export INIT_OPS_MANAGER_VERSION +DATABASE_VERSION="$(grep -o '"databaseImageVersion": "[^"]*' release.json | grep -o '[^"]*$')" +export DATABASE_VERSION +export MDB_DEFAULT_ARCHITECTURE=static +# For static smoke tests we nned the previous MDB version to have a ubi9 binary +export CUSTOM_MDB_PREV_VERSION=6.0.5 diff --git a/scripts/dev/contexts/evg-private-context b/scripts/dev/contexts/evg-private-context new file mode 100644 index 000000000..80e60df32 --- /dev/null +++ b/scripts/dev/contexts/evg-private-context @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/evergreen/e2e/lib.sh + +# This is the default cluster name to be used for single cluster. Multi-cluster variants need to override this. +export CLUSTER_NAME="kind-kind" +# In case of EVG tests, those variables need to be empty (in this case we assume an OpsManager test) or created by +# the setup_cloud_qa script. The latter case will create all credentials in $ENV_FILE. This case is handled below. +export OM_USER="" +export OM_API_KEY="" +export OM_ORGID="" +export OM_HOST="" +export OM_BASE_URL="" +# This file is used for injecting credentials when interacting with Cloud QA. Probably should be refactored +# into env vars in the future. +export ENV_FILE="${workdir:-}/.ops-manager-env" +if [ -f "${ENV_FILE}" ]; then + source "${ENV_FILE}" +fi + +export PROJECT_DIR="${script_dir:-}/../../../" + +export NAMESPACE_FILE="${workdir}/.namespace" +if [ -f "${NAMESPACE_FILE}" ]; then + echo "found existing namespace file" + NAMESPACE=$(cat "${NAMESPACE_FILE}") +else + echo "generating new namespace name" + NAMESPACE=$(generate_random_namespace) + echo "${NAMESPACE}" >"${NAMESPACE_FILE}" +fi +export NAMESPACE + +export BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" + +export REGISTRY="${BASE_REPO_URL}" +export QUAY_REGISTRY=quay.io/mongodb +export OPERATOR_REGISTRY=${REGISTRY} +export INIT_IMAGES_REGISTRY=${INIT_IMAGES_REGISTRY:-${REGISTRY}} +export INIT_APPDB_REGISTRY=${INIT_IMAGES_REGISTRY} +export INIT_OPS_MANAGER_REGISTRY=${INIT_IMAGES_REGISTRY-"${QUAY_REGISTRY}"} +export INIT_DATABASE_REGISTRY=${INIT_IMAGES_REGISTRY:-"${QUAY_REGISTRY}"} +export DATABASE_REGISTRY=${INIT_IMAGES_REGISTRY:-"${QUAY_REGISTRY}"} +export OPS_MANAGER_REGISTRY=${QUAY_REGISTRY} +export APPDB_REGISTRY=${QUAY_REGISTRY} +export MONGODB_ENTERPRISE_DATABASE_IMAGE="${INIT_IMAGES_REGISTRY}/mongodb-enterprise-database" +export INIT_DATABASE_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-enterprise-init-database" +export MDB_AGENT_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-agent-ubi" + +# these are needed to deploy OM +export INIT_APPDB_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-enterprise-init-appdb" +export OPS_MANAGER_IMAGE_REPOSITORY="${QUAY_REGISTRY}/mongodb-enterprise-ops-manager-ubi" +export INIT_OPS_MANAGER_IMAGE_REPOSITORY=${INIT_IMAGES_REGISTRY}/mongodb-enterprise-init-ops-manager + +export CLUSTER_TYPE="kind" +# Empty sting means that we're not using it. +export EVG_HOST_NAME="" +export GOROOT="/opt/golang/go1.24" + +if [[ ! ${PATH} =~ .*${workdir:-.}/bin.* ]]; then + export PATH=${PATH}:${workdir:-.}/bin +fi +if [[ ! ${PATH} =~ .*${GOROOT}/bin.* ]]; then + export PATH=${GOROOT}/bin:${PATH} +fi + +export LOCAL_OPERATOR="false" + +export AWS_ACCESS_KEY_ID="${mms_eng_test_aws_access_key}" +export AWS_SECRET_ACCESS_KEY="${mms_eng_test_aws_secret}" +export AWS_DEFAULT_REGION="${mms_eng_test_aws_region}" + +# setup_cloud_qa.py +export e2e_cloud_qa_orgid_owner="${e2e_cloud_qa_orgid_owner_ubi_cloudqa}" +export e2e_cloud_qa_apikey_owner="${e2e_cloud_qa_apikey_owner_ubi_cloudqa}" +export e2e_cloud_qa_user_owner="${e2e_cloud_qa_user_owner_ubi_cloudqa}" + +export e2e_cloud_qa_baseurl="https://cloud-qa.mongodb.com" + +export kubernetes_kind_version=1.22.0 + +# Note: this name is correct +export task_id="${EVR_TASK_ID}" + +export OTEL_TRACE_ID="${otel_trace_id:-"0xdeadbeef"}" +export OTEL_COLLECTOR_ENDPOINT="${otel_collector_endpoint:-"unknown"}" + +# This is given by an expansion from evg +export TASK_NAME="${task_name:-"unknown"}" +export task_name="${TASK_NAME}" +export IS_PATCH="${is_patch:-"unknown"}" +export BUILD_ID="${build_id:-"unknown"}" +export EXECUTION="${execution:-"unknown"}" +export BUILD_VARIANT="${build_variant:-"unknown"}" + +# var used in pipeline.py to determine if we're running on EVG host +# used to discern between local pipeline image build and the build in pipeline +export RUNNING_IN_EVG="true" + +export DELETE_KIND_NETWORK="true" diff --git a/scripts/dev/contexts/init_release_agents_on_ecr b/scripts/dev/contexts/init_release_agents_on_ecr new file mode 100644 index 000000000..606597585 --- /dev/null +++ b/scripts/dev/contexts/init_release_agents_on_ecr @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export all_agents="true" diff --git a/scripts/dev/contexts/init_test_run b/scripts/dev/contexts/init_test_run new file mode 100644 index 000000000..7f7f635e9 --- /dev/null +++ b/scripts/dev/contexts/init_test_run @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" diff --git a/scripts/dev/contexts/init_tests_with_olm b/scripts/dev/contexts/init_tests_with_olm new file mode 100644 index 000000000..c66fed75e --- /dev/null +++ b/scripts/dev/contexts/init_tests_with_olm @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + diff --git a/scripts/dev/contexts/init_tests_with_olm_openshift b/scripts/dev/contexts/init_tests_with_olm_openshift new file mode 100644 index 000000000..2696945a7 --- /dev/null +++ b/scripts/dev/contexts/init_tests_with_olm_openshift @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# This context is usable when generating OLM bundles locally that will be deployed on OpenShift. +# OpenShift must have managed security context enabled. + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export MANAGED_SECURITY_CONTEXT="true" diff --git a/scripts/dev/contexts/local-defaults-context b/scripts/dev/contexts/local-defaults-context new file mode 100644 index 000000000..46edddb64 --- /dev/null +++ b/scripts/dev/contexts/local-defaults-context @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# This context is the defacto local development default context, where we set sane defaults for everyone to be able to get started. +# This is the first context we source, therefore we can easily overwrite this. in the private-context or overrides. +# In case you don't agree with those defaults you can override them im your private-context without affecting +# others. + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +PROJECT_DIR=$(realpath "${script_dir}/../../../") +# DEPRECATED: usage of workdir is deprecated, use PROJECT_DIR instead +workdir=$(realpath "${script_dir}/../../../") +export PROJECT_DIR +export workdir + +# Here are all the required registries used by the Operator. We are defaulting to images ecr for most, some to quay +# The main registry to use for local test runs +# - global: /dev is the shared registry +# - dedicated: / +# Note: in our root-context we default the image tag to latest, if VERSION_ID is not preset on your machine +# Note2: For quick local development defaults we only use shared images +export QUAY_REGISTRY=quay.io/mongodb +export BASE_REPO_URL_SHARED="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev" +# Note: the Operator registry (OPERATOR_REGISTRY variable) is defined in the private context +export REGISTRY="${BASE_REPO_URL_SHARED}" + +export INIT_IMAGES_REGISTRY=${INIT_IMAGES_REGISTRY:-"${REGISTRY}"} + +export INIT_APPDB_REGISTRY="${BASE_REPO_URL_SHARED}" +export INIT_OPS_MANAGER_REGISTRY=${BASE_REPO_URL_SHARED:-"${QUAY_REGISTRY}"} +export INIT_DATABASE_REGISTRY=${BASE_REPO_URL_SHARED:-"${QUAY_REGISTRY}"} +export INIT_DATABASE_IMAGE_REPOSITORY="${BASE_REPO_URL_SHARED}/mongodb-enterprise-init-database" +export DATABASE_REGISTRY=${BASE_REPO_URL_SHARED:-"${QUAY_REGISTRY}"} +export OPS_MANAGER_REGISTRY="${QUAY_REGISTRY}" +export MONGODB_REPO_URL="${QUAY_REGISTRY}" +export APPDB_REGISTRY="${QUAY_REGISTRY}" +export MONGODB_ENTERPRISE_DATABASE_IMAGE="${INIT_IMAGES_REGISTRY}/mongodb-enterprise-database-ubi" +export MDB_AGENT_IMAGE_OPERATOR_VERSION=latest +export MDB_AGENT_IMAGE_REPOSITORY="${BASE_REPO_URL_SHARED}/mongodb-agent-ubi" +export AGENT_BASE_REGISTRY=${BASE_REPO_URL_SHARED} +export AGENT_IMAGE="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-agent-ubi:12.0.30.7791-1" + +# these are needed to deploy OM +export INIT_APPDB_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-enterprise-init-appdb" +export OPS_MANAGER_IMAGE_REPOSITORY="${QUAY_REGISTRY}/mongodb-enterprise-ops-manager-ubi" +export INIT_OPS_MANAGER_IMAGE_REPOSITORY="${INIT_IMAGES_REGISTRY}/mongodb-enterprise-init-ops-manager" + + +# Environment variable needed for local development +# Used in controllers/operator/mongodbopsmanager_controller.go +# The file is created by scripts/dev/prepare_local_e2e_run.sh +export MDB_OM_VERSION_MAPPING_PATH="${PROJECT_DIR}/release.json" +export ENV_FILE="${PROJECT_DIR}/.ops-manager-env" +export NAMESPACE_FILE="${PROJECT_DIR}/.namespace" + +# TO ensure we don't release by accident via pipeline.py +export skip_tags="release" + +# This setting is set if you want to remove your namespaces after running the tests +export ALWAYS_REMOVE_TESTING_NAMESPACE="true" diff --git a/scripts/dev/contexts/manual_ecr_release_agent b/scripts/dev/contexts/manual_ecr_release_agent new file mode 100644 index 000000000..606597585 --- /dev/null +++ b/scripts/dev/contexts/manual_ecr_release_agent @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export all_agents="true" diff --git a/scripts/dev/contexts/manual_telemetry_multi_cluster_dev b/scripts/dev/contexts/manual_telemetry_multi_cluster_dev new file mode 100644 index 000000000..5d3c2d604 --- /dev/null +++ b/scripts/dev/contexts/manual_telemetry_multi_cluster_dev @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-operator" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-operator +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" +export MDB_OPERATOR_TELEMETRY_SEND_ENABLED=true +export MDB_OPERATOR_TELEMETRY_SEND_BASEURL="https://cloud-dev.mongodb.com/" +export MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY="10m" # let's send frequently +export MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY="1m" # let's collect as often as we can diff --git a/scripts/dev/contexts/manual_telemetry_multi_cluster_prod b/scripts/dev/contexts/manual_telemetry_multi_cluster_prod new file mode 100644 index 000000000..93b1c86ee --- /dev/null +++ b/scripts/dev/contexts/manual_telemetry_multi_cluster_prod @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60" + +export KUBE_ENVIRONMENT_NAME=multi +export CLUSTER_NAME="kind-e2e-operator" +export MEMBER_CLUSTERS="kind-e2e-cluster-1 kind-e2e-cluster-2 kind-e2e-cluster-3" +export CENTRAL_CLUSTER=kind-e2e-operator +export TEST_POD_CLUSTER=kind-e2e-cluster-1 +export test_pod_cluster=kind-e2e-cluster-1 +export ops_manager_version="cloud_qa" +export MDB_OPERATOR_TELEMETRY_SEND_ENABLED=true +export MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY="10m" # let's send frequently +export MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY="1m" # let's collect as often as we can diff --git a/scripts/dev/contexts/periodic_build b/scripts/dev/contexts/periodic_build new file mode 100644 index 000000000..7f7f635e9 --- /dev/null +++ b/scripts/dev/contexts/periodic_build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" diff --git a/scripts/dev/contexts/periodic_teardown b/scripts/dev/contexts/periodic_teardown new file mode 100644 index 000000000..6ae9fe2c0 --- /dev/null +++ b/scripts/dev/contexts/periodic_teardown @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export ops_manager_version="cloud_qa" diff --git a/scripts/dev/contexts/preflight_om60_images b/scripts/dev/contexts/preflight_om60_images new file mode 100644 index 000000000..56226987c --- /dev/null +++ b/scripts/dev/contexts/preflight_om60_images @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60_image" + +export preflight_submit=true diff --git a/scripts/dev/contexts/preflight_om70_images b/scripts/dev/contexts/preflight_om70_images new file mode 100644 index 000000000..741902b05 --- /dev/null +++ b/scripts/dev/contexts/preflight_om70_images @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70_image" + +export preflight_submit=true diff --git a/scripts/dev/contexts/preflight_om80_images b/scripts/dev/contexts/preflight_om80_images new file mode 100644 index 000000000..041661c7f --- /dev/null +++ b/scripts/dev/contexts/preflight_om80_images @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om80_image" + +export preflight_submit=true diff --git a/scripts/dev/contexts/preflight_release_images b/scripts/dev/contexts/preflight_release_images new file mode 100644 index 000000000..bd0b34500 --- /dev/null +++ b/scripts/dev/contexts/preflight_release_images @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export preflight_submit=true diff --git a/scripts/dev/contexts/preflight_release_images_check b/scripts/dev/contexts/preflight_release_images_check new file mode 100644 index 000000000..94a7a9ead --- /dev/null +++ b/scripts/dev/contexts/preflight_release_images_check @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export preflight_submit=false diff --git a/scripts/dev/contexts/prerelease_gke_code_snippets b/scripts/dev/contexts/prerelease_gke_code_snippets new file mode 100644 index 000000000..8eb284746 --- /dev/null +++ b/scripts/dev/contexts/prerelease_gke_code_snippets @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# this context file is for code snippets running on GKE clusters +set -Eeou pipefail + +# overrides of public env_variables.sh +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export MDB_GKE_PROJECT="scratch-kubernetes-team" +export K8S_CLUSTER_SUFFIX="-${version_id}-${RANDOM}" +export COMMIT_OUTPUT=true + +# we reset evg host to use a default ~/.kube/config for GKE instead of the one from evg host +export EVG_HOST_NAME="" + +# ENV_VARIABLES.SH overrides +export OPERATOR_ADDITIONAL_HELM_VALUES="" +export OPERATOR_HELM_CHART=${PROJECT_DIR}/helm_chart diff --git a/scripts/dev/contexts/private-context-template b/scripts/dev/contexts/private-context-template new file mode 100644 index 000000000..835b558c6 --- /dev/null +++ b/scripts/dev/contexts/private-context-template @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +## This file contains properties that need to overridden by a user. +## Rename it to "private-context" and fill in with your data. + +# Kubernetes namespace for deploying a test +# Allowed values: +# - a valid Kubernetes namespace +# Sensible default: +# - mongodb-test +export NAMESPACE="mongodb-test" + +# An EVG host name. See https://wiki.corp.mongodb.com/display/MMS/Setting+up+local+development+and+E2E+testing#SettinguplocaldevelopmentandE2Etesting-RunningtestsagainstEvergreenhost +# Allowed values: +# a valid EVG Host +# Sensible default: +# Empty for start. +export EVG_HOST_NAME="" + +# ECR repo used for deploying the images +# Allowed values: +# - a full path to the ECR repo +# Sensible default: +# - 268558157000.dkr.ecr.us-east-1.amazonaws.com/ +export BASE_REPO_URL="268558157000.dkr.ecr.us-east-1.amazonaws.com/${USER}" +# The operator image should -in general- be pulled from the user repository +# In certain case, the /dev repo can be used as well (BASE_REPO_URL_SHARED) +export OPERATOR_REGISTRY="${BASE_REPO_URL}" + +# Set to true if running operator locally and not in a pod +# Allowed values: +# - true +# - false +# Sensible default: +# - false +export LOCAL_OPERATOR="false" + +# Set this to "local" if you wish to start the operator on some port +# other that 8080. This might be needed is use a tool like kubefwd which binds +# to port 8080 +export OPERATOR_ENV="dev" + +# Set to true for running cluster-wide operator +# Allowed values: +# - true +# - false +# Sensible default +# - false +export OPERATOR_CLUSTER_SCOPED="false" + +# Type of environment +# Allowed values: +# kops - for kops cluster +# openshift - for openshift cluster +# kind - for local/evg kind clusters +# Sensible default +# - kind +export CLUSTER_TYPE=${CLUSTER_TYPE-"###undefined-CLUSTER_TYPE"} + +# The main cluster name for setting the kubectl context +# Allowed values: +# kind-kind - for single cluster kind +# kind-e2e-operator - for multicluster +# Sensible default +# - kind-e2e-operator - when you're using EVG Host +# - kind-kind - otherwise +export CLUSTER_NAME="kind" + +# Your AWS credentials. Talk to your managed if you don't have them. +export AWS_ACCESS_KEY_ID="undefined" +export AWS_SECRET_ACCESS_KEY="undefined" +export AWS_DEFAULT_REGION="eu-central-1" + +# Ops Manager settings. Typically they need to be filled with your credentials from Cloud QA. +# Make sure your organization type is "Cloud Manager" and not Atlas, and that billing is setup with the fake credit card. +# OM_USER should be filled with your public API Key and not your username +export OM_USER="" +export OM_API_KEY="" +export OM_ORGID="" + + +# The settings below are used by teardown.sh and setup_cloud_qa.py. +# Typically, in a local environment they are empty but with provided default settings +# they can be used in your local dev as well. +# ENV_FILE and NAMESPACE_FILE might be used to emulate running tests the same way as EVG. +# Allowed values: +# Cloud QA credentials +# E2e Cloud QA credentials (taken from EVG) +# Sensible default +# As is +export e2e_cloud_qa_orgid_owner="${OM_ORGID}" +export e2e_cloud_qa_apikey_owner="${OM_API_KEY}" +export e2e_cloud_qa_user_owner="${OM_USER}" +export e2e_cloud_qa_orgid_owner_static_2="${OM_ORGID}" +export e2e_cloud_qa_apikey_owner_static_2="${OM_API_KEY}" +export e2e_cloud_qa_user_owner_static_2="${OM_USER}" diff --git a/scripts/dev/contexts/private_gke_code_snippets b/scripts/dev/contexts/private_gke_code_snippets new file mode 100644 index 000000000..50bc169ce --- /dev/null +++ b/scripts/dev/contexts/private_gke_code_snippets @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# this context file is for code snippets running on GKE clusters +set -Eeou pipefail + +# overrides of public env_variables.sh +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om80" + +export MDB_GKE_PROJECT="scratch-kubernetes-team" +export K8S_CLUSTER_SUFFIX="-${version_id}-${RANDOM}" + +# we reset evg host to use a default ~/.kube/config for GKE instead of the one from evg host +export EVG_HOST_NAME="" + +source scripts/funcs/operator_deployment +# ENV_VARIABLES.SH overrides +OPERATOR_ADDITIONAL_HELM_VALUES="$(get_operator_helm_values | tr ' ' ','),customEnvVars=OM_DEBUG_HTTP=true" +export OPERATOR_ADDITIONAL_HELM_VALUES +export OPERATOR_HELM_CHART="${PROJECT_DIR}/helm_chart" diff --git a/scripts/dev/contexts/public_gke_code_snippets b/scripts/dev/contexts/public_gke_code_snippets new file mode 100644 index 000000000..8f93b699b --- /dev/null +++ b/scripts/dev/contexts/public_gke_code_snippets @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# this context file is for code snippets running on GKE clusters +set -Eeou pipefail + +# overrides of public env_variables.sh +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export MDB_GKE_PROJECT="scratch-kubernetes-team" +export K8S_CLUSTER_SUFFIX="-${version_id}-${RANDOM}" + +# we reset evg host to use a default ~/.kube/config for GKE instead of the one from evg host +export EVG_HOST_NAME="" + +# ENV_VARIABLES.SH overrides +export OPERATOR_ADDITIONAL_HELM_VALUES="" +export OPERATOR_HELM_CHART="" +export COMMIT_OUTPUT=true diff --git a/scripts/dev/contexts/publish_om60_images b/scripts/dev/contexts/publish_om60_images new file mode 100644 index 000000000..56226987c --- /dev/null +++ b/scripts/dev/contexts/publish_om60_images @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om60_image" + +export preflight_submit=true diff --git a/scripts/dev/contexts/publish_om70_images b/scripts/dev/contexts/publish_om70_images new file mode 100644 index 000000000..741902b05 --- /dev/null +++ b/scripts/dev/contexts/publish_om70_images @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om70_image" + +export preflight_submit=true diff --git a/scripts/dev/contexts/publish_om80_images b/scripts/dev/contexts/publish_om80_images new file mode 100644 index 000000000..041661c7f --- /dev/null +++ b/scripts/dev/contexts/publish_om80_images @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" +source "${script_dir}/variables/om80_image" + +export preflight_submit=true diff --git a/scripts/dev/contexts/release_agent b/scripts/dev/contexts/release_agent new file mode 100644 index 000000000..865fd1732 --- /dev/null +++ b/scripts/dev/contexts/release_agent @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +# This will be removed once the main branch of the Static Containers will be merged. +source "${script_dir}/root-context" + +export preflight_submit=false +export include_tags=release diff --git a/scripts/dev/contexts/release_images b/scripts/dev/contexts/release_images new file mode 100644 index 000000000..55d5926cb --- /dev/null +++ b/scripts/dev/contexts/release_images @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +source "${script_dir}/root-context" + +export include_tags=release diff --git a/scripts/dev/contexts/root-context b/scripts/dev/contexts/root-context new file mode 100644 index 000000000..1475bc1a8 --- /dev/null +++ b/scripts/dev/contexts/root-context @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") +source "${script_dir}/private-context" + +export PROJECT_DIR="${PWD}" +export IMAGE_TYPE=ubi +export UBI_IMAGE_WITHOUT_SUFFIX=true +export WATCH_NAMESPACE=${WATCH_NAMESPACE:-${NAMESPACE}} + +# +# changing variables below should not be necessary +# + +# these are fixed when using scripts/dev/recreate_kind_clusters.sh +export TEST_POD_CLUSTER="${CLUSTER_NAME}" + +export CENTRAL_CLUSTER="${CLUSTER_NAME}" +export MULTI_CLUSTER_CREATE_SERVICE_ACCOUNT_TOKEN_SECRETS=true +export MULTI_CLUSTER_CONFIG_DIR=${PROJECT_DIR}/.multi_cluster_local_test_files +export MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH=${PROJECT_DIR}/docker/mongodb-enterprise-tests/multi-cluster-kube-config-creator + +# override for /etc/config/kubeconfig file mounted in operator's pod +if [[ "${LOCAL_OPERATOR}" == "true" ]]; then + # This env var is used by the local operator to load multi-cluster kubeconfig. + # Normally, the pod is loading it from a path that is mounted from the mongodb-enterprise-operator-multi-cluster-kubeconfig secret. + # When running locally, we use cli tool to generate that kubeconfig secret and write it to this file. + # This way, the local operator is using kubeconfig create by cli tool the same way as it's used when running in a pod. + export KUBE_CONFIG_PATH=~/.operator-dev/multicluster_kubeconfig + export PERFORM_FAILOVER=false +fi + +export OPERATOR_ENV=${OPERATOR_ENV:-"dev"} + +AGENT_VERSION="$(jq -r '.agentVersion' release.json)" +export AGENT_VERSION +export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" + +# Ops Manager +export OPS_MANAGER_NAMESPACE="operator-testing-50-current" + +# Moved from the old set_env_context.sh +export LOCAL_RUN=true + +# version_id is similar to version_id from Evergreen. Used to differentiate different builds. Can be constant +# for local run +version_id=${version_id:-"latest"} +if [[ "${OVERRIDE_VERSION_ID:-}" != "" ]]; then + version_id="${OVERRIDE_VERSION_ID}" +fi +export version_id +export VERSION_ID="${version_id}" + +export INIT_APPDB_VERSION="${VERSION_ID}" +export INIT_DATABASE_VERSION="${VERSION_ID}" +export INIT_OPS_MANAGER_VERSION="${VERSION_ID}" +export DATABASE_VERSION="${VERSION_ID}" + +export KUBE_ENVIRONMENT_NAME=kind + +# when using EVG ec2 instance, we copy kubeconfig locally and use it +if [[ "${EVG_HOST_NAME:-}" != "" ]]; then + KUBECONFIG=~/.operator-dev/evg-host.kubeconfig +else + KUBECONFIG=~/.kube/config +fi +export KUBECONFIG + +if [[ "$(uname)" == "Linux" ]]; then + export GOROOT=/opt/golang/go1.24 +fi + +export MONGODB_REPO_URL="quay.io/mongodb" + +export SIGNING_PUBLIC_KEY_URL="https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem" + +export CLUSTER_DOMAIN="cluster.local" + +export MDB_MAX_CONCURRENT_RECONCILES=10 +export MDB_IMAGE_TYPE="ubi9" + +# leaving them empty for now +export OM_HOST=https://cloud-qa.mongodb.com +export OM_BASE_URL=https://cloud-qa.mongodb.com + +export e2e_cloud_qa_baseurl="${OM_HOST}" +export e2e_cloud_qa_baseurl_static_2="${OM_HOST}" + +export OLM_VERSION=v0.31.0 + +# Python version we use locally and in CI +export PYTHON_VERSION=3.13 diff --git a/scripts/dev/contexts/variables/om60 b/scripts/dev/contexts/variables/om60 new file mode 100644 index 000000000..89736cb37 --- /dev/null +++ b/scripts/dev/contexts/variables/om60 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +CUSTOM_OM_PREV_VERSION=6.0.0 +export CUSTOM_OM_PREV_VERSION +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_60_latest\s+(\S+)\s+#" <"${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=6.0.16 +export CUSTOM_MDB_PREV_VERSION=5.0.7 + +export AGENT_VERSION=12.0.33.7866-1 +export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" + +export CUSTOM_APPDB_VERSION=6.0.5-ent +export TEST_MODE=opsmanager +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +export APPDB_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev diff --git a/scripts/dev/contexts/variables/om60_image b/scripts/dev/contexts/variables/om60_image new file mode 100644 index 000000000..1c4ee1ae6 --- /dev/null +++ b/scripts/dev/contexts/variables/om60_image @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +om_version=$(grep -E "^\s*-\s*&ops_manager_60_latest\s+(\S+)\s+#" < "${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export om_version diff --git a/scripts/dev/contexts/variables/om70 b/scripts/dev/contexts/variables/om70 new file mode 100644 index 000000000..b986248ff --- /dev/null +++ b/scripts/dev/contexts/variables/om70 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +CUSTOM_OM_PREV_VERSION=$(grep -E "^\s*-\s*&ops_manager_60_latest\s+(\S+)\s+#" <"${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_PREV_VERSION +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" <"${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=7.0.2 +export CUSTOM_MDB_PREV_VERSION=6.0.5 + +export AGENT_VERSION=107.0.11.8645-1 +export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" + +export CUSTOM_APPDB_VERSION=7.0.2-ent +export TEST_MODE=opsmanager +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +export APPDB_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev diff --git a/scripts/dev/contexts/variables/om70_image b/scripts/dev/contexts/variables/om70_image new file mode 100644 index 000000000..108fe158e --- /dev/null +++ b/scripts/dev/contexts/variables/om70_image @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +om_version=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" < "${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export om_version diff --git a/scripts/dev/contexts/variables/om80 b/scripts/dev/contexts/variables/om80 new file mode 100644 index 000000000..d6f573ae8 --- /dev/null +++ b/scripts/dev/contexts/variables/om80 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +CUSTOM_OM_PREV_VERSION=$(grep -E "^\s*-\s*&ops_manager_70_latest\s+(\S+)\s+#" < "${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_PREV_VERSION +CUSTOM_OM_VERSION=$(grep -E "^\s*-\s*&ops_manager_80_latest\s+(\S+)\s+#" < "${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export CUSTOM_OM_VERSION + +export CUSTOM_MDB_VERSION=8.0.0 +export CUSTOM_MDB_PREV_VERSION=7.0.9 + +export AGENT_VERSION=108.0.0.8694-1 +export AGENT_IMAGE="${MDB_AGENT_IMAGE_REPOSITORY}:${AGENT_VERSION}" + +export CUSTOM_APPDB_VERSION=8.0.0-ent +export TEST_MODE=opsmanager +export OPS_MANAGER_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev +export APPDB_REGISTRY=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev diff --git a/scripts/dev/contexts/variables/om80_image b/scripts/dev/contexts/variables/om80_image new file mode 100644 index 000000000..7821d0b01 --- /dev/null +++ b/scripts/dev/contexts/variables/om80_image @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") + +om_version=$(grep -E "^\s*-\s*&ops_manager_80_latest\s+(\S+)\s+#" < "${script_dir}"/../../../../.evergreen.yml | awk '{print $3}') +export om_version + +export om_download_url diff --git a/scripts/dev/delete_om_projects.sh b/scripts/dev/delete_om_projects.sh new file mode 100755 index 000000000..391d90f0e --- /dev/null +++ b/scripts/dev/delete_om_projects.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +### +# This is a cleanup script for preparing cloud-qa to e2e run. +# It deletes all projects that has been created in previous runs. +### + +set -euo pipefail + +source scripts/dev/set_env_context.sh + +delete_project() { + project_name=$1 + echo "Deleting project id of ${project_name} from ${OM_HOST}" + project_id=$(curl -s -u "${OM_USER}:${OM_API_KEY}" --digest "${OM_HOST}/api/public/v1.0/groups/byName/${project_name}" | jq -r .id) + if [[ "${project_id}" != "" && "${project_id}" != "null" ]]; then + echo "Removing controlledFeature policies for project ${project_name} (${project_id})" + curl -X PUT --digest -u "${OM_USER}:${OM_API_KEY}" "${OM_HOST}/api/public/v1.0/groups/${project_id}/controlledFeature" -H 'Content-Type: application/json' -d '{"externalManagementSystem": {"name": "mongodb-enterprise-operator"},"policies": []}' + echo + echo "Removing any existing automationConfig for project ${project_name} (${project_id})" + curl -X PUT --digest -u "${OM_USER}:${OM_API_KEY}" "${OM_HOST}/api/public/v1.0/groups/${project_id}/automationConfig" -H 'Content-Type: application/json' -d '{}' + echo + echo "Deleting project ${project_name} (${project_id})" + curl -X DELETE --digest -u "${OM_USER}:${OM_API_KEY}" "${OM_HOST}/api/public/v1.0/groups/${project_id}" + echo + else + echo "Project ${project_name} is already deleted" + fi +} + +delete_project "${NAMESPACE}" + +if [[ "${WATCH_NAMESPACE:-}" != "" && "${WATCH_NAMESPACE:-}" != "*" ]]; then + for ns in ${WATCH_NAMESPACE/,// }; do + delete_project "${ns}" || true + done +fi + + diff --git a/scripts/dev/evg_host.sh b/scripts/dev/evg_host.sh new file mode 100755 index 000000000..89e124a10 --- /dev/null +++ b/scripts/dev/evg_host.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +# This is a helper script for running tests on Evergreen Hosts. +# It allows to configure kind clusters and expose remote API servers on a local machine to +# enable local development while running Kind cluster on EC2 instance. +# Run "evg_host.sh help" command to see the full usage. +# See docs/dev/local_e2e_testing.md for a tutorial how to use it. + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +if [[ -z "${EVG_HOST_NAME}" ]]; then + echo "EVG_HOST_NAME env var is missing" + exit 1 +fi + +get_host_url() { + host=$(evergreen host list --json | jq -r ".[] | select (.name==\"${EVG_HOST_NAME}\") | .host_name ") + if [[ "${host}" == "" ]]; then + >&2 echo "Cannot find running EVG host with name ${EVG_HOST_NAME}. +Run evergreen host list --json or visit https://spruce.mongodb.com/spawn/host." + exit 1 + fi + echo "ubuntu@${host}" +} + +cmd=${1-""} + +if [[ "${cmd}" != "" && "${cmd}" != "help" ]]; then + host_url=$(get_host_url) +fi + +kubeconfig_path="${HOME}/.operator-dev/evg-host.kubeconfig" + +configure() { + shift 1 + arch=${1-"amd64"} + + echo "Configuring EVG host ${EVG_HOST_NAME} (${host_url}) with architecture ${arch}" + + if [[ "${cmd}" == "configure" && "${arch}" != "amd64" && "${arch}" != "arm64" ]]; then + echo "'configure' command supports the following architectures: 'amd64', 'arm64'" + exit 1 + fi + + ssh -T -q "${host_url}" "sudo chown ubuntu:ubuntu ~/.docker || true; mkdir -p ~/.docker" + if [[ -f "${HOME}/.docker/config.json" ]]; then + echo "Copying local ~/.docker/config.json authorization credentials to EVG host" + jq '. | with_entries(select(.key == "auths"))' "${HOME}/.docker/config.json" | ssh -T -q "${host_url}" 'cat > /home/ubuntu/.docker/config.json' + fi + + sync + + ssh -T -q "${host_url}" "cd ~/ops-manager-kubernetes; scripts/dev/switch_context.sh root-context; scripts/dev/setup_evg_host.sh ${arch}" +} + +sync() { + rsync --verbose --archive --compress --human-readable --recursive --progress \ + --delete --delete-excluded --max-size=1000000 --prune-empty-dirs \ + -e ssh \ + --include-from=.rsyncinclude \ + --exclude-from=.gitignore \ + --exclude-from=.rsyncignore \ + ./ "${host_url}:/home/ubuntu/ops-manager-kubernetes/" + + rsync --verbose --no-links --recursive --prune-empty-dirs --archive --compress --human-readable \ + --max-size=1000000 \ + -e ssh \ + ~/.operator-dev/ \ + "${host_url}:/home/ubuntu/.operator-dev" & + + wait +} + +remote-prepare-local-e2e-run() { + set -x + sync + cmd make switch context=appdb-multi + cmd scripts/dev/prepare_local_e2e_run.sh + rsync --verbose --no-links --recursive --prune-empty-dirs --archive --compress --human-readable \ + --max-size=1000000 \ + -e ssh \ + "${host_url}:/home/ubuntu/ops-manager-kubernetes/.multi_cluster_local_test_files" \ + ./ & + scp "${host_url}:/home/ubuntu/.operator-dev/multicluster_kubeconfig" "${KUBE_CONFIG_PATH}" & + + wait +} + +get-kubeconfig() { + remote_path="${host_url}:/home/ubuntu/.operator-dev/evg-host.kubeconfig" + echo "Copying remote kubeconfig from ${remote_path} to ${kubeconfig_path}" + scp "${remote_path}" "${kubeconfig_path}" +} + +recreate-kind-clusters() { + echo "Recreating kind clusters on ${EVG_HOST_NAME} (${host_url})..." + # shellcheck disable=SC2088 + ssh -T "${host_url}" "cd ~/ops-manager-kubernetes; scripts/dev/recreate_kind_clusters.sh" + echo "Copying kubeconfig to ${kubeconfig_path}" + get-kubeconfig +} + +recreate-kind-cluster() { + shift 1 + cluster_name=$1 + echo "Recreating kind cluster ${cluster_name} on ${EVG_HOST_NAME} (${host_url})..." + # shellcheck disable=SC2088 + ssh -T "${host_url}" "cd ~/ops-manager-kubernetes; scripts/dev/recreate_kind_cluster.sh ${cluster_name}" + echo "Copying kubeconfig to ${kubeconfig_path}" + get-kubeconfig +} + +tunnel() { + shift 1 + # shellcheck disable=SC2016 + api_servers=$(yq '.contexts[].context.cluster as $cluster | .clusters[] | select(.name == $cluster).cluster.server' < "${kubeconfig_path}" | sed 's/https:\/\///g') + echo "Extracted the following API server urls from ${kubeconfig_path}: ${api_servers}" + port_forwards=() + for api_server in ${api_servers}; do + host=$(echo "${api_server}" | cut -d ':' -f1) + port=$(echo "${api_server}" | cut -d ':' -f2) + if [[ "${port}" == "${host}" ]]; then + port="443" + fi + port_forwards+=("-L" "${port}:${host}:${port}") + done + + set -x + # shellcheck disable=SC2029 + ssh "${port_forwards[@]}" "${host_url}" "$@" + set +x +} + +ssh_to_host() { + shift 1 + # shellcheck disable=SC2029 + ssh "$@" "${host_url}" +} + +upload-my-ssh-private-key() { + ssh -T -q "${host_url}" "mkdir -p ~/.ssh" + scp "${HOME}/.ssh/id_rsa" "${host_url}:/home/ubuntu/.ssh/id_rsa" + scp "${HOME}/.ssh/id_rsa.pub" "${host_url}:/home/ubuntu/.ssh/id_rsa.pub" + ssh -T -q "${host_url}" "chmod 700 ~/.ssh && chown -R ubuntu:ubuntu ~/.ssh" +} + +cmd() { + if [[ "$1" == "cmd" ]]; then + shift 1 + fi + + cmd="cd ~/ops-manager-kubernetes; $*" + ssh -T -q "${host_url}" "${cmd}" +} + +usage() { + echo "USAGE: + evg_host.sh + +PREREQUISITES: + - create EVG host: https://spruce.mongodb.com/spawn/host + - install evergreen cli and set api-key in ~/.evergreen.yml: https://spruce.mongodb.com/preferences/cli + - define in context EVG_HOST_NAME + - VPN connection + +COMMANDS: + configure installs on a host: calls sync, switches context, installs necessary software + sync rsync of project directory + recreate-kind-clusters executes scripts/dev/recreate_kind_clusters.sh and executes get-kubeconfig + recreate-kind-cluster test-cluster executes scripts/dev/recreate_kind_cluster.sh test-cluster and executes get-kubeconfig + get-kubeconfig copies remote kubeconfig locally to ~/.operator-dev/evg-host.kubeconfig + tunnel [args] creates ssh session with tunneling to all API servers + ssh [args] creates ssh session passing optional arguments to ssh + cmd [command with args] execute command as if being on evg host + upload-my-ssh-private-key uploads your ssh keys (~/.ssh/id_rsa) to evergreen host + help this message +" +} + +case ${cmd} in +configure) configure "$@" ;; +recreate-kind-clusters) recreate-kind-clusters ;; +recreate-kind-cluster) recreate-kind-cluster "$@" ;; +get-kubeconfig) get-kubeconfig ;; +remote-prepare-local-e2e-run) remote-prepare-local-e2e-run ;; +ssh) ssh_to_host "$@" ;; +tunnel) tunnel "$@" ;; +sync) sync ;; +cmd) cmd "$@" ;; +upload-my-ssh-private-key) upload-my-ssh-private-key ;; +help) usage ;; +*) usage ;; +esac diff --git a/scripts/dev/install.sh b/scripts/dev/install.sh new file mode 100755 index 000000000..33d64a9c4 --- /dev/null +++ b/scripts/dev/install.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/funcs/printing +source scripts/dev/set_env_context.sh + +tools="kubectl helm coreutils kind jq shellcheck python@${PYTHON_VERSION}" +echo "The following tools will be installed using homebrew: ${tools}" +echo "Note, that you must download 'go' and Docker by yourself" + +grep -a "/usr/local/opt/coreutils/libexec/gnubin:\$PATH" ~/.bashrc || echo "PATH=\"/usr/local/opt/coreutils/libexec/gnubin:\$PATH\"" >> ~/.bashrc + +if [ "$(uname)" = "Darwin" ] ; then + # shellcheck disable=SC2086 + brew install ${tools} 2>&1 | prepend "brew install" +elif [ "$(uname)" = "Linux" ] ; then # Ubuntu only + sudo snap install kubectl --classic || true + + kops_version="$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)" + curl -Lo kops "https://github.com/kubernetes/kops/releases/download/${kops_version}/kops-linux-amd64" + echo "hi" + chmod +x kops + mv kops "${GOBIN}" || true + + sudo snap install helm --classic || true + + # Kind + go install sigs.k8s.io/kind + + sudo snap install --channel=edge shellcheck + +else + echo "This only works on OSX & Ubuntu - please install the tools yourself. Sorry!" + exit 1 +fi + +echo "Installing Python packages" +scripts/dev/recreate_python_venv.sh 2>&1 | prepend "recreate_python_venv.sh" + +title "Tools are installed" diff --git a/scripts/dev/install_csi_driver.sh b/scripts/dev/install_csi_driver.sh new file mode 100755 index 000000000..f46793868 --- /dev/null +++ b/scripts/dev/install_csi_driver.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -eux + +# Path to the deploy script +DEPLOY_SCRIPT_PATH="./deploy/kubernetes-latest/deploy.sh" + +# Function to deploy to a single cluster +deploy_to_cluster() { + local context="$1" + echo "Switching to context: ${context}" + + # Set the current context to the target cluster + if ! kubectl config use-context "${context}";then + echo "Failed to switch to context: ${context}" + fi + + echo "install resizable csi" + # Define variables + REPO_URL="https://github.com/kubernetes-csi/csi-driver-host-path/archive/refs/tags/v1.14.1.tar.gz" + TAR_FILE="csi-driver-host-path-v1.14.1.tar.gz" + EXTRACTED_DIR="csi-driver-host-path-1.14.1" + + # Download the tar.gz file + echo "Downloading ${REPO_URL}..." + curl -L -o "${TAR_FILE}" "${REPO_URL}" + + # Extract the tar.gz file + echo "Extracting ${TAR_FILE}..." + tar -xzf "${TAR_FILE}" + + # Navigate to the extracted directory + cd "${EXTRACTED_DIR}" + + # Change to the latest supported snapshotter release branch + SNAPSHOTTER_BRANCH=release-6.3 + + # Apply VolumeSnapshot CRDs + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_BRANCH}/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_BRANCH}/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_BRANCH}/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml + + # Change to the latest supported snapshotter version + SNAPSHOTTER_VERSION=v6.3.3 + + # Create snapshot controller + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + + # Run the deploy script + echo "Running deploy script on ${context}" + if ! bash "${DEPLOY_SCRIPT_PATH}"; then + echo "Failed to run deploy script on ${context}" + return 1 + fi + + echo "Installing csi storageClass" + kubectl apply -f ./examples/csi-storageclass.yaml + + echo "Deployment successful on ${context}" + return 0 +} + +CLUSTER_CONTEXTS=() + +# Set default values for the context variables if they are not set +CTX_CLUSTER="${CTX_CLUSTER:-}" +CTX_CLUSTER1="${CTX_CLUSTER1:-}" +CTX_CLUSTER2="${CTX_CLUSTER2:-}" +CTX_CLUSTER3="${CTX_CLUSTER3:-}" + +# Add to CLUSTER_CONTEXTS only if the environment variable is set and not empty +[[ -n "${CTX_CLUSTER}" ]] && CLUSTER_CONTEXTS+=("${CTX_CLUSTER}") +[[ -n "${CTX_CLUSTER1}" ]] && CLUSTER_CONTEXTS+=("${CTX_CLUSTER1}") +[[ -n "${CTX_CLUSTER2}" ]] && CLUSTER_CONTEXTS+=("${CTX_CLUSTER2}") +[[ -n "${CTX_CLUSTER3}" ]] && CLUSTER_CONTEXTS+=("${CTX_CLUSTER3}") + +# Main deployment loop +for context in "${CLUSTER_CONTEXTS[@]}"; do + deploy_to_cluster "${context}" +done + +echo "Deployment completed for all clusters." diff --git a/scripts/dev/interconnect_kind_clusters.sh b/scripts/dev/interconnect_kind_clusters.sh new file mode 100755 index 000000000..5283f5a89 --- /dev/null +++ b/scripts/dev/interconnect_kind_clusters.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +#### +## This script is based on https://gist.github.com/aojea/00bca6390f5f67c0a30db6acacf3ea91#multiple-clusters +#### + +function usage() { + echo "Interconnects Pod and Service networks between multiple Kind clusters. + +Usage: + interconnect_kind_clusters.sh + interconnect_kind_clusters.sh [-hv] + +Options: + -h (optional) Shows this screen. + -v (optional) Verbose mode. +" + exit 0 +} + +verbose=0 +while getopts ':h:v' opt; do + case ${opt} in + (v) verbose=1;; + (h) usage;; + (*) usage;; + esac +done +shift "$((OPTIND-1))" + +clusters=("$@") +echo "Interconnecting ${clusters[*]}" + +routes=() +kind_nodes_for_exec=() +for c in "${clusters[@]}"; do + # We need to explicitly ensure we're in a steady state. Kind often reports done while the API Server hasn't assigned Pod CIDRs yet + kubectl --context "kind-${c}" wait nodes --all --for=condition=ready > /dev/null + + pod_route=$(kubectl --context "kind-${c}" get nodes -o=jsonpath='{range .items[*]}{"ip route add "}{.spec.podCIDR}{" via "}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}') + # Is there a better way to do it? + service_cidr=$(kubectl --context "kind-${c}" --namespace kube-system get configmap kubeadm-config -o=jsonpath='{.data.ClusterConfiguration}' | grep "serviceSubnet" | cut -d\ -f4) + service_route=$(kubectl --context "kind-${c}" get nodes -o=jsonpath='{range .items[*]}{"ip route add "}'"${service_cidr}"'{" via "}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}') + # shellcheck disable=SC2086 + kind_node_in_docker=$(kind get nodes --name ${c}) + + if [ "${verbose}" -ne "0" ]; then + echo "[${c}] [${kind_node_in_docker}] Pod Route: ${pod_route}" + echo "[${c}] [${kind_node_in_docker}] Service Route: ${service_route}" + fi + + + routes+=("${pod_route}") + routes+=("${service_route}") + kind_nodes_for_exec+=("${kind_node_in_docker}") +done + +if [ "${verbose}" -ne "0" ]; then + echo "Injecting routes into the following Docker containers: ${clusters[*]}" + echo "Gathered the following Routes to inject:" + IFS=$'\n' eval 'echo "${routes[*]}"' +fi + +for c in "${kind_nodes_for_exec[@]}"; do + for r in "${routes[@]}"; do + error_code=0 + # shellcheck disable=SC2086 + docker exec ${c} ${r} || error_code=$? + if [ "${error_code}" -ne "0" ] && [ "${error_code}" -ne "2" ]; then + echo "Error while interconnecting Kind clusters. Try debugging it manually by calling:" + echo "docker exec ${c} ${r}" + exit 1 + fi + done +done diff --git a/scripts/dev/kind_clusters_check_interconnect.sh b/scripts/dev/kind_clusters_check_interconnect.sh new file mode 100755 index 000000000..778748fd1 --- /dev/null +++ b/scripts/dev/kind_clusters_check_interconnect.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash + +set -euo pipefail +source scripts/dev/set_env_context.sh + +function usage() { + echo "This scripts has been designed to work in conjunction with recreate_kind_clusters.sh and verifies if inter-cluster connectivity works fine. + +Usage: + kind_clusters_check_interconnect.sh [-h] [-r] + +Options: + -h (optional) Shows this screen. + -r (optional) Recreates namespaces before testing. Useful for iterative testing with cleanup. + -u (optional) Prevents undeploying services. Useful for iterative testing. +" + exit 0 +} + +install_echo() { + ctx=$1 + cluster_no=$2 + ns=$3 + recreate=$4 + + if [[ "${recreate}" == "true" ]]; then + kubectl --context "${ctx}" delete namespace "${ns}" --wait + fi + + kubectl --context "${ctx}" create namespace "${ns}" || true + kubectl --context "${ctx}" label namespace "${ns}" istio-injection=enabled || true + + kubectl apply --context "${ctx}" -n "${ns}" -f - </dev/null || true + kubectl config use-context "${current_context}" + echo "Current context: ${current_context}, namespace=${NAMESPACE}" + kubectl get nodes | grep "control-plane" +fi + +echo "Ensuring namespace ${NAMESPACE}" +ensure_namespace "${NAMESPACE}" + +echo "Deleting ~/.docker/.config.json and re-creating it" +rm ~/.docker/config.json || true +scripts/dev/configure_docker_auth.sh + +echo "Configuring operator" +scripts/evergreen/e2e/configure_operator.sh 2>&1 | prepend "configure_operator: " + +echo "Preparing operator config map" +prepare_operator_config_map "$(kubectl config current-context)" 2>&1 | prepend "prepare_operator_config_map: " + +rm -rf docker/mongodb-enterprise-tests/helm_chart +cp -rf helm_chart docker/mongodb-enterprise-tests/helm_chart + +# shellcheck disable=SC2154 +if [[ "${KUBE_ENVIRONMENT_NAME}" == "multi" ]]; then + prepare_multi_cluster_e2e_run 2>&1 | prepend "prepare_multi_cluster_e2e_run" + run_multi_cluster_kube_config_creator 2>&1 | prepend "run_multi_cluster_kube_config_creator" +fi + +make install 2>&1 | prepend "make install: " +test -f "docker/mongodb-enterprise-tests/.test_identifiers" && rm "docker/mongodb-enterprise-tests/.test_identifiers" +scripts/dev/delete_om_projects.sh + +if [[ "${DEPLOY_OPERATOR:-"false"}" == "true" ]]; then + echo "installing operator helm chart to create the necessary sa and roles" + # shellcheck disable=SC2178 + helm_values=$(get_operator_helm_values) + # shellcheck disable=SC2179 + if [[ "${LOCAL_OPERATOR}" == true ]]; then + helm_values+=" operator.replicas=0" + fi + + # shellcheck disable=SC2128 + helm upgrade --install mongodb-enterprise-operator helm_chart --set "$(echo "${helm_values}" | tr ' ' ',')" +fi + +if [[ "${KUBE_ENVIRONMENT_NAME}" == "kind" ]]; then + echo "patching all default sa with imagePullSecrets to ensure we can deploy without setting it for each pod" + + service_accounts=$(kubectl get serviceaccounts -n "${NAMESPACE}" -o jsonpath='{.items[*].metadata.name}') + + for service_account in ${service_accounts}; do + kubectl patch serviceaccount "${service_account}" -n "${NAMESPACE}" -p "{\"imagePullSecrets\": [{\"name\": \"image-registries-secret\"}]}" + done +fi diff --git a/scripts/dev/print_automation_config.sh b/scripts/dev/print_automation_config.sh new file mode 100755 index 000000000..67e6815f9 --- /dev/null +++ b/scripts/dev/print_automation_config.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -Eeou pipefail +source scripts/dev/set_env_context.sh + +# shellcheck disable=SC1090 +source ~/.operator-dev/om +# shellcheck disable=SC1090 +source ~/.operator-dev/context + +file_name="$(mktemp).json" +group_id=$(curl -u "${OM_USER}:${OM_API_KEY}" "${OM_HOST}/api/public/v1.0/groups/byName/${NAMESPACE}" --digest -sS | jq -r .id) +curl -u "${OM_USER}:${OM_API_KEY}" "${OM_HOST}/api/public/v1.0/groups/${group_id}/automationConfig" --digest -sS | jq 'del(.mongoDbVersions)' > "${file_name}" +${EDITOR:-vi} "${file_name}" diff --git a/scripts/dev/print_operator_env.sh b/scripts/dev/print_operator_env.sh new file mode 100755 index 000000000..84fc95f9a --- /dev/null +++ b/scripts/dev/print_operator_env.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# NOTE: these are the env vars which are required to run the operator, either via a pod or locally +# This is not used by E2E evergreen, only when running tests "locally" + +UBI_IMAGE_SUFFIX="-ubi" + +# Convert context variables to variables required by the operator binary +function print_operator_env() { + echo "OPERATOR_ENV=\"${OPERATOR_ENV}\" +WATCH_NAMESPACE=\"${WATCH_NAMESPACE}\" +NAMESPACE=\"${NAMESPACE}\" +IMAGE_PULL_POLICY=\"Always\" +MONGODB_ENTERPRISE_DATABASE_IMAGE=\"${MONGODB_ENTERPRISE_DATABASE_IMAGE:-${DATABASE_REGISTRY}/mongodb-enterprise-database${UBI_IMAGE_SUFFIX}}\" +INIT_DATABASE_IMAGE_REPOSITORY=\"${INIT_DATABASE_REGISTRY}/mongodb-enterprise-init-database${UBI_IMAGE_SUFFIX}\" +INIT_DATABASE_VERSION=\"${INIT_DATABASE_VERSION}\" +DATABASE_VERSION=\"${DATABASE_VERSION}\" +OPS_MANAGER_IMAGE_REPOSITORY=\"${OPS_MANAGER_REGISTRY}/mongodb-enterprise-ops-manager${UBI_IMAGE_SUFFIX}\" +INIT_OPS_MANAGER_IMAGE_REPOSITORY=\"${INIT_OPS_MANAGER_REGISTRY}/mongodb-enterprise-init-ops-manager${UBI_IMAGE_SUFFIX}\" +INIT_OPS_MANAGER_VERSION=\"${INIT_OPS_MANAGER_VERSION}\" +INIT_APPDB_IMAGE_REPOSITORY=\"${INIT_APPDB_REGISTRY}/mongodb-enterprise-init-appdb${UBI_IMAGE_SUFFIX}\" +INIT_APPDB_VERSION=\"${INIT_APPDB_VERSION}\" +OPS_MANAGER_IMAGE_PULL_POLICY=\"Always\" +MONGODB_IMAGE=\"mongodb-enterprise-server\" +MONGODB_AGENT_VERSION=\"${MONGODB_AGENT_VERSION:-}\" +MONGODB_REPO_URL=\"${MONGODB_REPO_URL:-}\" +IMAGE_PULL_SECRETS=\"image-registries-secret\" +MDB_DEFAULT_ARCHITECTURE=\"${MDB_DEFAULT_ARCHITECTURE:-non-static}\" +MDB_IMAGE_TYPE=\"${MDB_IMAGE_TYPE:-ubi8}\" +MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY=\"${MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY:-1m}\" +MDB_OPERATOR_TELEMETRY_SEND_ENABLED=\"${MDB_OPERATOR_TELEMETRY_SEND_ENABLED:-false}\" +" + +if [[ "${AGENT_IMAGE:-}" != "" ]]; then + echo "AGENT_IMAGE=${AGENT_IMAGE}" +else + echo "AGENT_IMAGE=\"quay.io/mongodb/mongodb-agent${UBI_IMAGE_SUFFIX}:${AGENT_VERSION:-}\"" +fi + +if [[ "${KUBECONFIG:-""}" != "" ]]; then + echo "KUBECONFIG=${KUBECONFIG}" +fi + +if [[ "${MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY:-""}" != "" ]]; then + echo "MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY=${MDB_OPERATOR_TELEMETRY_SEND_FREQUENCY}" +fi + + +if [[ "${MDB_OPERATOR_TELEMETRY_SEND_BASEURL:-""}" != "" ]]; then + echo "MDB_OPERATOR_TELEMETRY_SEND_BASEURL=${MDB_OPERATOR_TELEMETRY_SEND_BASEURL}" +fi + +if [[ "${MDB_AGENT_VERSION:-""}" != "" ]]; then + echo "MDB_AGENT_VERSION=${MDB_AGENT_VERSION}" +fi + +if [[ "${MDB_AGENT_DEBUG:-""}" != "" ]]; then + echo "MDB_AGENT_DEBUG=${MDB_AGENT_DEBUG}" +fi + +if [[ "${KUBE_CONFIG_PATH:-""}" != "" ]]; then + echo "KUBE_CONFIG_PATH=${KUBE_CONFIG_PATH}" +fi + +if [[ "${PERFORM_FAILOVER:-""}" != "" ]]; then + echo "PERFORM_FAILOVER=${PERFORM_FAILOVER}" +fi + +if [[ "${OM_DEBUG_HTTP:-""}" != "" ]]; then + echo "OM_DEBUG_HTTP=${OM_DEBUG_HTTP}" +fi + +if [[ "${OPS_MANAGER_MONITOR_APPDB:-""}" != "" ]]; then + echo "OPS_MANAGER_MONITOR_APPDB=${OPS_MANAGER_MONITOR_APPDB}" +fi + +if [[ "${OPERATOR_ENV:-""}" != "" ]]; then + echo "OPERATOR_ENV=${OPERATOR_ENV}" +fi + +if [[ "${MDB_OM_VERSION_MAPPING_PATH:-""}" != "" ]]; then + echo "MDB_OM_VERSION_MAPPING_PATH=${MDB_OM_VERSION_MAPPING_PATH}" +fi + +if [[ "${MDB_AGENT_IMAGE_REPOSITORY:-""}" != "" ]]; then + echo "MDB_AGENT_IMAGE_REPOSITORY=${MDB_AGENT_IMAGE_REPOSITORY}" +fi + +if [[ "${MDB_MAX_CONCURRENT_RECONCILES:-""}" != "" ]]; then + echo "MDB_MAX_CONCURRENT_RECONCILES=${MDB_MAX_CONCURRENT_RECONCILES}" +fi +} + +print_operator_env diff --git a/scripts/dev/recreate_e2e_kops.sh b/scripts/dev/recreate_e2e_kops.sh new file mode 100755 index 000000000..d18e00edb --- /dev/null +++ b/scripts/dev/recreate_e2e_kops.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/errors +source scripts/funcs/kubernetes +source scripts/funcs/printing + +recreate="${1-}" +CLUSTER="${2:-e2e.mongokubernetes.com}" + +title "Deleting kops cluster ${CLUSTER}" + +if [[ "${recreate}" != "yes" ]]; then + fatal "Exiting as \"imsure=yes\" parameter is not specified" +fi + +# make sure kops version is >= 1.14.0 +kops_version=$(kops version | awk '{ print $2 }') +major=$(echo "${kops_version}" | cut -d "." -f 1) +minor=$(echo "${kops_version}" | cut -d "." -f 2) +if (( major != 1 || minor < 14 )); then + fatal "kops must be of version >= 1.14.0!" +fi + + +# wait until the cluster is removed (could be removed already) +kops delete cluster "${CLUSTER}" --yes || true + +title "Cluster deleted" + +# Note, that for e2e cluster we use us-east-2 region as us-east-1 most of all has reached max number of VPCs (5) +if [[ "${CLUSTER}" = "e2e.mongokubernetes.com" ]]; then + # todo 2xlarge can be too big - this is a fix for the "2 OMs on one node" problem which should be solved by + # pod anti affinity rule + create_kops_cluster "${CLUSTER}" 4 64 "t2.2xlarge" "t2.medium" "us-east-2a,us-east-2b,us-east-2c" +elif [[ "${CLUSTER}" = "e2e.om.mongokubernetes.com" ]]; then + # this one is for Ops Manager e2e tests + create_kops_cluster "${CLUSTER}" 4 32 "t2.2xlarge" "t2.medium" "us-west-2a" +else [[ "${CLUSTER}" = "e2e.legacy.mongokubernetes.com" ]]; + # we're recreating a "legacy" cluster on K8s 1.11 to perform basic check. + # This version is used by Openshift 3.11 and allows to more or less emulate 3.11 environment + # Dev note: if you need to deploy Operator to this cluster you'll need to make two things before calling 'make' + # 1. remove "subresources" field from each CRD + # 2. remove "kubeVersion" field from Chart.yaml + # TODO Ideally we should automatically run some tests on this cluster + create_kops_cluster "${CLUSTER}" 2 16 "t2.medium" "t2.medium" "us-west-2a,us-west-2b,us-west-2c" "v1.11.10" +fi diff --git a/scripts/dev/recreate_kind_cluster.sh b/scripts/dev/recreate_kind_cluster.sh new file mode 100755 index 000000000..a0393f9b6 --- /dev/null +++ b/scripts/dev/recreate_kind_cluster.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/kubernetes + +cluster_name=$1 +if [[ -z ${cluster_name} ]]; then + echo "Usage: recreate_kind_cluster.sh " + exit 1 +fi + +if [[ "${DELETE_KIND_NETWORK:-"false"}" == "true" ]]; then + delete_kind_network +fi + +scripts/dev/setup_kind_cluster.sh -r -e -n "${cluster_name}" -l "172.18.255.200-172.18.255.250" -c "${CLUSTER_DOMAIN}" +CTX_CLUSTER1=${cluster_name}-kind scripts/dev/install_csi_driver.sh diff --git a/scripts/dev/recreate_kind_clusters.sh b/scripts/dev/recreate_kind_clusters.sh new file mode 100755 index 000000000..55622c393 --- /dev/null +++ b/scripts/dev/recreate_kind_clusters.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/kubernetes + +if [[ "${DELETE_KIND_NETWORK:-"false"}" == "true" ]]; then + delete_kind_network +fi + +if [[ "${DELETE_KIND_NETWORK:-"false"}" == "true" ]]; then + delete_kind_network +fi + +# first script prepares registry, so to avoid race it have to finish running before we execute subsequent ones in parallel +# To future maintainers: whenever modifying this bit, make sure you also update coredns.yaml +scripts/dev/setup_kind_cluster.sh -n "e2e-operator" -p "10.244.0.0/16" -s "10.96.0.0/16" -l "172.18.255.200-172.18.255.210" -c "${CLUSTER_DOMAIN}" +scripts/dev/setup_kind_cluster.sh -n "e2e-cluster-1" -p "10.245.0.0/16" -s "10.97.0.0/16" -l "172.18.255.210-172.18.255.220" -c "${CLUSTER_DOMAIN}" & +scripts/dev/setup_kind_cluster.sh -n "e2e-cluster-2" -p "10.246.0.0/16" -s "10.98.0.0/16" -l "172.18.255.220-172.18.255.230" -c "${CLUSTER_DOMAIN}" & +scripts/dev/setup_kind_cluster.sh -n "e2e-cluster-3" -p "10.247.0.0/16" -s "10.99.0.0/16" -l "172.18.255.230-172.18.255.240" -c "${CLUSTER_DOMAIN}" & + +echo "Waiting for recreate_kind_cluster.sh to complete" +wait + +# we do exports sequentially as setup_kind_cluster.sh is run in parallel and we hit kube config locks +kind export kubeconfig --name "e2e-operator" +kind export kubeconfig --name "e2e-cluster-1" +kind export kubeconfig --name "e2e-cluster-2" +kind export kubeconfig --name "e2e-cluster-3" + +echo "Interconnecting Kind clusters" +scripts/dev/interconnect_kind_clusters.sh -v e2e-cluster-1 e2e-cluster-2 e2e-cluster-3 e2e-operator + +export VERSION=${VERSION:-1.16.1} + +source multi_cluster/tools/download_istio.sh + +VERSION=1.16.1 CTX_CLUSTER1=kind-e2e-cluster-1 CTX_CLUSTER2=kind-e2e-cluster-2 CTX_CLUSTER3=kind-e2e-cluster-3 multi_cluster/tools/install_istio.sh & +VERSION=1.16.1 CTX_CLUSTER=kind-e2e-operator multi_cluster/tools/install_istio_central.sh & + +CTX_CLUSTER=kind-e2e-operator CTX_CLUSTER1=kind-e2e-cluster-1 CTX_CLUSTER2=kind-e2e-cluster-2 CTX_CLUSTER3=kind-e2e-cluster-3 scripts/dev/install_csi_driver.sh & +wait diff --git a/scripts/dev/recreate_python_venv.sh b/scripts/dev/recreate_python_venv.sh new file mode 100755 index 000000000..fb1f9ab8f --- /dev/null +++ b/scripts/dev/recreate_python_venv.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This scripts recreates local python venv in the ${PROJECT_DIR} directory from the current context. + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +if [[ -d "${PROJECT_DIR}"/venv ]]; then + echo "Removing venv..." + cd "${PROJECT_DIR}" + rm -rf "venv" +fi + +# in our EVG hosts, python versions are always in /opt/python +python_bin="/opt/python/${PYTHON_VERSION}/bin/python3" +if [[ "$(uname)" == "Darwin" ]]; then + python_bin="python${PYTHON_VERSION}" +fi + +echo "Using python from the following path: ${python_bin}" + +"${python_bin}" -m venv venv +source venv/bin/activate +pip install --upgrade pip +pip install -r requirements.txt +echo "Python venv was recreated successfully." +echo "Current python path: $(which python)" +python --version diff --git a/scripts/dev/reset.go b/scripts/dev/reset.go new file mode 100644 index 000000000..2ef6646cb --- /dev/null +++ b/scripts/dev/reset.go @@ -0,0 +1,433 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "time" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + kerrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/10gen/ops-manager-kubernetes/pkg/util/env" +) + +type DynamicResource struct { + GVR schema.GroupVersionResource + ResourceName string +} + +var ( + zero int64 = 0 + deleteOptionsNoGrace = v1.DeleteOptions{ + GracePeriodSeconds: &zero, + } +) + +// waitForBackupPodDeletion waits for the backup daemon pod to be deleted +func waitForBackupPodDeletion(kubeClient *kubernetes.Clientset, namespace string) error { + podName := "backup-daemon-0" + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Wait until pods named "backup-daemon-0" are deleted + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctxWithTimeout.Done(): + fmt.Println("Warning: failed to remove backup daemon statefulset") + return ctxWithTimeout.Err() // error will be context.DeadlineExceeded + + case <-ticker.C: + // Periodically check if the pod is deleted + _, err := kubeClient.CoreV1().Pods(namespace).Get(ctxWithTimeout, podName, v1.GetOptions{}) + if kerrors.IsNotFound(err) { + // Pod has been deleted + return nil + } else if err != nil { + return err + } + } + } +} + +// deleteDynamicResources deletes a list of dynamic resources +func deleteDynamicResources(ctx context.Context, dynamicClient dynamic.Interface, namespace string, resources []DynamicResource, collectError func(error, string)) { + for _, resource := range resources { + err := dynamicClient.Resource(resource.GVR).Namespace(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, fmt.Sprintf("failed to delete %s", resource.ResourceName)) + } +} + +// deleteCRDs deletes a list of CustomResourceDefinitions +func deleteCRDs(ctx context.Context, dynamicClient dynamic.Interface, crdNames []string, collectError func(error, string)) { + crdGVR := schema.GroupVersionResource{ + Group: "apiextensions.k8s.io", + Version: "v1", + Resource: "customresourcedefinitions", + } + for _, crdName := range crdNames { + err := dynamicClient.Resource(crdGVR).Delete(ctx, crdName, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete CRD %s", crdName)) + } +} + +// deleteRolesAndBindings deletes roles and rolebindings containing 'mongodb' in their names +func deleteRolesAndBindings(ctx context.Context, kubeClient *kubernetes.Clientset, namespace string, collectError func(error, string)) { + // Delete roles + roleList, err := kubeClient.RbacV1().Roles(namespace).List(ctx, v1.ListOptions{}) + collectError(err, "failed to list roles") + if err == nil { + for _, role := range roleList.Items { + if strings.Contains(role.Name, "mongodb") { + err = kubeClient.RbacV1().Roles(namespace).Delete(ctx, role.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete role %s", role.Name)) + } + } + } + + // Delete rolebindings + rbList, err := kubeClient.RbacV1().RoleBindings(namespace).List(ctx, v1.ListOptions{}) + collectError(err, "failed to list rolebindings") + if err == nil { + for _, rb := range rbList.Items { + if strings.Contains(rb.Name, "mongodb") { + err = kubeClient.RbacV1().RoleBindings(namespace).Delete(ctx, rb.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete rolebinding %s", rb.Name)) + } + } + } +} + +// resetContext deletes cluster-scoped resources in the given context +func resetContext(ctx context.Context, contextName string, deleteCRD bool, collectError func(error, string)) { + fmt.Printf("Resetting context %s\n", contextName) + + kubeClient, _, err := initKubeClient(contextName) + if err != nil { + collectError(err, fmt.Sprintf("failed to initialize Kubernetes client for context %s", contextName)) + return + } + + // Delete ClusterRoleBindings with names containing "mongodb" + crbList, err := kubeClient.RbacV1().ClusterRoleBindings().List(ctx, v1.ListOptions{}) + collectError(err, fmt.Sprintf("failed to list ClusterRoleBindings in context %s", contextName)) + if err == nil { + for _, crb := range crbList.Items { + if strings.Contains(crb.Name, "mongodb") { + err = kubeClient.RbacV1().ClusterRoleBindings().Delete(ctx, crb.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete ClusterRoleBinding %s in context %s", crb.Name, contextName)) + } + } + } + + // Delete ClusterRoles with names containing "mongodb" + crList, err := kubeClient.RbacV1().ClusterRoles().List(ctx, v1.ListOptions{}) + collectError(err, fmt.Sprintf("failed to list ClusterRoles in context %s", contextName)) + if err == nil { + for _, cr := range crList.Items { + if strings.Contains(cr.Name, "mongodb") { + err = kubeClient.RbacV1().ClusterRoles().Delete(ctx, cr.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete ClusterRole %s in context %s", cr.Name, contextName)) + } + } + } + + clusterNamespaces, err := getTestNamespaces(ctx, kubeClient, contextName) + if err != nil { + collectError(err, fmt.Sprintf("failed to list TestNamespaces in context %s", contextName)) + return + } + if len(clusterNamespaces) == 0 { + // This env variable is used for single cluster tests + namespace := os.Getenv("NAMESPACE") // nolint:forbidigo + clusterNamespaces = append(clusterNamespaces, namespace) + } + fmt.Printf("%s: resetting namespaces: %v\n", contextName, clusterNamespaces) + for _, ns := range clusterNamespaces { + resetNamespace(ctx, contextName, ns, deleteCRD, collectError) + } + + fmt.Printf("Finished resetting context %s\n", contextName) +} + +// resetNamespace cleans up the namespace in the given context +func resetNamespace(ctx context.Context, contextName string, namespace string, deleteCRD bool, collectError func(error, string)) { + kubeClient, dynamicClient, err := initKubeClient(contextName) + if err != nil { + collectError(err, fmt.Sprintf("failed to initialize Kubernetes client for context %s", contextName)) + return + } + + // Hack: remove the statefulset for backup daemon first - otherwise it may get stuck on removal if AppDB is removed first + stsList, err := kubeClient.AppsV1().StatefulSets(namespace).List(ctx, v1.ListOptions{}) + collectError(err, "failed to list statefulsets") + if err == nil { + for _, sts := range stsList.Items { + if strings.Contains(sts.Name, "backup-daemon") { + err = kubeClient.AppsV1().StatefulSets(namespace).Delete(ctx, sts.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete statefulset %s", sts.Name)) + } + } + } + + err = waitForBackupPodDeletion(kubeClient, namespace) + collectError(err, "failed to delete backup daemon pod") + + // Delete all statefulsets + err = kubeClient.AppsV1().StatefulSets(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete statefulsets") + + // Delete all pods + err = kubeClient.CoreV1().Pods(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete pods") + + // Delete all deployments + err = kubeClient.AppsV1().Deployments(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete deployments") + + // Delete all services + services, err := kubeClient.CoreV1().Services(namespace).List(ctx, v1.ListOptions{}) + collectError(err, "failed to list services") + if err == nil { + for _, service := range services.Items { + err = kubeClient.CoreV1().Services(namespace).Delete(ctx, service.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete service %s", service.Name)) + } + } + + // Delete opsmanager resources + opsManagerGVR := schema.GroupVersionResource{ + Group: "mongodb.com", + Version: "v1", + Resource: "opsmanagers", + } + err = dynamicClient.Resource(opsManagerGVR).Namespace(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete opsmanager resources") + + // Delete CSRs matching the namespace + csrList, err := kubeClient.CertificatesV1().CertificateSigningRequests().List(ctx, v1.ListOptions{}) + collectError(err, "failed to list CSRs") + if err == nil { + for _, csr := range csrList.Items { + if strings.Contains(csr.Name, namespace) { + err = kubeClient.CertificatesV1().CertificateSigningRequests().Delete(ctx, csr.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete CSR %s", csr.Name)) + } + } + } + + // Delete secrets + err = kubeClient.CoreV1().Secrets(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete secrets") + + // Delete configmaps + err = kubeClient.CoreV1().ConfigMaps(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete configmaps") + + // Delete validating webhook configuration + err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, "mdbpolicy.mongodb.com", deleteOptionsNoGrace) + collectError(err, "failed to delete validating webhook configuration") + + // Define dynamic resources to delete + dynamicResources := []DynamicResource{ + { + GVR: schema.GroupVersionResource{ + Group: "cert-manager.io", + Version: "v1", + Resource: "certificates", + }, + ResourceName: "certificates", + }, + { + GVR: schema.GroupVersionResource{ + Group: "cert-manager.io", + Version: "v1", + Resource: "issuers", + }, + ResourceName: "issuers", + }, + { + GVR: schema.GroupVersionResource{ + Group: "operators.coreos.com", + Version: "v1alpha1", + Resource: "catalogsources", + }, + ResourceName: "catalogsources", + }, + { + GVR: schema.GroupVersionResource{ + Group: "operators.coreos.com", + Version: "v1alpha1", + Resource: "subscriptions", + }, + ResourceName: "subscriptions", + }, + { + GVR: schema.GroupVersionResource{ + Group: "operators.coreos.com", + Version: "v1alpha1", + Resource: "clusterserviceversions", + }, + ResourceName: "clusterserviceversions", + }, + } + + // Delete dynamic resources + deleteDynamicResources(ctx, dynamicClient, namespace, dynamicResources, collectError) + + // Delete PVCs + err = kubeClient.CoreV1().PersistentVolumeClaims(namespace).DeleteCollection(ctx, deleteOptionsNoGrace, v1.ListOptions{}) + collectError(err, "failed to delete PVCs") + + // Delete CRDs if specified + if deleteCRD { + crdNames := []string{ + "mongodb.mongodb.com", + "mongodbmulti.mongodb.com", + "mongodbmulticluster.mongodb.com", + "mongodbusers.mongodb.com", + "opsmanagers.mongodb.com", + } + deleteCRDs(ctx, dynamicClient, crdNames, collectError) + } + + // Delete serviceaccounts excluding 'default' + saList, err := kubeClient.CoreV1().ServiceAccounts(namespace).List(ctx, v1.ListOptions{}) + collectError(err, "failed to list serviceaccounts") + if err == nil { + for _, sa := range saList.Items { + if sa.Name != "default" { + err = kubeClient.CoreV1().ServiceAccounts(namespace).Delete(ctx, sa.Name, deleteOptionsNoGrace) + collectError(err, fmt.Sprintf("failed to delete serviceaccount %s", sa.Name)) + } + } + } + + // Delete roles and rolebindings + deleteRolesAndBindings(ctx, kubeClient, namespace, collectError) + + fmt.Printf("Finished resetting namespace %s in context %s\n", namespace, contextName) +} + +// Replaces our get_test_namespaces bash function, get the list of namespaces with evergreen label selector +func getTestNamespaces(ctx context.Context, kubeClient *kubernetes.Clientset, contextName string) ([]string, error) { + labelSelector := "evg=task" + + namespaces, err := kubeClient.CoreV1().Namespaces().List(ctx, v1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return nil, fmt.Errorf("failed to list namespaces for context %s: %v", contextName, err) + } + + var namespaceNames []string + for _, ns := range namespaces.Items { + namespaceNames = append(namespaceNames, ns.Name) + } + + return namespaceNames, nil +} + +func main() { + ctx := context.Background() + + kubeEnvNameVar := "KUBE_ENVIRONMENT_NAME" + kubeEnvironmentName, found := os.LookupEnv(kubeEnvNameVar) // nolint:forbidigo + if !found { + fmt.Println(kubeEnvNameVar, "must be set. Make sure you sourced your env file") + os.Exit(1) + } + + deleteCRD := env.ReadOrDefault("DELETE_CRD", "true") == "true" // nolint:forbidigo + + // Cluster is a set because central cluster can be part of member clusters + clusters := make(map[string]bool) + if kubeEnvironmentName == "multi" { + memberClusters := strings.Fields(os.Getenv("MEMBER_CLUSTERS")) // nolint:forbidigo + for _, cluster := range memberClusters { + clusters[cluster] = true + } + centralClusterName := os.Getenv("CENTRAL_CLUSTER") // nolint:forbidigo + clusters[centralClusterName] = true + } else { + clusterName := os.Getenv("CLUSTER_NAME") // nolint:forbidigo + clusters[clusterName] = true + } + + fmt.Println("Resetting clusters:") + fmt.Println(clusters) + + // For each call to resetContext, we collect errors in a slice, and display them at the end + errorMap := make(map[string][]error) + var errorMapMu sync.Mutex // Secure concurrent access to the map + var wg sync.WaitGroup + + for cluster := range clusters { + wg.Add(1) + go func(cluster string) { + defer wg.Done() + var localErrs []error + collectError := func(err error, msg string) { + // Ignore any "not found" error + if err != nil && !kerrors.IsNotFound(err) { + localErrs = append(localErrs, fmt.Errorf("%s: %v", msg, err)) + } + } + resetContext(ctx, cluster, deleteCRD, collectError) + errorMapMu.Lock() + if len(localErrs) > 0 { + errorMap[cluster] = localErrs + } + errorMapMu.Unlock() + }(cluster) + } + + wg.Wait() + + // Print out errors for each cluster + if len(errorMap) > 0 { + fmt.Fprintf(os.Stderr, "Errors occurred during reset:\n") + for cluster, errs := range errorMap { + fmt.Fprintf(os.Stderr, "Cluster %s:\n", cluster) + for _, err := range errs { + fmt.Fprintf(os.Stderr, " %v\n", err) + } + } + os.Exit(1) + } + + fmt.Println("Done") +} + +func initKubeClient(contextName string) (*kubernetes.Clientset, dynamic.Interface, error) { + kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{CurrentContext: contextName}, + ) + + config, err := kubeconfig.ClientConfig() + if err != nil { + return nil, nil, fmt.Errorf("failed to get Kubernetes client config for context %s: %v", contextName, err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, nil, fmt.Errorf("failed to create Kubernetes client for context %s: %v", contextName, err) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, nil, fmt.Errorf("failed to create dynamic client for context %s: %v", contextName, err) + } + + return kubeClient, dynamicClient, nil +} diff --git a/scripts/dev/set_env_context.sh b/scripts/dev/set_env_context.sh new file mode 100755 index 000000000..890f746fd --- /dev/null +++ b/scripts/dev/set_env_context.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# shellcheck disable=1091 +source scripts/funcs/errors + +script_name=$(readlink -f "${BASH_SOURCE[0]}") +script_dir=$(dirname "${script_name}") +context_file="${script_dir}/../../.generated/context.export.env" + +if [[ ! -f ${context_file} ]]; then + fatal "File ${context_file} not found! Make sure to follow this guide to get started: https://wiki.corp.mongodb.com/display/MMS/Setting+up+local+development+and+E2E+testing#SettinguplocaldevelopmentandE2Etesting-GettingStartedGuide(VariantSwitching)" +fi + +# shellcheck disable=SC1090 +source "${context_file}" + +export PATH="${PROJECT_DIR}/bin:${PATH}" diff --git a/scripts/dev/setup_evg_host.sh b/scripts/dev/setup_evg_host.sh new file mode 100755 index 000000000..ce29e3157 --- /dev/null +++ b/scripts/dev/setup_evg_host.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# this script downloads necessary tooling in EVG host + +set -Eeou pipefail + +echo "Increasing fs.inotify.max_user_instances" +sudo sysctl -w fs.inotify.max_user_instances=8192 + +echo "Increasing fs.inotify.max_user_watches" +sudo sysctl -w fs.inotify.max_user_watches=10485760 + +# retrieve arch variable off the shell command line +ARCH=${1-"amd64"} + +download_kind() { + scripts/evergreen/setup_kind.sh /usr/local +} + +download_curl() { + echo "Downloading curl..." + curl -s -o kubectl -L https://dl.k8s.io/release/"$(curl -L -s https://dl.k8s.io/release/stable.txt)"/bin/linux/"${ARCH}"/kubectl + chmod +x kubectl + sudo mv kubectl /usr/local/bin/kubectl +} + +download_helm() { + echo "Downloading helm..." + curl -s -o helm.tar.gz -L https://get.helm.sh/helm-v3.10.3-linux-"${ARCH}"tar.gz + tar -xf helm.tar.gz 2>/dev/null + sudo mv linux-"${ARCH}"helm /usr/local/bin/helm + rm helm.tar.gz + rm -rf linux-"${ARCH}/" +} + +download_kind & +download_curl & +download_helm & + +wait diff --git a/scripts/dev/setup_kind_cluster.sh b/scripts/dev/setup_kind_cluster.sh index 3178f2878..c1325a249 100755 --- a/scripts/dev/setup_kind_cluster.sh +++ b/scripts/dev/setup_kind_cluster.sh @@ -1,11 +1,17 @@ #!/usr/bin/env bash set -Eeou pipefail +source scripts/dev/set_env_context.sh + #### # This file is copy-pasted from https://github.com/mongodb/mongodb-kubernetes-operator/blob/master/scripts/dev/setup_kind_cluster.sh # Do not edit !!! #### +run_docker() { + docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" registry:2 +} + function usage() { echo "Deploy local registry and create kind cluster configured to use this registry. Local Docker registry is deployed at localhost:5000. @@ -21,42 +27,70 @@ Options: -r (optional) Recreate cluster if needed -p (optional) Network reserved for Pods, e.g. 10.244.0.0/16 -s (optional) Network reserved for Services, e.g. 10.96.0.0/16 + -l (optional) MetalLB IP range, e.g. 172.18.255.200-172.18.255.250 + -c (optional) Cluster domain. If not supplied, cluster.local will be used " exit 0 } cluster_name=${CLUSTER_NAME:-"kind"} +cluster_domain="cluster.local" export_kubeconfig=0 recreate=0 pod_network="10.244.0.0/16" service_network="10.96.0.0/16" -while getopts ':p:s:n:her' opt; do - case $opt in - (n) cluster_name=$OPTARG;; - (e) export_kubeconfig=1;; - (r) recreate=1;; - (p) pod_network=$OPTARG;; - (s) service_network=$OPTARG;; - (h) usage;; - (*) usage;; - esac +metallb_ip_range="172.18.255.200-172.18.255.250" +while getopts ':c:l:p:s:n:her' opt; do + case ${opt} in + n) cluster_name=${OPTARG} ;; + e) export_kubeconfig=1 ;; + r) recreate=1 ;; + p) pod_network=${OPTARG} ;; + s) service_network=${OPTARG} ;; + l) metallb_ip_range=${OPTARG} ;; + c) cluster_domain=${OPTARG} ;; + h) usage ;; + *) usage ;; + esac done -shift "$((OPTIND-1))" +shift "$((OPTIND - 1))" -kubeconfig_path="$HOME/.kube/${cluster_name}" +kubeconfig_path="${HOME}/.kube/${cluster_name}" -# create the kind network early unless it already exists. -# it would normally be created automatically by kind but we -# need it earlier to get the IP address of our registry. -docker network create kind || true +# We create docker network primarily so that all kind clusters use the same network. +# We hardcode 172.18/16 subnet in few places, so we must ensure the network uses that subnet: +# - we set IP ranges for MetalLB +# - we write into coredns config map with IP used for exposing pods externally and for no-mesh test +# In case of any errors, verify if any other subnet is not already using that subnet. +docker network create --subnet=172.18.0.0/16 kind || true # adapted from https://kind.sigs.k8s.io/docs/user/local-registry/ # create registry container unless it already exists reg_name='kind-registry' reg_port='5000' +kind_node_version='v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f' running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" + +max_retries=3 +retry_count=0 + +success=false + if [ "${running}" != 'true' ]; then - docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network kind --name "${reg_name}" registry:2 + while [ "${retry_count}" -lt "${max_retries}" ]; do + if run_docker; then + echo "Docker container started successfully." + success=true + break + else + echo "Docker run failed. Attempting to restart Docker service and retrying" + fi + done + + if [ "${success}" = false ]; then + echo "Docker run command failed after ${max_retries} attempts!" + exit 1 + fi fi if [ "${recreate}" != 0 ]; then @@ -64,39 +98,95 @@ if [ "${recreate}" != 0 ]; then fi # create a cluster with the local registry enabled in containerd -cat </dev/null + +echo "installing metallb" +kubectl get --kubeconfig "${kubeconfig_path}" nodes -owide +kubectl apply --kubeconfig "${kubeconfig_path}" --timeout=600s -f https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml + +echo "waiting metallb to be ready" +kubectl wait --kubeconfig "${kubeconfig_path}" --timeout=3000s --namespace metallb-system \ + --for=condition=ready pod \ + --selector=app=metallb + +echo "install metallb to be ready" +cat < "${destination_envs_file}.env" +# shellcheck disable=SC2129 +echo -e "## Regenerated at $(date)\n" >> "${destination_envs_file}.env" +echo "${current_envs}" >> "${destination_envs_file}.env" + +# Below is a list of special cases of variables that are called the same in EVG and local context. +# This piece should probably be refactored as it's super easy to make a mistake and shadow the proper variable. +echo "workdir=\"${workdir:-.}\"" >> "${destination_envs_file}.env" + +# We need to tail +5 lines due to above generated comment. +awk '{print "export " $0}' < "${destination_envs_file}".env | tail -n +5 > "${destination_envs_file}".export.env + +scripts/dev/print_operator_env.sh | sort | uniq >"${destination_envs_file}.operator.env" +awk '{print "export " $0}' < "${destination_envs_file}".operator.env > "${destination_envs_file}".operator.export.env + +echo -n "${context}" > "${destination_envs_dir}/.current_context" + +echo "Generated env files in $(readlink -f "${destination_envs_dir}"):" +# shellcheck disable=SC2010 +ls -l1 "${destination_envs_dir}" | grep "context" + +if which kubectl > /dev/null; then + if [ "${CLUSTER_NAME-}" ]; then + # The convention: the cluster name must match the name of kubectl context + # We expect this not to be true if kubernetes cluster is still to be created (minikube/kops) + if ! kubectl config use-context "${CLUSTER_NAME}"; then + echo "Warning: failed to switch kubectl context to: ${CLUSTER_NAME}" + echo "Does a matching Kubernetes context exist?" + fi + + # Setting the default namespace for current context + kubectl config set-context "$(kubectl config current-context)" "--namespace=${NAMESPACE}" &>/dev/null || true + + # shellcheck disable=SC2153 + echo "Current context: ${context} (kubectl context: ${CLUSTER_NAME}), namespace=${NAMESPACE}" + fi +else + echo "Kubectl doesn't exist, skipping setting the context" +fi diff --git a/scripts/dev/update_docs_snippets.sh b/scripts/dev/update_docs_snippets.sh new file mode 100755 index 000000000..1dfbb407f --- /dev/null +++ b/scripts/dev/update_docs_snippets.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# script to update code snippets file from MEKO in docs repository +# Usage: +# cd +# ./scripts/dev/update_docs_snippets.sh +# +# To customize directories run +# MEKO_DIR= DOCS_DIR= ./update_docs_snippets.sh +# Example: +# MEKO_DIR=~/mdb/ops-manager-kubernetes DOCS_DIR=~/mdb/docs-k8s-operator ./update_docs_snippets.sh + +set -eou pipefail + +MEKO_DIR=${MEKO_DIR:-"ops-manager-kubernetes"} +MEKO_BRANCH=${MEKO_BRANCH:-"om-mc-gke"} +DOCS_DIR=${DOCS_DIR:-"docs-k8s-operator"} +DOCS_BRANCH=${DOCS_BRANCH:-"master"} + +function prepare_repositories() { + pushd "${MEKO_DIR}" + git fetch + git checkout "${MEKO_BRANCH}" + + if [[ -n "$(git status --porcelain)" ]]; then + echo "${MEKO_DIR} has modified files, stashing..." + git stash + fi + + git reset --hard "origin/${MEKO_BRANCH}" + + popd + + pushd "${DOCS_DIR}" + git fetch + if [[ -n "$(git status --porcelain)" ]]; then + echo "${DOCS_DIR} has modified files, stashing..." + git stash + fi + + git checkout "${DOCS_BRANCH}" + git reset --hard "origin/${DOCS_BRANCH}" + + git checkout -b "meko-snippets-update-$(date "+%Y%m%d%H%M%S")" + popd +} + +function copy_files() { + samples_dir=$1 + dst_dir="${DOCS_DIR}/source/includes/code-examples/reference-architectures/${samples_dir}" + src_dir="${MEKO_DIR}/public/architectures/${samples_dir}" + + rm -rf "${dst_dir}" + mkdir -p "${dst_dir}" + + cp -r "${src_dir}/code_snippets" "${dst_dir}" + cp -r "${src_dir}/output" "${dst_dir}" + cp "${src_dir}/env_variables.sh" "${dst_dir}" || true + cp -r "${src_dir}/yamls" "${dst_dir}" || true +} + +function prepare_docs_pr() { + pushd "${DOCS_DIR}" + if [[ -z "$(git status --porcelain)" ]]; then + echo "No changes to push" + return 1 + fi + + git add "source/" + git commit -m "Update sample files from MEKO" + git push + popd +} + +pushd ../ +prepare_repositories +copy_files "ops-manager-multi-cluster" +copy_files "ops-manager-mc-no-mesh" +copy_files "mongodb-sharded-multi-cluster" +copy_files "mongodb-sharded-mc-no-mesh" +copy_files "mongodb-replicaset-multi-cluster" +copy_files "mongodb-replicaset-mc-no-mesh" +copy_files "setup-multi-cluster/verify-connectivity" +copy_files "setup-multi-cluster/setup-gke" +copy_files "setup-multi-cluster/setup-istio" +copy_files "setup-multi-cluster/setup-operator" +copy_files "setup-multi-cluster/setup-cert-manager" +copy_files "setup-multi-cluster/setup-externaldns" +prepare_docs_pr +popd diff --git a/scripts/dev/update_go.sh b/scripts/dev/update_go.sh new file mode 100755 index 000000000..ba9313194 --- /dev/null +++ b/scripts/dev/update_go.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -x + +# This script updates Go version in the Enterprise Operator repo. The implementation is very naive and +# just massively replaces one to another. Please carefully review the Pull Request. +# This script doesn't validate any parameters. +# +# Usage: +# ./scripts/dec/update_go.sh 1.23 1.23 + + +prev_ver=$1 +next_ver=$2 + +# Ensure we match dot correctly with \. +prev_ver="${prev_ver/\./\\.}" +next_ver="${next_ver/\./\\.}" + +for i in $(git grep --files-with-matches "[go :]${prev_ver}" | grep -v RELEASE_NOTES.md) +do + perl -p -i -e "s/${prev_ver}/${next_ver}/g" "${i}" +done diff --git a/scripts/evergreen/add_evergreen_task.sh b/scripts/evergreen/add_evergreen_task.sh new file mode 100755 index 000000000..5f41d2700 --- /dev/null +++ b/scripts/evergreen/add_evergreen_task.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script is used in evergreen yaml for dynamically adding task to be executed in given variant. + +variant=$1 +task=$2 + +if [[ -z "${variant}" || -z "${task}" ]]; then + echo "usage: add_evergreen_task.sh {variant} {task}" + exit 1 +fi + +cat >evergreen_tasks.json </dev/null | grep "Docker Root Dir" | grep -q "/var/lib/docker"; then + # we need to reconfigure Docker so its image storage points to a + # directory with enough space, in this case /data + echo "Trying with /etc/docker/daemon.json file" + sudo mkdir -p /etc/docker + sudo chmod o+w /etc/docker + cat <"${HOME}/daemon.json" +{ + "data-root": "/data/docker", + "storage-driver": "overlay2" +} +EOF + sudo mv "${HOME}/daemon.json" /etc/docker/daemon.json + + sudo systemctl restart docker + if docker info 2>/dev/null | grep "Docker Root Dir" | grep -q "/data/docker"; then + echo "Docker storage configured correctly" + else + # The change didn't went through, we are not failing the test, but it might + # fail because of no free space left in device. + echo "Docker storage was not configured properly" + fi +fi + +mkdir -p logs/ +docker info >logs/docker-info.text diff --git a/scripts/evergreen/deployments/multi-cluster-roles/Chart.yaml b/scripts/evergreen/deployments/multi-cluster-roles/Chart.yaml new file mode 100644 index 000000000..f3311660f --- /dev/null +++ b/scripts/evergreen/deployments/multi-cluster-roles/Chart.yaml @@ -0,0 +1,2 @@ +name: mongodb-enterprise-multi-cluster-test +version: 0.1.0 diff --git a/scripts/evergreen/deployments/multi-cluster-roles/templates/mongodb-enterprise-tests.yaml b/scripts/evergreen/deployments/multi-cluster-roles/templates/mongodb-enterprise-tests.yaml new file mode 100644 index 000000000..f852b0f7e --- /dev/null +++ b/scripts/evergreen/deployments/multi-cluster-roles/templates/mongodb-enterprise-tests.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator-tests-multi-cluster-service-account + namespace: {{ .Values.namespace }} +{{- if .Values.imagePullSecrets}} +imagePullSecrets: + - name: {{ .Values.imagePullSecrets }} +{{- end }} + + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: operator-multi-cluster-tests-role-binding-{{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: operator-tests-multi-cluster-service-account + namespace: {{ .Values.namespace }} diff --git a/scripts/evergreen/deployments/multi-cluster-roles/values.yaml b/scripts/evergreen/deployments/multi-cluster-roles/values.yaml new file mode 100644 index 000000000..b0f39e89d --- /dev/null +++ b/scripts/evergreen/deployments/multi-cluster-roles/values.yaml @@ -0,0 +1,5 @@ +namespace: ${NAMESPACE} + +# Set this when the ECR registry is not accessible from the testing cluster. +# A Secret with type kubernetes.io/dockerconfigjson needs to exist. +imagePullSecrets: diff --git a/scripts/evergreen/deployments/ops-manager-vanilla.yaml b/scripts/evergreen/deployments/ops-manager-vanilla.yaml new file mode 100644 index 000000000..b4da39538 --- /dev/null +++ b/scripts/evergreen/deployments/ops-manager-vanilla.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: mongodb.com/v1 +kind: MongoDBOpsManager +metadata: + name: ops-manager +spec: + # the number of Ops Manager instances to run. Set to value bigger + # than 1 to get high availability and upgrades without downtime + replicas: 1 + + # We put a dummy version that does not exist to make sure that we actually patch it + version: 10.0.0 + + # optional. Specify the custom cluster domain of the Kubernetes cluster if it's different from the default one ('cluster.local'). + # This affects the urls generated by the Operator. + # This field is also used for Application Database url + # clusterDomain: mycompany.net + + # the name of the secret containing admin user credentials. + # Either remove the secret or change the password using Ops Manager UI after the Ops Manager + # resource is created! + adminCredentials: ops-manager-admin-secret + + backup: + enabled: false + # the application database backing Ops Manager. Replica Set is the only supported type + # Application database has the SCRAM-SHA authentication mode always enabled + applicationDatabase: + members: 3 + version: 4.2.6-ent + + configuration: + mms.limits.maxGroupsPerOrg: "5000" + mms.limits.maxGroupsPerUser: "5000" + mms.limits.maxOrgsPerUser: "5000" diff --git a/scripts/evergreen/deployments/test-app/Chart.yaml b/scripts/evergreen/deployments/test-app/Chart.yaml new file mode 100644 index 000000000..1c63a0e2a --- /dev/null +++ b/scripts/evergreen/deployments/test-app/Chart.yaml @@ -0,0 +1,2 @@ +name: mongodb-enterprise-test +version: 0.1.0 diff --git a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml new file mode 100644 index 000000000..516f4b6ad --- /dev/null +++ b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml @@ -0,0 +1,197 @@ +# grant all permissions in all namespaces to the tests +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator-tests-service-account + namespace: {{ .Values.namespace }} +{{- if .Values.imagePullSecrets}} +imagePullSecrets: + - name: {{ .Values.imagePullSecrets }} +{{- end }} + + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: operator-tests-role-binding-{{ .Values.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: operator-tests-service-account + namespace: {{ .Values.namespace }} + +--- +apiVersion: v1 +kind: Pod +metadata: + name: mongodb-enterprise-operator-tests + namespace: {{ .Values.namespace }} + labels: + role: operator-tests +spec: + serviceAccountName: operator-tests-service-account + restartPolicy: Never + terminationGracePeriodSeconds: 0 + volumes: + - name: results + emptyDir: { } + - name: diagnostics + emptyDir: { } + {{ if .Values.multiCluster.memberClusters }} + - name: kube-config-volume + secret: + defaultMode: 420 + secretName: test-pod-kubeconfig + - name: multi-cluster-config + secret: + defaultMode: 420 + secretName: test-pod-multi-cluster-config + {{ end }} + containers: + - image: busybox + name: keepalive + command: [ "/bin/sh", "-c", "sleep inf" ] + volumeMounts: + - name: results + mountPath: /tmp/results + - name: diagnostics + mountPath: /tmp/diagnostics + - name: mongodb-enterprise-operator-tests + env: + # OTEL env vars can either be used to construct custom spans or are used by pytest opentelemetry dynamic instrumentation + {{ if .Values.otel_trace_id }} + - name: OTEL_TRACE_ID + value: {{ .Values.otel_trace_id }} + {{ end }} + {{ if .Values.otel_endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel_endpoint}} + {{ end }} + {{ if .Values.otel_parent_id }} + - name: OTEL_PARENT_ID + value: {{ .Values.otel_parent_id }} + {{ end }} + {{ if .Values.otel_resource_attributes }} + - name: OTEL_RESOURCE_ATTRIBUTES + value: {{ .Values.otel_resource_attributes }} + {{ end }} + - name: OTEL_SERVICE_NAME + value: evergreen-agent + - name: PYTEST_RUN_NAME + value: {{ .Values.taskName }} + - name: TASK_ID + value: {{ .Values.taskId }} + - name: OM_HOST + value: {{ .Values.baseUrl }} + - name: OM_API_KEY + value: {{ .Values.apiKey }} + - name: OM_USER + value: {{ .Values.apiUser }} + - name: OM_ORGID + value: {{ .Values.orgId }} + - name: NAMESPACE + value: {{ .Values.namespace }} + # We can pass additional options to pytest. For instance -s + - name: PYTEST_ADDOPTS + value: {{ .Values.pytest.addopts }} + - name: SKIP_EXECUTION + value: {{ .Values.skipExecution}} + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.aws.accessKey}} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.aws.secretAccessKey}} + - name: MANAGED_SECURITY_CONTEXT + value: '{{ .Values.managedSecurityContext }}' + - name: PYTHONUNBUFFERED + value: 'true' + - name: PYTHONWARNINGS + value: 'ignore:yaml.YAMLLoadWarning,ignore:urllib3.InsecureRequestWarning' + - name: POD_NAME + value: 'mongodb-enterprise-operator-testing-pod' + - name: VERSION_ID + value: {{ .Values.tag }} + - name: REGISTRY + value: {{ .Values.registry }} + {{- if .Values.multiCluster.memberClusters }} + - name: KUBECONFIG + value: /etc/config/kubeconfig + - name: MEMBER_CLUSTERS + value: {{ .Values.multiCluster.memberClusters }} + - name: CENTRAL_CLUSTER + value: {{ .Values.multiCluster.centralCluster }} + - name: TEST_POD_CLUSTER + value: {{ .Values.multiCluster.testPodCluster }} + {{- end }} + {{ if .Values.customOmVersion }} + - name: CUSTOM_OM_VERSION + value: {{ .Values.customOmVersion }} + {{ end }} + {{ if .Values.taskReplicas }} + - name: PERF_TASK_REPLICAS + value: "{{ .Values.taskReplicas }}" + {{ end }} + {{ if .Values.taskDeployments }} + - name: PERF_TASK_DEPLOYMENTS + value: "{{ .Values.taskDeployments }}" + {{ end }} + {{ if .Values.customOmPrevVersion }} + - name: CUSTOM_OM_PREV_VERSION + value: {{ .Values.customOmPrevVersion }} + {{ end }} + {{ if .Values.customOmMdbVersion }} + - name: CUSTOM_MDB_VERSION + value: {{ .Values.customOmMdbVersion }} + {{ end }} + {{ if .Values.customOmMdbPrevVersion }} + - name: CUSTOM_MDB_PREV_VERSION + value: {{ .Values.customOmMdbPrevVersion }} + {{ end }} + {{ if .Values.customAppDbVersion }} + - name: CUSTOM_APPDB_VERSION + value: {{ .Values.customAppDbVersion }} + {{ end }} + {{ if .Values.projectDir }} + - name: PROJECT_DIR + value: {{ .Values.projectDir }} + {{ end }} + {{ if .Values.localOperator }} + - name: LOCAL_OPERATOR + value: "{{ .Values.localOperator }}" + {{ end }} + - name: MDB_DEFAULT_ARCHITECTURE + value: {{ .Values.mdbDefaultArchitecture }} + - name: MDB_IMAGE_TYPE + value: {{ .Values.mdbImageType }} + - name: CLUSTER_DOMAIN + value: {{ .Values.clusterDomain }} + {{ if .Values.omDebugHttp }} + - name: OM_DEBUG_HTTP + value: "{{ .Values.omDebugHttp }}" + {{ end }} + - name: ops_manager_version + value: "{{ .Values.opsManagerVersion }}" + image: {{ .Values.repo }}/mongodb-enterprise-tests:{{ .Values.tag }} + # Options to pytest command should go in the pytest.ini file. + command: ["pytest"] + {{ if .Values.otel_endpoint }} + args: ["-vv", "-m", "{{ .Values.taskName }}", "--trace-parent", "00-{{ .Values.otel_trace_id }}-{{ .Values.otel_parent_id }}-01", "--export-traces"] + {{ else }} + args: ["-vv", "-m", "{{ .Values.taskName }}"] + {{ end }} + imagePullPolicy: Always + volumeMounts: + - name: results + mountPath: /tmp/results + - name: diagnostics + mountPath: /tmp/diagnostics + {{ if .Values.multiCluster.memberClusters }} + - mountPath: /etc/config + name: kube-config-volume + - mountPath: /etc/multicluster + name: multi-cluster-config + {{ end }} diff --git a/scripts/evergreen/deployments/test-app/values.yaml b/scripts/evergreen/deployments/test-app/values.yaml new file mode 100644 index 000000000..fe171c27a --- /dev/null +++ b/scripts/evergreen/deployments/test-app/values.yaml @@ -0,0 +1,43 @@ +namespace: ${NAMESPACE} + +registry: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/ubi + +managedSecurityContext: false + +apiKey: omApiKey +orgId: "" +projectId: omProjectId +tag: +taskName: ${TASK_NAME} + +pytest: + addopts: "-s" + +aws: + accessKey: "none" + secretAccessKey: "none" + +skipExecution: "false" + +# Set this when the ECR registry is not accessible from the testing cluster. +# A Secret with type kubernetes.io/dockerconfigjson needs to exist. +imagePullSecrets: + +localOperator: "false" + +clusterDomain: "cluster.local" + +multiCluster: + memberClusters: "" # Set this to a space separated list of member clusters to configure the test pod for running a multi cluster test. + centralCluster: "" # Set this to the name of the central cluster to configure the test pod for running a multi cluster test. + testPodCluster: "" + +otel_trace_id: +otel_parent_id: +otel_endpoint: +otel_resource_attributes: +mdbDefaultArchitecture: "non-static" +mdbImageType: "ubi8" + +# set to "true" to set OM_DEBUG_HTTP=true for the operator +omDebugHttp: diff --git a/scripts/evergreen/deployments/values-ops-manager.yaml b/scripts/evergreen/deployments/values-ops-manager.yaml new file mode 100644 index 000000000..82494e3dc --- /dev/null +++ b/scripts/evergreen/deployments/values-ops-manager.yaml @@ -0,0 +1,6 @@ +registry: + repository: 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev + pullPolicy: Always + imagePullSecrets: +opsManager: + name: mongodb-enterprise-ops-manager \ No newline at end of file diff --git a/scripts/evergreen/e2e/configure_operator.sh b/scripts/evergreen/e2e/configure_operator.sh new file mode 100755 index 000000000..e13187f3f --- /dev/null +++ b/scripts/evergreen/e2e/configure_operator.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/funcs/printing +source scripts/funcs/kubernetes + +# Configuration of the resources for Ops Manager +title "Creating admin secret for the new Ops Manager instance" +kubectl --namespace "${NAMESPACE}" delete secrets ops-manager-admin-secret --ignore-not-found +kubectl --namespace "${NAMESPACE}" create secret generic ops-manager-admin-secret \ + --from-literal=Username="jane.doe@example.com" \ + --from-literal=Password="Passw0rd." \ + --from-literal=FirstName="Jane" \ + --from-literal=LastName="Doe" -n "${NAMESPACE}" + +# Configuration of the resources for MongoDB +title "Creating project and credentials Kubernetes object..." + +if [ -n "${OM_BASE_URL-}" ]; then + BASE_URL="${OM_BASE_URL}" +else + BASE_URL="http://ops-manager-svc.${OPS_MANAGER_NAMESPACE:-}.svc.cluster.local:8080" +fi + +# We always create the image pull secret from the docker config.json which gives access to all necessary image repositories +create_image_registries_secret + +# delete `my-project` if it exists +kubectl --namespace "${NAMESPACE}" delete configmap my-project --ignore-not-found +# Configuring project +kubectl --namespace "${NAMESPACE}" create configmap my-project \ + --from-literal=projectName="${NAMESPACE}" --from-literal=baseUrl="${BASE_URL}" \ + --from-literal=orgId="${OM_ORGID:-}" + +if [[ -z ${OM_USER-} ]] || [[ -z ${OM_API_KEY-} ]]; then + echo "OM_USER and/or OM_API_KEY env variables are not provided - assuming this is an" + echo "e2e test for MongoDbOpsManager, skipping creation of the credentials secret" +else + # delete `my-credentials` if it exists + kubectl --namespace "${NAMESPACE}" delete secret my-credentials --ignore-not-found + # Configure the Kubernetes credentials for Ops Manager + if [[ -z ${OM_PUBLIC_API_KEY:-} ]]; then + kubectl --namespace "${NAMESPACE}" create secret generic my-credentials \ + --from-literal=user="${OM_USER:-admin}" --from-literal=publicApiKey="${OM_API_KEY}" + else + kubectl --namespace "${NAMESPACE}" create secret generic my-credentials \ + --from-literal=user="${OM_PUBLIC_API_KEY}" --from-literal=publicApiKey="${OM_API_KEY}" + fi +fi + +title "All necessary ConfigMaps and Secrets have been created" diff --git a/scripts/evergreen/e2e/dump_diagnostic_information.sh b/scripts/evergreen/e2e/dump_diagnostic_information.sh new file mode 100755 index 000000000..4760c4a6e --- /dev/null +++ b/scripts/evergreen/e2e/dump_diagnostic_information.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +## We need to make sure this script does not fail if one of +## the kubectl commands fails. +set +e + +source scripts/funcs/printing + +dump_all () { + [[ "${MODE-}" = "dev" ]] && return + + # TODO: provide a cleaner way of handling this. For now we run the same command with kubectl configured + # with a different context. + local original_context + original_context="$(kubectl config current-context)" + kubectl config use-context "${1:-${original_context}}" &> /dev/null + prefix="${1:-${original_context}}_" + # shellcheck disable=SC2154 + if [[ "${KUBE_ENVIRONMENT_NAME:-}" != "multi" ]]; then + prefix="" + fi + + # The dump process usually happens for a single namespace (the one the test and the operator are installed to) + # but in some exceptional cases (e.g. clusterwide operator) there can be more than 1 namespace to print diagnostics + # In this case the python test app may create the test namespace and add necessary labels and annotations so they + # would be dumped for diagnostics as well + for ns in $(kubectl get namespace -l "evg=task" --output=jsonpath={.items..metadata.name}); do + if kubectl get namespace "${ns}" -o jsonpath='{.metadata.annotations}' | grep -q "${task_id:?}"; then + echo "Dumping all diagnostic information for namespace ${ns}" + dump_namespace "${ns}" "${prefix}" + fi + done + + if kubectl get namespace "olm" &>/dev/null; then + echo "Dumping olm namespace" + dump_namespace "olm" "olm" + fi + + kubectl config use-context "${original_context}" &> /dev/null + + kubectl -n "kube-system" get configmap coredns -o yaml > "logs/${prefix}coredns.yaml" +} + +dump_objects() { + local object=$1 + local msg=$2 + local namespace=${3} + local action=${4:-get -o yaml} + + if [ "$(kubectl get "${object}" --no-headers -o name -n "${namespace}" | wc -l)" = "0" ]; then + # if no objects of this type, return + return + fi + + header "${msg}" + # shellcheck disable=SC2086 + kubectl -n "${namespace}" ${action} "${object}" 2>&1 +} + +# get_operator_managed_pods returns a list of names of the Pods that are managed +# by the Operator. +get_operator_managed_pods() { + local namespace=${1} + kubectl get pods --namespace "${namespace}" --selector=controller=mongodb-enterprise-operator --no-headers -o custom-columns=":metadata.name" +} + +get_all_pods() { + local namespace=${1} + kubectl get pods --namespace "${namespace}" --no-headers -o custom-columns=":metadata.name" +} + +is_mdb_resource_pod() { + local pod="${1}" + local namespace="${2}" + + kubectl exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-verbose.log &>/dev/null +} + +# dump_pod_logs dumps agent and mongodb logs. +dump_pod_logs() { + local pod="${1}" + local namespace="${2}" + local prefix="${3}" + + if is_mdb_resource_pod "${pod}" "${namespace}"; then + # for MDB resource Pods, we dump the log files from the file system + echo "Writing agent and mongodb logs for pod ${pod} to logs" + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/automation-agent-verbose.log" "logs/${prefix}${pod}-agent-verbose.log" &> /dev/null + tail -n 500 "logs/${pod}-agent-verbose.log" > "logs/${prefix}${pod}-agent.log" || true + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/monitoring-agent-verbose.log" "logs/${prefix}${pod}-monitoring-agent-verbose.log" &> /dev/null + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/monitoring-agent.log" "logs/${prefix}${pod}-monitoring-agent.log" &> /dev/null + kubectl logs -n "${namespace}" "${pod}" -c "mongodb-agent-monitoring" > "logs/${prefix}${pod}-monitoring-agent-stdout.log" || true + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/mongodb.log" "logs/${prefix}${pod}-mongodb.log" &> /dev/null || true + # note that this file may get empty if the logs have already grew too much - seems it's better to have it explicitly empty then just omit + kubectl logs -n "${namespace}" "${pod}" | jq -c -r 'select( .logType == "agent-launcher-script") | .contents' 2> /dev/null > "logs/${prefix}${pod}-launcher.log" + else + # for all other pods we want each log per container from kubectl + for container in $(kubectl get pods -n "${namespace}" "${pod}" -o jsonpath='{.spec.containers[*].name}'); do + echo "Writing log file for pod ${pod} - container ${container} to logs/${pod}-${container}.log" + kubectl logs -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}.log" + + # Check if the container has restarted by examining its restart count + restartCount=$(kubectl get pod -n "${namespace}" "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].restartCount}") + + if [ "${restartCount}" -gt 0 ]; then + echo "Writing log file for restarted ${pod} - container ${container} to logs/${pod}-${container}-previous.log" + kubectl logs --previous -n "${namespace}" "${pod}" -c "${container}" > "logs/${pod}-${container}-previous.log" || true + fi + + done + fi + + if kubectl exec "${pod}" -n "${namespace}" -- ls /var/log/mongodb-mms-automation/automation-agent-stderr.log &>/dev/null; then + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/automation-agent-stderr.log" "logs/${prefix}${pod}-agent-stderr.log" &> /dev/null + fi +} + +# dump_pod_readiness_state dumps readiness and agent-health-status files. +dump_pod_readiness_state() { + local pod="${1}" + local namespace="${2}" + local prefix="${3}" + + # kubectl cp won't create any files if the file doesn't exist in the container + agent_health_status="logs/${prefix}${pod}-agent-health-status.json" + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/agent-health-status.json" "${agent_health_status}" &> /dev/null + ([[ -f "${agent_health_status}" ]] && jq . < "${agent_health_status}" > tmpfile && mv tmpfile "${agent_health_status}") + + kubectl cp "${namespace}/${pod}:var/log/mongodb-mms-automation/readiness.log" "logs/${prefix}${pod}-readiness.log" &> /dev/null +} + +# dump_pod_config dumps mongod configuration and cluster-config. +dump_pod_config() { + local pod="${1}" + local namespace="${2}" + local prefix="${3}" + + # cluster-config.json is a mounted volume and the actual file is located in the "..data" directory + pod_cluster_config="logs/${prefix}${pod}-cluster-config.json" + kubectl cp "${namespace}/${pod}:var/lib/mongodb-automation/..data/cluster-config.json" "${pod_cluster_config}" &> /dev/null + ([[ -f "${pod_cluster_config}" ]] && jq . < "${pod_cluster_config}" > tmpfile && mv tmpfile "${pod_cluster_config}") + + # Mongodb Configuration + kubectl cp "${namespace}/${pod}:data/automation-mongod.conf" "logs/${prefix}${pod}-automation-mongod.conf" &> /dev/null +} + +dump_configmaps() { + local namespace="${1}" + local prefix="${2}" + kubectl -n "${namespace}" get configmaps -o yaml > "logs/${prefix}z_configmaps.txt" +} + +decode_secret() { + local secret=${1} + local namespace=${2} + + kubectl get secret "${secret}" -o json -n "${namespace}" | jq -r '.data | with_entries(.value |= @base64d)' 2> /dev/null +} + +dump_secrets() { + local namespace="${1}" + local prefix="${2}" + for secret in $(kubectl get secrets -n "${namespace}" --no-headers | grep -v service-account-token | grep -v dockerconfigjson | awk '{ print $1 }'); do + decode_secret "${secret}" "${namespace}" > "logs/${prefix}z_secret_${secret}.txt" + done +} + +dump_services() { + local namespace="${1}" + local prefix="${2}" + kubectl -n "${namespace}" get svc -o yaml > "logs/${prefix}z_services.txt" +} + +dump_metrics() { + local namespace="${1}" + local operator_pod="${2}" + kubectl exec -it "${operator_pod}" -n "${namespace}" -- curl localhost:8080/metrics > "logs/metrics_${operator_pod}.txt" +} + +# dump_pods writes logs for each relevant Pod in the namespace: agent, mongodb +# logs, etc. +dump_pods() { + local namespace="${1}" + local prefix="${2}" + + pods=$(get_all_pods "${namespace}") + operator_managed_pods=$(get_operator_managed_pods "${namespace}") + + # we only have readiness and automationconfig in mdb pods + for pod in ${operator_managed_pods}; do + dump_pod_readiness_state "${pod}" "${namespace}" "${prefix}" + dump_pod_config "${pod}" "${namespace}" "${prefix}" + done + + # for all pods in the namespace we want to have logs and describe output + for pod in ${pods}; do + kubectl describe "pod/${pod}" -n "${namespace}" > "logs/${prefix}${pod}-pod-describe.txt" + dump_pod_logs "${pod}" "${namespace}" "${prefix}" + done + + if (kubectl get pod -n "${namespace}" -l app.kubernetes.io/name=controller ) &> /dev/null ; then + operator_pod=$(kubectl get pod -n "${namespace}" -l app.kubernetes.io/component=controller --no-headers -o custom-columns=":metadata.name") + if [ -n "${operator_pod}" ]; then + kubectl describe "pod/${operator_pod}" -n "${namespace}" > "logs/z_${operator_pod}-pod-describe.txt" + dump_metrics "${namespace}" "${operator_pod}" + fi + + fi +} + +# dump_diagnostics writes only the *most important information* for debugging +# tests, no more. Ideally the diagnostics file is as small as possible. Avoid +# high density of information; the main objective of this file is to direct you +# to a place where to find your problem, not to tell you what the problem is. +dump_diagnostics() { + local namespace="${1}" + + dump_objects mongodb "MongoDB Resources" "${namespace}" + dump_objects mongodbusers "MongoDBUser Resources" "${namespace}" + dump_objects opsmanagers "MongoDBOpsManager Resources" "${namespace}" + dump_objects mongodbmulticluster "MongoDB Multi Resources" "${namespace}" + + header "All namespace resources" + kubectl get all -n "${namespace}" +} + +# dump_namespace dumps a namespace, diagnostics, logs and generic Kubernetes +# resources. +dump_namespace() { + local namespace=${1} + local prefix="${2}" + + # do not fail for any reason + set +e + + # 1. Dump diagnostic information + # gathers the information about K8s objects and writes it to the file which will be attached to Evergreen job + mkdir -p logs + + # 2. Write diagnostics file + dump_diagnostics "${namespace}" > "logs/${prefix}0_diagnostics.txt" + + # 3. Print Pod logs + dump_pods "${namespace}" "${prefix}" + + # 4. Print other Kubernetes resources + dump_configmaps "${namespace}" "${prefix}" + dump_secrets "${namespace}" "${prefix}" + dump_services "${namespace}" "${prefix}" + + dump_objects pvc "Persistent Volume Claims" "${namespace}" > "logs/${prefix}z_persistent_volume_claims.txt" + dump_objects deploy "Deployments" "${namespace}" > "logs/${prefix}z_deployments.txt" + dump_objects deploy "Deployments" "${namespace}" "describe" > "logs/${prefix}z_deployments_describe.txt" + dump_objects sts "StatefulSets" "${namespace}" describe > "logs/${prefix}z_statefulsets.txt" + dump_objects sts "StatefulSets Yaml" "${namespace}" >> "logs/${prefix}z_statefulsets.txt" + dump_objects serviceaccounts "ServiceAccounts" "${namespace}" > "logs/${prefix}z_service_accounts.txt" + dump_objects validatingwebhookconfigurations "Validating Webhook Configurations" "${namespace}" > "logs/${prefix}z_validatingwebhookconfigurations.txt" + dump_objects certificates.cert-manager.io "Cert-manager certificates" "${namespace}" 2> /dev/null > "logs/${prefix}z_certificates_certmanager.txt" + dump_objects catalogsources "OLM CatalogSources" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_catalogsources.txt" + dump_objects operatorgroups "OLM OperatorGroups" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_operatorgroups.txt" + dump_objects subscriptions "OLM Subscriptions" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_subscriptions.txt" + dump_objects installplans "OLM InstallPlans" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_installplans.txt" + dump_objects clusterserviceversions "OLM ClusterServiceVersions" "${namespace}" 2> /dev/null > "logs/${prefix}z_olm_clusterserviceversions.txt" + dump_objects pods "Pods" "${namespace}" 2> /dev/null > "logs/${prefix}z_pods.txt" + + # shellcheck disable=SC2046 + kubectl describe $(kubectl get crd -o name | grep mongodb.com) > "logs/${prefix}z_mongodb_crds.log" +} diff --git a/scripts/evergreen/e2e/e2e.sh b/scripts/evergreen/e2e/e2e.sh new file mode 100755 index 000000000..76c62c0d5 --- /dev/null +++ b/scripts/evergreen/e2e/e2e.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +start_time=$(date +%s) + +source scripts/funcs/checks +source scripts/funcs/kubernetes +source scripts/funcs/printing +source scripts/evergreen/e2e/dump_diagnostic_information.sh +source scripts/evergreen/e2e/lib.sh +source scripts/dev/set_env_context.sh + +if [[ -n "${KUBECONFIG:-}" && ! -f "${KUBECONFIG}" ]]; then + echo "Kube configuration: ${KUBECONFIG} file does not exist!" + exit 1 +fi + +# +# This is the main entry point for running e2e tests. It can be used both for simple e2e tests (running a single test +# application) and for the Operator upgrade ones involving two steps (deploy previous Operator version, run test, deploy +# a new Operator - run verification tests) +# All the preparation work (fetching OM information, configuring resources) is done before running tests but +# it should be moved to e2e tests themselves (TODO) +# +check_env_var "TASK_NAME" "The 'TASK_NAME' must be specified for the Operator e2e tests" + +# 1. Ensure the namespace exists - it should be created during the private-context switch + +current_context=$(kubectl config current-context) +# shellcheck disable=SC2154 +if [[ "${KUBE_ENVIRONMENT_NAME}" == "multi" ]]; then + current_context="${CENTRAL_CLUSTER}" + kubectl config set-context "${current_context}" "--namespace=${NAMESPACE}" &>/dev/null || true + kubectl config use-context "${current_context}" + echo "Current context: ${current_context}, namespace=${NAMESPACE}" + kubectl get nodes | grep "control-plane" +fi + +ensure_namespace "${NAMESPACE}" + +# 2. Fetch OM connection information - it will be saved to environment variables +. scripts/evergreen/e2e/fetch_om_information.sh + +# 3. Configure Operator resources +. scripts/evergreen/e2e/configure_operator.sh + +if [[ "${RUNNING_IN_EVG:-false}" == "true" ]]; then + # 4. install honeycomb observability + . scripts/evergreen/e2e/performance/honeycomb/install-hc.sh +fi + +if [ -n "${TEST_NAME_OVERRIDE:-}" ]; then + echo "Running test with override: ${TEST_NAME_OVERRIDE}" + TEST_NAME="${TEST_NAME_OVERRIDE}" +else + TEST_NAME="${TASK_NAME:?}" +fi + +export TEST_NAME +echo "TEST_NAME is set to: ${TEST_NAME}" + +delete_operator "${NAMESPACE}" + +# 4. Main test run. + +# We'll have the task running for the alloca ted time, minus the time it took us +# to get all the way here, assuming configuring and deploying the operator can +# take a bit of time. This is needed because Evergreen kills the process *AND* +# Docker containers running on the host when it hits a timeout. Under these +# circumstances and in Kind based environments, it is impossible to fetch the +# results from the Kubernetes cluster running on top of Docker. +# +current_time=$(date +%s) +elapsed_time=$((current_time - start_time)) + +task_timeout=$(get_timeout_for_task "${TASK_NAME:?}") + +timeout_sec=$((task_timeout - elapsed_time - 60)) +echo "This task is allowed to run for ${timeout_sec} seconds" +TEST_RESULTS=0 +timeout --foreground "${timeout_sec}" scripts/evergreen/e2e/single_e2e.sh || TEST_RESULTS=$? +# Dump information from all clusters. +# TODO: ensure cluster name is included in log files so there is no overwriting of cross cluster files. +# shellcheck disable=SC2154 +if [[ "${KUBE_ENVIRONMENT_NAME:-}" = "multi" ]]; then + echo "Dumping diagnostics for context ${CENTRAL_CLUSTER}" + dump_all "${CENTRAL_CLUSTER}" || true + + for member_cluster in ${MEMBER_CLUSTERS}; do + echo "Dumping diagnostics for context ${member_cluster}" + dump_all "${member_cluster}" || true + done +else + # Dump all the information we can from this namespace + dump_all || true +fi + +# we only have static cluster in openshift, otherwise there is no need to mark and clean them up here +if [[ ${CLUSTER_TYPE} == "openshift" ]]; then + if [[ "${TEST_RESULTS}" -ne 0 ]]; then + # Mark namespace as failed to be cleaned later + kubectl label "namespace/${NAMESPACE}" "evg/state=failed" --overwrite=true + + if [ "${ALWAYS_REMOVE_TESTING_NAMESPACE-}" = "true" ]; then + # Failed namespaces might cascade into more failures if the namespaces + # are not being removed soon enough. + reset_namespace "$(kubectl config current-context)" "${NAMESPACE}" || true + fi + else + if [[ "${KUBE_ENVIRONMENT_NAME}" = "multi" ]]; then + echo "Tearing down cluster ${CENTRAL_CLUSTER}" + reset_namespace "${CENTRAL_CLUSTER}" "${NAMESPACE}" || true + + for member_cluster in ${MEMBER_CLUSTERS}; do + echo "Tearing down cluster ${member_cluster}" + reset_namespace "${member_cluster}" "${NAMESPACE}" || true + done + else + # If the test pass, then the namespace is removed + reset_namespace "$(kubectl config current-context)" "${NAMESPACE}" || true + fi + fi +fi + +# We exit with the test result to surface status to Evergreen. +exit ${TEST_RESULTS} diff --git a/scripts/evergreen/e2e/fetch_om_information.sh b/scripts/evergreen/e2e/fetch_om_information.sh new file mode 100755 index 000000000..b101ce290 --- /dev/null +++ b/scripts/evergreen/e2e/fetch_om_information.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/funcs/checks +source scripts/funcs/printing + +[[ "${MODE-}" = "dev" ]] && return + +if [[ "${TEST_MODE:-}" = "opsmanager" ]]; then + echo "Skipping Ops Manager connection configuration as current test is for Ops Manager" + return +fi + +if [[ "${OM_EXTERNALLY_CONFIGURED:-}" = "true" ]]; then + echo "Skipping Ops Manager connection configuration as the connection details are already provided" + return +fi + +title "Reading Ops Manager environment variables..." + +check_env_var "OPS_MANAGER_NAMESPACE" "The 'OPS_MANAGER_NAMESPACE' must be specified to fetch Ops Manager connection details" + +OPERATOR_TESTING_FRAMEWORK_NS=${OPS_MANAGER_NAMESPACE} +if ! kubectl get "namespace/${OPERATOR_TESTING_FRAMEWORK_NS}" &> /dev/null; then + error "Ops Manager is not installed in this cluster. Make sure the Ops Manager installation script is called beforehand. Exiting..." + + exit 1 +else + echo "Ops Manager is already installed in this cluster. Will reuse it now." +fi + +echo "Getting credentials from secrets" + + +OM_USER="$(kubectl get secret ops-manager-admin-secret -n "${OPERATOR_TESTING_FRAMEWORK_NS}" -o json | jq -r '.data | with_entries(.value |= @base64d)' | jq '.Username' -r)" +OM_PASSWORD="$(kubectl get secret ops-manager-admin-secret -n "${OPERATOR_TESTING_FRAMEWORK_NS}" -o json | jq -r '.data | with_entries(.value |= @base64d)' | jq '.Password' -r)" + +OM_PUBLIC_API_KEY="$(kubectl get secret "${OPERATOR_TESTING_FRAMEWORK_NS}"-ops-manager-admin-key -n "${OPERATOR_TESTING_FRAMEWORK_NS}" -o json | jq -r '.data | with_entries(.value |= @base64d)' | jq '.publicKey' -r)" +OM_API_KEY="$(kubectl get secret "${OPERATOR_TESTING_FRAMEWORK_NS}"-ops-manager-admin-key -n "${OPERATOR_TESTING_FRAMEWORK_NS}" -o json | jq -r '.data | with_entries(.value |= @base64d)' | jq '.privateKey' -r)" + +export OM_USER +export OM_PASSWORD +export OM_API_KEY +export OM_PUBLIC_API_KEY + + +title "Ops Manager environment is successfully read" diff --git a/scripts/evergreen/e2e/lib.sh b/scripts/evergreen/e2e/lib.sh new file mode 100755 index 000000000..ac46eaa04 --- /dev/null +++ b/scripts/evergreen/e2e/lib.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Finds the exec_timeout_secs for this task in Evergreen. If it can't find it, will return the general +# exec_timeout_secs attribute set in top-level evergreen.yaml. +get_timeout_for_task () { + local task_name=${1} + + local exec_timeout + # OMG: really??? + exec_timeout=$(grep "name: ${task_name}$" .evergreen.yml -A 3 | grep exec_timeout_secs | awk '{ print $2 }') + if [[ ${exec_timeout} = "" ]]; then + exec_timeout=$(grep "^exec_timeout_secs:" .evergreen.yml | head -1 | awk '{ print $2 }') + fi + + echo "${exec_timeout}" +} + +# Returns a random string that can be used as a namespace. +generate_random_namespace() { + local random_namespace + random_namespace=$(LC_ALL=C tr -dc 'a-z0-9' HTTPDigestAuth: + """Builds a HTTPDigestAuth from user and api_key""" + return HTTPDigestAuth(user, api_key) + + +def get_auth(auth_type: str = "org_owner") -> HTTPDigestAuth: + """Builds an Authentication object depending on the type required.""" + if auth_type == "org_owner": + api_key = os.getenv(APIKEY_OWNER) + assert api_key is not None, f"no {APIKEY_OWNER} env variable defined" + user = os.getenv(USER_OWNER) + assert user is not None, f"no {USER_OWNER} env variable defined" + return _get_auth(api_key, user) + if auth_type == "project_owner": + env = read_env_file() + return _get_auth(env["OM_API_KEY"], env["OM_USER"]) + assert False, "wrong auth_type" + + +def create_api_key(org: str, description: str, roles: List[str] = None): + """Creates an Organization level API Key object.""" + if roles is None: + roles = ["ORG_GROUP_CREATOR"] + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/orgs/{}/apiKeys".format(base_url, org) + data = {"roles": roles, "desc": description} + response = requests.post(url, auth=get_auth(), json=data) + if response.status_code != 200: + raise Exception("Could not create Programmatic API Key", response.text) + + return response.json() + + +def create_group(org: str, name: str): + """Creates a group in an organization. + + note: this is not needed for now, I use it for local testing. + """ + auth = get_auth("project_owner") + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/groups".format(base_url) + data = {"orgId": org, "name": name} + response = requests.post(url, auth=auth, json=data) + + print(response.text) + + +def delete_api_key(org: str, key_id: str): + """Deletes an Organization level API Key object.""" + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/orgs/{}/apiKeys/{}".format(base_url, org, key_id) + response = requests.delete(url, auth=get_auth()) + if response.status_code != 204: + raise Exception("Could not remove the Programmatic API Key", response.text) + + return response + + +def whitelist_key(org: str, key_id: str, whitelist: List[str] = None): + """Whitelists an Organization level API Key object.""" + if whitelist is None: + whitelist = ["0.0.0.0/1", "128.0.0.0/1"] + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/orgs/{}/apiKeys/{}/whitelist".format(base_url, org, key_id) + data = [{"cidrBlock": cb} for cb in whitelist] + response = requests.post(url, auth=get_auth(), json=data) + if response.status_code != 200: + raise Exception("Could not add whitelist", response.text) + + return response + + +def get_group_id_by_name(name: str, retry=3) -> str: + """Returns the 'id' that corresponds to this Project name.""" + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/groups/byName/{}".format(base_url, name) + + while retry > 0: + groups = requests.get(url, auth=get_auth("project_owner")) + + response = groups.json() + if "id" not in response: + print("Id not in the response, this is what we got") + print(response) + retry -= 1 + time.sleep(3 + random.random()) + continue + + break + + return groups.json()["id"] + + +def project_was_created_before(group_name: str, minutes_interval: int) -> bool: + """Returns True if the group was created before 'current_time() - minutes_interval'""" + try: + group_seconds_epoch = int(group_name.split("-")[1]) # a-1598972093-yr3jzt3v7bsl -> 1598972093 + except ValueError: + print( + f"group_name is: {group_name}, and the second part is not convertible to a timestamp this is unexpected " + f"and shouldn't happen. Will delete it now! This might cause" + f"failures in test until the test is fixed to not use wrong name patterns." + ) + return True + except IndexError: + print( + f"group_name is: {group_name}, and there is no '-' in the name to identify the timestamp. This is unexpected " + f"and shouldn't happen. Will delete it now! This might cause" + f"failures in test until the test is fixed to not use wrong name patterns." + ) + return True + return is_before(group_seconds_epoch, minutes_interval) + + +def key_is_older_than(key_description: str, minutes_interval: int) -> bool: + """Returns True if the key was created before 'current_time() - minutes_interval'""" + try: + key_seconds_epoch = int(key_description) + except ValueError: + print( + f"deleting keys with wrong description since its not convertible to an int, " + f"it should not be the case; key description name {key_description}" + ) + # any keys with the wrong description format (old/manual?) need to be removed as well + return True + return is_before(key_seconds_epoch, minutes_interval) + + +def is_before(time_since_epoch: int, minutes_interval: int) -> bool: + current_seconds_epoch = time.time() + return time_since_epoch + (minutes_interval * 60) <= current_seconds_epoch + + +def generate_key_description() -> str: + """Returns the programmatic key description: it's the seconds after Unix epoch""" + return str(int(time.time())) + + +def remove_group_by_id(group_id: str, retry=3): + """Removes a group with a given Id.""" + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/groups/{}".format(base_url, group_id) + while retry > 0: + controlled_features_data = { + "externalManagementSystem": {"name": "mongodb-enterprise-operator"}, + "policies": [], + } + + result = requests.put( + f"{url}/controlledFeature", + auth=get_auth("org_owner"), + json=controlled_features_data, + ) + print(result) + result = requests.put(f"{url}/automationConfig", auth=get_auth("org_owner"), json={}) + print(result) + result = requests.delete(url, auth=get_auth("org_owner")) + print(result) + if result.status_code != 202: + retry -= 1 + time.sleep(3 + random.random()) + continue + + break + + return result + + +def remove_group_by_name(name: str): + """Removes a group by its name.""" + _id = get_group_id_by_name(name) + result = remove_group_by_id(_id) + + status = "OK" if result.status_code == 202 else "FAILED" + print("Removing group id: {} and name: {} -> {}".format(_id, name, status)) + return result + + +def read_namespace_from_file(): + """Reads a testing namespace name from a file.""" + namespace_file = os.getenv(NAMESPACE_FILE) + with open(namespace_file) as fd: + return fd.read().strip() + + +def get_key_value_from_line(line: str) -> Tuple[str, str]: + """Returns a key, value from a line with the format 'export key=value""" + matcher = re.compile(r"^export\s+([A-Z_]+)\s*=\s*(\S+)$") + match = matcher.match(line) + assert match, "Unrecognised pattern in ENV_FILE" + return match.group(1), match.group(2) + + +def read_env_file(): + """Returns the env file (in ENV_FILE env variable) as a key=value dict.""" + data = {} + + env_file = os.getenv(ENV_FILE) + with open(env_file) as fd: + for line in fd.readlines(): + try: + key, value = get_key_value_from_line(line) + except IndexError: + pass + data[key] = value + + return data + + +def configure(): + """Creates a programmatic API Key, and whitelist it. This function also + creates a sourceable file with the Cloud QA configuration, + unfortunately, that's the only way of passing data from one command to + the next. + """ + org = os.getenv(ORG_ID) + response = create_api_key(org, generate_key_description()) + + # we will use key_id to remove this key + key_id = response["id"] + whitelist_key(org, key_id) + + public = response["publicKey"] + private = response["privateKey"] + + env_file = os.getenv(ENV_FILE) + base_url = os.getenv(BASE_URL) + with open(env_file, "w") as fd: + fd.write("export OM_BASE_URL={}\n".format(base_url)) + fd.write("export OM_USER={}\n".format(public)) + fd.write("export OM_API_KEY={}\n".format(private)) + fd.write("export OM_ORGID={}\n".format(org)) + fd.write("export OM_KEY_ID={}\n".format(key_id)) + fd.write("export OM_EXTERNALLY_CONFIGURED=true\n") + + +def get_projects_older_than(org_id: str, minutes_interval: int = 0) -> Tuple[List[Dict], List[Dict]]: + """Returns the project ids which are older than 'age' days ago""" + query_params = {"includeCount": True, "itemsPerPage": 500, "pageNum": 1} + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/orgs/{}/groups".format(base_url, org_id) + old_groups = [] + new_groups = [] + while True: + response = requests.get(url, auth=get_auth(), params=query_params) + print(f"Queried page {query_params['pageNum']}, URL: {response.request.url}") + results = response.json().get("results", []) + for group in results: + if project_was_created_before(group["name"], minutes_interval): + old_groups.append(group) + else: + new_groups.append(group) + + if len(results) == 0 or query_params["pageNum"] > PAGINATION_LIMIT: + break + query_params["pageNum"] += 1 + + return old_groups, new_groups + + +def get_keys_older_than(org_id: str, minutes_interval: int = 0) -> Tuple[List[Dict], List[Dict]]: + """Returns the programmatic keys which are older than 'minutes_interval' ago""" + query_params = {"includeCount": True, "itemsPerPage": 500, "pageNum": 1} + base_url = os.getenv(BASE_URL) + url = "{}/api/public/v1.0/orgs/{}/apiKeys".format(base_url, org_id) + old_keys = [] + new_keys = [] + while True: + response = requests.get(url, auth=get_auth(), params=query_params) + print(f"Queried page {query_params['pageNum']}, URL: {response.request.url}") + results = response.json().get("results", []) + for key in results: + if key_is_older_than(key["desc"], minutes_interval): + old_keys.append(key) + else: + new_keys.append(key) + + if len(results) == 0 or query_params["pageNum"] > PAGINATION_LIMIT: + break + + query_params["pageNum"] += 1 + + return old_keys, new_keys + + +def clean_unused_keys(org_id: str): + """Iterates over all existing keys in the organization and removes the leftovers. + Keeps keys: + - older than minutes_interval + - currently used by the script (USER_OWNER env variable) + - containing "EVG" or "NOT_DELETE" in their description + """ + keys, newer_keys = get_keys_older_than(org_id, minutes_interval=70) + print(f"found {len(keys)} keys for potential removal") + deleted_keys = [] + kept_keys = [] + + for key in keys: + if not keep_the_key(key): + print("Removing the key {} ({})".format(key["publicKey"], key["desc"])) + delete_api_key(org_id, key["id"]) + deleted_keys.append(key["id"]) + else: + print("Keeping the key {} ({})".format(key["publicKey"], key["desc"])) + kept_keys.append(key) + print(f"KEY REMOVAL SUMMARY\n----------") + print(f"Ignored {len(newer_keys)} keys because they are too new.") + print(f"Removed {len(deleted_keys)} keys.") + print(f"Kept {len(kept_keys)}:") + for key in kept_keys: + print(f"\t{key['publicKey']}\t{key['desc']}") + print("----------") + + +def keep_the_key(key: Dict) -> bool: + """Returns True if the key shouldn't be removed""" + return key["publicKey"] == os.getenv(USER_OWNER).lower() or "EVG" in key["desc"] or "NOT_DELETE" in key["desc"] + + +def clean_unused_projects(org_id: str): + """Iterates over all existing projects in the organization and removes the leftovers""" + projects, newer_projects = get_projects_older_than(org_id, minutes_interval=70) + print(f"found {len(projects)} projects for potential removal") + deleted_projects = [] + kept_projects = [] + + for project in projects: + print("Removing the project {} ({})".format(project["id"], project["name"])) + response = remove_group_by_id(project["id"]) + if response.status_code == 202: + deleted_projects.append(project["id"]) + else: + kept_projects.append(project) + print(f"PROJECT REMOVAL SUMMARY\n----------") + print(f"Ignored {len(newer_projects)} projects because they are too new.") + print(f"Removed {len(deleted_projects)} projects.") + print(f"Kept {len(kept_projects)} projects:") + for project in kept_projects: + print(f"\t{project['name']}\t{project['id']}") + print("----------") + + +def unconfigure_all(): + """Tries to remove the project and API Key from Cloud-QA""" + env_file_exists = True + try: + env = read_env_file() + except Exception as e: + print("Got an exception trying to read env-file", e) + env_file_exists = False + + namespace = None + try: + namespace = read_namespace_from_file() + except Exception as e: + print("Got an exception trying to read namespace", e) + + # The "group" needs to be removed using the user's API credentials + if namespace is not None: + print("Got namespace:", namespace) + try: + remove_group_by_name(namespace) + except Exception as e: + print("Got an exception trying to remove group", e) + + org = os.getenv(ORG_ID) + + # The API Key needs to be removed using the Owner's API credentials + if env_file_exists: + key_id = env["OM_KEY_ID"] + try: + delete_api_key(org, key_id) + except Exception as e: + print("Got an exception trying to remove Api Key", e) + + clean_unused_projects(org) + clean_unused_keys(org) + + +def unconfigure_from_used_project(): + """Tries to remove the project and API Key from Cloud-QA""" + env_file_exists = True + try: + env = read_env_file() + except Exception as e: + print("Got an exception trying to read env-file", e) + env_file_exists = False + + namespace = None + try: + namespace = read_namespace_from_file() + except Exception as e: + print("Got an exception trying to read namespace", e) + + # The "group" needs to be removed using the user's API credentials + if namespace is not None: + print("Got namespace:", namespace) + try: + remove_group_by_name(namespace) + print(f"Removing Namespace file: {os.getenv(NAMESPACE_FILE)}") + os.remove(os.getenv(NAMESPACE_FILE)) + except Exception as e: + print("Got an exception trying to remove group", e) + + org = os.getenv(ORG_ID) + + # The API Key needs to be removed using the Owner's API credentials + if env_file_exists: + key_id = env["OM_KEY_ID"] + try: + delete_api_key(org, key_id) + print(f"Removing ENV_FILE file: {os.getenv(ENV_FILE)}") + os.remove(os.getenv(ENV_FILE)) + except Exception as e: + print("Got an exception trying to remove Api Key", e) + + +def argv_error() -> int: + print("This script can only be called with create or delete") + return 1 + + +def check_env_variables() -> bool: + status = True + for var in REQUIRED_ENV_VARIABLES: + if not os.getenv(var): + print("Missing env variable: {}".format(var)) + status = False + return status + + +def main() -> int: + global ORG_ID, USER_OWNER, APIKEY_OWNER + if not check_env_variables(): + print("Please define all required env variables") + return 1 + om_version = os.getenv(OPS_MANAGER_VERSION) + if om_version is None or om_version != ALLOWED_OPS_MANAGER_VERSION: + print( + "ops_manager_version env variable is not correctly defined: " + "only '{}' is allowed".format(ALLOWED_OPS_MANAGER_VERSION) + ) + # Should not run if not using Cloud-QA + return 1 + + if len(sys.argv) < 2: + return argv_error() + if sys.argv[1] == "delete": + print("Removing project and api key from Cloud QA") + unconfigure_from_used_project() + elif sys.argv[1] == "create": + print("Configuring Cloud QA") + configure() + elif sys.argv[1] == "delete_all": + for i, _ in enumerate(ORG_IDS): + ORG_ID = ORG_IDS[i] + USER_OWNER = USER_OWNERS[i] + APIKEY_OWNER = APIKEY_OWNERS[i] + print(f"Removing all project and api key from Cloud QA which are older than X for {ORG_ID}") + unconfigure_all() + else: + return argv_error() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh new file mode 100755 index 000000000..5e6fd2746 --- /dev/null +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +## +## The script deploys a single test application and waits until it finishes. +## All the Operator deployment, configuration and teardown work is done in 'e2e' script +## + +source scripts/funcs/checks +source scripts/funcs/printing +source scripts/funcs/errors +source scripts/funcs/multicluster +source scripts/funcs/operator_deployment + +check_env_var "TEST_NAME" "The 'TEST_NAME' must be specified to run the Operator single e2e test" + + +deploy_test_app() { + printenv + title "Deploying test application" + local context=${1} + local helm_template_file + helm_template_file=$(mktemp) + tag="${VERSION_ID:-latest}" + if [[ "${OVERRIDE_VERSION_ID:-}" != "" ]]; then + tag="${OVERRIDE_VERSION_ID}" + fi + + IS_PATCH="${IS_PATCH:-default_patch}" + TASK_NAME="${TASK_NAME:-default_task}" + EXECUTION="${EXECUTION:-default_execution}" + BUILD_ID="${BUILD_ID:-default_build_id}" + BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}" + + # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager + helm_params=( + "--set" "taskId=${task_id:-'not-specified'}" + "--set" "repo=${BASE_REPO_URL:=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev}" + "--set" "namespace=${NAMESPACE}" + "--set" "taskName=${task_name}" + "--set" "tag=${tag}" + "--set" "aws.accessKey=${AWS_ACCESS_KEY_ID}" + "--set" "aws.secretAccessKey=${AWS_SECRET_ACCESS_KEY}" + "--set" "skipExecution=${SKIP_EXECUTION:-'false'}" + "--set" "baseUrl=${OM_BASE_URL:-http://ops-manager-svc.${OPS_MANAGER_NAMESPACE}.svc.cluster.local:8080}" + "--set" "apiKey=${OM_API_KEY:-}" + "--set" "apiUser=${OM_USER:-admin}" + "--set" "orgId=${OM_ORGID:-}" + "--set" "imagePullSecrets=image-registries-secret" + "--set" "managedSecurityContext=${MANAGED_SECURITY_CONTEXT:-false}" + "--set" "registry=${REGISTRY:-${BASE_REPO_URL}/${IMAGE_TYPE}}" + "--set" "mdbDefaultArchitecture=${MDB_DEFAULT_ARCHITECTURE:-'non-static'}" + "--set" "mdbImageType=${MDB_IMAGE_TYPE:-'ubi8'}" + "--set" "clusterDomain=${CLUSTER_DOMAIN:-'cluster.local'}" + ) + + # shellcheck disable=SC2154 + if [[ ${KUBE_ENVIRONMENT_NAME} = "multi" ]]; then + helm_params+=("--set" "multiCluster.memberClusters=${MEMBER_CLUSTERS}") + helm_params+=("--set" "multiCluster.centralCluster=${CENTRAL_CLUSTER}") + helm_params+=("--set" "multiCluster.testPodCluster=${test_pod_cluster}") + fi + + if [[ -n "${CUSTOM_OM_VERSION:-}" ]]; then + # The test needs to create an OM resource with specific version + helm_params+=("--set" "customOmVersion=${CUSTOM_OM_VERSION}") + fi + if [[ -n "${pytest_addopts:-}" ]]; then + # The test needs to create an OM resource with specific version + helm_params+=("--set" "pytest.addopts=${pytest_addopts:-}") + fi + # As soon as we are having one OTEL expansion it means we want to trace and send everything to our trace provider. + # otel_parent_id is a special case (hence lower cased) since it is directly coming from evergreen and not via our + # make switch mechanism. We need the "freshest" parent_id otherwise we are attaching to the wrong parent span. + if [[ -n "${otel_parent_id:-}" ]]; then + otel_resource_attributes="evergreen.version.id=${VERSION_ID:-},evergreen.version.requester=${requester:-},meko_git_branch=${branch_name:-},evergreen.version.pr_num=${github_pr_number:-},meko_git_commit=${github_commit:-},meko_revision=${revision:-},is_patch=${IS_PATCH},evergreen.task.name=${TASK_NAME},evergreen.task.execution=${EXECUTION},evergreen.build.id=${BUILD_ID},evergreen.build.name=${BUILD_VARIANT},evergreen.task.id=${task_id},evergreen.project.id=${project_identifier:-}" + # shellcheck disable=SC2001 + escaped_otel_resource_attributes=$(echo "${otel_resource_attributes}" | sed 's/,/\\,/g') + # The test needs to create an OM resource with specific version + helm_params+=("--set" "otel_parent_id=${otel_parent_id:-"unknown"}") + helm_params+=("--set" "otel_trace_id=${OTEL_TRACE_ID:-"unknown"}") + helm_params+=("--set" "otel_endpoint=${OTEL_COLLECTOR_ENDPOINT:-"unknown"}") + helm_params+=("--set" "otel_resource_attributes=${escaped_otel_resource_attributes}") + fi + if [[ -n "${CUSTOM_OM_PREV_VERSION:-}" ]]; then + # The test needs to create an OM resource with specific version + helm_params+=("--set" "customOmPrevVersion=${CUSTOM_OM_PREV_VERSION}") + fi + if [[ -n "${PERF_TASK_DEPLOYMENTS:-}" ]]; then + # The test needs to create an OM resource with specific version + helm_params+=("--set" "taskDeployments=${PERF_TASK_DEPLOYMENTS}") + fi + if [[ -n "${PERF_TASK_REPLICAS:-}" ]]; then + # The test needs to create an OM resource with specific version + helm_params+=("--set" "taskReplicas=${PERF_TASK_REPLICAS}") + fi + if [[ -n "${CUSTOM_MDB_VERSION:-}" ]]; then + # The test needs to test MongoDB of a specific version + helm_params+=("--set" "customOmMdbVersion=${CUSTOM_MDB_VERSION}") + fi + if [[ -n "${CUSTOM_MDB_PREV_VERSION:-}" ]]; then + # The test needs to test MongoDB of a previous version + helm_params+=("--set" "customOmMdbPrevVersion=${CUSTOM_MDB_PREV_VERSION}") + fi + if [[ -n "${CUSTOM_APPDB_VERSION:-}" ]]; then + helm_params+=("--set" "customAppDbVersion=${CUSTOM_APPDB_VERSION}") + fi + + if [[ -n "${PROJECT_DIR:-}" ]]; then + helm_params+=("--set" "projectDir=/ops-manager-kubernetes") + fi + + if [[ "${LOCAL_OPERATOR}" == true ]]; then + helm_params+=("--set" "localOperator=true") + fi + + if [[ "${OM_DEBUG_HTTP}" == "true" ]]; then + helm_params+=("--set" "omDebugHttp=true") + fi + + helm_params+=("--set" "opsManagerVersion=${ops_manager_version}") + + helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || exit 1 + + cat "${helm_template_file}" + + kubectl --context "${context}" -n "${NAMESPACE}" delete -f "${helm_template_file}" 2>/dev/null || true + + kubectl --context "${context}" -n "${NAMESPACE}" apply -f "${helm_template_file}" +} + +wait_until_pod_is_running_or_failed_or_succeeded() { + local context=${1} + # Do wait while the Pod is not yet running or failed (can be in Pending or ContainerCreating state) + # Note that the pod may jump to Failed/Completed state quickly - so we need to give up waiting on this as well + echo "Waiting until the test application gets to Running state..." + + is_running_cmd="kubectl --context '${context}' -n ${NAMESPACE} get pod ${TEST_APP_PODNAME} -o jsonpath={.status.phase} | grep -q 'Running'" + + # test app usually starts instantly but sometimes (quite rarely though) may require more than a min to start + # in Evergreen so let's wait for 2m + timeout --foreground "2m" bash -c "while ! ${is_running_cmd}; do printf .; sleep 1; done;" + echo + + if ! eval "${is_running_cmd}"; then + error "Test application failed to start on time!" + kubectl --context "${context}" -n "${NAMESPACE}" describe pod "${TEST_APP_PODNAME}" + fatal "Failed to run test application - exiting" + fi +} + +test_app_ended() { + local context="${1}" + local status + status="$(kubectl --context "${context}" get pod mongodb-enterprise-operator-tests -n "${NAMESPACE}" -o jsonpath="{.status}" | jq -r '.containerStatuses[] | select(.name == "mongodb-enterprise-operator-tests")'.state.terminated.reason)" + [[ "${status}" = "Error" || "${status}" = "Completed" ]] +} + +# Will run the test application and wait for its completion. +run_tests() { + local task_name=${1} + local operator_context + local test_pod_context + operator_context="$(kubectl config current-context)" + + test_pod_context="${operator_context}" + if [[ "${KUBE_ENVIRONMENT_NAME}" = "multi" ]]; then + operator_context="${CENTRAL_CLUSTER}" + # shellcheck disable=SC2154,SC2269 + test_pod_context="${test_pod_cluster:-${operator_context}}" + fi + + echo "Operator running in cluster ${operator_context}" + echo "Test pod running in cluster ${test_pod_context}" + + TEST_APP_PODNAME=mongodb-enterprise-operator-tests + + if [[ "${KUBE_ENVIRONMENT_NAME}" = "multi" ]]; then + configure_multi_cluster_environment + fi + + prepare_operator_config_map "${operator_context}" + + deploy_test_app "${test_pod_context}" + + wait_until_pod_is_running_or_failed_or_succeeded "${test_pod_context}" + + title "Running e2e test ${task_name} (tag: ${TEST_IMAGE_TAG})" + + # we don't output logs to file when running tests locally + if [[ "${MODE-}" == "dev" ]]; then + kubectl --context "${test_pod_context}" -n "${NAMESPACE}" logs "${TEST_APP_PODNAME}" -c mongodb-enterprise-operator-tests -f + else + output_filename="logs/test_app.log" + + # At this time ${TEST_APP_PODNAME} has finished running, so we don't follow (-f) it + # Similarly, the operator deployment has finished with our tests, so we print whatever we have + # until this moment and go continue with our lives + kubectl --context "${test_pod_context}" -n "${NAMESPACE}" logs "${TEST_APP_PODNAME}" -c mongodb-enterprise-operator-tests -f | tee "${output_filename}" || true + fi + + + # Waiting a bit until the pod gets to some end + while ! test_app_ended "${test_pod_context}"; do printf .; sleep 1; done; + echo + + # We need to make sure to access this file after the test has finished + kubectl --context "${test_pod_context}" -n "${NAMESPACE}" cp "${TEST_APP_PODNAME}":/tmp/results/myreport.xml logs/myreport.xml + kubectl --context "${test_pod_context}" -n "${NAMESPACE}" cp "${TEST_APP_PODNAME}":/tmp/diagnostics logs + + status="$(kubectl --context "${test_pod_context}" get pod "${TEST_APP_PODNAME}" -n "${NAMESPACE}" -o jsonpath="{ .status }" | jq -r '.containerStatuses[] | select(.name == "mongodb-enterprise-operator-tests")'.state.terminated.reason)" + [[ "${status}" == "Completed" ]] +} + +mkdir -p logs/ + +TESTS_OK=0 +# shellcheck disable=SC2153 +run_tests "${TEST_NAME}" || TESTS_OK=1 + +echo "Tests have finished with the following exit code: ${TESTS_OK}" + +[[ "${TESTS_OK}" -eq 0 ]] diff --git a/scripts/evergreen/flakiness-report.py b/scripts/evergreen/flakiness-report.py new file mode 100644 index 000000000..406bcb2f1 --- /dev/null +++ b/scripts/evergreen/flakiness-report.py @@ -0,0 +1,72 @@ +import json +import os +import sys + +import requests + +EVERGREEN_API = "https://evergreen.mongodb.com/api" + + +def print_usage(): + print( + """Set EVERGREEN_USER and EVERGREEN_API_KEY env. variables +Obtain the version from either Evergreen UI or Github checks +Call + python flakiness-report.py + python flakiness-report.py 62cfba5957e85a64e1f801fa""" + ) + + +def get_variants_with_retried_tasks() -> dict[str, list[dict]]: + evg_user = os.environ.get("EVERGREEN_USER", "") + api_key = os.environ.get("API_KEY", "") + + if len(sys.argv) != 2 or evg_user == "" or api_key == "": + print_usage() + exit(1) + + version = sys.argv[1] + + headers = {"Api-User": evg_user, "Api-Key": api_key} + print("Fetching build variants...", file=sys.stderr) + build_ids = requests.get(url=f"{EVERGREEN_API}/rest/v2/versions/{version}", headers=headers).json() + build_statuses = [build_status for build_status in build_ids["build_variants_status"]] + + variants_with_retried_tasks: dict[str, list[dict]] = {} + print(f"Fetching tasks for build variants: ", end="", file=sys.stderr) + for build_status in build_statuses: + tasks = requests.get( + url=f"{EVERGREEN_API}/rest/v2/builds/{build_status['build_id']}/tasks", + headers=headers, + ).json() + retried_tasks = [task for task in tasks if task["execution"] > 1] + if len(retried_tasks) > 0: + variants_with_retried_tasks[build_status["build_variant"]] = sorted( + retried_tasks, key=lambda task: task["execution"], reverse=True + ) + print(f"{build_status['build_variant']}, ", end="", file=sys.stderr) + + print("", file=sys.stderr) + return variants_with_retried_tasks + + +def print_retried_tasks(retried_tasks: dict[str, list[dict]]): + if len(retried_tasks) == 0: + print("No retried tasks found") + return + + print("Number of retries in tasks grouped by build variant:") + for build_variant, tasks in retried_tasks.items(): + print(f"{build_variant}:") + for task in tasks: + print(f"\t{task['display_name']}: {task['execution']}") + + +def main(): + variants_with_retried_tasks = get_variants_with_retried_tasks() + print("\n") + print_retried_tasks(variants_with_retried_tasks) + + +if __name__ == "__main__": + main() diff --git a/scripts/evergreen/generate_evergreen_expansions.sh b/scripts/evergreen/generate_evergreen_expansions.sh new file mode 100755 index 000000000..3af0afb01 --- /dev/null +++ b/scripts/evergreen/generate_evergreen_expansions.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This file converts release.json values to flat evergreen_expansions.yaml file +# to be source in evergreen's expansion.update function. +# +# Important: expansion.update can only source yaml file as simple key:value lines. + +yaml_file=evergreen_expansions.yaml + +# add additional variables when needed +cat <"${yaml_file}" +mongodbOperator: $(jq -r .mongodbOperator < release.json) +EOF + +echo "Generated ${yaml_file} file from release.json" +cat "${yaml_file}" + diff --git a/scripts/evergreen/lint_code.sh b/scripts/evergreen/lint_code.sh new file mode 100755 index 000000000..d0a070166 --- /dev/null +++ b/scripts/evergreen/lint_code.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +# Set required version +required_version="v2.0.2" + +# Install or update golangci-lint if not installed or version is incorrect +if ! [[ -x "$(command -v golangci-lint)" ]]; then + echo "Installing/updating golangci-lint to version ${required_version}..." + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)"/bin "${required_version}" +else + echo "golangci-lint is already installed" +fi + +echo "Go Version: $(go version)" + +echo "Running golangci-lint..." +if PATH=$(go env GOPATH)/bin:${PATH} golangci-lint run --fix; then + echo "No issues found by golangci-lint." +else + echo "golangci-lint found issues or made changes." + exit 1 +fi diff --git a/scripts/evergreen/operator-sdk/install-olm.sh b/scripts/evergreen/operator-sdk/install-olm.sh new file mode 100755 index 000000000..0c7c732f7 --- /dev/null +++ b/scripts/evergreen/operator-sdk/install-olm.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +operator-sdk olm install --version="${OLM_VERSION}" diff --git a/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh b/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh new file mode 100755 index 000000000..1f8b47f1e --- /dev/null +++ b/scripts/evergreen/operator-sdk/prepare-openshift-bundles-for-e2e.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +# This scripts prepares bundles and catalog source for operator upgrade tests using OLM. +# It builds and publishes the following docker images: +# - certified bundle from the last published version of the operator, from RedHat's certified operators repository +# - bundle from the current branch, referencing images built as part of EVG pipeline +# - catalog source with two channels: stable - referencing last published version, fast - referencing bundle built from the current branch +# +# Required env vars: +# - BASE_REPO_URL (or REGISTRY for EVG run) +# - VERSION_ID (set in EVG as patch-id) + +# for get_operator_helm_values function +source scripts/funcs/operator_deployment +source scripts/funcs/printing +source scripts/dev/set_env_context.sh + +increment_version() { + IFS=. read -r major minor patch <<< "$1" + ((patch++)) + echo "${major}.${minor}.${patch}" +} + +if [[ "${REGISTRY:-""}" != "" ]]; then + base_repo_url="${REGISTRY}" +else + base_repo_url="${BASE_REPO_URL:-"268558157000.dkr.ecr.us-east-1.amazonaws.com/dev"}" +fi + +# Generates helm charts the same way they are generates part of pre-commit hook. +# We also provide helm values override the same way it's done when installing the operator helm chart in e2e tests. +generate_helm_charts() { + read -ra helm_values < <(get_operator_helm_values) + echo "Passing overrides to helm values: " + tr ' ' '\n' <<< "${helm_values[*]}" + + declare -a helm_set_values=() + for param in "${helm_values[@]}"; do + helm_set_values+=("--set" "${param}") + done + + .githooks/pre-commit generate_standalone_yaml "${helm_set_values[@]}" +} + +function clone_git_repo_into_temp() { + git_repo_url=$1 + tmpdir=$(mktemp -d) + git clone --depth 1 "${git_repo_url}" "${tmpdir}" + echo "${tmpdir}" +} + +function find_the_latest_certified_operator() { + certified_operators_cloned_repo=$1 + # In this specific case, we don't want to use find as ls sorts things lexicographically - we need this! + # shellcheck disable=SC2012 + ls -r "${certified_operators_cloned_repo}/operators/mongodb-enterprise" | sed -n '2 p' +} + +# Clones git_repo_url, builds the bundle in the given version and publishes it to bundle_image_url. +function build_bundle_from_git_repo() { + tmpdir=$1 + version=$2 + bundle_image_url=$3 + + pushd "${tmpdir}" + mkdir -p "bundle" + + mv "operators/mongodb-enterprise/${version}" "bundle/" + docker build --platform "${DOCKER_PLATFORM}" -f "./bundle/${version}/bundle.Dockerfile" -t "${bundle_image_url}" . + docker push "${bundle_image_url}" + popd +} + +# Builds and publishes catalog source with two channels: stable and fast. +build_and_publish_catalog_with_two_channels() { + temp_dir=$1 + latest_bundle_version=$2 + latest_bundle_image=$3 + current_bundle_version=$4 + current_bundle_image=$5 + catalog_image=$6 + echo "Building catalog with latest released bundle and the current one" + + # The OPM tool generates the Dockerfile one directory higher then specified + catalog_dir="${temp_dir}/mongodb-enterprise-operator-catalog/mongodb-enterprise-operator-catalog" + mkdir -p "${catalog_dir}" + + echo "Generating the dockerfile" + opm generate dockerfile "${catalog_dir}" + echo "Generating the catalog" + + # Stable - latest release, fast - current version + opm init mongodb-enterprise \ + --default-channel="stable" \ + --output=yaml \ + > "${catalog_dir}"/operator.yaml + + echo "Adding latest release ${latest_bundle_image} to the catalog" + opm render "${latest_bundle_image}" --output=yaml >> "${catalog_dir}"/operator.yaml + + echo "Adding current unreleased ${current_bundle_image} to the catalog" + opm render "${current_bundle_image}" --output=yaml >> "${catalog_dir}"/operator.yaml + + echo "Adding previous release channel as STABLE to ${catalog_dir}/operator.yaml" + echo "--- +schema: olm.channel +package: mongodb-enterprise +name: stable +entries: + - name: mongodb-enterprise.v${latest_bundle_version}" >> "${catalog_dir}"/operator.yaml + + echo "Adding current version channel as FAST to ${catalog_dir}/operator.yaml" + echo "--- +schema: olm.channel +package: mongodb-enterprise +name: fast +entries: + - name: mongodb-enterprise.v${current_bundle_version} + replaces: mongodb-enterprise.v${latest_bundle_version}" >> "${catalog_dir}"/operator.yaml + + echo "Validating catalog" + opm validate "${catalog_dir}" + echo "Catalog is valid" + echo "Building catalog image" + cd "${catalog_dir}" && cd ../ + docker build --platform "${DOCKER_PLATFORM}" . -f mongodb-enterprise-operator-catalog.Dockerfile -t "${catalog_image}" + docker push "${catalog_image}" + echo "Catalog has been build and published" + cd - +} + +title "Executing prepare-openshift-bundles-for-e2e.sh" + +export BUILD_DOCKER_IMAGES=true +export DOCKER_PLATFORM=${DOCKER_PLATFORM:-"linux/amd64"} +CERTIFIED_OPERATORS_REPO="https://github.com/redhat-openshift-ecosystem/certified-operators.git" + +certified_repo_cloned="$(clone_git_repo_into_temp ${CERTIFIED_OPERATORS_REPO})" +latest_released_operator_version="$(find_the_latest_certified_operator "${certified_repo_cloned}")" +current_operator_version_from_release_json=$(jq -r .mongodbOperator < release.json) +current_incremented_operator_version_from_release_json=$(increment_version "${current_operator_version_from_release_json}") +current_incremented_operator_version_from_release_json_with_version_id="${current_incremented_operator_version_from_release_json}-${VERSION_ID:-"latest"}" +certified_catalog_image="${base_repo_url}/mongodb-enterprise-operator-certified-catalog:${current_incremented_operator_version_from_release_json_with_version_id}" + +export LATEST_CERTIFIED_BUNDLE_IMAGE="${base_repo_url}/mongodb-enterprise-operator-certified-bundle:${latest_released_operator_version}" +export CURRENT_BUNDLE_IMAGE="${base_repo_url}/mongodb-enterprise-operator-certified-bundle:${current_incremented_operator_version_from_release_json_with_version_id}" + + +header "Configuration:" +echo "certified_repo_cloned: ${certified_repo_cloned}" +echo "latest_released_operator_version: ${latest_released_operator_version}" +echo "current_incremented_operator_version_from_release_json: ${current_incremented_operator_version_from_release_json}" +echo "current_incremented_operator_version_from_release_json_with_version_id: ${current_incremented_operator_version_from_release_json_with_version_id}" +echo "certified_catalog_image: ${certified_catalog_image}" +echo "LATEST_CERTIFIED_BUNDLE_IMAGE: ${LATEST_CERTIFIED_BUNDLE_IMAGE}" +echo "CURRENT_BUNDLE_IMAGE: ${CURRENT_BUNDLE_IMAGE}" +echo "BUILD_DOCKER_IMAGES: ${BUILD_DOCKER_IMAGES}" +echo "DOCKER_PLATFORM: ${DOCKER_PLATFORM}" +echo "CERTIFIED_OPERATORS_REPO: ${CERTIFIED_OPERATORS_REPO}" + +# Build latest published bundle form RedHat's certified operators repository. +header "Building bundle:" +build_bundle_from_git_repo "${certified_repo_cloned}" "${latest_released_operator_version}" "${LATEST_CERTIFIED_BUNDLE_IMAGE}" + +# Generate helm charts providing overrides for images to reference images build in EVG pipeline. +header "Building Helm charts:" +generate_helm_charts + +# prepare openshift bundles the same way it's built in release process from the current sources and helm charts. +export CERTIFIED_BUNDLE_IMAGE=${CURRENT_BUNDLE_IMAGE} +export VERSION="${current_incremented_operator_version_from_release_json}" +export OPERATOR_IMAGE="${OPERATOR_REGISTRY:-${REGISTRY}}/mongodb-enterprise-operator-ubi:${VERSION_ID}" +header "Preparing OpenShift bundles:" +scripts/evergreen/operator-sdk/prepare-openshift-bundles.sh + +# publish two-channel catalog source to be used in e2e test. +header "Building and pushing the catalog:" +build_and_publish_catalog_with_two_channels "${certified_repo_cloned}" "${latest_released_operator_version}" "${LATEST_CERTIFIED_BUNDLE_IMAGE}" "${current_incremented_operator_version_from_release_json}" "${CURRENT_BUNDLE_IMAGE}" "${certified_catalog_image}" + +header "Cleaning up tmp directory" +rm -rf "${certified_repo_cloned}" diff --git a/scripts/evergreen/operator-sdk/prepare-openshift-bundles.sh b/scripts/evergreen/operator-sdk/prepare-openshift-bundles.sh new file mode 100755 index 000000000..ffa5bcc84 --- /dev/null +++ b/scripts/evergreen/operator-sdk/prepare-openshift-bundles.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +# This script prepares two openshift bundles tgz: certified and community. + +RELEASE_JSON_PATH=${RELEASE_JSON_PATH:-"release.json"} +VERSION=${VERSION:-$(jq -r .mongodbOperator < "${RELEASE_JSON_PATH}")} +BUILD_DOCKER_IMAGES=${BUILD_DOCKER_IMAGES:-"false"} +OPERATOR_IMAGE=${OPERATOR_IMAGE:-"quay.io/mongodb/mongodb-enterprise-operator-ubi:${VERSION}"} +DOCKER_PLATFORM=${DOCKER_PLATFORM:-"linux/amd64"} + +mkdir -p bundle + +echo "Generating openshift bundle for version ${VERSION}" +make bundle VERSION="${VERSION}" + +mv bundle.Dockerfile "./bundle/${VERSION}/bundle.Dockerfile" + +minimum_supported_openshift_version=$(jq -r .openshift.minimumSupportedVersion < "${RELEASE_JSON_PATH}") +bundle_annotations_file="bundle/${VERSION}/metadata/annotations.yaml" +bundle_dockerfile="bundle/${VERSION}/bundle.Dockerfile" +bundle_csv_file="bundle/${VERSION}/manifests/mongodb-enterprise.clusterserviceversion.yaml" + +echo "Aligning metadata.annotations.containerImage version with deployment's image in ${bundle_csv_file}" +operator_deployment_image=$(yq '.spec.install.spec.deployments[0].spec.template.spec.containers[0].image' < "${bundle_csv_file}") +yq e ".metadata.annotations.containerImage = \"${operator_deployment_image}\"" -i "${bundle_csv_file}" + +echo "Edited CSV: ${bundle_csv_file}" +cat "${bundle_csv_file}" + +echo "Adding minimum_supported_openshift_version annotation to ${bundle_annotations_file}" +yq e ".annotations.\"com.redhat.openshift.versions\" = \"v${minimum_supported_openshift_version}\"" -i "${bundle_annotations_file}" + +echo "Adding minimum_supported_openshift_version annotation to ${bundle_dockerfile}" +echo "LABEL com.redhat.openshift.versions=\"v${minimum_supported_openshift_version}\"" >> "${bundle_dockerfile}" + +PATH="${PATH}:bin" + +echo "Running digest pinning for certified bundle" +# This can fail during the release because the latest image is not available yet and will be available the next day/next daily rebuild. +# We decided to skip digest pinning during the as it is a post-processing step and it should be fine to skip it when testing OLM during the release. +if [[ "${DIGEST_PINNING_ENABLED:-"true"}" == "true" ]]; then + operator_image=$(yq ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image" < ./bundle/"${VERSION}"/manifests/mongodb-enterprise.clusterserviceversion.yaml) + operator_annotation_image=$(yq ".metadata.annotations.containerImage" < ./bundle/"${VERSION}"/manifests/mongodb-enterprise.clusterserviceversion.yaml) + if [[ "${operator_image}" != "${operator_annotation_image}" ]]; then + echo "Inconsistent operator images in CSV (.spec.install.spec.deployments[0].spec.template.spec.containers[0].image=${operator_image}, .metadata.annotations.containerImage=${operator_annotation_image})" + cat ./bundle/"${VERSION}"/manifests/mongodb-enterprise.clusterserviceversion.yaml + exit 1 + fi + + if docker manifest inspect "${operator_image}" > /dev/null 2>&1; then + echo "Running digest pinning, since the operator image: ${operator_image} exists" + operator-manifest-tools pinning pin -v --resolver skopeo "bundle/${VERSION}/manifests" + else + echo "Skipping pinning tools, since the operator image: ${operator_image} or ${operator_annotation_image} are missing and we are most likely in a release" + fi +fi + +certified_bundle_file="./bundle/operator-certified-${VERSION}.tgz" +echo "Generating certified bundle" +tar -czvf "${certified_bundle_file}" "./bundle/${VERSION}" + +if [[ "${BUILD_DOCKER_IMAGES}" == "true" ]]; then + docker build --platform "${DOCKER_PLATFORM}" -f "./bundle/${VERSION}/bundle.Dockerfile" -t "${CERTIFIED_BUNDLE_IMAGE}" . + docker push "${CERTIFIED_BUNDLE_IMAGE}" +fi diff --git a/scripts/evergreen/periodic-cleanup-aws.py b/scripts/evergreen/periodic-cleanup-aws.py new file mode 100755 index 000000000..68d9b70b0 --- /dev/null +++ b/scripts/evergreen/periodic-cleanup-aws.py @@ -0,0 +1,179 @@ +import argparse +import re +from datetime import datetime, timedelta, timezone +from typing import List + +import boto3 + +REPOSITORIES_NAMES = [ + "dev/mongodb-agent-ubi", + "dev/mongodb-enterprise-init-appdb-ubi", + "dev/mongodb-enterprise-database-ubi", + "dev/mongodb-enterprise-init-database-ubi", + "dev/mongodb-enterprise-init-ops-manager-ubi", + "dev/mongodb-enterprise-operator-ubi", +] +REGISTRY_ID = "268558157000" +REGION = "us-east-1" +DEFAULT_AGE_THRESHOLD_DAYS = 1 # Number of days to consider as the age threshold +BOTO_MAX_PAGE_SIZE = 1000 + +ecr_client = boto3.client("ecr", region_name=REGION) + + +def describe_all_ecr_images(repository: str) -> List[dict]: + """Retrieve all ECR images from the repository.""" + images = [] + + # Boto3 can only return a maximum of 1000 images per request, we need a paginator to retrieve all images + # from the repository + paginator = ecr_client.get_paginator("describe_images") + + page_iterator = paginator.paginate( + repositoryName=repository, registryId=REGISTRY_ID, PaginationConfig={"PageSize": BOTO_MAX_PAGE_SIZE} + ) + + for page in page_iterator: + details = page.get("imageDetails", []) + images.extend(details) + + return images + + +def filter_tags_to_delete(images: List[dict]) -> List[dict]: + """Filter the image list to only delete tags matching the pattern, signatures, or untagged images.""" + filtered_images = [] + untagged_count = 0 + for image_detail in images: + if "imageTags" in image_detail: + for tag in image_detail["imageTags"]: + # The Evergreen patch id we use for building the test images tags uses an Object ID + # https://www.mongodb.com/docs/v6.2/reference/bson-types/#std-label-objectid + # The first 4 bytes are based on the timestamp, so it will always have the same prefix for a while (_6 in that case) + # This is valid until and must be changed before: July 2029 + # 70000000 -> decimal -> 1879048192 => Wednesday, July 18, 2029 + # Note that if the operator ever gets to major version 6, some tags can unintentionally match '_6' + # It is an easy and relatively reliable way of identifying our test images tags + if "_6" in tag or ".sig" in tag or contains_timestamped_tag(tag): + filtered_images.append( + { + "imageTag": tag, + "imagePushedAt": image_detail["imagePushedAt"], + "imageDigest": image_detail["imageDigest"], + } + ) + else: + filtered_images.append( + { + "imageTag": "", + "imagePushedAt": image_detail["imagePushedAt"], + "imageDigest": image_detail["imageDigest"], + } + ) + untagged_count += 1 + print(f"found {untagged_count} untagged images") + return filtered_images + + +# match 107.0.0.8502-1-b20241125T000000Z-arm64 +def contains_timestamped_tag(s: str) -> bool: + if "b" in s and "T" in s and "Z" in s: + pattern = r"b\d{8}T\d{6}Z" + return bool(re.search(pattern, s)) + return False + + +def get_images_with_dates(repository: str) -> List[dict]: + """Retrieve the list of patch images, corresponding to the regex, with push dates""" + ecr_images = describe_all_ecr_images(repository) + print(f"Found {len(ecr_images)} images in repository {repository}") + images_matching_tag = filter_tags_to_delete(ecr_images) + + return images_matching_tag + + +def batch_delete_images(repository: str, images: List[dict]) -> None: + print(f"Deleting {len(images)} images in repository {repository}") + digests_to_delete = [{"imageDigest": image["imageDigest"]} for image in images] + # batch_delete_image only support a maximum of 100 images at a time + for i in range(0, len(digests_to_delete), 100): + batch = digests_to_delete[i : i + 100] + print(f"Deleting batch {i//100 + 1} with {len(batch)} images...") + ecr_client.batch_delete_image(repositoryName=repository, registryId=REGISTRY_ID, imageIds=batch) + print(f"Deleted images") + + +def delete_image(repository: str, image_tag: str) -> None: + ecr_client.batch_delete_image(repositoryName=repository, registryId=REGISTRY_ID, imageIds=[{"imageTag": image_tag}]) + print(f"Deleted image with tag: {image_tag}") + + +def delete_images( + repository: str, + images_with_dates: List[dict], + age_threshold: int = DEFAULT_AGE_THRESHOLD_DAYS, + dry_run: bool = False, +) -> None: + # Get the current time in UTC + current_time = datetime.now(timezone.utc) + + # Process the images, deleting those older than the threshold + delete_count = 0 + age_threshold_timedelta = timedelta(days=age_threshold) + images_to_delete = [] + for image in images_with_dates: + tag = image["imageTag"] + push_date = image["imagePushedAt"] + image_age = current_time - push_date + + log_message_base = f"Image {tag if tag else 'UNTAGGED'} was pushed at {push_date.isoformat()}" + delete_message = "should be cleaned up" if dry_run else "deleting..." + if image_age > age_threshold_timedelta: + print(f"{log_message_base}, older than {age_threshold} day(s), {delete_message}") + images_to_delete.append(image) + delete_count += 1 + else: + print(f"{log_message_base}, not older than {age_threshold} day(s)") + if not dry_run: + batch_delete_images(repository, images_to_delete) + deleted_message = "need to be cleaned up" if dry_run else "deleted" + print(f"{delete_count} images {deleted_message}") + + +def cleanup_repository(repository: str, age_threshold: int = DEFAULT_AGE_THRESHOLD_DAYS, dry_run: bool = False): + print(f"Cleaning up images older than {age_threshold} day(s) from repository {repository}") + print("Getting list of images...") + images_with_dates = get_images_with_dates(repository) + print(f"Images matching the pattern: {len(images_with_dates)}") + + # Sort the images by their push date (oldest first) + images_with_dates.sort(key=lambda x: x["imagePushedAt"]) + + delete_images(repository, images_with_dates, age_threshold, dry_run) + print(f"Repository {repository} cleaned up") + + +def main(): + parser = argparse.ArgumentParser(description="Process and delete old ECR images.") + parser.add_argument( + "--age-threshold", + type=int, + default=DEFAULT_AGE_THRESHOLD_DAYS, + help="Age threshold in days for deleting images (default: 1 day)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="If specified, only display what would be deleted without actually deleting.", + ) + args = parser.parse_args() + + if args.dry_run: + print("Dry run - not deleting images") + + for repository in REPOSITORIES_NAMES: + cleanup_repository(repository, age_threshold=args.age_threshold, dry_run=args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/scripts/evergreen/precommit_bump.sh b/scripts/evergreen/precommit_bump.sh new file mode 100755 index 000000000..68e4287be --- /dev/null +++ b/scripts/evergreen/precommit_bump.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -Eeou pipefail +source scripts/dev/set_env_context.sh + +export GOLANGCI_LINT_CACHE="${HOME}/.cache/golangci-lint" + +ORIGINAL_BRANCH="" +# Detect the original branch (same commit, but not the evg-pr-test-* branch which evg creates) +ORIGINAL_BRANCH=$(git for-each-ref --format='%(refname:short) %(objectname)' refs/remotes/origin | grep "$(git rev-parse HEAD)" | grep -v "evg-pr-test-" | awk '{print $1}' | sed 's|^origin/||' | head -n 1 || true) + +if [[ -z "${ORIGINAL_BRANCH}" ]]; then + echo "Fork: Could not determine the original branch. Running in a fork" + exit 0 +fi +echo "Detected original branch: ${ORIGINAL_BRANCH}" + +REQUIRED_PATTERNS=( + "^dependabot/" + "_version_bump$" + "^enterprise-operator-release-" +) + +echo "Checking branch '${ORIGINAL_BRANCH}' against required patterns:" + +MATCH_FOUND=false +for pattern in "${REQUIRED_PATTERNS[@]}"; do + if [[ "${ORIGINAL_BRANCH}" =~ ${pattern} ]]; then + MATCH_FOUND=true + echo "✅ Match found: '${ORIGINAL_BRANCH}' matches pattern '${pattern}'" + break + fi +done + +if [[ "${MATCH_FOUND}" == false ]]; then + echo "❌ Branch '${ORIGINAL_BRANCH}' does not match any required patterns. Exiting." + printf " - %s\n" "${REQUIRED_PATTERNS[@]}" + exit 0 +fi + +echo "Detected a branch that should be bumped." + +git checkout "${ORIGINAL_BRANCH}" + +EVERGREEN_MODE=true .githooks/pre-commit + +git add . + +if [[ -z $(git diff --name-only --cached) ]]; then + echo "No staged changes to commit. Exiting." + exit 0 +fi + +git commit -m "Run pre-commit hook" +git remote set-url origin https://x-access-token:"${GH_TOKEN}"@github.com/10gen/ops-manager-kubernetes.git + +echo "changes detected, pushing them" +git push origin "${ORIGINAL_BRANCH}" diff --git a/scripts/evergreen/prepare_aws.sh b/scripts/evergreen/prepare_aws.sh new file mode 100755 index 000000000..db49eba06 --- /dev/null +++ b/scripts/evergreen/prepare_aws.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +calculate_hours_since_creation() { + if [[ $(uname) == "Darwin" ]]; then + creation_timestamp=$(gdate -d "$1" +%s) + current_timestamp=$(TZ="UTC" gdate +%s) + else + creation_timestamp=$(date -d "$1" +%s) + current_timestamp=$(TZ="UTC" date +%s) + fi + echo $(( (current_timestamp - creation_timestamp) / 3600 )) +} + +delete_buckets_from_file() { + list_file=$1 + + echo "Deleting buckets from file: ${list_file}" + + while IFS= read -r bucket_entry; do + bucket_name=$(echo "${bucket_entry}" | jq -r '.Name') + bucket_creation_date=$(echo "${bucket_entry}" | jq -r '.CreationDate') + echo "[${list_file}/${bucket_name}] Processing bucket name=${bucket_name}, creationDate=${bucket_creation_date})" + + tags=$(aws s3api get-bucket-tagging --bucket "${bucket_name}" --output json || true) + operatorTagExists=$(echo "${tags}" | jq -r 'select(.TagSet | map({(.Key): .Value}) | add | .evg_task and .environment == "mongodb-enterprise-operator-tests")') + if [[ -n "${operatorTagExists}" ]]; then + # Bucket created by the test run in EVG, check if it's older than 2 hours + hours_since_creation=$(calculate_hours_since_creation "${bucket_creation_date}") + + if [[ ${hours_since_creation} -ge 2 ]]; then + aws_cmd="aws s3 rb s3://${bucket_name} --force" + echo "[${list_file}/${bucket_name}] Deleting e2e bucket: ${bucket_name}/${bucket_creation_date}; age in hours: ${hours_since_creation}; (${aws_cmd}), tags: Tags: $(echo "${tags}" | jq -cr .)" + ${aws_cmd} || true + else + echo "[${list_file}/${bucket_name}] Bucket ${bucket_name} is not older than 2 hours, skipping deletion." + fi + else + # Bucket not created by the test run in EVG, check if it's older than 24 hours and owned by us + hours_since_creation=$(calculate_hours_since_creation "${bucket_creation_date}") + operatorOwnedTagExists=$(echo "${tags}" | jq -r 'select(.TagSet | map({(.Key): .Value}) | add | .environment == "mongodb-enterprise-operator-tests")') + if [[ ${hours_since_creation} -ge 24 && -n "${operatorOwnedTagExists}" ]]; then + aws_cmd="aws s3 rb s3://${bucket_name} --force" + echo "[${list_file}/${bucket_name}] Deleting manual bucket: ${bucket_name}/${bucket_creation_date}; age in hours: ${hours_since_creation}; (${aws_cmd}), tags: Tags: $(echo "${tags}" | jq -cr .)" + ${aws_cmd} || true + else + echo "[${list_file}/${bucket_name}] Bucket ${bucket_name} is not older than 24 hours or not owned by us, skipping deletion." + fi + fi + done <<< "$(cat "${list_file}")" +} + +remove_old_buckets() { + echo "##### Removing old s3 buckets" + + if [[ $(uname) == "Darwin" ]]; then + # Use gdate on macOS + bucket_date=$(TZ="UTC" gdate +%Y-%m-%dT%H:%M:%S%z -d "24 hour ago") + else + # Use date on Linux + bucket_date=$(TZ="UTC" date +%Y-%m-%dT%H:%M:%S%z -d "24 hour ago") + fi + + echo "Removing buckets older than ${bucket_date}" + + bucket_list=$(aws s3api list-buckets --query "sort_by(Buckets[?starts_with(Name,'test-')], &CreationDate)" | jq -c --raw-output '.[]') + if [[ -z ${bucket_list} ]]; then + echo "Bucket list is empty, nothing to do" + return 0 + fi + + # here we split bucket_list jsons (each bucket json on its own line) to multiple files using split -l + tmp_dir=$(mktemp -d) + pushd "${tmp_dir}" + bucket_list_file="bucket_list.json" + echo "${bucket_list}" >${bucket_list_file} + + # Get the number of lines in the file + num_lines=$(wc -l < "${bucket_list_file}") + + # Check if file is empty + if [ "${num_lines}" -eq 0 ]; then + echo "Error: ${bucket_list_file} is empty." + exit 1 + fi + + # Calculate the number of lines per split file + if [ "${num_lines}" -lt 30 ]; then + # If the file has fewer than 30 lines, split it into the number of lines + lines_per_split="${num_lines}" + else + # Otherwise, set the max at 30 + lines_per_split=30 + fi + + echo "Splitting bucket list ($(wc -l <"${bucket_list_file}") lines) into ${lines_per_split} files in dir: ${tmp_dir}. Processing them in parallel." + # we split to lines_per_split files, so we execute lines_per_split delete processes in parallel + split -l $(( $(wc -l <"${bucket_list_file}") / lines_per_split)) "${bucket_list_file}" splitted- + splitted_files_list=$(ls -1 splitted-*) + + # for each file containing slice of buckets, we execute delete_buckets_from_file + while IFS= read -r list_file; do + echo "Deleting buckets from file: ${list_file}" + echo "Bucket list: ${list_file}: $(cat "${list_file}")}" + delete_buckets_from_file "${list_file}" & + done <<< "${splitted_files_list}" + wait + + popd + rm -rf "${tmp_dir}" +} + +# Does general cleanup work for external sources (currently only aws). Must be called once per Evergreen run +prepare_aws() { + echo "##### Detaching the EBS Volumes that are stuck" + for v in $(aws ec2 describe-volumes --filters Name=attachment.status,Values=attaching | grep VolumeId | cut -d "\"" -f 4); do + set -v + aws ec2 detach-volume --volume-id "${v}" --force + set +v + done + + echo "##### Removing ESB volumes which are not used any more" + # Seems Openshift (sometimes?) doesn't remove the volumes but marks them as available - we need to clean these volumes + # manually + for v in $(aws ec2 describe-volumes --filters Name=status,Values=available | grep VolumeId | cut -d "\"" -f 4); do + set -v + aws ec2 delete-volume --volume-id "${v}" + set +v + done + + remove_old_buckets +} + +prepare_aws diff --git a/scripts/evergreen/release/agent_matrix.py b/scripts/evergreen/release/agent_matrix.py new file mode 100644 index 000000000..dc05bc387 --- /dev/null +++ b/scripts/evergreen/release/agent_matrix.py @@ -0,0 +1,59 @@ +import json +from typing import Dict, List + +DEFAULT_SUPPORTED_OPERATOR_VERSIONS = 3 +LATEST_OPERATOR_VERSION = 1 + + +def get_release() -> Dict[str, str]: + return json.load(open("release.json")) + + +def build_agent_gather_versions(release: Dict[str, str]): + # This is a list of a tuples - agent version and corresponding tools version + agent_versions_to_be_build = list() + agent_versions_to_be_build.append( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], + ) + for _, om in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].items(): + agent_versions_to_be_build.append(om["agent_version"]) + return agent_versions_to_be_build + + +def get_supported_version_for_image_matrix_handling( + image: str, supported_versions: int = DEFAULT_SUPPORTED_OPERATOR_VERSIONS +) -> List[str]: + # if we are a certifying mongodb-agent, we will need to also certify the + # static container images which are a matrix of _ + if image == "mongodb-agent": + # officially, we start the support with 1.25.0, but we only support the last three versions + last_supported_operator_versions = get_supported_operator_versions(supported_versions) + + agent_version_with_static_support_without_operator_suffix = build_agent_gather_versions(get_release()) + agent_version_with_static_support_with_operator_suffix = list() + for agent in agent_version_with_static_support_without_operator_suffix: + for version in last_supported_operator_versions: + agent_version_with_static_support_with_operator_suffix.append(agent + "_" + version) + agent_versions_no_static_support = get_release()["supportedImages"][image]["versions"] + agents = sorted( + list( + set( + agent_version_with_static_support_with_operator_suffix + + agent_version_with_static_support_without_operator_suffix + + list(agent_versions_no_static_support) + ) + ) + ) + return agents + + return sorted(get_release()["supportedImages"][image]["versions"]) + + +def get_supported_operator_versions(supported_versions: int = DEFAULT_SUPPORTED_OPERATOR_VERSIONS): + operator_versions = list(get_release()["supportedImages"]["operator"]["versions"]) + operator_versions.sort(key=lambda s: list(map(int, s.split(".")))) + + if len(operator_versions) <= supported_versions: + return operator_versions + + return operator_versions[-supported_versions:] diff --git a/scripts/evergreen/release/agent_matrix_test.py b/scripts/evergreen/release/agent_matrix_test.py new file mode 100644 index 000000000..f38671c57 --- /dev/null +++ b/scripts/evergreen/release/agent_matrix_test.py @@ -0,0 +1,58 @@ +from unittest import mock + +from scripts.evergreen.release.agent_matrix import get_supported_operator_versions + +empty_release = {"supportedImages": {"operator": {"versions": []}}} + + +@mock.patch("scripts.evergreen.release.agent_matrix.get_release", return_value=empty_release) +def test_get_supported_operator_versions_empty(_): + supported_versions = get_supported_operator_versions() + assert len(supported_versions) == 0 + + +single_release = {"supportedImages": {"operator": {"versions": ["1.30.0"]}}} + + +@mock.patch("scripts.evergreen.release.agent_matrix.get_release", return_value=single_release) +def test_get_supported_operator_versions_single_release(_): + supported_versions = get_supported_operator_versions() + assert len(supported_versions) == 1 + assert supported_versions[0] == "1.30.0" + + +three_releases_not_ordered = {"supportedImages": {"operator": {"versions": ["1.30.0", "1.28.0", "2.0.2"]}}} + + +@mock.patch("scripts.evergreen.release.agent_matrix.get_release", return_value=three_releases_not_ordered) +def test_get_supported_operator_versions_three_releases_not_ordered(_): + supported_versions = get_supported_operator_versions() + assert len(supported_versions) == 3 + assert supported_versions == ["1.28.0", "1.30.0", "2.0.2"] + + +many_releases_not_ordered = { + "supportedImages": { + "operator": { + "versions": [ + "1.32.0", + "1.25.0", + "1.26.0", + "1.27.1", + "1.27.0", + "1.30.0", + "1.300.0", + "1.28.123", + "0.0.1", + "2.0.2", + ] + } + } +} + + +@mock.patch("scripts.evergreen.release.agent_matrix.get_release", return_value=many_releases_not_ordered) +def test_get_supported_operator_versions_many_releases_not_ordered(_): + supported_versions = get_supported_operator_versions() + assert len(supported_versions) == 3 + assert supported_versions == ["1.32.0", "1.300.0", "2.0.2"] diff --git a/scripts/evergreen/release/helm_files_handler.py b/scripts/evergreen/release/helm_files_handler.py new file mode 100644 index 000000000..69b8b9927 --- /dev/null +++ b/scripts/evergreen/release/helm_files_handler.py @@ -0,0 +1,94 @@ +from typing import Any + +import ruamel.yaml + + +def update_all_helm_values_files(chart_key: str, new_release: str): + """Updates all values.yaml files setting chart_key.'version' field to new_release""" + update_single_helm_values_file("helm_chart/values.yaml", key=chart_key, new_release=new_release) + + +def update_single_helm_values_file(values_yaml_path: str, key: str, new_release: str): + yaml = ruamel.yaml.YAML() + with open(values_yaml_path, "r") as fd: + doc = yaml.load(fd) + doc[key]["version"] = new_release + # Make sure we are writing a valid values.yaml file. + assert "operator" in doc + assert "registry" in doc + with open(values_yaml_path, "w") as fd: + yaml.dump(doc, fd) + print(f'Set "{values_yaml_path} {key}.version to {new_release}"') + + +def set_value_in_doc(yaml_doc: Any, dotted_path: str, new_value: Any): + """Sets the value at the given dotted path in the given yaml document.""" + + path = dotted_path.split(".") + doc = yaml_doc + for key in path[:-1]: + doc = doc[key] + doc[path[-1]] = new_value + + +def get_value_in_doc(yaml_doc: Any, dotted_path: str): + """Gets the value at the given dotted path in the given yaml document.""" + + path = dotted_path.split(".") + doc = yaml_doc + for key in path[:-1]: + doc = doc[key] + return doc[path[-1]] + + +def set_value_in_yaml_file(yaml_file_path: str, key: str, new_value: Any, preserve_quotes: bool = False): + """Sets one value under key in yaml_file. Key could be passed as a dotted path, e.g. relatedImages.mongodb.""" + + yaml = ruamel.yaml.YAML() + if preserve_quotes: + yaml.preserve_quotes = True + + with open(yaml_file_path, "r") as fd: + doc = yaml.load(fd) + + set_value_in_doc(doc, key, new_value) + + with open(yaml_file_path, "w") as fd: + yaml.dump(doc, fd) + + print(f'Setting in "{yaml_file_path} value {key}') + + +def get_value_in_yaml_file(yaml_file_path: str, key: str): + + yaml = ruamel.yaml.YAML() + with open(yaml_file_path, "r") as fd: + doc = yaml.load(fd) + + return get_value_in_doc(doc, key) + + +def update_standalone_installer(yaml_file_path: str, version: str): + """ + Updates a bundle of manifests with the correct image version for + the operator deployment. + """ + yaml = ruamel.yaml.YAML() + + yaml.explicit_start = True # Ensure explicit `---` in the output + yaml.indent(mapping=2, sequence=4, offset=2) # Align with tab width produced by Helm + yaml.preserve_quotes = True # Preserve original quotes in the YAML file + + with open(yaml_file_path, "r") as fd: + data = list(yaml.load_all(fd)) # Convert the generator to a list + + for doc in data: + # We're only interested in the Deployments of the operator, where + # we change the image version to the one provided in the release. + if doc["kind"] == "Deployment": + full_image = doc["spec"]["template"]["spec"]["containers"][0]["image"] + image = full_image.rsplit(":", 1)[0] + doc["spec"]["template"]["spec"]["containers"][0]["image"] = image + ":" + version + + with open(yaml_file_path, "w") as fd: + yaml.dump_all(data, fd) diff --git a/scripts/evergreen/release/images_signing.py b/scripts/evergreen/release/images_signing.py new file mode 100644 index 000000000..dfa8771e9 --- /dev/null +++ b/scripts/evergreen/release/images_signing.py @@ -0,0 +1,255 @@ +import os +import random +import subprocess +import sys +import time +from typing import List, Optional + +import requests +from opentelemetry import trace + +from lib.base_logger import logger + +SIGNING_IMAGE_URI = os.environ.get( + "SIGNING_IMAGE_URI", "artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-cosign" +) + +RETRYABLE_ERRORS = [500, 502, 503, 504, 429, "timeout", "WARNING"] + +TRACER = trace.get_tracer("evergreen-agent") + + +def is_retryable_error(stderr: str) -> bool: + """ + Determines if the error message is retryable. + + :param stderr: The standard error output from the subprocess. + :return: True if the error is retryable, False otherwise. + """ + return any(str(error) in stderr for error in RETRYABLE_ERRORS) + + +@TRACER.start_as_current_span("run_command_with_retries") +def run_command_with_retries(command, retries=6, base_delay=10): + """ + Runs a subprocess command with retries and exponential backoff. + 6 retries and 10 seconds delays sums up to be around 10 minutes. + Delays: 10,20,40,80,160,320 + :param command: The command to run. + :param retries: Number of retries before failing. + :param base_delay: Base delay in seconds for exponential backoff. + :raises subprocess.CalledProcessError: If the command fails after retries. + """ + span = trace.get_current_span() + span.set_attribute(f"meko.command.command", command) + for attempt in range(retries): + try: + result = subprocess.run(command, capture_output=True, text=True, check=True) + span.set_attribute(f"meko.command.retries", attempt) + span.set_attribute(f"meko.command.failure", False) + span.set_attribute(f"meko.command.result", result.stdout) + return result + except subprocess.CalledProcessError as e: + logger.error(f"Attempt {attempt + 1} failed: {e.stderr}") + if is_retryable_error(e.stderr): + logger.error(f"Attempt {attempt + 1} failed with retryable error: {e.stderr}") + if attempt + 1 < retries: + delay = base_delay * (2**attempt) + random.uniform(0, 1) + logger.info(f"Retrying in {delay:.2f} seconds...") + time.sleep(delay) + else: + logger.error(f"All {retries} attempts failed for command: {command}") + span.set_attribute(f"meko.command.failure", "no_retries") + raise + else: + logger.error(f"Non-retryable error occurred: {e.stderr}") + span.set_attribute(f"meko.command.failure", e.stderr) + raise + + +def mongodb_artifactory_login() -> None: + command = [ + "docker", + "login", + "--password-stdin", + "--username", + os.environ.get("ARTIFACTORY_USERNAME", "mongodb-enterprise-kubernetes-operator"), + "artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-cosign", + ] + subprocess.run(command, input=os.environ["ARTIFACTORY_PASSWORD"].encode("utf-8"), check=True) + + +def get_ecr_login_password(region: str) -> Optional[str]: + """ + Retrieves the login password from aws CLI, the secrets need to be stored in ~/.aws/credentials or equivalent. + :param region: Registry's AWS region + :return: The password as a string + """ + try: + result = subprocess.run( + ["aws", "ecr", "get-login-password", "--region", region], capture_output=True, text=True, check=True + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger.error(f"Failed to get ECR login password: {e.stderr}") + return None + + +def is_ecr_registry(image_name: str) -> bool: + return "amazonaws.com" in image_name + + +def get_image_digest(image_name: str) -> Optional[str]: + """ + Retrieves the digest of an image from its tag. Uses the skopeo container to be able to retrieve manifests tags as well. + :param image_name: The full image name with its tag. + :return: the image digest, or None in case of failure. + """ + + transport_protocol = "docker://" + # Get digest + digest_command = [ + "docker", + "run", + "--rm", + f"--volume={os.path.expanduser('~')}/.aws:/root/.aws:ro", + "quay.io/skopeo/stable:latest", + "inspect", + "--format={{.Digest}}", + ] + + # Specify ECR credentials if necessary + if is_ecr_registry(image_name): + aws_region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") + ecr_password = get_ecr_login_password(aws_region) + digest_command.append(f"--creds=AWS:{ecr_password}") + + digest_command.append(f"{transport_protocol}{image_name}") + + try: + result = run_command_with_retries(digest_command) + digest = result.stdout.strip() + return digest + except subprocess.CalledProcessError as e: + logger.error(f"Failed to get digest for {image_name}: {e.stderr}") + + +def build_cosign_docker_command(additional_args: List[str], cosign_command: List[str]) -> List[str]: + """ + Common logic to build a cosign command with the garasign cosign image provided by DevProd. + :param additional_args: additional arguments passed to the docker container, e.g mounted volume or env + :param cosign_command: actual command executed with cosign such as `sign` or `verify` + :return: the full command as a List of strings + """ + home_dir = os.path.expanduser("~") + base_command = [ + "docker", + "run", + "--platform", + "linux/amd64", + "--rm", + f"--volume={home_dir}/.docker/config.json:/root/.docker/config.json:ro", + ] + return base_command + additional_args + [SIGNING_IMAGE_URI, "cosign"] + cosign_command + + +@TRACER.start_as_current_span("sign_image") +def sign_image(repository: str, tag: str) -> None: + start_time = time.time() + span = trace.get_current_span() + + image = repository + ":" + tag + logger.debug(f"Signing image {image}") + + working_directory = os.getcwd() + container_working_directory = "/usr/local/kubernetes" + + # Referring to the image via its tag is deprecated in cosign + # We fetch the digest from the registry + digest = get_image_digest(image) + if digest is None: + logger.error("Impossible to get image digest, exiting...") + sys.exit(1) + image_ref = f"{repository}@{digest}" + + # Read secrets from environment and put them in env file for container + grs_username = os.environ["GRS_USERNAME"] + grs_password = os.environ["GRS_PASSWORD"] + pkcs11_uri = os.environ["PKCS11_URI"] + env_file_content = [ + f"GRS_CONFIG_USER1_USERNAME={grs_username}", + f"GRS_CONFIG_USER1_PASSWORD={grs_password}", + f"COSIGN_REPOSITORY={repository}", + ] + env_file_content = "\n".join(env_file_content) + temp_file = "./env-file" + with open(temp_file, "w") as f: + f.write(env_file_content) + + additional_args = [ + f"--env-file={temp_file}", + f"--volume={working_directory}:{container_working_directory}", + f"--workdir={container_working_directory}", + ] + cosign_command = [ + "sign", + f"--key={pkcs11_uri}", + f"--sign-container-identity={image}", + f"--tlog-upload=false", + image_ref, + ] + command = build_cosign_docker_command(additional_args, cosign_command) + + try: + run_command_with_retries(command) + except subprocess.CalledProcessError as e: + # Fail the pipeline if signing fails + logger.error(f"Failed to sign image {image}: {e.stderr}") + raise + + end_time = time.time() + duration = end_time - start_time + span.set_attribute(f"meko.signing.duration", duration) + span.set_attribute(f"meko.signing.repository", repository) + logger.debug("Signing successful") + + +@TRACER.start_as_current_span("verify_signature") +def verify_signature(repository: str, tag: str) -> bool: + start_time = time.time() + span = trace.get_current_span() + + image = repository + ":" + tag + logger.debug(f"Verifying signature of {image}") + public_key_url = os.environ.get( + "SIGNING_PUBLIC_KEY_URL", "https://cosign.mongodb.com/mongodb-enterprise-kubernetes-operator.pem" + ) + r = requests.get(public_key_url) + # Ensure the request was successful + if r.status_code == 200: + # Access the content of the file + kubernetes_operator_public_key = r.text + else: + logger.error(f"Failed to retrieve the public key: Status code {r.status_code}") + return False + + public_key_var_name = "OPERATOR_PUBLIC_KEY" + additional_args = [ + "--env", + f"{public_key_var_name}={kubernetes_operator_public_key}", + ] + cosign_command = ["verify", "--insecure-ignore-tlog=true", f"--key=env://{public_key_var_name}", image] + command = build_cosign_docker_command(additional_args, cosign_command) + + try: + run_command_with_retries(command, retries=10) + except subprocess.CalledProcessError as e: + # Fail the pipeline if verification fails + logger.error(f"Failed to verify signature for image {image}: {e.stderr}") + raise + + end_time = time.time() + duration = end_time - start_time + span.set_attribute(f"meko.verification.duration", duration) + span.set_attribute(f"meko.verification.repository", repository), + logger.debug("Successful verification") diff --git a/scripts/evergreen/release/purl_creator.sh b/scripts/evergreen/release/purl_creator.sh new file mode 100755 index 000000000..8b878dc7e --- /dev/null +++ b/scripts/evergreen/release/purl_creator.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +### +# This is an internal script which is part of the sbom.py file. Do not call it directly. +### + +go version -m "$1" | sed -E -n 's% *(dep|=>) *([^ ]*) *([^ ]*)( .*)?%pkg:golang/\2@\3%p' > "$2" diff --git a/scripts/evergreen/release/sbom.py b/scripts/evergreen/release/sbom.py new file mode 100644 index 000000000..611fc58eb --- /dev/null +++ b/scripts/evergreen/release/sbom.py @@ -0,0 +1,368 @@ +"""SBOM manipulation library + +This file contains all necessary functions for manipulating SBOMs for MCO and MEKO. The intention is to run +generate_sbom and generate_sbom_for_cli on a daily basis per each shipped image and the CLI. + +The SSDLC reporting doesn't strictly require to follow the daily rebuild flow. However, triggering it is part of the +release process, and it might be used in the future (perceived security vs real security). More information about the +report generation might be found in https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Enterprise+Operator+Release+Guide#KubernetesEnterpriseOperatorReleaseGuide-SSDLC + +On a typical daily run, the workflow is the following: + +- Generate SBOM Lite +- Uploading SBOM Lite to Kondukto +- Generate Augmented SBOM +- Uploading SBOM Lite and Augmented SBOM to S3 + +In addition to this, there are special steps done only for the initial upload of a newly released images: + +- Generate SBOM Lite +- Uploading SBOM Lite to Kondukto +- Generate Augmented SBOM +- Uploading the SBOM Lite and Augmented SBOM to a special path to S3, it's never updated - we want it to stay the same + +""" + +import os +import random +import subprocess +import tempfile +import time +import urllib + +import boto3 +import botocore + +from lib.base_logger import logger + +S3_BUCKET = "kubernetes-operators-sboms" +SILK_BOMB_IMAGE = "artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" +KONDUKTO_REPO = "10gen/ops-manager-kubernetes" +WORKDIR = os.getenv("workdir") + + +def get_image_sha(image_pull_spec: str): + logger.debug(f"Finding image SHA for {image_pull_spec}") + # Because of the manifest generation workflow, the Docker Daemon might be confused what have been built + # locally and what not. We need re-pull the image to ensure it's fresh every time we obtain the SHA Digest. + command = [ + "docker", + "pull", + image_pull_spec, + ] + subprocess.run(command, check=True, capture_output=True, text=True) + # See https://stackoverflow.com/a/55626495 + command = [ + "docker", + "inspect", + "--format={{index .Id}}", + image_pull_spec, + ] + result = subprocess.run(command, check=True, capture_output=True, text=True) + logger.debug(f"Found image SHA") + return result.stdout.strip() + + +def parse_image_pull_spec(image_pull_spec: str): + logger.debug(f"Parsing image pull spec {image_pull_spec}") + + parts = image_pull_spec.split("/") + + registry = parts[0] + organization = parts[1] + image_name = parts[2] + + image_parts = image_name.split(":") + image_name = image_parts[0] + tag = image_parts[1] + sha = get_image_sha(image_pull_spec) + + logger.debug( + f"Parsed image spec, registry: {registry}, org: {organization}, image: {image_name}, tag: {tag}, sha: {sha}" + ) + return registry, organization, image_name, tag, sha + + +def create_sbom_lite_for_image(image_pull_spec: str, directory: str, file_name: str, platform: str): + logger.debug(f"Creating SBOM for {image_pull_spec} to {directory}/{file_name}") + command = [ + "docker", + "sbom", + "--platform", + platform, + "-o", + f"{directory}/{file_name}", + "--format", + "cyclonedx-json", + image_pull_spec, + ] + subprocess.run(command, check=True) + logger.debug(f"Created SBOM") + + +def upload_to_s3(directory: str, file_name: str, s3_bucket: str, s3_path: str): + file_on_disk = f"{directory}/{file_name}" + logger.debug(f"Uploading file {file_on_disk} to S3 {s3_bucket}/{s3_path}") + s3 = boto3.resource("s3") + versioning = s3.BucketVersioning(s3_bucket) + versioning.enable() + s3.meta.client.upload_file(file_on_disk, S3_BUCKET, s3_path) + logger.debug(f"Uploading file done") + + +def get_silkbomb_env_file_path() -> str: + if not WORKDIR: + raise EnvironmentError("'workdir' environment variable is not set") + + silkbomb_env_path = os.path.join(WORKDIR, "silkbomb.env") + if not os.path.exists(silkbomb_env_path): + raise FileNotFoundError(f"{silkbomb_env_path} does not exist") + return silkbomb_env_path + + +def augment_sbom( + silkbomb_env_file: str, + directory: str, + sbom_lite_file_name: str, + sbom_augmented_file_name: str, + kondukto_repo: str, + kondukto_branch: str, +) -> bool: + logger.debug(f"Augmenting SBOM {directory}/{sbom_lite_file_name} with Kondukto scan results") + + command = [ + "docker", + "run", + "--platform", + "linux/amd64", + "--rm", + "-v", + f"{directory}:/sboms", + "--env-file", + silkbomb_env_file, + SILK_BOMB_IMAGE, + "augment", + "--sbom_in", + f"sboms/{sbom_lite_file_name}", + "--repo", + kondukto_repo, + "--branch", + kondukto_branch, + "--sbom_out", + f"sboms/{sbom_augmented_file_name}", + ] + + logger.debug(f"Calling Silkbomb augment: {' '.join(command)}") + if retry(lambda: subprocess.run(command, check=True)): + logger.debug(f"Augmenting SBOM done") + return True + else: + logger.error(f"Failed to augment SBOM") + return False + + +def retry(f, max_retries=5) -> bool: + for attempt in range(max_retries): + try: + logger.debug(f"Calling function with retries") + f() + logger.debug(f"Calling function with retries done") + return True + except subprocess.CalledProcessError as e: + err = e + wait_time = (2**attempt) + random.uniform(0, 1) + logger.warning(f"Rate limited. Retrying in {wait_time:.2f} seconds...") + time.sleep(wait_time) + logger.error(f"Calling function with retries failed with error: {err}") + return False + + +def download_file(url: str, directory: str, file_path: str): + logger.info(f"Downloading file {directory}/{file_path} from {url}") + urllib.request.urlretrieve(url, f"{directory}/{file_path}") + logger.info("Downloading file done") + + +def unpack(directory: str, file_path: str): + logger.info(f"Unpacking {directory}/{file_path}") + subprocess.check_output(f"tar -zxf {directory}/{file_path} -C {directory}", shell=True) + logger.info("Unpacking done") + + +def create_sbom_lite_for_binary(directory: str, file_path: str, sbom_light_path: str): + logger.info(f"Creating SBOM Lite for {directory}/{file_path}") + + purl_file_name = f"{sbom_light_path}.purl" + + subprocess.check_call( + f"./scripts/evergreen/release/purl_creator.sh {directory}/{file_path} {directory}/{purl_file_name}", shell=True + ) + + command = [ + "docker", + "run", + "--platform", + "linux/amd64", + "--rm", + "-v", + f"{directory}:/sboms", + SILK_BOMB_IMAGE, + "update", + "--purls", + f"/sboms/{purl_file_name}", + "--sbom_out", + f"/sboms/{sbom_light_path}", + ] + logger.debug(f"Calling update purls: {' '.join(command)}") + subprocess.run(command, check=True) + + logger.info(f"Creating SBOM Lite done") + + +def generate_sbom_for_cli(cli_version: str = "1.25.0", platform: str = "linux/amd64"): + logger.info(f"Generating SBOM for CLI for version {cli_version} and platform {platform}") + try: + silkbomb_env_file = get_silkbomb_env_file_path() + platform_sanitized = platform.replace("/", "-") + platform_sanitized_with_underscores = platform.replace("/", "_") + + with tempfile.TemporaryDirectory() as directory: + sbom_lite_file_name = f"kubectl-mongodb-{cli_version}-{platform_sanitized}.json" + sbom_augmented_file_name = f"kubectl-mongodb-{cli_version}-{platform_sanitized}-augmented.json" + product_name = "mongodb-enterprise-cli" + kondukto_project_repo = "mongodb/mongodb-enterprise-kubernetes" + kondukto_branch_id = f"{product_name}-release-{cli_version}-{platform_sanitized}" + s3_release_sbom_lite_path = f"sboms/release/lite/{product_name}/{cli_version}/{platform_sanitized}" + s3_release_sbom_augmented_path = ( + f"sboms/release/augmented/{product_name}/{cli_version}/{platform_sanitized}" + ) + binary_file_name = f"kubectl-mongodb_{cli_version}_{platform_sanitized_with_underscores}.tar.gz" + download_binary_url = f"https://github.com/mongodb/mongodb-enterprise-kubernetes/releases/download/{cli_version}/{binary_file_name}" + unpacked_binary_file_name = "kubectl-mongodb" + + if not s3_path_exists(s3_release_sbom_augmented_path): + download_file(download_binary_url, directory, binary_file_name) + unpack(directory, binary_file_name) + create_sbom_lite_for_binary(directory, unpacked_binary_file_name, sbom_lite_file_name) + logger.info("Augmenting SBOM Lite and uploading SBOM Lite and Augmented SBOM for the first release") + if augment_sbom( + silkbomb_env_file, + directory, + sbom_lite_file_name, + sbom_augmented_file_name, + kondukto_project_repo, + kondukto_branch_id, + ): + upload_to_s3(directory, sbom_lite_file_name, S3_BUCKET, s3_release_sbom_lite_path) + upload_to_s3(directory, sbom_augmented_file_name, S3_BUCKET, s3_release_sbom_augmented_path) + else: + logger.exception(f"Could not augment release SBOM with Kondukto scan results") + except: + logger.exception("Skipping SBOM Generation because of an error") + + logger.info(f"Generating SBOM done") + + +def get_kondukto_sbom_data(image_name: str, tag: str, platform_sanitized: str): + daily_project_branch_id = f"{image_name}-daily-{tag}-{platform_sanitized}" + release_project_branch_id = f"{image_name}-release-{tag}-{platform_sanitized}" + if image_name.startswith("mongodb-enterprise"): + return daily_project_branch_id, release_project_branch_id, "10gen/ops-manager-kubernetes" + return daily_project_branch_id, release_project_branch_id, "mongodb/mongodb-kubernetes-operator" + + +def s3_path_exists(s3_path): + logger.debug(f"Checking if path exists {s3_path} ?") + pathExists = False + s3 = boto3.client("s3") + try: + response = s3.list_objects(Bucket=S3_BUCKET, Prefix=s3_path, MaxKeys=1) + logger.debug(f"Response from S3: {response}") + if "Contents" in response: + logger.debug(f"Content found, assuming the path exists") + pathExists = True + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] != "404": + logger.exception("Could not determine if the path exists. Assuming it is not.") + logger.debug(f"Checking done ({pathExists})") + return pathExists + + +def generate_sbom(image_pull_spec: str, platform: str = "linux/amd64"): + logger.info(f"Generating SBOM for {image_pull_spec} {platform}") + + registry: str + organization: str + image_name: str + tag: str + try: + silkbomb_env_file = get_silkbomb_env_file_path() + registry, organization, image_name, tag, sha = parse_image_pull_spec(image_pull_spec) + platform_sanitized = platform.replace("/", "-") + daily_project_branch_id, release_project_branch_id, kondukto_project_repo = get_kondukto_sbom_data( + image_name, tag, platform_sanitized + ) + + with tempfile.TemporaryDirectory() as directory: + sbom_lite_file_name = f"{image_name}_{tag}_{platform_sanitized}.json" + sbom_augmented_file_name = f"{image_name}_{tag}_{platform_sanitized}-augmented.json" + + create_sbom_lite_for_image(image_pull_spec, directory, sbom_lite_file_name, platform) + + ### Daily SBOM generation ### + s3_daily_sbom_lite_path = ( + f"sboms/daily/lite/{registry}/{organization}/{image_name}/{tag}/{platform_sanitized}/{sha}" + ) + s3_daily_sbom_augmented_path = ( + f"sboms/daily/augmented/{registry}/{organization}/{image_name}/{tag}/{platform_sanitized}/{sha}" + ) + + # produce Augmented SBOM with Silkbomb and upload SBOM Lite and Augmented SBOM to S3 + if augment_sbom( + silkbomb_env_file, + directory, + sbom_lite_file_name, + sbom_augmented_file_name, + kondukto_project_repo, + daily_project_branch_id, + ): + upload_to_s3(directory, sbom_lite_file_name, S3_BUCKET, s3_daily_sbom_lite_path) + upload_to_s3(directory, sbom_augmented_file_name, S3_BUCKET, s3_daily_sbom_augmented_path) + else: + logger.exception(f"Could not augment daily SBOM with Kondukto scan results. Continuing...") + + ### Release SBOM generation ### + # Then checking for path, we don't want to include SHA Digest. + # We just want to keep there the initial one. Nothing more. + s3_release_sbom_augmented_path_for_specific_tag = ( + f"sboms/release/augmented/{registry}/{organization}/{image_name}/{tag}/{platform_sanitized}/" + ) + + s3_release_sbom_lite_path = ( + f"sboms/release/lite/{registry}/{organization}/{image_name}/{tag}/{platform_sanitized}/{sha}" + ) + s3_release_sbom_augmented_path = ( + f"sboms/release/augmented/{registry}/{organization}/{image_name}/{tag}/{platform_sanitized}/{sha}" + ) + + # This path is only executed when there's a first rebuild of the release artifacts. + # Then, we upload the SBOM Lite and Augmented SBOM this single time only. + if not s3_path_exists(s3_release_sbom_augmented_path_for_specific_tag): + logger.info("Augmenting SBOM Lite and uploading SBOM Lite and Augmented SBOM for the first release") + if augment_sbom( + silkbomb_env_file, + directory, + sbom_lite_file_name, + sbom_augmented_file_name, + kondukto_project_repo, + release_project_branch_id, + ): + upload_to_s3(directory, sbom_lite_file_name, S3_BUCKET, s3_release_sbom_lite_path) + upload_to_s3(directory, sbom_augmented_file_name, S3_BUCKET, s3_release_sbom_augmented_path) + else: + logger.exception(f"Could not augment release SBOM with Kondukto scan results") + + except Exception as err: + logger.exception(f"Skipping SBOM Generation because of an error: {err}") + + logger.info(f"Generating SBOM done") diff --git a/scripts/evergreen/release/test_update_release.py b/scripts/evergreen/release/test_update_release.py new file mode 100644 index 000000000..244ef109a --- /dev/null +++ b/scripts/evergreen/release/test_update_release.py @@ -0,0 +1,261 @@ +from update_release import trim_ops_manager_mapping, trim_ops_manager_versions + + +def test_trim_ops_manager_mapping(): + mock_release = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "ops_manager": { + "1.0.0": {"tools_version": "100.0.0"}, + "1.5.0": {"tools_version": "100.0.5"}, + "2.0.0": {"tools_version": "100.1.0"}, + "2.5.0": {"tools_version": "100.1.5"}, + "3.0.0": {"tools_version": "100.2.0"}, + } + } + } + } + } + + # Expected result: Latest 3 versions per major version (1.x, 2.x, 3.x) + # Since we have fewer than 3 versions for each major, all should be kept + expected_mapping = { + "1.0.0": {"tools_version": "100.0.0"}, + "1.5.0": {"tools_version": "100.0.5"}, + "2.0.0": {"tools_version": "100.1.0"}, + "2.5.0": {"tools_version": "100.1.5"}, + "3.0.0": {"tools_version": "100.2.0"}, + } + + trim_ops_manager_mapping(mock_release) + + ops_manager_mapping = mock_release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"] + + assert len(ops_manager_mapping) == 5 + assert ops_manager_mapping == expected_mapping + + # Test with multiple versions in the same major version + complex_release = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "ops_manager": { + "1.0.0": {"tools_version": "100.0.0"}, + "1.5.0": {"tools_version": "100.0.5"}, + "1.9.2": {"tools_version": "100.0.9"}, + "1.10.0": {"tools_version": "100.0.10"}, + "2.0.0": {"tools_version": "100.1.0"}, + "2.5.0": {"tools_version": "100.1.5"}, + "4.0.0": {"tools_version": "100.3.0"}, + } + } + } + } + } + + # The function keeps the latest 3 versions per major version using semver + # For major version 1, keep: 1.10.0, 1.9.2, 1.5.0 + # For major version 2, keep: 2.5.0, 2.0.0 + # For major version 4, keep: 4.0.0 + expected_complex_mapping = { + "1.10.0": {"tools_version": "100.0.10"}, + "1.9.2": {"tools_version": "100.0.9"}, + "1.5.0": {"tools_version": "100.0.5"}, + "2.5.0": {"tools_version": "100.1.5"}, + "2.0.0": {"tools_version": "100.1.0"}, + "4.0.0": {"tools_version": "100.3.0"}, + } + + trim_ops_manager_mapping(complex_release) + assert ( + complex_release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"] + == expected_complex_mapping + ) + + # Test with fewer than 3 major versions (should keep all latest per major) + small_release = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "ops_manager": { + "1.0.0": {"tools_version": "100.0.0"}, + "1.2.0": {"tools_version": "100.0.2"}, + "2.0.0": {"tools_version": "100.1.0"}, + } + } + } + } + } + + expected_small_mapping = { + "1.2.0": {"tools_version": "100.0.2"}, + "1.0.0": {"tools_version": "100.0.0"}, + "2.0.0": {"tools_version": "100.1.0"}, + } + + trim_ops_manager_mapping(small_release) + assert ( + small_release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"] == expected_small_mapping + ) + + +def test_trim_ops_manager_mapping_multiple_versions_per_major(): + """Test that only the latest 3 versions per major version are kept.""" + mock_release = { + "supportedImages": { + "mongodb-agent": { + "opsManagerMapping": { + "ops_manager": { + # Five versions for major version 1 + "1.1.0": {"tools_version": "100.1.0"}, + "1.2.0": {"tools_version": "100.1.1"}, + "1.3.0": {"tools_version": "100.1.2"}, + "1.4.0": {"tools_version": "100.1.3"}, + "1.5.0": {"tools_version": "100.1.4"}, + # Five versions for major version 2 + "2.1.0": {"tools_version": "100.2.0"}, + "2.2.0": {"tools_version": "100.2.1"}, + "2.3.0": {"tools_version": "100.2.2"}, + "2.4.0": {"tools_version": "100.2.3"}, + "2.5.0": {"tools_version": "100.2.4"}, + } + } + } + } + } + + # Expected result: Keep the 3 latest versions for each major version + expected_mapping = { + "1.5.0": {"tools_version": "100.1.4"}, + "1.4.0": {"tools_version": "100.1.3"}, + "1.3.0": {"tools_version": "100.1.2"}, + "2.5.0": {"tools_version": "100.2.4"}, + "2.4.0": {"tools_version": "100.2.3"}, + "2.3.0": {"tools_version": "100.2.2"}, + } + + trim_ops_manager_mapping(mock_release) + assert mock_release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"] == expected_mapping + + +def test_trim_ops_manager_versions(): + """Test that only the latest 3 versions per major version are kept in ops-manager versions.""" + mock_release = { + "supportedImages": { + "ops-manager": { + "versions": [ + # Five versions for major version 6 + "6.5.0", + "6.4.0", + "6.3.0", + "6.2.0", + "6.1.0", + # Five versions for major version 7 + "7.5.0", + "7.4.0", + "7.3.0", + "7.2.0", + "7.1.0", + ] + } + } + } + + # Expected result: Keep the 3 latest versions for each major version + expected_versions = [ + "7.5.0", + "7.4.0", + "7.3.0", # Latest 3 from major version 7 + "6.5.0", + "6.4.0", + "6.3.0", # Latest 3 from major version 6 + ] + + trim_ops_manager_versions(mock_release) + assert set(mock_release["supportedImages"]["ops-manager"]["versions"]) == set(expected_versions) + assert len(mock_release["supportedImages"]["ops-manager"]["versions"]) == 6 + + +def test_trim_ops_manager_versions_semver_sorting(): + """Test semantic version sorting in ops-manager versions.""" + mock_release = { + "supportedImages": { + "ops-manager": { + "versions": [ + # Major version 6 with version that would sort incorrectly with string sorting + "6.1.0", + "6.2.0", + "6.3.0", + "6.10.0", + "6.11.0", + # Major version 7 with version that would sort incorrectly with string sorting + "7.1.0", + "7.10.0", + "7.2.0", + ] + } + } + } + + # Expected result with correct semver sorting: + expected_versions = [ + "7.10.0", + "7.2.0", + "7.1.0", # Latest 3 from major version 7 (semver order) + "6.11.0", + "6.10.0", + "6.3.0", # Latest 3 from major version 6 (semver order) + ] + + trim_ops_manager_versions(mock_release) + assert mock_release["supportedImages"]["ops-manager"]["versions"] == expected_versions + + +def test_trim_ops_manager_versions_fewer_than_three(): + """Test when there are fewer than 3 versions for a major version.""" + mock_release = { + "supportedImages": { + "ops-manager": { + "versions": [ + # Two versions for major version 5 + "5.2.0", + "5.1.0", + # One version for major version 6 + "6.0.0", + ] + } + } + } + + # Expected result: Keep all versions since each major has fewer than 3 + expected_versions = ["6.0.0", "5.2.0", "5.1.0"] + + trim_ops_manager_versions(mock_release) + assert set(mock_release["supportedImages"]["ops-manager"]["versions"]) == set(expected_versions) + + # Verify the versions are sorted in descending order by semver + assert mock_release["supportedImages"]["ops-manager"]["versions"] == ["6.0.0", "5.2.0", "5.1.0"] + + +def test_trim_ops_manager_versions_missing_keys(): + """Test that the function handles missing keys gracefully.""" + mock_release = { + "supportedImages": { + # No ops-manager key + } + } + + # Should not raise an exception + trim_ops_manager_versions(mock_release) + + mock_release = { + "supportedImages": { + "ops-manager": { + # No versions key + } + } + } + + # Should not raise an exception + trim_ops_manager_versions(mock_release) diff --git a/scripts/evergreen/release/update_helm_values_files.py b/scripts/evergreen/release/update_helm_values_files.py new file mode 100755 index 000000000..24ed11b40 --- /dev/null +++ b/scripts/evergreen/release/update_helm_values_files.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +""" +Performs the update of release fields in all relevant files in the project +Note, that the script must be called from the root of the project + +Usage: + update_helm_values_files.py +""" +import json +import sys +from typing import List + +from agent_matrix import get_supported_version_for_image_matrix_handling +from helm_files_handler import ( + get_value_in_yaml_file, + set_value_in_yaml_file, + update_all_helm_values_files, + update_standalone_installer, +) + +RELEASE_JSON_TO_HELM_KEY = { + "mongodbOperator": "operator", + "initDatabaseVersion": "initDatabase", + "initOpsManagerVersion": "initOpsManager", + "initAppDbVersion": "initAppDb", + "databaseImageVersion": "database", + "agentVersion": "agent", +} + + +def load_release(): + with open("release.json", "r") as fd: + return json.load(fd) + + +def filterNonReleaseOut(versions: List[str]) -> List[str]: + """Filters out all Release Candidate versions""" + return list(filter(lambda x: "-rc" not in x, versions)) + + +def main() -> int: + release = load_release() + + operator_version = release["mongodbOperator"] + + for k in release: + if k in RELEASE_JSON_TO_HELM_KEY: + update_all_helm_values_files(RELEASE_JSON_TO_HELM_KEY[k], release[k]) + + update_helm_charts(operator_version, release) + update_standalone(operator_version) + update_cluster_service_version(operator_version) + + return 0 + + +def update_standalone(operator_version): + update_standalone_installer("public/mongodb-enterprise.yaml", operator_version), + update_standalone_installer("public/mongodb-enterprise-openshift.yaml", operator_version), + update_standalone_installer("public/mongodb-enterprise-multi-cluster.yaml", operator_version), + + +def update_helm_charts(operator_version, release): + set_value_in_yaml_file( + "helm_chart/values-openshift.yaml", + "relatedImages.opsManager", + filterNonReleaseOut(release["supportedImages"]["ops-manager"]["versions"]), + ) + set_value_in_yaml_file( + "helm_chart/values-openshift.yaml", + "relatedImages.mongodbLegacyAppDb", + filterNonReleaseOut(release["supportedImages"]["appdb-database"]["versions"]), + ) + set_value_in_yaml_file( + "helm_chart/values-openshift.yaml", + "relatedImages.mongodb", + filterNonReleaseOut(release["supportedImages"]["mongodb-enterprise-server"]["versions"]), + ) + set_value_in_yaml_file( + "helm_chart/values-openshift.yaml", + "relatedImages.agent", + filterNonReleaseOut(get_supported_version_for_image_matrix_handling("mongodb-agent")), + ) + set_value_in_yaml_file("helm_chart/values-openshift.yaml", "operator.version", operator_version) + set_value_in_yaml_file("helm_chart/values.yaml", "operator.version", operator_version) + set_value_in_yaml_file("helm_chart/Chart.yaml", "version", operator_version) + + +def update_cluster_service_version(operator_version): + old_operator_version = get_value_in_yaml_file( + "config/manifests/bases/mongodb-enterprise.clusterserviceversion.yaml", "metadata.annotations.containerImage" + ).split(":")[-1] + + if old_operator_version != operator_version: + set_value_in_yaml_file( + "config/manifests/bases/mongodb-enterprise.clusterserviceversion.yaml", + "spec.replaces", + f"mongodb-enterprise.v{old_operator_version}", + preserve_quotes=True, + ) + + set_value_in_yaml_file( + "config/manifests/bases/mongodb-enterprise.clusterserviceversion.yaml", + "metadata.annotations.containerImage", + f"quay.io/mongodb/mongodb-enterprise-operator-ubi:{operator_version}", + preserve_quotes=True, + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/evergreen/release/update_release.py b/scripts/evergreen/release/update_release.py new file mode 100755 index 000000000..c76874e0a --- /dev/null +++ b/scripts/evergreen/release/update_release.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +import json +import logging +import os +from collections import defaultdict + +import yaml +from packaging import version + +logger = logging.getLogger(__name__) + + +def get_latest_om_versions_from_evergreen_yml(): + # Define a custom constructor to preserve the anchors in the YAML file + evergreen_file = os.path.join(os.getcwd(), ".evergreen.yml") + with open(evergreen_file) as f: + data = yaml.safe_load(f) + return data["variables"][0], data["variables"][1] + + +def update_release_json(): + # Define a custom constructor to preserve the anchors in the YAML file + release = os.path.join(os.getcwd(), "release.json") + with open(release, "r") as fd: + data = json.load(fd) + + # Trim ops_manager_mapping to keep only the latest 3 versions + trim_ops_manager_mapping(data) + + # Trim ops-manager versions to keep only the latest 3 versions per major + trim_ops_manager_versions(data) + + # PCT already bumps the release.json, such that the last element contains the newest version, since they are sorted + newest_om_version = data["supportedImages"]["ops-manager"]["versions"][-1] + update_mongodb_tools_bundle(data, newest_om_version) + + # PCT bumps this field, and we can use this as a base to set the version for everything else in release.json + newest_operator_version = data["mongodbOperator"] + update_operator_related_versions(data, newest_operator_version) + + with open(release, "w") as f: + json.dump( + data, + f, + indent=2, + ) + f.write("\n") + + +def trim_ops_manager_mapping(release: dict): + """ + Keep only the latest 3 versions per major version in opsManagerMapping.ops_manager. + """ + if ( + "mongodb-agent" in release["supportedImages"] + and "opsManagerMapping" in release["supportedImages"]["mongodb-agent"] + ): + ops_manager_mapping = release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"] + + major_version_groups = defaultdict(list) + for v in ops_manager_mapping.keys(): + major_version = v.split(".")[0] + major_version_groups[major_version].append(v) + + trimmed_mapping = {} + + for major_version, versions in major_version_groups.items(): + versions.sort(key=lambda x: version.parse(x), reverse=True) + latest_versions = versions[:3] + + for v in latest_versions: + trimmed_mapping[v] = ops_manager_mapping[v] + + trimmed_mapping = dict(sorted(trimmed_mapping.items(), key=lambda x: version.parse(x[0]))) + + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"] = trimmed_mapping + + +def trim_ops_manager_versions(release: dict): + """ + Keep only the latest 3 versions per major version in supportedImages.ops-manager.versions. + """ + if "ops-manager" in release["supportedImages"] and "versions" in release["supportedImages"]["ops-manager"]: + versions = release["supportedImages"]["ops-manager"]["versions"] + + major_version_groups = defaultdict(list) + for v in versions: + major_version = v.split(".")[0] + major_version_groups[major_version].append(v) + + trimmed_versions = [] + + for major_version, versions in major_version_groups.items(): + versions.sort(key=lambda x: version.parse(x), reverse=True) + latest_versions = versions[:3] + trimmed_versions.extend(latest_versions) + + # Sort the final list in ascending order + trimmed_versions.sort(key=lambda x: version.parse(x)) + release["supportedImages"]["ops-manager"]["versions"] = trimmed_versions + + +def update_operator_related_versions(release: dict, version: str): + """ + Updates version on `source`, that corresponds to `release.json`. + """ + + logger.debug(f"Updating release.json for version: {version}") + + keys_to_update_with_current_version = [ + "initDatabaseVersion", + "initOpsManagerVersion", + "initAppDbVersion", + "databaseImageVersion", + ] + + for key in keys_to_update_with_current_version: + release[key] = version + + keys_to_add_supported_versions = [ + "operator", + "init-ops-manager", + "init-database", + "init-appdb", + "database", + ] + + for key in keys_to_add_supported_versions: + if version not in release["supportedImages"][key]["versions"]: + release["supportedImages"][key]["versions"].append(version) + + logger.debug(f"Updated content {release}") + + +def update_mongodb_tools_bundle(data, newest_om_version): + om_mapping = data["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][newest_om_version] + mongo_tool_version = om_mapping["tools_version"] + + version_name = f"mongodb-database-tools-rhel88-x86_64-{mongo_tool_version}.tgz" + data["mongodbToolsBundle"]["ubi"] = version_name + + +if __name__ == "__main__": + update_release_json() diff --git a/scripts/evergreen/retry-evergreen.sh b/scripts/evergreen/retry-evergreen.sh new file mode 100755 index 000000000..e09baacd6 --- /dev/null +++ b/scripts/evergreen/retry-evergreen.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +set -x + +### +## This script automatically retriggers failed tasks from Evergreen based on the `version_id` passed in from Evergreen. +## +## usage: +## Set EVERGREEN_USER and EVERGREEN_API_KEY env. variables +## Obtain the version from either Evergreen UI or Github checks +## Call +## ./retry-evergreen.sh 62cfba5957e85a64e1f801fa +### +echo "EVERGREEN_RETRY=${EVERGREEN_RETRY:-"true"}" +if [[ "${EVERGREEN_RETRY:-"true"}" != "true" ]]; then + echo "Skipping evergreen retry" + exit 0 +fi + +if [ $# -eq 0 ]; then + echo "Details URL not passed in, exiting..." + exit 1 +else + VERSION=$1 +fi +if [ -z "${EVERGREEN_USER}" ]; then + echo "$$EVERGREEN_USER not set" + exit 1 +fi +if [ -z "${EVERGREEN_API_KEY}" ]; then + echo "$$EVERGREEN_API_KEY not set" + exit 1 +fi + +EVERGREEN_API="https://evergreen.mongodb.com/api" +MAX_RETRIES="${EVERGREEN_MAX_RETRIES:-3}" + +# shellcheck disable=SC2207 +BUILD_IDS=($(curl -s -H "Api-User: ${EVERGREEN_USER}" -H "Api-Key: ${EVERGREEN_API_KEY}" ${EVERGREEN_API}/rest/v2/versions/"${VERSION}" | jq -r '.build_variants_status[] | select(.build_variant != "unit_tests" and .build_variant != "run_pre_commit") | .build_id')) + +for BUILD_ID in "${BUILD_IDS[@]}"; do + echo "Finding failed tasks in BUILD ID: ${BUILD_ID}" + # shellcheck disable=SC2207 + TASK_IDS=($(curl -s -H "Api-User: ${EVERGREEN_USER}" -H "Api-Key: ${EVERGREEN_API_KEY}" ${EVERGREEN_API}/rest/v2/builds/"${BUILD_ID}"/tasks | jq ".[] | select(.status == \"failed\" and .execution <= ${MAX_RETRIES})" | jq -r '.task_id')) + + for TASK_ID in "${TASK_IDS[@]}"; do + echo "Retriggering TASK ID: ${TASK_ID}" + curl -H "Api-User: ${EVERGREEN_USER}" -H "Api-Key: ${EVERGREEN_API_KEY}" -X POST ${EVERGREEN_API}/rest/v2/tasks/"${TASK_ID}"/restart + done +done diff --git a/scripts/evergreen/run_python.sh b/scripts/evergreen/run_python.sh new file mode 100755 index 000000000..640ebb89a --- /dev/null +++ b/scripts/evergreen/run_python.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/printing + +# shellcheck disable=SC2154 +if [ -f "${PROJECT_DIR}/venv/bin/activate" ]; then + source "${PROJECT_DIR}/venv/bin/activate" +else + echo "Cannot find python venv in ${PROJECT_DIR}" + ls -al "${PROJECT_DIR}" + exit 1 +fi + +export PYTHONPATH="${PROJECT_DIR}" + +current_python_version=$(python --version 2>&1 | awk '{split($2, a, "."); print a[1] "." a[2]}') +if [[ "${current_python_version}" != "${PYTHON_VERSION}" ]]; then + echo -e "${RED}Detected mismatched version of python in your venv (detected version: ${current_python_version}).${NO_COLOR}" + echo -e "${RED}Please re-run scripts/dev/install.sh or recreate venv using Python ${PYTHON_VERSION} manually by running (scripts/dev/recreate_python_venv.sh).${NO_COLOR}" + echo "which python: $(which python)" + echo "python --version:" + python --version + exit 1 +fi + +python "$@" diff --git a/scripts/evergreen/setup_aws.sh b/scripts/evergreen/setup_aws.sh new file mode 100755 index 000000000..931eb0a36 --- /dev/null +++ b/scripts/evergreen/setup_aws.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +INSTALL_DIR="${workdir:?}/.local/lib/aws" +BIN_LOCATION="${workdir}/bin" + +mkdir -p "${BIN_LOCATION}" + +tmpdir=$(mktemp -d) +cd "${tmpdir}" + +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip &> /dev/null + +docker_dir="/home/${USER}/.docker" +if [[ ! -d "${docker_dir}" ]]; then + mkdir -p "${docker_dir}" +fi + +sudo chown "${USER}":"${USER}" "${docker_dir}" -R +sudo chmod g+rwx "${docker_dir}" -R +sudo ./aws/install --bin-dir "${BIN_LOCATION}" --install-dir "${INSTALL_DIR}" --update +cd - + +rm -rf "${tmpdir}" diff --git a/scripts/evergreen/setup_docker_sbom.sh b/scripts/evergreen/setup_docker_sbom.sh new file mode 100755 index 000000000..efd2c5f96 --- /dev/null +++ b/scripts/evergreen/setup_docker_sbom.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +if [ -f ~/.docker/cli-plugins/docker-sbom ]; then + echo "Docker sbom exists. Skipping the installation." +else + echo "Installing Docker sbom plugin." + + docker_dir="/home/${USER}/.docker" + if [[ ! -d "${docker_dir}" ]]; then + mkdir -p "${docker_dir}" + sudo chown "${USER}":"${USER}" "${docker_dir}" -R + sudo chmod g+rwx "${docker_dir}" -R + fi + + plugins_dir="/home/${USER}/.docker/cli-plugins" + mkdir -p "${plugins_dir}" + sudo chown "${USER}":"${USER}" "${plugins_dir}" -R + sudo chmod g+rwx "${plugins_dir}" -R + wget "https://github.com/docker/sbom-cli-plugin/releases/download/v0.6.1/sbom-cli-plugin_0.6.1_linux_amd64.tar.gz" + tar -zxf sbom-cli-plugin_0.6.1_linux_amd64.tar.gz + chmod +x ./docker-sbom + mv ./docker-sbom "${plugins_dir}" + rm -rf sbom-cli-plugin_0.6.1_linux_amd64.tar.gz +fi + diff --git a/scripts/evergreen/setup_gcloud_cli.sh b/scripts/evergreen/setup_gcloud_cli.sh new file mode 100755 index 000000000..15b127dd5 --- /dev/null +++ b/scripts/evergreen/setup_gcloud_cli.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +curl -s --retry 3 -LO "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz" +tar xvf google-cloud-cli-linux-x86_64.tar.gz -C "${workdir}" +"${workdir}"/google-cloud-sdk/install.sh --quiet +source "${workdir}/google-cloud-sdk/path.bash.inc" + +gcloud components install gke-gcloud-auth-plugin +echo "${GCP_SERVICE_ACCOUNT_JSON_FOR_SNIPPETS_TESTS}" > gcp_keyfile.json +gcloud auth activate-service-account --key-file gcp_keyfile.json +gcloud auth list diff --git a/scripts/evergreen/setup_jq.sh b/scripts/evergreen/setup_jq.sh new file mode 100755 index 000000000..e21d4a07e --- /dev/null +++ b/scripts/evergreen/setup_jq.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# A script Evergreen will use to setup jq +# +# This should be executed from root of the evergreen build dir +# + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/install + +download_and_install_binary "${PROJECT_DIR:-.}/bin" jq "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" diff --git a/scripts/evergreen/setup_kind.sh b/scripts/evergreen/setup_kind.sh new file mode 100755 index 000000000..96b315a78 --- /dev/null +++ b/scripts/evergreen/setup_kind.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +# Store the lowercase name of Operating System +os=$(uname | tr '[:upper:]' '[:lower:]') +# This should be changed when needed +latest_version="v0.27.0" + +mkdir -p "${PROJECT_DIR}/bin/" +echo "Saving kind to ${PROJECT_DIR}/bin" +curl --retry 3 --silent -L "https://github.com/kubernetes-sigs/kind/releases/download/${latest_version}/kind-${os}-amd64" -o kind + +chmod +x kind +sudo mv kind "${PROJECT_DIR}/bin" +echo "Installed kind in ${PROJECT_DIR}/bin" diff --git a/scripts/evergreen/setup_kubectl.sh b/scripts/evergreen/setup_kubectl.sh new file mode 100755 index 000000000..45f382fec --- /dev/null +++ b/scripts/evergreen/setup_kubectl.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +bindir="${PROJECT_DIR}/bin" +tmpdir="${PROJECT_DIR}/tmp" +mkdir -p "${bindir}" "${tmpdir}" + +echo "Downloading latest kubectl" +curl -s --retry 3 -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x kubectl +echo "kubectl version --client" +./kubectl version --client +mv kubectl "${bindir}" + +echo "Downloading helm" +helm_archive="${tmpdir}/helm.tgz" +helm_version="v3.17.1" +curl -s https://get.helm.sh/helm-${helm_version}-linux-amd64.tar.gz --output "${helm_archive}" + +tar xfz "${helm_archive}" -C "${tmpdir}" &> /dev/null +mv "${tmpdir}/linux-amd64/helm" "${bindir}" diff --git a/scripts/evergreen/setup_kubernetes_environment.sh b/scripts/evergreen/setup_kubernetes_environment.sh new file mode 100755 index 000000000..707231c9f --- /dev/null +++ b/scripts/evergreen/setup_kubernetes_environment.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh +source scripts/funcs/kubernetes + +# shellcheck disable=SC2154 +bindir="${PROJECT_DIR}/bin" + +if [[ "${KUBE_ENVIRONMENT_NAME}" == "vanilla" || ("${KUBE_ENVIRONMENT_NAME}" == "multi" && "${CLUSTER_TYPE}" == "kops") ]]; then + export AWS_ACCESS_KEY_ID="${mms_eng_test_aws_access_key:?}" + export AWS_SECRET_ACCESS_KEY="${mms_eng_test_aws_secret:?}" + export AWS_DEFAULT_REGION="${mms_eng_test_aws_region:?}" +fi + +if [ "${KUBE_ENVIRONMENT_NAME}" = "openshift_4" ]; then + echo "Downloading OC & setting up Openshift 4 cluster" + OC_PKG=oc-linux.tar.gz + + # Source of this file is https://access.redhat.com/downloads/content/290/ver=4.12/rhel---8/4.12.8/x86_64/product-software + # But it has been copied to S3 to avoid authentication issues in the future. + curl --fail --retry 3 -s -L 'https://operator-kubernetes-build.s3.amazonaws.com/oc-4.12.8-linux.tar.gz' \ + --output "${OC_PKG}" + tar xfz "${OC_PKG}" &>/dev/null + mv oc "${bindir}" + + # https://stackoverflow.com/c/private-cloud-kubernetes/questions/15 + oc login --token="${OPENSHIFT_TOKEN}" --server="${OPENSHIFT_URL}" +elif [ "${KUBE_ENVIRONMENT_NAME}" = "kind" ] || [ "${KUBE_ENVIRONMENT_NAME}" = "performance" ]; then + scripts/dev/recreate_kind_cluster.sh "kind" +elif [[ "${KUBE_ENVIRONMENT_NAME}" = "multi" && "${CLUSTER_TYPE}" == "kind" ]]; then + scripts/dev/recreate_kind_clusters.sh +else + echo "KUBE_ENVIRONMENT_NAME not recognized" + echo "value is <<${KUBE_ENVIRONMENT_NAME}>>. If empty it means it was not set" + + # Fail if there's no Kubernetes environment set + exit 1 +fi diff --git a/scripts/evergreen/setup_mongosh.sh b/scripts/evergreen/setup_mongosh.sh new file mode 100755 index 000000000..614da0b33 --- /dev/null +++ b/scripts/evergreen/setup_mongosh.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +bindir="${workdir:?}/bin" +mkdir -p "${bindir}" + +curl -s --retry 3 -LO "https://downloads.mongodb.com/compass/mongosh-2.3.8-linux-x64.tgz" +tar -zxvf mongosh-2.3.8-linux-x64.tgz +cd mongosh-2.3.8-linux-x64/bin +./mongosh --version +mv mongosh "${bindir}" diff --git a/scripts/evergreen/setup_preflight.sh b/scripts/evergreen/setup_preflight.sh new file mode 100755 index 000000000..8221b1bee --- /dev/null +++ b/scripts/evergreen/setup_preflight.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# A script Evergreen will use to setup openshift-preflight +set -Eeou pipefail +source scripts/dev/set_env_context.sh + +bindir="${PROJECT_DIR:?}/bin" +mkdir -p "${bindir}" + +echo "Downloading preflight binary" +preflight_version="1.11.1" +curl -s --retry 3 --fail-with-body -o preflight -LO "https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/${preflight_version}/preflight-linux-amd64" +chmod +x preflight +mv preflight "${bindir}" +echo "Installed preflight to ${bindir}" diff --git a/scripts/evergreen/setup_prepare_openshift_bundles.sh b/scripts/evergreen/setup_prepare_openshift_bundles.sh new file mode 100755 index 000000000..cdb6d0160 --- /dev/null +++ b/scripts/evergreen/setup_prepare_openshift_bundles.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Script for evergreen to setup necessary software for generating openshift bundles. +# +# This should be executed from root of the evergreen build dir + +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +source scripts/funcs/install + +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m | tr '[:upper:]' '[:lower:]') +if [[ "${ARCH}" == "x86_64" ]]; then + ARCH="amd64" +fi + +download_and_install_binary "${PROJECT_DIR:-.}/bin" operator-sdk "https://github.com/operator-framework/operator-sdk/releases/download/v1.26.1/operator-sdk_${OS}_${ARCH}" +download_and_install_binary "${PROJECT_DIR:-.}/bin" operator-manifest-tools "https://github.com/operator-framework/operator-manifest-tools/releases/download/v0.2.2/operator-manifest-tools_0.2.2_${OS}_amd64" + +if [[ "${OS}" == "darwin" ]]; then + brew install skopeo +else + sudo apt install -y skopeo +fi + +opm_os="linux" +if [[ "${OS}" == "darwin" ]]; then + opm_os="mac" +fi + +# there is no mac build in for arm64 +opm_arch="amd64" +curl -L --retry 3 -o opm.tar.gz "https://mirror.openshift.com/pub/openshift-v4/${opm_arch}/clients/ocp/latest-4.12/opm-${opm_os}.tar.gz" + +# TODO: Sometimes tar is failing for unknown reasons in EVG. This is left intentionally. Remove if not causing problems anymore. +ls -al opm.tar.gz +head -c 50 < opm.tar.gz | xxd + +tar xvf opm.tar.gz +chmod +x opm && mv opm "${PROJECT_DIR:-.}/bin" +rm -rf opm.tar.gz diff --git a/scripts/evergreen/setup_shellcheck.sh b/scripts/evergreen/setup_shellcheck.sh new file mode 100755 index 000000000..577188e2c --- /dev/null +++ b/scripts/evergreen/setup_shellcheck.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -Eeou pipefail + +source scripts/dev/set_env_context.sh + +bindir="${PROJECT_DIR:?}/bin" +tmpdir="${PROJECT_DIR:?}/tmp" +mkdir -p "${bindir}" "${tmpdir}" + +echo "Downloading shellcheck" +shellcheck_archive="${tmpdir}/shellcheck.tar.xz" +shellcheck_version="v0.9.0" +curl --retry 3 --silent -L "https://github.com/koalaman/shellcheck/releases/download/${shellcheck_version}/shellcheck-${shellcheck_version}.linux.x86_64.tar.xz" -o "${shellcheck_archive}" +tar -xf "${shellcheck_archive}" -C "${tmpdir}" +mv "${tmpdir}/shellcheck-${shellcheck_version}/shellcheck" "${bindir}" +rm "${shellcheck_archive}" diff --git a/scripts/evergreen/setup_yq.sh b/scripts/evergreen/setup_yq.sh new file mode 100755 index 000000000..caa0ae88b --- /dev/null +++ b/scripts/evergreen/setup_yq.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# A script Evergreen will use to setup yq +# +# This should be executed from root of the evergreen build dir + +set -Eeou pipefail + +source scripts/funcs/install +source scripts/dev/set_env_context.sh + +download_and_install_binary "${PROJECT_DIR:-.}/bin" yq "https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_amd64" diff --git a/scripts/evergreen/should_prepare_openshift_bundles.sh b/scripts/evergreen/should_prepare_openshift_bundles.sh new file mode 100755 index 000000000..c85ed5e0b --- /dev/null +++ b/scripts/evergreen/should_prepare_openshift_bundles.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# This file is a condition script used for conditionally executing evergreen task for generating openshift bundles (prepare_and_upload_openshift_bundles). + +set -Eeou pipefail +source scripts/dev/set_env_context.sh + +check_file_exists() { + url=$1 + stderr_out=$(mktemp) + echo "Checking if file exists: ${url}..." + http_error_code=$(curl -o /dev/stderr --head --write-out '%{http_code}' --silent "${url}" 2>"${stderr_out}") + echo "http error code=${http_error_code}" + cat "${stderr_out}" + rm "${stderr_out}" + if [[ "${http_error_code}" == "200" ]]; then + return 0 + else + return 1 + fi +} + +version=$(jq -r .mongodbOperator licenses_full.csv 2> licenses_stderr || true + + # Filter and sort the licenses report + grep -v 10gen licenses_full.csv | grep -v "github.com/mongodb" | grep -v "^golang.org" | sort > licenses.csv || true + + # Return to the repo root directory + cd "${REPO_DIR}" || exit +} + +process_licenses "${REPO_DIR}" & +process_licenses "${REPO_DIR}/public/tools/multicluster" & + +wait + +echo "License processing complete for all modules." diff --git a/scripts/evergreen/update_licenses.tpl b/scripts/evergreen/update_licenses.tpl new file mode 100644 index 000000000..2eb3d77b5 --- /dev/null +++ b/scripts/evergreen/update_licenses.tpl @@ -0,0 +1,3 @@ +{{ range . }} +{{.Name}},{{.Version}},{{.LicenseURL}},{{.LicenseName}} +{{- end }} \ No newline at end of file diff --git a/scripts/funcs/checks b/scripts/funcs/checks new file mode 100644 index 000000000..6115c46da --- /dev/null +++ b/scripts/funcs/checks @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +pushd "${PWD}" > /dev/null || return +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "${DIR}" || return +source errors +popd > /dev/null || return + +check_env_var() { + local var_name="$1" + local msg="$2" + set +u + if [[ -z "${!var_name}" ]]; then + echo "${msg}" + exit 1 + fi +} + +check_app() { + local var="$1" + local msg="$2" + if ! which "${var}" > /dev/null; then + echo "${msg}" + exit 1 + fi +} + +check_mandatory_param() { + local param="${1-}" + local param_name="${2-}" + if [[ -z "${param}" ]]; then + fatal "Parameter ${param_name} must be specified!" + fi +} diff --git a/scripts/funcs/errors b/scripts/funcs/errors new file mode 100644 index 000000000..c1cb17755 --- /dev/null +++ b/scripts/funcs/errors @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +fatal() { + error "$1" + exit 1 +} + +error() { + echo "(!!) $1" + return +} + diff --git a/scripts/funcs/install b/scripts/funcs/install new file mode 100644 index 000000000..fee7fc657 --- /dev/null +++ b/scripts/funcs/install @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Downloads a binary from and moves it into directory. +# Example usage: download_and_install_binary ${workdir}/bin jq "https://..." +download_and_install_binary() { + dir=$1 + bin=$2 + url=$3 + + mkdir -p "${dir}" + echo "Downloading ${url}" + curl --retry 3 --silent -L "${url}" -o "${bin}" + chmod +x "${bin}" + mv "${bin}" "${dir}" + echo "Installed ${bin} to ${dir}" +} diff --git a/scripts/funcs/kubernetes b/scripts/funcs/kubernetes new file mode 100644 index 000000000..be016be28 --- /dev/null +++ b/scripts/funcs/kubernetes @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +pushd "${PWD}" > /dev/null || return +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "${DIR}" || return +# shellcheck source=scripts/funcs/checks +source checks +# shellcheck source=scripts/funcs/errors +source errors +# shellcheck source=scripts/funcs/printing +source printing +popd > /dev/null || return + +ensure_namespace() { + local namespace="${1}" + local tmp_file + tmp_file=$(mktemp) + cat < "${tmp_file}" +apiVersion: v1 +kind: Namespace +metadata: + name: ${namespace} + labels: + evg: task + annotations: + evg/version: "https://evergreen.mongodb.com/version/${version_id:-'not-specified'}" + evg/task-name: ${TASK_NAME:-'not-specified'} + evg/task: "https://evergreen.mongodb.com/task/${task_id:-'not-specified'}" + evg/mms-version: "${ops_manager_version:-'not-specified'}" +EOF + + if kubectl get "ns/${namespace}" -o name &> /dev/null; then + echo "Namespace ${namespace} already exists!" + else + echo "Creating new namespace: ${namespace}" + cat "${tmp_file}" + kubectl create -f "${tmp_file}" + fi +} + +delete_operator() { + local ns="$1" + local name=${OPERATOR_NAME:=mongodb-enterprise-operator} + + title "Removing the Operator deployment ${name}" + ! kubectl --namespace "${ns}" get deployments | grep -q "${name}" \ + || kubectl delete deployment "${name}" -n "${ns}" || true +} + +# wait_for_operator waits for the Operator to start +wait_for_operator_start() { + local ns="$1" + local timeout="${2:-2m}" + echo "Waiting until the Operator gets to Running state..." + export OPERATOR_NAME + + local cmd + + # Waiting until there is only one pod left as this could be the upgrade operation (very fast in practice) + # shellcheck disable=SC2016 + cmd='while [[ $(kubectl -n '"${ns}"' get pods -l app.kubernetes.io/name=${OPERATOR_NAME} --no-headers 2>/dev/null | wc -l) -gt 1 ]] ; do printf .; sleep 1; done' + timeout --foreground "1m" bash -c "${cmd}" || true + + cmd="while ! kubectl -n ${ns} get pods -l app.kubernetes.io/name=${OPERATOR_NAME} -o jsonpath={.items[0].status.phase} 2>/dev/null | grep -q Running ; do printf .; sleep 1; done" + timeout --foreground "${timeout}" bash -c "${cmd}" || true + + # In the end let's check again and print the state + if ! kubectl -n "${ns}" get pods -l "app.kubernetes.io/name=${OPERATOR_NAME}" -o jsonpath="{.items[0].status.phase}" | grep -q "Running"; then + error "Operator hasn't reached RUNNING state after ${timeout}. The full yaml configuration for the pod is:" + kubectl -n "${ns}" get pods -l "app.kubernetes.io/name=${OPERATOR_NAME}" -o yaml + + title "Operator failed to start, exiting" + return 1 + fi + echo "" + + title "The Operator successfully installed to the Kubernetes cluster" +} + +# recreates docker credentials secret in the cluster +create_image_registries_secret() { + echo "Started creating image-registries-secret in cluster(s)" + + if ! kubectl cluster-info > /dev/null 2>&1; then + echo -e "\033[31mCannot get cluster info - does the cluster exist and is reachable?" + exit 0 + fi + + secret_name="image-registries-secret" + + if [[ "${NAMESPACE:-""}" == "" ]]; then + echo -e "\033[31mNAMESPACE env var is not set, skipping creating image-registries-secret" + exit 0 + fi + + if which kubectl > /dev/null; then + echo "Kubectl command is there, proceeding..." + else + echo -e "\033[31mKubectl doesn't exist, skipping setting the context" + exit 0 + fi + + create_pull_secret() { + context=$1 + namespace=$2 + secret_name=$3 + # shellcheck disable=SC2154 + if kubectl --context "${context}" get namespace "${namespace}"; then + kubectl --context "${context}" -n "${namespace}" delete secret "${secret_name}" --ignore-not-found + echo "${context}: Creating ${namespace}/${secret_name} pull secret" + kubectl --context "${context}" -n "${namespace}" create secret generic "${secret_name}" \ + --from-file=.dockerconfigjson="${HOME}/.docker/config.json" --type=kubernetes.io/dockerconfigjson + else + echo "Skipping creating pull secret in ${context}/${namespace}. The namespace doesn't exist yet." + fi + } + + echo "Creating/updating pull secret from docker configured file" + if [[ "${KUBE_ENVIRONMENT_NAME:-}" == "multi" ]]; then + create_pull_secret "${CENTRAL_CLUSTER}" "${NAMESPACE}" "${secret_name}" & + for member_cluster in ${MEMBER_CLUSTERS}; do + for ns in ${WATCH_NAMESPACE//,/ }; do + create_pull_secret "${member_cluster}" "${ns}" "${secret_name}" & + done + done + wait + else + current_ctx=$(kubectl config current-context) + create_pull_secret "${current_ctx}" "${NAMESPACE}" "${secret_name}" & + for ns in ${WATCH_NAMESPACE//,/ }; do + create_pull_secret "${current_ctx}" "${ns}" "${secret_name}" & + done + fi +} + +reset_namespace() { + context=$1 + namespace=$2 + if [[ "${context}" == "" ]]; then + echo "context cannot be empty" + exit 1 + fi + + set +e + + helm uninstall --kube-context="${context}" mongodb-enterprise-operator || true & + helm uninstall --kube-context="${context}" mongodb-enterprise-operator-multi-cluster || true & + + # Cleans the namespace. Note, that fine-grained cleanup is performed instead of just deleting the namespace as it takes + # considerably less time + title "Cleaning Kubernetes resources in context: ${context}" + + ensure_namespace "${namespace}" + + kubectl delete --context "${context}" mdb --all -n "${namespace}" || true + kubectl delete --context "${context}" mdbu --all -n "${namespace}" || true + kubectl delete --context "${context}" mdbmc --all -n "${namespace}" || true + kubectl delete --context "${context}" om --all -n "${namespace}" || true + + # Openshift variant runs all tests sequentially. In order to avoid clashes between tests, we need to wait till + # the namespace is gone. This trigger OpenShift Project deletion, which is a "Namespace on Steroids" and it takes + # a while to delete it. + should_wait="false" + # shellcheck disable=SC2153 + if [[ ${CURRENT_VARIANT_CONTEXT} == e2e_mdb_openshift_ubi_cloudqa || ${CURRENT_VARIANT_CONTEXT} == e2e_openshift_static_mdb_ubi_cloudqa ]]; then + should_wait="true" + echo "Removing the test namespace ${namespace}, should_wait=${should_wait}" + kubectl --context "${context}" delete "namespace/${namespace}" --wait="${should_wait}" || true + fi + + echo "Removing CSRs" + kubectl --context "${context}" delete "$(kubectl get csr -o name | grep "${NAMESPACE}")" &> /dev/null || true +} + +delete_kind_network() { + if ! docker network ls | grep -q "kind"; then + echo "Docker network 'kind' does not exist." + return 0 + fi + echo "Docker network 'kind' exists." + + # Stop all containers in the "kind" network + containers=$(docker network inspect -f '{{range .Containers}}{{.Name}} {{end}}' kind) + if [[ -n "${containers}" ]]; then + echo "Stopping containers ${containers} using kind network..." + # shellcheck disable=SC2086 + docker stop ${containers} + docker container prune -f + fi + + docker network rm kind +} diff --git a/scripts/funcs/multicluster b/scripts/funcs/multicluster new file mode 100644 index 000000000..c19e6284e --- /dev/null +++ b/scripts/funcs/multicluster @@ -0,0 +1,292 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +source scripts/funcs/kubernetes + +ensure_test_namespace() { + local context=${1} + kubectl create ns --context "${context}" "${NAMESPACE}" &>/dev/null || true + kubectl label ns "${NAMESPACE}" --context "${context}" "evg=task" &>/dev/null || true + # shellcheck disable=SC2154 + kubectl annotate ns "${NAMESPACE}" --context "${context}" "evg/task=https://evergreen.mongodb.com/task/${task_id:-}" &>/dev/null || true +} + +get_test_namespaces() { + local context=${1} + kubectl get namespaces --context "${context}" --selector=evg=task -o jsonpath='{.items[*].metadata.name}' +} + +create_service_account_token_secret() { + context=$1 + service_account_name=$2 + secret_name=$3 + kubectl --context "${context}" apply -n "${NAMESPACE}" -f - <"${helm_template_file}" || exit 1 + + echo "Creating KubeConfig secret for test pod in namespace ${NAMESPACE}" + secret_kubeconfig=${KUBECONFIG} + if [[ "${CLUSTER_TYPE}" == "kind" ]]; then + secret_kubeconfig=${kind_kubeconfig} + echo "Using temporary kubeconfig with changed api servers: ${secret_kubeconfig}" + fi + # shellcheck disable=SC2154 + kubectl --context "${test_pod_cluster}" delete secret test-pod-kubeconfig -n "${NAMESPACE}" --ignore-not-found + kubectl --context "${test_pod_cluster}" create secret generic test-pod-kubeconfig --from-file=kubeconfig="${secret_kubeconfig}" --namespace "${NAMESPACE}" || true + + echo "Creating project configmap" + # delete `my-project` if it exists + kubectl --context "${CENTRAL_CLUSTER}" --namespace "${NAMESPACE}" delete configmap my-project --ignore-not-found + # Configuring project + kubectl --context "${CENTRAL_CLUSTER}" --namespace "${NAMESPACE}" create configmap my-project \ + --from-literal=projectName="${NAMESPACE}" --from-literal=baseUrl="${OM_BASE_URL}" \ + --from-literal=orgId="${OM_ORGID:-}" + + create_image_registries_secret + + echo "Creating credentials secret" + # delete `my-credentials` if it exists + kubectl --context "${CENTRAL_CLUSTER}" --namespace "${NAMESPACE}" delete secret my-credentials --ignore-not-found + # Configure the Kubernetes credentials for Ops Manager + kubectl --context "${CENTRAL_CLUSTER}" --namespace "${NAMESPACE}" create secret generic my-credentials \ + --from-literal=user="${OM_USER:=admin}" --from-literal=publicApiKey="${OM_API_KEY}" + + echo "Creating required roles and service accounts." + kubectl --context "${CENTRAL_CLUSTER}" -n "${NAMESPACE}" apply -f "${helm_template_file}" + for member_cluster in ${MEMBER_CLUSTERS}; do + kubectl --context "${member_cluster}" -n "${NAMESPACE}" apply -f "${helm_template_file}" & + done + wait + + rm "${helm_template_file}" + # wait some time for service account token secrets to appear. + sleep 1 + + local service_account_name="operator-tests-multi-cluster-service-account" + + local secret_name + secret_name="$(kubectl --context "${CENTRAL_CLUSTER}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')" + if [[ "${secret_name}" == "" ]]; then + secret_name="${service_account_name}-token-secret" + create_service_account_token_secret "${CENTRAL_CLUSTER}" "${service_account_name}" "${secret_name}" + fi + + local central_cluster_token + central_cluster_token="$(kubectl --context "${CENTRAL_CLUSTER}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)" + echo "Creating Multi Cluster configuration secret" + + configuration_params=( + "--from-literal=central_cluster=${CENTRAL_CLUSTER}" + ) + + configuration_params+=( + "--from-literal=${CENTRAL_CLUSTER}=${central_cluster_token}" + ) + + local secret_name + secret_name="$(kubectl --context "${CENTRAL_CLUSTER}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')" + if [[ "${secret_name}" == "" ]]; then + secret_name="${service_account_name}-token-secret" + create_service_account_token_secret "${CENTRAL_CLUSTER}" "${service_account_name}" "${secret_name}" + fi + + local central_cluster_token + central_cluster_token="$(kubectl --context "${CENTRAL_CLUSTER}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)" + echo "Creating Multi Cluster configuration secret" + + configuration_params=( + "--from-literal=central_cluster=${CENTRAL_CLUSTER}" + ) + + configuration_params+=( + "--from-literal=${CENTRAL_CLUSTER}=${central_cluster_token}" + ) + + INDEX=1 + for member_cluster in ${MEMBER_CLUSTERS}; do + secret_name="$(kubectl --context "${member_cluster}" get secret -n "${NAMESPACE}" | { grep "${service_account_name}" || test $? = 1; } | awk '{ print $1 }')" + if [[ "${secret_name}" == "" ]]; then + secret_name="${service_account_name}-token-secret" + create_service_account_token_secret "${member_cluster}" "${service_account_name}" "${secret_name}" + fi + + member_cluster_token="$(kubectl --context "${member_cluster}" get secret "${secret_name}" -o jsonpath='{ .data.token}' -n "${NAMESPACE}" | base64 -d)" + # for 2 cluster tests central cluster is the first member, so we cannot add this as it will result in duplicate key and error in create secret + if [[ "${member_cluster}" != "${CENTRAL_CLUSTER}" ]]; then + configuration_params+=( + "--from-literal=${member_cluster}=${member_cluster_token}" + ) + fi + configuration_params+=( + "--from-literal=member_cluster_${INDEX}=${member_cluster}" + ) + ((INDEX++)) + done + + if [[ "${CLUSTER_TYPE}" == "kind" ]]; then + if [[ -f "${kind_kubeconfig}" ]]; then + rm "${kind_kubeconfig}" + fi + fi + + kubectl --context "${test_pod_cluster}" delete secret test-pod-multi-cluster-config -n "${NAMESPACE}" --ignore-not-found + kubectl --context "${test_pod_cluster}" create secret generic test-pod-multi-cluster-config -n "${NAMESPACE}" "${configuration_params[@]}" +} + +prepare_multi_cluster_e2e_run() { + # shellcheck disable=SC2034 + operator_context="${CENTRAL_CLUSTER}" + configure_multi_cluster_environment + + if [[ "$(uname)" == "Darwin" ]]; then + goarch="amd64" + if [[ "$(uname -m)" == "arm64" ]]; then + goarch="arm64" + fi + + ( + cd public/tools/multicluster/ + GOOS=darwin GOARCH="${goarch}" go build -o "${MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH}" main.go + ) + PATH=${PATH}:docker/mongodb-enterprise-tests + else + ( + cd public/tools/multicluster/ + # shellcheck disable=SC2030 + export PATH=${GOROOT}:${PATH} + GOOS=linux GOARCH=amd64 go build -o "${MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH}" main.go + ) + # shellcheck disable=SC2031 + PATH=${PATH}:docker/mongodb-enterprise-tests + fi + + test_pod_secret_name="test-pod-multi-cluster-config" + echo "Creating local configuration for multi cluster test in ${MULTI_CLUSTER_CONFIG_DIR}" + mkdir -p "${MULTI_CLUSTER_CONFIG_DIR}" + + # escape "." sign from cluster names + # shellcheck disable=SC2001,SC2086 + central_cluster_escaped=$(echo ${CENTRAL_CLUSTER} | sed 's/\./\\./g') + # shellcheck disable=SC2206 + member_cluster_list=(${MEMBER_CLUSTERS}) + + kubectl --context "${test_pod_cluster}" get secret "${test_pod_secret_name}" -n "${NAMESPACE}" -o jsonpath="{ .data.central_cluster }" | base64 -d >"${MULTI_CLUSTER_CONFIG_DIR}/central_cluster" + kubectl --context "${test_pod_cluster}" get secret "${test_pod_secret_name}" -n "${NAMESPACE}" -o jsonpath="{ .data.${central_cluster_escaped} }" | base64 -d >"${MULTI_CLUSTER_CONFIG_DIR}/${CENTRAL_CLUSTER}" + + INDEX=1 + for member_cluster in "${member_cluster_list[@]}"; do + # shellcheck disable=SC2001,SC2086 + member_cluster_escaped=$(echo ${member_cluster} | sed 's/\./\\./g') + kubectl --context "${test_pod_cluster}" get secret "${test_pod_secret_name}" -n "${NAMESPACE}" -o jsonpath="{ .data.member_cluster_${INDEX} }" | base64 -d >"${MULTI_CLUSTER_CONFIG_DIR}/member_cluster_${INDEX}" & + kubectl --context "${test_pod_cluster}" get secret "${test_pod_secret_name}" -n "${NAMESPACE}" -o jsonpath="{ .data.${member_cluster_escaped} }" | base64 -d >"${MULTI_CLUSTER_CONFIG_DIR}/${member_cluster}" & + ((INDEX++)) + done + + wait + +} + +# TODO: unify with scripts/funcs/operator_deployment +run_multi_cluster_kube_config_creator() { + if [[ "${LOCAL_OPERATOR}" != "true" ]]; then + echo "Skipping configuring multi-cluster with kubectl mongodb cli tool due to LOCAL_OPERATOR=false" + return + fi + + # shellcheck disable=SC2153 + # convert space separated to comma separated + comma_separated_list="$(echo "${MEMBER_CLUSTERS}" | tr ' ' ',')" + + params=( + "--member-clusters" "${comma_separated_list}" + "--central-cluster" "${CENTRAL_CLUSTER}" + "--member-cluster-namespace" "${NAMESPACE}" + "--central-cluster-namespace" "${NAMESPACE}" + "--service-account" "mongodb-enterprise-operator-multi-cluster" + ) + + # This is used to skip the database roles when the context configures clusters so that the operator cluster is also used as one of the member clusters. + # In case of this overlap we cannot install those roles here as otherwise multi-cluster cli tool + if [[ ! "${MEMBER_CLUSTERS}" == *"${CLUSTER_NAME}"* ]]; then + >&2 echo "WARNING: When running e2e tests for multicluster locally, installing database roles by cli tool might cause failures when installing the operator in e2e test." + params+=("--install-database-roles") + fi + + if [[ "${OPERATOR_CLUSTER_SCOPED}" == "true" ]]; then + params+=("--cluster-scoped") + fi + if [[ "${MULTI_CLUSTER_CREATE_SERVICE_ACCOUNT_TOKEN_SECRETS}" == "true" ]]; then + params+=("--create-service-account-secrets") + fi + + echo "Executing multi cluster cli setup:" + echo "${MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH} multicluster setup ${params[*]}" + ${MULTI_CLUSTER_KUBE_CONFIG_CREATOR_PATH} multicluster setup "${params[@]}" + + kubectl get secret --context "${CENTRAL_CLUSTER}" -n "${NAMESPACE}" mongodb-enterprise-operator-multi-cluster-kubeconfig -o json | jq -rc '.data.kubeconfig' | base64 -d >"${KUBE_CONFIG_PATH}" +} diff --git a/scripts/funcs/operator_deployment b/scripts/funcs/operator_deployment new file mode 100644 index 000000000..6cf6b070a --- /dev/null +++ b/scripts/funcs/operator_deployment @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -Eeou pipefail + +get_operator_helm_values() { + # shellcheck disable=SC2153 + local database_registry=${DATABASE_REGISTRY} + local database_name=${DATABASE_NAME:=mongodb-enterprise-database-ubi} + + declare -a config=( + "managedSecurityContext=${MANAGED_SECURITY_CONTEXT:-false}" + "registry.operator=${OPERATOR_REGISTRY:-${REGISTRY}}" + "registry.imagePullSecrets=image-registries-secret" + "registry.initOpsManager=${INIT_OPS_MANAGER_REGISTRY}" + "registry.initAppDb=${INIT_APPDB_REGISTRY}" + "registry.initDatabase=${INIT_DATABASE_REGISTRY}" + "registry.agent=${AGENT_BASE_REGISTRY:-${REGISTRY}}" + "registry.opsManager=${OPS_MANAGER_REGISTRY}" + "registry.appDb=${APPDB_REGISTRY}" + "registry.database=${database_registry}" + "opsManager.name=${OPS_MANAGER_NAME:=mongodb-enterprise-ops-manager-ubi}" + "database.name=${database_name:=mongodb-enterprise-database-ubi}" + "operator.version=${OPERATOR_VERSION-${VERSION_ID}}" + "initOpsManager.version=${INIT_OPS_MANAGER_VERSION:-${VERSION_ID}}" + "initAppDb.version=${INIT_APPDB_VERSION:-${VERSION_ID}}" + "initDatabase.version=${INIT_DATABASE_VERSION:-${VERSION_ID}}" + "database.version=${DATABASE_VERSION:-${VERSION_ID}}" + "agent.version=${AGENT_VERSION}" + "mongodb.name=mongodb-enterprise-server" + "mongodb.imageType=${MDB_IMAGE_TYPE:-ubi8}" + "operator.mdbDefaultArchitecture=${MDB_DEFAULT_ARCHITECTURE:-non-static}" + "operator.enablePVCResize=${MDB_ENABLE_PVC_RESIZE:-true}" + # only send the telemetry to the backend on a specific variant, thus default to false + "operator.telemetry.send.enabled=${MDB_OPERATOR_TELEMETRY_SEND_ENABLED:-false}" + # lets collect and save in the configmap as frequently as we can + "operator.telemetry.collection.frequency=${MDB_OPERATOR_TELEMETRY_COLLECTION_FREQUENCY:-1m}" + ) + + if [[ "${MDB_OPERATOR_TELEMETRY_INSTALL_CLUSTER_ROLE_INSTALLATION:-}" != "" ]]; then + config+=("operator.telemetry.installClusterRole=${MDB_OPERATOR_TELEMETRY_INSTALL_CLUSTER_ROLE_INSTALLATION}") + fi + + if [[ "${MDB_OPERATOR_TELEMETRY_ENABLED:-true}" == "false" ]]; then + config+=("operator.telemetry.enabled=false") + config+=("operator.telemetry.collection.clusters.enabled=false") + config+=("operator.telemetry.collection.deployments.enabled=false") + config+=("operator.telemetry.collection.operators.enabled=false") + fi + + # shellcheck disable=SC2154 + if [[ "${KUBE_ENVIRONMENT_NAME-}" = "multi" ]]; then + comma_separated_list="$(echo "${MEMBER_CLUSTERS}" | tr ' ' ',')" + # shellcheck disable=SC2154 + config+=("multiCluster.clusters={${comma_separated_list}}") + fi + + if [[ "${KUBE_ENVIRONMENT_NAME:-}" == "multi" ]]; then + config+=("operator.createOperatorServiceAccount=false") + fi + + if [[ "${BUILD_WITH_RACE_DETECTION:-}" == "true" ]]; then + config+=("operator.build=-race") + fi + + if [[ "${MDB_MAX_CONCURRENT_RECONCILES:-}" != "" ]]; then + config+=("operator.maxConcurrentReconciles=${MDB_MAX_CONCURRENT_RECONCILES}") + fi + + # change this locally or as changed in variant e2e_operator_race_ubi_with_telemetry which also sends telemetry + if [[ "${MDB_OPERATOR_TELEMETRY_SEND_BASEURL:-}" != "" ]]; then + config+=("operator.telemetry.send.baseUrl=${MDB_OPERATOR_TELEMETRY_SEND_BASEURL}") + fi + + if [[ "${MDB_HELM_OPERATOR_WEBHOOK_INSTALL_CLUSTER_ROLE:-}" != "" ]]; then + config+=("operator.webhook.installClusterRole=${MDB_HELM_OPERATOR_WEBHOOK_INSTALL_CLUSTER_ROLE}") + fi + + echo "${config[@]}" +} + +prepare_operator_config_map() { + local context=${1} + kubectl --context "${context}" delete configmap operator-installation-config --ignore-not-found + title "Preparing the ConfigMap with Operator installation configuration" + + read -ra helm_values < <(get_operator_helm_values) + declare -a config_map_values=() + for param in "${helm_values[@]}"; do + config_map_values+=("--from-literal" "${param}") + done + # shellcheck disable=SC2086,SC2048 + kubectl --context "${context}" create configmap operator-installation-config -n "${NAMESPACE}" ${config_map_values[*]} || true +} diff --git a/scripts/funcs/printing b/scripts/funcs/printing new file mode 100644 index 000000000..84fc88b97 --- /dev/null +++ b/scripts/funcs/printing @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +title() { + echo "=> $1" +} + +header() { + echo + echo "--------------------------------------------------" + echo "$1" + echo "--------------------------------------------------" +} + +# Function to prepend every line with a prefix passed as an argument. Useful when spawning +# multiple jobs in the background to identify to which job the logs belong. +# Pipe output to it, e.g. | prepend "job prefix: " +prepend() { + prefix=$1 + awk -v prefix="${prefix}" '{printf "%s: %s\n", prefix, $0}' +} + +export RED='\033[0;31m' +export NO_COLOR='\033[0m' diff --git a/scripts/preflight_images.py b/scripts/preflight_images.py new file mode 100755 index 000000000..b5492596f --- /dev/null +++ b/scripts/preflight_images.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +import argparse +import concurrent +import json +import logging +import os +import platform +import random +import re +import subprocess +import sys +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor +from typing import Dict, Tuple + +import requests +from evergreen.release.agent_matrix import ( + get_supported_version_for_image_matrix_handling, +) + +LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper() +logging.basicConfig(level=LOGLEVEL) + + +def image_config( + image: str, + rh_cert_project_id: str, + name_prefix: str = "mongodb-enterprise-", + name_suffix: str = "-ubi", +) -> Tuple[str, Dict[str, str]]: + args = { + "registry": f"quay.io/mongodb/{name_prefix}{image}{name_suffix}", + "image": f"mongodb/{name_prefix}{image}{name_suffix}", + "rh_cert_project_id": rh_cert_project_id, + } + return image, args + + +def official_server_image( + image: str, + rh_cert_project_id: str, +) -> Tuple[str, Dict[str, str]]: + args = { + "registry": f"quay.io/mongodb/mongodb-enterprise-server", + "image": f"mongodb/mongodb-enterprise-server", + "rh_cert_project_id": rh_cert_project_id, + } + return image, args + + +def args_for_image(image: str) -> Dict[str, str]: + image_configs = [ + image_config( + "database", + "633fc9e582f7934b1ad3be45", + ), + official_server_image( + "mongodb-enterprise-server", # official server images + "643daaa56da4ecc48795693a", + ), + image_config( + "init-appdb", + "633fcb576f43719c9df9349f", + ), + image_config( + "init-database", + "633fcc2982f7934b1ad3be46", + ), + image_config( + "init-ops-manager", + "633fccb16f43719c9df934a0", + ), + image_config( + "operator", + "633fcdfaade0e891294196ac", + ), + image_config( + "ops-manager", + "633fcd36c4ee7ff29edff589", + ), + image_config( + "mongodb-agent", + "633fcfd482f7934b1ad3be47", + name_prefix="", + ), + ] + images = {k: v for k, v in image_configs} + return images[image] + + +def get_api_token(): + token = os.environ.get("rh_pyxis", "") + return token + + +def get_release() -> Dict[str, str]: + return json.load(open("release.json")) + + +def create_auth_file(): + # In theory, we could remove this as our container images reside in public repo + # However, due to https://github.com/redhat-openshift-ecosystem/openshift-preflight/issues/685 + # we need to supply a non-empty --docker-config + public_auth = """ + { + "auths": { + "quay.io": { + "auth": "" + } + } + } + """ + with open("./temp-authfile.json", "w") as file: + file.write(public_auth) + + +def run_preflight_check(image: str, version: str, submit: bool = False) -> int: + arch = "amd64" if platform.machine() == "x86_64" else "arm64" + + with tempfile.TemporaryDirectory() as tmpdir: + preflight_command = [ + "preflight", + "check", + "container", + f"{args_for_image(image)['registry']}:{version}", + "--artifacts", + f"{tmpdir}", + ] + + if submit: + preflight_command.extend( + [ + "--submit", + f"--pyxis-api-token={get_api_token()}", + f"--certification-project-id={args_for_image(image)['rh_cert_project_id']}", + ] + ) + preflight_command.append("--docker-config=./temp-authfile.json") + logging.info(f'Running command: {" ".join(preflight_command)}') + + run_preflight_with_retries(preflight_command, version) + + result_file = os.path.join(f"{tmpdir}", arch, "results.json") + + if os.path.exists(result_file): + with open(result_file, "r") as f: + result_data = json.load(f).get("results", "") + failed = result_data.get("failed") + errors = result_data.get("errors") + if failed or errors: + logging.error( + f"Following errors or failures found for image: {args_for_image(image)['registry']}:{version}, failures: {failed}, {errors}" + ) + return 1 + else: + logging.info("Preflight check passed") + return 0 + else: + logging.info( + f"Result file not found, counting as failed for image: {args_for_image(image)['registry']}:{version}" + ) + return 1 + + +def run_preflight_with_retries(preflight_command, version, max_retries=5): + for attempt in range(max_retries): + try: + subprocess.run(preflight_command, capture_output=True, check=True) + return + except subprocess.CalledProcessError as e: + if attempt + 1 < max_retries: + delay = (2**attempt) + random.uniform(0, 1) + logging.error(f"Attempt {attempt + 1} failed for version {version}: {e.stderr}") + logging.info(f"Retrying in {delay:.2f} seconds...") + time.sleep(delay) + else: + logging.error(f"All {max_retries} attempts failed for version: {version}") + raise + + +def fetch_tags(page, image, regex_filter): + """Fetch a single page of tags from Quay API.""" + url = f"https://quay.io/api/v1/repository/{image}/tag/?page={page}&limit=100" + response = requests.get(url) + + if response.status_code != 200: + return [] + + tags = response.json().get("tags", []) + + filtered_tags = [tag["name"] for tag in tags if re.match(regex_filter, tag["name"])] + + return filtered_tags + + +def get_filtered_tags_parallel(image, max_pages=5, regex_filter=""): + """retrieves all tags in parallel from the quay endpoint. If not done in parallel it takes around 5 minutes.""" + all_tags = set() + futures = [] + with ThreadPoolExecutor() as executor: + for page in range(1, max_pages + 1): + futures.append(executor.submit(fetch_tags, page, image, regex_filter)) + + for future in concurrent.futures.as_completed(futures): + all_tags.update(future.result()) + + return list(all_tags) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--image", help="image to run preflight checks on", type=str, required=True) + parser.add_argument( + "--submit", + help="submit image for certification (true|false)", + type=str, + required=True, + ) + parser.add_argument("--version", help="specific version to check", type=str, default=None) + args = parser.parse_args() + submit = args.submit.lower() == "true" + image_version = os.environ.get("image_version", args.version) + image_args = args_for_image(args.image) + + # mongodb-enterprise-server are externally provided. We preflight for all of them. + if args.image == "mongodb-enterprise-server": + versions = get_filtered_tags_parallel( + image=image_args["image"], max_pages=10, regex_filter=r"^[0-9]+\.[0-9]+\.[0-9]+-ubi[89]$" + ) + else: + # these are the images we own, we preflight all of them as long as we officially support them in release.json + versions = get_supported_version_for_image_matrix_handling(args.image) + + # only preflight the current agent version and the subset of agent images suffixed with the current operator version + if args.image == "mongodb-agent": + release = get_release() + operator_version = release["mongodbOperator"] + versions = list(filter(lambda version: version.endswith(f"_{operator_version}"), versions)) + versions.append(release["agentVersion"]) + + # Attempt to run a pre-flight check on a single version of the image + if image_version is not None: + return preflight_single_image(args, image_version, submit, versions) + + # Attempt to run pre-flight checks on all the supported and unpublished versions of the image + logging.info(f"preflight for image: {image_args['image']}") + logging.info(f"preflight for versions: {versions}") + + create_auth_file() + + # Note: if running preflight on image tag (not daily tag) we in turn preflight the corresponding sha it is pointing to. + return_codes_version = preflight_parallel(args, versions, submit) + logging.info("preflight complete, printing summary") + found_error = False + for return_code, version in return_codes_version: + if return_code != 0: + found_error = True + logging.error(f"failed image: {args.image}:{version} with exit code: {return_code}") + else: + logging.info(f"succeeded image: {args.image}:{version}") + + if found_error: + return 1 + return 0 + + +def preflight_parallel(args, versions, submit): + with ThreadPoolExecutor() as executor: + futures = [] + return_codes = [] + + for version in versions: + logging.info(f"Running preflight check for image: {args.image}:{version}") + future = executor.submit(run_preflight_check, args.image, version, submit) + futures.append(future) + + # Collect results as they complete + for future in concurrent.futures.as_completed(futures): + index = futures.index(future) + version = versions[index] # Get the version from the original list + try: + result = future.result() + return_codes.append((result, version)) + except Exception as e: + return_codes.append((1, version)) + logging.error(f"Preflight check failed with exception: {e}") + + return return_codes + + +def preflight_single_image(args, image_version, submit, versions): + logging.info("Submitting preflight check for a single image version") + if image_version not in versions: + logging.error( + f"Version {image_version} for image {args.image} is not supported. Supported versions: {versions}" + ) + return 1 + else: + create_auth_file() + return_code = run_preflight_check(args.image, image_version, submit=submit) + if return_code != 0: + logging.error( + f"Running preflight check for image: {args.image}:{image_version} failed with exit code: {return_code}" + ) + return return_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ssdlc/templates/SSDLC Containerized MongoDB Agent ${VERSION}.md b/scripts/ssdlc/templates/SSDLC Containerized MongoDB Agent ${VERSION}.md new file mode 100644 index 000000000..d5b29e8ad --- /dev/null +++ b/scripts/ssdlc/templates/SSDLC Containerized MongoDB Agent ${VERSION}.md @@ -0,0 +1,45 @@ +SSDLC Compliance Report: MongoDB Agent (shipped with MongoDB Kubernetes Enterprise Operator ${VERSION}) +======================================================================================================= + +- Release Creators: ${AUTHOR} +- Created On: ${DATE} + +Overview: + +- **Product and Release Name** + + - MongoDB Enterprise Operator ${VERSION}, ${DATE}. + - Release Type: ${RELEASE_TYPE} + +- **Process Document** + - http://go/how-we-develop-software-doc + +- **Tool used to track third party vulnerabilities** + - Snyk + +- **Dependency Information** + - See SBOMS Lite manifests (CycloneDX in JSON format for the SBOM and JSON for the supplementary report on CVEs): + ${SBOMS} + +- **Static Analysis Report** + - We use GoSec for static analysis scanning on our CI tests. There are no findings (neither critical nor high) unresolved. + +- **Release Signature Report** + - Image signatures enforced by CI pipeline. + - Signatures verification: documentation in-progress: https://jira.mongodb.org/browse/DOCSP-39646 + +- **Security Testing Report** + - Sast: https://jira.mongodb.org/browse/CLOUDP-251553 + - Pentest: (Same as the others) https://jira.mongodb.org/browse/CLOUDP-251555 + - Dast: We decided not to do per https://jira.mongodb.org/browse/CLOUDP-251554 and the linked scope + +- **Security Assessment Report** + - https://jira.mongodb.org/browse/CLOUDP-251555 + +Assumptions and attestations: + +1. Internal processes are used to ensure CVEs are identified and mitigated within SLAs. + +2. The Dependency document does not specify third party OSS CVEs fixed by the release and the date we discovered them. + +3. There is no CycloneDX field for original/modified CVSS score or discovery date. The `x-` prefix indicates this. diff --git a/scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise Kubernetes Operator ${VERSION}.md b/scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise Kubernetes Operator ${VERSION}.md new file mode 100644 index 000000000..17b8242e5 --- /dev/null +++ b/scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise Kubernetes Operator ${VERSION}.md @@ -0,0 +1,45 @@ +SSDLC Compliance Report: MongoDB Enterprise Operator ${VERSION} +================================================================= + +- Release Creators: ${AUTHOR} +- Created On: ${DATE} + +Overview: + +- **Product and Release Name** + + - MongoDB Enterprise Operator ${VERSION}, ${DATE}. + - Release Type: ${RELEASE_TYPE} + +- **Process Document** + - http://go/how-we-develop-software-doc + +- **Tool used to track third party vulnerabilities** + - Snyk + +- **Dependency Information** + - See SBOMS Lite manifests (CycloneDX in JSON format for the SBOM and JSON for the supplementary report on CVEs): + ${SBOMS} + +- **Static Analysis Report** + - We use GoSec for static analysis scanning on our CI tests. There are no findings (neither critical nor high) unresolved. + +- **Release Signature Report** + - Image signatures enforced by CI pipeline. + - Signatures verification: documentation in-progress: https://jira.mongodb.org/browse/DOCSP-39646 + +- **Security Testing Report** + - Sast: https://jira.mongodb.org/browse/CLOUDP-251553 + - Pentest: (Same as the others) https://jira.mongodb.org/browse/CLOUDP-251555 + - Dast: We decided not to do per https://jira.mongodb.org/browse/CLOUDP-251554 and the linked scope + +- **Security Assessment Report** + - https://jira.mongodb.org/browse/CLOUDP-251555 + +Assumptions and attestations: + +1. Internal processes are used to ensure CVEs are identified and mitigated within SLAs. + +2. The Dependency document does not specify third party OSS CVEs fixed by the release and the date we discovered them. + +3. There is no CycloneDX field for original/modified CVSS score or discovery date. The `x-` prefix indicates this. diff --git a/scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise OpsManager ${VERSION}.md b/scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise OpsManager ${VERSION}.md new file mode 100644 index 000000000..63614e021 --- /dev/null +++ b/scripts/ssdlc/templates/SSDLC Containerized MongoDB Enterprise OpsManager ${VERSION}.md @@ -0,0 +1,45 @@ +SSDLC Compliance Report: MongoDB Ops Manager (shipped with MongoDB Kubernetes Enterprise Operator ${VERSION}) +============================================================================================================= + +- Release Creators: ${AUTHOR} +- Created On: ${DATE} + +Overview: + +- **Product and Release Name** + + - MongoDB Enterprise Operator ${VERSION}, ${DATE}. + - Release Type: ${RELEASE_TYPE} + +- **Process Document** + - http://go/how-we-develop-software-doc + +- **Tool used to track third party vulnerabilities** + - Snyk + +- **Dependency Information** + - See SBOMS Lite manifests (CycloneDX in JSON format for the SBOM and JSON for the supplementary report on CVEs): + ${SBOMS} + +- **Static Analysis Report** + - We use GoSec for static analysis scanning on our CI tests. There are no findings (neither critical nor high) unresolved. + +- **Release Signature Report** + - Image signatures enforced by CI pipeline. + - Signatures verification: documentation in-progress: https://jira.mongodb.org/browse/DOCSP-39646 + +- **Security Testing Report** + - Sast: https://jira.mongodb.org/browse/CLOUDP-251553 + - Pentest: (Same as the others) https://jira.mongodb.org/browse/CLOUDP-251555 + - Dast: We decided not to do per https://jira.mongodb.org/browse/CLOUDP-251554 and the linked scope + +- **Security Assessment Report** + - https://jira.mongodb.org/browse/CLOUDP-251555 + +Assumptions and attestations: + +1. Internal processes are used to ensure CVEs are identified and mitigated within SLAs. + +2. The Dependency document does not specify third party OSS CVEs fixed by the release and the date we discovered them. + +3. There is no CycloneDX field for original/modified CVSS score or discovery date. The `x-` prefix indicates this. diff --git a/scripts/ssdlc/templates/SSDLC MongoDB Enterprise Operator Testing Report ${VERSION}.md b/scripts/ssdlc/templates/SSDLC MongoDB Enterprise Operator Testing Report ${VERSION}.md new file mode 100644 index 000000000..d23e43fa8 --- /dev/null +++ b/scripts/ssdlc/templates/SSDLC MongoDB Enterprise Operator Testing Report ${VERSION}.md @@ -0,0 +1,29 @@ +MongoDB Enterprise Kubernetes Operator Security Testing Summary +== + +This document lists specific instances of security-relevant testing that is being performed for the MongoDB Enterprise Kubernetes Operator. All parts of the MongoDB Enterprise Kubernetes source code are subject to unit and end-to-end testing on every change made to the project, including the specific instances listed below. Additionally, smoke tests (end-to-end) are performed every time we release a new tagged version of Docker images used by the operator. + +Authentication End-to-End Tests +=== + +Our authentication tests verify that multiple authentication mechanisms are supported and configurable for the MongoDB instances deployed by the operator and that changes in the MongoDB custom resources result in correct authentication changes in the underlying automation config. Our tests cover SCRAM, LDAP and X509 authentication. This extends internal cluster authentication. For X509 we also cover certificate rotation. + +Vault integration End-to-End Tests +=== + +Our operator relies on the availability of possibly sensitive data on the customer premise (TLS certificates, database admin passwords, etc.) and the operator provides an integration that allows those secrets to be sourced from Vault. Our end-to-end tests cover this integration to ensure that a customer could run the operator while keeping control of the sensitive data in an external system. + +TLS End-to-End Tests +=== + +Our TLS tests verify that core security properties of TLS connections are applied appropriately for the MongoDB resources. The tests also verify that certificate rotation and database upgrades when TLS is enabled do not introduce downtime in live workloads. These tests cover TLS connections to the mongod servers, OpsManager and the underlying Application Database instances. This means ensuring secure connections within the Kubernetes cluster and secure external connectivity to all the resources deployed in the cluster. The tests also verify that certificate rotation and upgrades when TLS is enabled do not introduce downtime in live workloads. + +Static Container Architecture End-to-End Tests +=== + +All the end-to-end tests that we run for the MongoDB functionality using dynamic containers that pull mongod and agent binaries from OpsManager/CloudManager are replicated in a separate variant for the static architecture to ensure the same functionality is available with this secure model that assures customers that the Docker images they run on-prem do not dynamically change on runtime. + +Data encryption End-to-End Tests +=== + +The enterprise operator supports configuring Automatic Queryable Encryption with KMIP. Our end-to-end tests cover the configuration of MongoDB resources to connect to a KMIP server and encrypt data and related backups. diff --git a/scripts/update_dockerfiles_in_s3.py b/scripts/update_dockerfiles_in_s3.py new file mode 100644 index 000000000..1e764ff4b --- /dev/null +++ b/scripts/update_dockerfiles_in_s3.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# + +""" +This script is used to push the dockerfiles from this repo (found in public/dockerfiles folder) to S3 bucket. +The main purpose of this sync is to keep the dockerfiles in S3 up to date with the latest security fixes for our periodic re-builds. + +required environment vars (can also be added to /.operator-dev/om): + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + +run the script: + PYTHONPATH=":/docker/mongodb-enterprise-tests" python ./scripts/update_dockerfiles_in_s3.py +""" + +import os +import subprocess + +from kubetester.awss3client import AwsS3Client + + +def get_repo_root(): + output = subprocess.check_output("git rev-parse --show-toplevel".split()) + + return output.decode("utf-8").strip() + + +AWS_REGION = "eu-west-1" +S3_BUCKET = "enterprise-operator-dockerfiles" +S3_FOLDER = "dockerfiles" +S3_PUBLIC_READ = True + +LOCAL_DOCKERFILE_LOCATION = "public/dockerfiles" + +DOCKERFILE_NAME = "Dockerfile" + +public_dir = os.path.join(get_repo_root(), LOCAL_DOCKERFILE_LOCATION) +client = AwsS3Client(AWS_REGION) + +for root, _, files in os.walk(public_dir): + for file_name in filter(lambda f: f == DOCKERFILE_NAME, files): + file_path = os.path.join(root, file_name) + object_name = file_path.replace(f"{public_dir}", S3_FOLDER, 1) + client.upload_file(os.path.join(root, file_name), S3_BUCKET, object_name, S3_PUBLIC_READ) + print(f" > {object_name}") + +print("Done!") diff --git a/scripts/update_supported_dockerfiles.py b/scripts/update_supported_dockerfiles.py new file mode 100755 index 000000000..df38ab359 --- /dev/null +++ b/scripts/update_supported_dockerfiles.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# + +import json +import os +import subprocess +import sys +from typing import Dict, List + +import requests +from evergreen.release.agent_matrix import ( + get_supported_version_for_image_matrix_handling, +) +from git import Repo +from requests import Response + + +def get_repo_root(): + output = subprocess.check_output("git rev-parse --show-toplevel".split()) + + return output.decode("utf-8").strip() + + +SUPPORTED_IMAGES = ( + "mongodb-agent", + "mongodb-enterprise-database", + "mongodb-enterprise-init-database", + "mongodb-enterprise-init-appdb", + "mongodb-enterprise-ops-manager", + "mongodb-enterprise-init-ops-manager", + "mongodb-enterprise-operator", +) + +URL_LOCATION_BASE = "https://enterprise-operator-dockerfiles.s3.amazonaws.com/dockerfiles" + +LOCAL_DOCKERFILE_LOCATION = "public/dockerfiles" +DOCKERFILE_NAME = "Dockerfile" + + +def get_release() -> Dict[str, str]: + return json.load(open("release.json")) + + +def get_supported_variants_for_image(image: str) -> List[str]: + splitted_image_name = image.split("mongodb-enterprise-", 1) + if len(splitted_image_name) == 2: + image = splitted_image_name[1] + + return get_release()["supportedImages"][image]["variants"] + + +def get_supported_version_for_image(image: str) -> List[str]: + splitted_image_name = image.split("mongodb-enterprise-", 1) + if len(splitted_image_name) == 2: + image = splitted_image_name[1] + + return get_supported_version_for_image_matrix_handling(image) + + +def download_dockerfile_from_s3(image: str, version: str, distro: str) -> Response: + url = f"{URL_LOCATION_BASE}/{image}/{version}/{distro}/Dockerfile" + return requests.get(url) + + +def git_add_dockerfiles(base_directory: str): + """Looks for all of the `Dockerfile`s in the public/dockerfiles + directory and stages them in git.""" + repo = Repo() + public_dir = os.path.join(get_repo_root(), LOCAL_DOCKERFILE_LOCATION) + + for root, _, files in os.walk(public_dir): + for fname in files: + if fname != DOCKERFILE_NAME: + continue + + repo.index.add(os.path.join(root, fname)) + + +def save_supported_dockerfiles(): + """ + Finds every supported release in the release.json and downloads the corresponding + Dockerfile. + """ + for image in SUPPORTED_IMAGES: + print("Image:", image) + versions = get_supported_version_for_image(image) + for version in versions: + for variant in get_supported_variants_for_image(image): + response = download_dockerfile_from_s3(image, version, variant) + if response.ok: + dockerfile = response.text + docker_dir = f"{LOCAL_DOCKERFILE_LOCATION}/{image}/{version}/{variant}" + os.makedirs(docker_dir, exist_ok=True) + docker_path = os.path.join(docker_dir, DOCKERFILE_NAME) + with open(docker_path, "w") as fd: + fd.write(dockerfile) + print("* {} - {}: {}".format(version, variant, docker_path)) + else: + print("* {} - {}: does not exist".format(version, variant)) + + +def main() -> int: + save_supported_dockerfiles() + git_add_dockerfiles(LOCAL_DOCKERFILE_LOCATION) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools.go b/tools.go index 2403594c4..7e8331071 100644 --- a/tools.go +++ b/tools.go @@ -1,6 +1,14 @@ //go:build tools // +build tools -// Place any runtime dependencies as imports in this file. -// Go modules will be forced to download and install them. package tools + +// forcing these packages to be imported in vendor by `go mod vendor` +import ( + // test code for unit tests + _ "k8s.io/client-go/discovery/fake" + _ "k8s.io/client-go/testing" + + // code-generator that does not support go modules yet + _ "k8s.io/code-generator" +)