CLOUDP-278184 - om_ops_manager_backup_sharded_cluster test multi-cluster adoption (#3913)

# Summary

CLOUDP-278184 - om_ops_manager_backup_sharded_cluster test multi-cluster
adoption

## Documentation changes

* [ ] Add an entry to [release notes](.../RELEASE_NOTES.md).
* [ ] When needed, make sure you create a new [DOCSP
ticket](https://jira.mongodb.org/projects/DOCSP) that documents your
change.

## Changes to CRDs

* [ ] Add `slaskawi`(Sebastian) and `@giohan` (George) as reviewers.
* [ ] Make sure any changes are reflected on `/public/samples`
directory.

Author: MaciejKaras

docker/mongodb-enterprise-tests/tests/opsmanager/om_ops_manager_backup_sharded_cluster.py

@@ -14,12 +14,16 @@
 from kubetester.kubetester import run_periodically
 from kubetester.mongodb import MongoDB, Phase
 from kubetester.mongodb_user import MongoDBUser
+from kubetester.mongotester import MongoTester
 from kubetester.opsmanager import MongoDBOpsManager
 from pytest import fixture, mark
 from tests.conftest import is_multi_cluster
-from tests.opsmanager.backup_snapshot_schedule_tests import BackupSnapshotScheduleTests
 from tests.opsmanager.om_ops_manager_backup import create_aws_secret, create_s3_bucket
 from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment
+from tests.shardedcluster.conftest import (
+    enable_multi_cluster_deployment as enable_multi_cluster_deployment_mdb,
+)
+from tests.shardedcluster.conftest import get_mongos_service_names
 
 HEAD_PATH = "/head/"
 S3_SECRET_NAME = "my-s3-secret"
@@ -65,8 +69,7 @@ def oplog_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoD
     # mongoURI not being updated unless pod is killed. This is documented in CLOUDP-60443, once resolved this skip & comment can be deleted
     resource["spec"]["security"] = {"authentication": {"enabled": True, "modes": ["SCRAM"]}}
 
-    resource.update()
-    return resource
+    return resource.update()
 
 
 @fixture(scope="module")
@@ -78,8 +81,7 @@ def s3_replica_set(ops_manager, namespace, custom_mdb_version: str) -> MongoDB:
     ).configure(ops_manager, "s3metadata")
     resource.set_version(custom_mdb_version)
 
-    resource.update()
-    return resource
+    return resource.update()
 
 
 @fixture(scope="module")
@@ -90,8 +92,8 @@ def blockstore_replica_set(ops_manager, namespace, custom_mdb_version: str) -> M
         name=BLOCKSTORE_RS_NAME,
     ).configure(ops_manager, "blockstore")
     resource.set_version(custom_mdb_version)
-    resource.update()
-    return resource
+
+    return resource.update()
 
 
 @fixture(scope="module")
@@ -259,11 +261,23 @@ def mdb_latest(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_versi
             namespace=namespace,
             name="mdb-four-two",
         ).configure(ops_manager, "firstProject")
-        resource.set_version(ensure_ent_version(custom_mdb_version))
+
+        if try_load(resource):
+            return resource
+
+        if is_multi_cluster():
+            enable_multi_cluster_deployment_mdb(
+                resource,
+                shard_members_array=[1, 1, 1],
+                mongos_members_array=[1, None, None],
+                configsrv_members_array=[None, 1, None],
+            )
+
         resource.configure_backup(mode="disabled")
+        resource.set_version(ensure_ent_version(custom_mdb_version))
+        resource.set_architecture_annotation()
 
-        try_load(resource)
-        return resource
+        return resource.update()
 
     @fixture(scope="class")
     def mdb_prev(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_prev_version: str):
@@ -272,23 +286,37 @@ def mdb_prev(self, ops_manager: MongoDBOpsManager, namespace, custom_mdb_prev_ve
             namespace=namespace,
             name="mdb-four-zero",
         ).configure(ops_manager, "secondProject")
-        resource.set_version(ensure_ent_version(custom_mdb_prev_version))
+
+        if try_load(resource):
+            return resource
+
+        if is_multi_cluster():
+            enable_multi_cluster_deployment_mdb(
+                resource,
+                shard_members_array=[1, 1, 1],
+                mongos_members_array=[1, None, None],
+                configsrv_members_array=[None, 1, None],
+            )
+
         resource.configure_backup(mode="disabled")
+        resource.set_version(ensure_ent_version(custom_mdb_prev_version))
+        resource.set_architecture_annotation()
 
-        try_load(resource)
-        return resource
+        return resource.update()
 
-    def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB):
-        mdb_latest.update()
-        mdb_prev.update()
-        mdb_latest.assert_reaches_phase(Phase.Running, timeout=1000)
-        mdb_prev.assert_reaches_phase(Phase.Running, timeout=1000)
+    @fixture(scope="class")
+    def mdb_latest_tester(self, mdb_latest: MongoDB) -> MongoTester:
+        service_names = get_mongos_service_names(mdb_latest)
 
-    def test_mdbs_enable_backup(self, mdb_latest: MongoDB, mdb_prev: MongoDB):
-        mdb_latest.load()
+        return mdb_latest.tester(service_names=service_names)
 
+    def test_mdbs_created(self, mdb_latest: MongoDB, mdb_prev: MongoDB):
+        mdb_latest.assert_reaches_phase(Phase.Running, timeout=1200)
+        mdb_prev.assert_reaches_phase(Phase.Running, timeout=1200)
+
+    def test_mdbs_enable_backup(self, mdb_latest: MongoDB, mdb_prev: MongoDB, mdb_latest_tester: MongoTester):
         def until_shards_are_here():
-            shards = mdb_latest.tester().client.admin.command("listShards")
+            shards = mdb_latest_tester.client.admin.command("listShards")
             if len(shards["shards"]) == 2:
                 return True
             else:
@@ -299,26 +327,27 @@ def until_shards_are_here():
         # we need to sleep here to give OM some time to recognize the shards.
         # otherwise, if you start a backup during a topology change will lead the backup to be aborted.
         time.sleep(30)
+        mdb_latest.load()
         mdb_latest.configure_backup(mode="enabled")
         mdb_latest.update()
 
         mdb_prev.load()
         mdb_prev.configure_backup(mode="enabled")
         mdb_prev.update()
 
-        mdb_prev.assert_reaches_phase(Phase.Running, timeout=600)
-        mdb_latest.assert_reaches_phase(Phase.Running, timeout=600)
+        mdb_prev.assert_reaches_phase(Phase.Running, timeout=1200)
+        mdb_latest.assert_reaches_phase(Phase.Running, timeout=1200)
 
     def test_mdbs_backuped(self, ops_manager: MongoDBOpsManager):
         om_tester_first = ops_manager.get_om_tester(project_name="firstProject")
         om_tester_second = ops_manager.get_om_tester(project_name="secondProject")
 
         # wait until a first snapshot is ready for both
         om_tester_first.wait_until_backup_snapshots_are_ready(
-            expected_count=1, expected_config_count=4, is_sharded_cluster=True
+            expected_count=1, expected_config_count=4, is_sharded_cluster=True, timeout=1600
         )
         om_tester_second.wait_until_backup_snapshots_are_ready(
-            expected_count=1, expected_config_count=4, is_sharded_cluster=True
+            expected_count=1, expected_config_count=4, is_sharded_cluster=True, timeout=1600
         )
 
     def test_can_transition_from_started_to_stopped(self, mdb_latest: MongoDB, mdb_prev: MongoDB):
@@ -327,22 +356,22 @@ def test_can_transition_from_started_to_stopped(self, mdb_latest: MongoDB, mdb_p
         mdb_prev.assert_backup_reaches_status("STARTED", timeout=100)
         mdb_prev.configure_backup(mode="disabled")
         mdb_prev.update()
-        mdb_prev.assert_backup_reaches_status("STOPPED", timeout=600)
+        mdb_prev.assert_backup_reaches_status("STOPPED", timeout=1600)
 
     def test_can_transition_from_started_to_terminated_0(self, mdb_latest: MongoDB, mdb_prev: MongoDB):
         # a direct transition from enabled to terminated is not possible
         # the operator should handle the transition from STARTED -> STOPPED -> TERMINATING
         mdb_latest.assert_backup_reaches_status("STARTED", timeout=100)
         mdb_latest.configure_backup(mode="terminated")
         mdb_latest.update()
-        mdb_latest.assert_backup_reaches_status("TERMINATING", timeout=600)
+        mdb_latest.assert_backup_reaches_status("TERMINATING", timeout=1600)
 
     def test_backup_terminated_for_deleted_resource(self, ops_manager: MongoDBOpsManager, mdb_prev: MongoDB):
         # re-enable backup
         mdb_prev.configure_backup(mode="enabled")
         mdb_prev["spec"]["backup"]["autoTerminateOnDeletion"] = True
         mdb_prev.update()
-        mdb_prev.assert_backup_reaches_status("STARTED", timeout=600)
+        mdb_prev.assert_backup_reaches_status("STARTED", timeout=1600)
         mdb_prev.delete()
 
         def resource_is_deleted() -> bool:

docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py

@@ -68,11 +68,12 @@ def get_member_cluster_clients_using_cluster_mapping(resource_name: str, namespa
     return get_member_cluster_clients(cluster_mapping)
 
 
-def get_mongos_service_names(sc) -> [str]:
+def get_mongos_service_names(resource: MongoDB):
     service_names = []
-    for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace):
-        for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)):
-            service_names.append(sc.mongos_service_name(member_idx, cluster_member_client.cluster_index))
+    for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(resource.name, resource.namespace):
+        for member_idx in range(resource.mongos_members_in_cluster(cluster_member_client.cluster_name)):
+            service_name = resource.mongos_service_name(member_idx, cluster_member_client.cluster_index)
+            service_names.append(service_name)
 
     return service_names
 

docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster.py

@@ -12,6 +12,7 @@
 from tests.shardedcluster.conftest import (
     enable_multi_cluster_deployment,
     get_member_cluster_clients_using_cluster_mapping,
+    get_mongos_service_names,
 )
 
 SCALED_SHARD_COUNT = 2
@@ -81,21 +82,17 @@ def test_mongod_sharded_cluster_service(self, sc: MongoDB):
     # When testing locally make sure you have kubefwd forwarding all cluster hostnames
     # kubefwd does not contain fix for multiple cluster, use https://github.com/lsierant/kubefwd fork instead
     def test_shards_were_configured_and_accessible(self, sc: MongoDB):
-        for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace):
-            for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)):
-                service_name = sc.mongos_service_name(member_idx, cluster_member_client.cluster_index)
-                tester = sc.tester(service_names=[service_name])
-                tester.assert_connectivity()
+        for service_name in get_mongos_service_names(sc):
+            tester = sc.tester(service_names=[service_name])
+            tester.assert_connectivity()
 
     @skip_if_local()  # Local machine DNS don't contain K8s CoreDNS SRV records which are required
     @skip_if_multi_cluster()  # srv option does not work for multi-cluster tests as each cluster DNS contains entries
     # related only to that cluster. Additionally, we don't pass srv option when building multi-cluster conn string
     def test_shards_were_configured_with_srv_and_accessible(self, sc: MongoDB):
-        for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace):
-            for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)):
-                service_name = sc.mongos_service_name(member_idx, cluster_member_client.cluster_index)
-                tester = sc.tester(service_names=[service_name], srv=True)
-                tester.assert_connectivity()
+        for service_name in get_mongos_service_names(sc):
+            tester = sc.tester(service_names=[service_name], srv=True)
+            tester.assert_connectivity()
 
     def test_monitoring_versions(self, sc: MongoDB):
         """Verifies that monitoring agent is configured for each process in the deployment"""

docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_pv.py

@@ -11,6 +11,7 @@
 from tests.shardedcluster.conftest import (
     enable_multi_cluster_deployment,
     get_member_cluster_clients_using_cluster_mapping,
+    get_mongos_service_names,
 )
 
 
@@ -119,11 +120,9 @@ def test_pvc_are_bound(self, sc: MongoDB):
                 self.check_pvc_labels(pvc)
 
     def test_mongos_are_reachable(self, sc: MongoDB):
-        for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace):
-            for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)):
-                service_name = sc.mongos_service_name(member_idx, cluster_member_client.cluster_index)
-                tester = sc.tester(service_names=[service_name])
-                tester.assert_connectivity()
+        for service_name in get_mongos_service_names(sc):
+            tester = sc.tester(service_names=[service_name])
+            tester.assert_connectivity()
 
 
 @mark.e2e_sharded_cluster_pv

docker/mongodb-enterprise-tests/tests/shardedcluster/sharded_cluster_upgrade_downgrade.py

@@ -10,6 +10,7 @@
 from tests.shardedcluster.conftest import (
     enable_multi_cluster_deployment,
     get_member_cluster_clients_using_cluster_mapping,
+    get_mongos_service_names,
 )
 
 
@@ -36,11 +37,7 @@ def sc(namespace: str, custom_mdb_prev_version: str) -> MongoDB:
 
 @fixture(scope="module")
 def mongod_tester(sc: MongoDB) -> MongoTester:
-    service_names = []
-    for cluster_member_client in get_member_cluster_clients_using_cluster_mapping(sc.name, sc.namespace):
-        for member_idx in range(sc.mongos_members_in_cluster(cluster_member_client.cluster_name)):
-            service_name = sc.mongos_service_name(member_idx, cluster_member_client.cluster_index)
-            service_names.append(service_name)
+    service_names = get_mongos_service_names(sc)
 
     return sc.tester(service_names=service_names)