Fix flakiness of e2e_appdb_tls_operator_upgrade_v1_32_to_mck (#100)

# Summary

This fix should at least reduce the flakiness of the mentioned test. The
flakiness comes from waiting for database measurements of the "admin"
db. Measurements from this db are rare, especially with no operations
happening. Therefore, we now wait for host measurements (cpu, memory,
etc.) which are constant regardless of db operations.

## Proof of Work

There are no failures due to waiting for db measurements now.

https://spruce.mongodb.com/task/mongodb_kubernetes_e2e_multi_cluster_om_appdb_e2e_appdb_tls_operator_upgrade_v1_32_to_mck_patch_6a1393da1c85ce7f02cc31986bc547bfd7adf03f_682b1e248ca26800084d419a_25_05_19_12_03_57/tests?execution=4&sortBy=STATUS&sortDir=ASC

## Checklist
- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you checked for release_note changes?

Author: lucian-tosa

docker/mongodb-kubernetes-tests/kubetester/omtester.py

@@ -542,7 +542,7 @@ def api_read_backup_snapshot_schedule(self) -> Dict:
             f"/groups/{self.context.project_id}/backupConfigs/{backup_configs['clusterId']}/snapshotSchedule",
         ).json()
 
-    def api_read_monitoring_measurements(
+    def api_read_measurements(
         self,
         host_id: str,
         database_name: Optional[str] = None,
@@ -554,11 +554,13 @@ def api_read_monitoring_measurements(
 
         https://docs.opsmanager.mongodb.com/v4.4/reference/api/measures/get-host-process-system-measurements/
         """
-        if database_name is None:
-            database_name = "admin"
+        database_path = ""
+        if database_name is not None:
+            database_path = f"/databases/{database_name}"
+
         return self.om_request(
             "get",
-            f"/groups/{project_id}/hosts/{host_id}/databases/{database_name}/measurements?granularity=PT30S&period={period}",
+            f"/groups/{project_id}/hosts/{host_id}{database_path}/measurements?granularity=PT30S&period={period}",
         ).json()["measurements"]
 
     def api_read_monitoring_agents(self) -> List:

docker/mongodb-kubernetes-tests/kubetester/opsmanager.py

@@ -174,7 +174,11 @@ def no_automation_agents_have_registered() -> bool:
     TRACER.start_as_current_span("assert_monitoring_data_exists")
 
     def assert_monitoring_data_exists(
-        self, database_name: str = "admin", period: str = "P1DT12H", timeout: int = 120, all_hosts: bool = True
+        self,
+        database_name: Optional[str] = None,
+        period: str = "P1DT12H",
+        timeout: int = 600,
+        all_hosts: bool = True,
     ):
         """
         Asserts the existence of monitoring measurements in this Ops Manager instance.
@@ -186,7 +190,7 @@ def assert_monitoring_data_exists(
 
         def agent_is_showing_metrics():
             for host_id in host_ids:
-                measurements = tester.api_read_monitoring_measurements(
+                measurements = tester.api_read_measurements(
                     host_id,
                     database_name=database_name,
                     project_id=project_id,

docker/mongodb-kubernetes-tests/tests/opsmanager/withMonitoredAppDB/om_ops_manager_appdb_monitoring_tls.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 import pymongo
-from kubetester import create_or_update_secret
+from kubetester import create_or_update_secret, try_load
 from kubetester.certs import create_ops_manager_tls_certs
 from kubetester.kubetester import fixture as yaml_fixture
 from kubetester.mongodb import Phase
@@ -36,13 +36,16 @@ def ops_manager(
     custom_appdb_version: str,
     issuer_ca_filepath: str,
 ) -> MongoDBOpsManager:
-    create_or_update_secret(namespace, "appdb-secret", {"password": "Hello-World!"})
-
     print("Creating OM object")
     om = MongoDBOpsManager.from_yaml(yaml_fixture("om_ops_manager_appdb_monitoring_tls.yaml"), namespace=namespace)
     om.set_version(custom_version)
     om.set_appdb_version(custom_appdb_version)
 
+    if try_load(om):
+        return om
+
+    create_or_update_secret(namespace, "appdb-secret", {"password": "Hello-World!"})
+
     # ensure the requests library will use this CA when communicating with Ops Manager
     os.environ["REQUESTS_CA_BUNDLE"] = issuer_ca_filepath
 
@@ -78,9 +81,9 @@ def test_appdb_password_can_be_changed(ops_manager: MongoDBOpsManager):
     )
 
     # We know that Ops Manager will detect the changes and be restarted
-    ops_manager.appdb_status().assert_reaches_phase(Phase.Pending, timeout=120)
+    ops_manager.appdb_status().assert_reaches_phase(Phase.Pending, timeout=300)
     ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600)
-    ops_manager.om_status().assert_reaches_phase(Phase.Pending, timeout=120)
+    ops_manager.om_status().assert_reaches_phase(Phase.Pending, timeout=300)
     ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=800)