From b1791df7c13c1574f1068e03214f1d7487be84bc Mon Sep 17 00:00:00 2001 From: Simon Baeumer Date: Thu, 15 May 2025 17:31:02 +0200 Subject: [PATCH 1/2] Add pipeline.py env var and func documentations. Moved some func documentations back to their original locations. --- PIPELINE.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ pipeline.py | 77 ++++++++++++++++++++++++++++------------------------- 2 files changed, 110 insertions(+), 37 deletions(-) create mode 100644 PIPELINE.md diff --git a/PIPELINE.md b/PIPELINE.md new file mode 100644 index 000000000..7f77dc80d --- /dev/null +++ b/PIPELINE.md @@ -0,0 +1,70 @@ +# Pipeline + +## Environment Variables (env vars) + +This listing contains all environment variables used in `pipeline.py`. +Default evergreen-ci expansions can be looked up [here](https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#expansions). + +| Environment Variable | Usage / Description | +|-------------------------------|------------------------------------------------------------------------------------| +| `otel_trace_id` | OpenTelemetry tracing: trace ID. Default evergreen-ci expansion. | +| `otel_parent_id` | OpenTelemetry tracing: parent span ID. Default evergreen-ci expansion. | +| `otel_collector_endpoint` | OpenTelemetry tracing: collector endpoint. Default evergreen-ci expansion. | +| `distro` | Image type (defaults to `ubi`) | +| `BASE_REPO_URL` | Base repository URL for images | +| `namespace` | Kubernetes namespace (defaults to `default`) | +| `skip_tags` | Tags to skip during build | +| `include_tags` | Tags to include during build | +| `all_agents` | Whether to build all agent images | +| `RUNNING_IN_EVG` | Whether running in Evergreen pipeline | +| `is_patch` | Whether running as a patch build. Default evergreen-ci expansion. | +| `pin_tag_at` | Time to pin image tag (format: `HH:MM`) | +| `created_at` | Build creation time (format: `%y_%m_%d_%H_%M_%S`). Default evergreen-ci expansion. | +| `triggered_by_git_tag` | Git tag that triggered the build. Default evergreen-ci expansion. Default evergreen-ci expansion. | +| `version_id` | Patch ID or version for non-release builds. Default evergreen-ci expansion. | +| `test_suffix` | Suffix for test images | +| `LOG_AUTOMATION_CONFIG_DIFF` | Whether to log automation config diff | +| `PYTHON_VERSION` | Python version for test images | +| `GOLANG_VERSION` | Go version for community images and tests | +| `QUAY_REGISTRY` | Quay registry URL (defaults to `quay.io/mongodb`) | +| `REGISTRY` | ECR registry URL (defaults to `268558157000.dkr.ecr.us-east-1.amazonaws.com/dev`) | +| `om_version` | Ops Manager version for OM image builds | +| `om_download_url` | Download URL for Ops Manager (optional, can be auto-detected) | + +## Context Image Build Process + +``` + ┌─────────────────────────────┐ + │ Release Pipeline │ + └────────────┬────────────────┘ + │ + ▼ + ┌─────────────────────────────────┐ + │ Build context image │ + │ Tag: opsmanager-context:1.33.0 │ + └────────────┬────────────────────┘ + │ + ▼ + ┌───────────────────────────────┐ + │ Daily Build │ + │ Base: opsmanager-context │ + │ Input tag: 1.33.0 │ + └────────────┬──────────────────┘ + │ + ▼ + ┌────────────────────────────────────┐ + │ Push Two Image Tags │ + └────────────┬───────────────┬───────┘ + ▼ ▼ + ┌────────────────────────┐ ┌──────────────────────────────┐ + │ Rolling Tag (latest) │ │ Immutable Tag (daily stamp) │ + │ opsmanager:1.33.0 │ │ opsmanager:1.33.0-2025-01-01 │ + └────────────────────────┘ └──────────────────────────────┘ + + ▼ (next day build) + ┌────────────────────────┐ ┌──────────────────────────────┐ + │ opsmanager:1.33.0 │ │ opsmanager:1.33.0-2025-01-02 │ + └────────────────────────┘ └──────────────────────────────┘ + ↑ now updated to point ↑ new image pushed + to the 2025-01-02 build +``` diff --git a/pipeline.py b/pipeline.py index e05e5e5a0..e9603c37a 100755 --- a/pipeline.py +++ b/pipeline.py @@ -264,6 +264,7 @@ def get_release() -> Dict: def get_git_release_tag() -> tuple[str, bool]: + """Returns the git tag of the current run on releases, on non-release returns the patch id.""" release_env_var = os.getenv("triggered_by_git_tag") # that means we are in a release and only return the git_tag; otherwise we want to return the patch_id @@ -290,22 +291,19 @@ def copy_into_container(client, src, dst): container.put_archive(os.path.dirname(dst), fd.read()) -""" -Generates docker manifests by running the following commands: -1. Clear existing manifests -docker manifest rm config.repo_url/image:tag -2. Create the manifest -docker manifest create config.repo_url/image:tag --amend config.repo_url/image:tag-amd64 --amend config.repo_url/image:tag-arm64 -3. Push the manifest -docker manifest push config.repo_url/image:tag -""" - - -# This method calls docker directly on the command line, this is different from the rest of the code which uses -# Sonar as an interface to docker. We decided to keep this asymmetry for now, as Sonar will be removed soon. - - def create_and_push_manifest(image: str, tag: str, architectures: list[str]) -> None: + """ + Generates docker manifests by running the following commands: + 1. Clear existing manifests + docker manifest rm config.repo_url/image:tag + 2. Create the manifest + docker manifest create config.repo_url/image:tag --amend config.repo_url/image:tag-amd64 --amend config.repo_url/image:tag-arm64 + 3. Push the manifest + docker manifest push config.repo_url/image:tag + + This method calls docker directly on the command line, this is different from the rest of the code which uses + Sonar as an interface to docker. We decided to keep this asymmetry for now, as Sonar will be removed soon. + """ final_manifest = image + ":" + tag args = [ @@ -343,14 +341,12 @@ def try_get_platform_data(client, image): return None -""" -Checks if a docker image supports AMD and ARM platforms by inspecting the registry data. - -:param str image: The image name and tag -""" - - def check_multi_arch(image: str, suffix: str) -> bool: + """ + Checks if a docker image supports AMD and ARM platforms by inspecting the registry data. + + :param str image: The image name and tag + """ client = docker.from_env() platforms = ["linux/amd64", "linux/arm64"] @@ -741,17 +737,6 @@ def submit(self, fn, *args, **kwargs): return super().submit(lambda: fn(*args, **kwargs)) -""" -Starts the daily build process for an image. This function works for all images we support, for community and -enterprise operator. The list of supported image_name is defined in get_builder_function_for_image_name. -Builds an image for each version listed in ./release.json -The registry used to pull base image and output the daily build is configured in the image_config function, it is passed -as an argument to the inventories/daily.yaml file. - -If the context image supports both ARM and AMD architectures, both will be built. -""" - - def should_skip_arm64(): """ Determines if arm64 builds should be skipped based on environment. @@ -766,7 +751,15 @@ def build_image_daily( max_version: str = None, operator_version: str = None, ): - """Builds a daily image.""" + """ + Starts the daily build process for an image. This function works for all images we support, for community and + enterprise operator. The list of supported image_name is defined in get_builder_function_for_image_name. + Builds an image for each version listed in ./release.json + The registry used to pull base image and output the daily build is configured in the image_config function, it is passed + as an argument to the inventories/daily.yaml file. + + If the context image supports both ARM and AMD architectures, both will be built. + """ def get_architectures_set(build_configuration, args): """Determine the set of architectures to build for""" @@ -992,6 +985,11 @@ def build_image_generic( multi_arch_args_list: list = None, is_run_in_parallel: bool = False, ): + """Build image generic builds context images and is used for triggering release. During releases + it signs and verifies the context image. + The release process uses the daily images build process. + """ + if not multi_arch_args_list: multi_arch_args_list = [extra_args or {}] @@ -1011,8 +1009,12 @@ def build_image_generic( # But since we don't run daily rebuilds on ecr image builds, we can do that step instead here. # We only need to push manifests for multi-arch images. create_and_push_manifest(registry_address, version, architectures=architectures) + + # Sign and verify the context image if on releases if requied. if config.sign and config.is_release_step_executed(): sign_and_verify_context_image(registry, version) + + # Release step. Release images via the daily image process. if config.is_release_step_executed() and version and QUAY_REGISTRY_URL in registry: logger.info( f"finished building context images, releasing them now via daily builds process for" @@ -1546,11 +1548,12 @@ def calculate_images_to_build( def main(): _setup_tracing() + _setup_tracing() parser = argparse.ArgumentParser() - parser.add_argument("--include", action="append") - parser.add_argument("--exclude", action="append") - parser.add_argument("--builder", default="docker", type=str) + parser.add_argument("--include", action="append", help="list of images to include") + parser.add_argument("--exclude", action="append", help="list of images to exclude") + parser.add_argument("--builder", default="docker", type=str, help="docker or podman") parser.add_argument("--list-images", action="store_true") parser.add_argument("--parallel", action="store_true", default=False) parser.add_argument("--debug", action="store_true", default=False) From 047178f2dd130718d70c8f04f6e1119eb27d743f Mon Sep 17 00:00:00 2001 From: Simon Baeumer Date: Fri, 16 May 2025 11:23:15 +0200 Subject: [PATCH 2/2] fix md --- PIPELINE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PIPELINE.md b/PIPELINE.md index 7f77dc80d..e6383446c 100644 --- a/PIPELINE.md +++ b/PIPELINE.md @@ -35,7 +35,7 @@ Default evergreen-ci expansions can be looked up [here](https://docs.devprod.pro ``` ┌─────────────────────────────┐ - │ Release Pipeline │ + │ Release Pipeline │ └────────────┬────────────────┘ │ ▼