ci: [NGOv2.X] Corrections of CI setup issues

.yamato/_run-all.yml

@@ -1,6 +1,20 @@
-{% metadata_file .yamato/project.metafile %}
+{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
 ---
 
+# DESCRIPTION--------------------------------------------------------------------------
+  # Those jobs group together related test as dependencies to allow to easily manage running a given set of tests.
+  # This enables efficient test execution for different validation scenarios
+
+# CONFIGURATION STRUCTURE--------------------------------------------------------------
+  # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations
+  # Jobs are generated using nested loops through:
+
+  # It's important to ensure that all generated in such way dependencies exist (this can be verified in Yamato)
+  # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations.
+  # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.
+
+#-----------------------------------------------------------------------------------
+
 # Runs all package tests
 run_all_package_tests:
   name: Run All Package Tests
@@ -73,7 +87,7 @@ run_all_webgl_builds:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.all -%}
-    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}
+    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -86,7 +100,7 @@ run_all_webgl_builds_trunk:
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
 {% for editor in validation_editors.default -%}
-    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}
+    - .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -128,7 +142,7 @@ run_all_project_tests_mobile_standalone:
 {% for project in projects.default -%}
 {% for platform in test_platforms.mobile_test -%}
 {% for editor in validation_editors.all -%}
-    - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
+    - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -141,7 +155,7 @@ run_all_project_tests_mobile_standalone_trunk:
 {% for project in projects.default -%}
 {% for platform in test_platforms.mobile_test -%}
 {% for editor in validation_editors.default -%}
-    - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
+    - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -154,7 +168,7 @@ run_all_project_tests_console_standalone:
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_test -%}
 {% for editor in validation_editors.all -%}
-    - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
+    - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -167,7 +181,7 @@ run_all_project_tests_console_standalone_trunk:
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_test -%}
 {% for editor in validation_editors.default -%}
-    - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
+    - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}

.yamato/_triggers.yml

@@ -1,6 +1,40 @@
-{% metadata_file .yamato/project.metafile %}
+{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
 ---
 
+# DESCRIPTION--------------------------------------------------------------------------
+  # This configuration defines three main CI trigger patterns:
+    # 1. Pull Request Validation: Validation performed on PR basis
+    # 2. Nightly Development: Test set run nightly (validates most important test cases)
+    # 3. Weekly Full Validation: Test set run weekly (validates all test cases to prevent any suprises)
+  # Each pattern represents different balance between validation depth, execution time and CI resource usage
+
+# TRIGGER PATTERNS-------------------------------------------------------------------
+  # Pull Request:
+    # This test validaes Standards, Package tests, Project tests and Desktop standalone tests to ensure that main platforms are covered
+    # Triggers on PRs to develop, develop-2.0.0, and release branches
+    # Focuses on critical validation paths that we should validate before merging PRs
+    # Cancels previous runs on new commits
+    # Excludes draft PRs
+
+  # Nightly:
+    # This test validaes same subset as pull_request_trigger with addition of mobile/console tests and webgl builds
+    # Runs daily on develop-2.0.0 (local configuration)
+    # Includes all test types but only on trunk. TODO: Add validation for minimum supported editor in nightly
+    # Adds platform-specific and APV validation
+
+  # Weekly:
+  # This test validaes same subset as develop_nightly but runs per all supported editors as well as executes code coverage test and runs project standards per project
+    # Runs across all supported editor versions
+    # Includes code coverage analysis
+    # Validates all projects and standards
+
+# CONFIGURATION STRUCTURE--------------------------------------------------------------
+  # Jobs configurations are generated by ensuring that all dependencies are successful.
+  # The dependencies are taken from _run-all.yml file where we can gather multiple tests into proper sets
+
+
+#-----------------------------------------------------------------------------------
+
 # Run all relevant tasks when a pull request targeting the develop or release branch is created or updated.
 pull_request_trigger:
   name: Pull Request Trigger (develop, develop-2.0.0, & release branches)

.yamato/code-coverage.yml

@@ -1,10 +1,37 @@
-{% metadata_file .yamato/project.metafile %}
+{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
 ---
-
-# Runs package tests in order to determine code coverage of the NGO package.
-# In essence it's performing the same task as .yamato/package-test jobs with the overhead being the measured code coverage
-# It's ok for code coverage to be performed only on one platform (default) since code coverage won't change much between those.
-# Default platform was chosen (ubuntu) since it's the fastest and most resource friendly with default editor.
+
+# DESCRIPTION--------------------------------------------------------------------------
+  # This job is responsible for executing package tests with code coverage analysis enabled.
+  # Coverage analysis provides insights into:
+    # Test coverage metrics for NGO assemblies
+    # Line and branch coverage statistics
+    # Generated HTML reports for coverage visualization
+    # Additional metrics for coverage analysis
+
+# CONFIGURATION STRUCTURE--------------------------------------------------------------
+  # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations
+  # Jobs are generated using nested loops through:
+    # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms)
+    # 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors)
+
+  # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations.
+  # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.
+
+  # The jobs name being generated would be for example "Code Coverage - NGO [testproject, 6000.0]" (name visible in Yamato) or "code_coverage_testproject_6000.0" (job name when querying for it)
+
+#TECHNICAL CONSIDERATIONS---------------------------------------------------------------
+  # In theory this job also runs package tests, but we don't want to use it as default since is heavier (because of added coverage analysis) and coverage is not changing that often
+  # Requires Unity Editor installation
+  # Burst compilation is disabled to ensure accurate coverage measurement
+  # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible
+
+# QUALITY CONSIDERATIONS--------------------------------------------------------------------
+  # TODO: somewhere in 2025 we will be able to upload resuls to CodeCov from public repos
+  # To see where this job is included (in trigger job definitions) look into _triggers.yml file
+
+
+
 {% for platform in test_platforms.default -%}
 {% for editor in validation_editors.default -%}
 code_coverage_{{ platform.name }}_{{ editor }}:
@@ -14,15 +41,14 @@ code_coverage_{{ platform.name }}_{{ editor }}:
     image: {{ platform.image }}
     flavor: {{ platform.flavor }}
   commands:
-    - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm
-    - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait
-    - DISPLAY=:0 upm-ci package test -u {{ editor }} --package-path com.unity.netcode.gameobjects --enable-code-coverage --code-coverage-options 'generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime' --extra-utr-arg="--extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun"
+    - unity-downloader-cli -u {{ editor }} -c Editor
+    - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor
+    - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results
   artifacts:
     logs:
       paths:
-        - "upm-ci~/test-results/**/*"
-        - "upm-ci~/test-results/CoverageResults/**/*"
+        - "test-results/**/*"
   dependencies:
-    - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}_upmCI
+    - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}
 {% endfor -%}
 {% endfor -%}

.yamato/console-standalone-test.yml

@@ -1,35 +1,60 @@
-{% metadata_file .yamato/project.metafile %}
+{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
 ---
 
-# Builds a player on console standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build.
-# Default project (testptoject) in this case is used as a context.
-# Builds/Tests are made on each console platform (PS4, PS5, Switch, XboxOne, XboxSeriesX) as in project.metafile declaration
-# Builds/Tests are made on each supported editor as in project.metafile declaration
+# DESCRIPTION--------------------------------------------------------------------------
+  # This job is responsible for Console platform test validation.
+  # Those tests cover both PlayMode and EditMode tests from package test assemblies.
 
-# For SOME of the console devices it's necessary to split build and run phases so it was split for all
-# For all consoles we need to use il2cpp scripting backend (so no testing with mono)
-# Switch works only with ARM64 and the rest with x64 architectures
-# For now all platforms used for building are windows based
+# CONFIGURATION STRUCTURE--------------------------------------------------------------
+  # It may seem that there is only 1 job defined, but we are using "for" loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations
+  # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency:
+    # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne)
+    # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors)
+  ` # 3. For the default project.
 
-# Builds are made with x64 architecture machines since those are compatible to run on ARM64 devices
+  # In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations.
+  # The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.
+
+  # The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "console_standalone_build_testproject_win_6000.0" (job name when querying for it)   
+
+# TECHNICAL CONSIDERATIONS---------------------------------------------------------------
+  # For console devices a split is required into two phases:
+    # 1. Build Phase: Creates standalone players for console platforms
+    # 2. Run Phase: Executes runtime tests on actual console devices
+  # The Run phase uses build job as dependency
+
+  # Note: More of a Unity specific but test assemblies need to be included in the build phase command
+  # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices
+
+# PLATFORM SPECIFICS-----------------------------------------------------------------
+  # Common Requirements:
+    # All consoles require IL2CPP scripting backend
+    # Platform-specific SDK environment variables
+
+  # Platform-Specific:
+    # Switch: ARM64 architecture only
+    # Other Consoles: x64 architecture
+    # Each console requires specific SDK paths and tools
+
+# QUALITY THOUGHTS--------------------------------------------------------------------
+  # TODO: consider adding all projects that have tests
+  # To see where this job is included (in trigger job definitions) look into _triggers.yml file
+
+
+
+# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_build -%}
 {% for editor in validation_editors.all -%}
-build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
+console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
   name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp{% if platform.name == "switch" %}, arm64{% endif %}]
   agent:
     type: {{ platform.type }}
     image: {{ platform.image }}
     flavor: {{ platform.flavor }}
   commands:
-    # Installing tools. unity-downloader-cli and utr should be already preinstalled on the image
-    - sudo pip install unity-downloader-cli
-    - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait
-    - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat
-
-    # Platform specific Build
-    - utr --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800
-
+    - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
+    - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -53,11 +78,11 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
 
 
 
-# Executes RuntimeTests of the NGO package in the Standalone build for consoles
+# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_test -%}
 {% for editor in validation_editors.all -%}
-run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
+console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
   name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp]
   agent:
     type: {{ platform.type }}
@@ -67,14 +92,8 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
     model: {{ platform.model }}
 {% endif %}
   commands:
-    # Installing tools.
-    - sudo pip install unity-downloader-cli
-    - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait
-    - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat
-
-    # Platform specific Execution
-    - utr --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
-
+    - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
+    - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -90,7 +109,7 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
       paths:
         - "test-results/**/*"
   dependencies:
-    - .yamato/console-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }}
+    - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}