From acb11351cd089a366a4a494876337154c16697b3 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Tue, 11 Apr 2023 17:16:13 -0500 Subject: [PATCH 1/2] Remove build_cache_metrics_only and all_data --- .../04-validate-remote-build-caching-ci-ci.sh | 2 +- components/scripts/lib/build-scan-offline.sh | 2 +- components/scripts/lib/build-scan-online.sh | 19 +++---- components/scripts/lib/build-scan-parse.sh | 57 ++++++++++++------- .../03-validate-remote-build-caching-ci-ci.sh | 2 +- 5 files changed, 50 insertions(+), 32 deletions(-) diff --git a/components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh b/components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh index 6df904b9..ee4cbda9 100755 --- a/components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh +++ b/components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh @@ -116,7 +116,7 @@ validate_required_args() { } fetch_build_scans() { - fetch_build_scans_and_build_time_metrics 'all_data' "${build_scan_urls[@]}" + fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}" } # Overrides summary.sh#print_experiment_specific_summary_info diff --git a/components/scripts/lib/build-scan-offline.sh b/components/scripts/lib/build-scan-offline.sh index 1bcd8528..1507a524 100644 --- a/components/scripts/lib/build-scan-offline.sh +++ b/components/scripts/lib/build-scan-offline.sh @@ -40,5 +40,5 @@ read_build_scan_dumps() { build_scan_data="$(invoke_java "$BUILD_SCAN_SUPPORT_TOOL_JAR" extract "0,${build_scan_dumps[0]}" "1,${build_scan_dumps[1]}")" echo ", done." - parse_build_scans_and_build_time_metrics 'build_cache_metrics_only' "$build_scan_data" + parse_build_scans_and_build_time_metrics "$build_scan_data" } diff --git a/components/scripts/lib/build-scan-online.sh b/components/scripts/lib/build-scan-online.sh index c33574ed..7d59ae2a 100644 --- a/components/scripts/lib/build-scan-online.sh +++ b/components/scripts/lib/build-scan-online.sh @@ -7,7 +7,7 @@ readonly FETCH_BUILD_SCAN_DATA_JAR="${LIB_DIR}/export-api-clients/fetch-build-sc # Enterprise API. process_build_scan_data_online() { read_build_scan_metadata - fetch_build_scans_and_build_time_metrics 'build_cache_metrics_only' "${build_scan_urls[@]}" + fetch_build_scans_and_build_time_metrics 'brief_logging' "${build_scan_urls[@]}" } read_build_scan_metadata() { @@ -53,28 +53,27 @@ fetch_single_build_scan() { parse_single_build_scan "${build_scan_data}" } +# The value of logging_level should be either 'brief_logging' or +# 'verbose_logging' fetch_build_scans_and_build_time_metrics() { - local build_cache_metrics_only="$1" + local logging_level="$1" shift local build_scan_urls=("$@") - local brief_logging - if [[ "${build_cache_metrics_only}" == 'build_cache_metrics_only' ]]; then - brief_logging="brief_logging" - else + if [[ "${logging_level}" != 'brief_logging' ]]; then info "Fetching build scan data" fi local build_scan_data - build_scan_data="$(fetch_build_scan_data "${brief_logging}" "${build_scan_urls[@]}")" + build_scan_data="$(fetch_build_scan_data "${logging_level}" "${build_scan_urls[@]}")" - parse_build_scans_and_build_time_metrics "${build_cache_metrics_only}" "${build_scan_data}" + parse_build_scans_and_build_time_metrics "${build_scan_data}" } # Note: Callers of this function require stdout to be clean. No logging can be # done inside this function. fetch_build_scan_data() { - local brief_logging="$1" + local logging_level="$1" shift local build_scan_urls=("$@") @@ -90,7 +89,7 @@ fetch_build_scan_data() { args+=("--network-settings-file" "${SCRIPT_DIR}/network.settings") fi - if [[ "${brief_logging}" == "brief_logging" ]]; then + if [[ "${logging_level}" == "brief_logging" ]]; then args+=("--brief-logging") fi diff --git a/components/scripts/lib/build-scan-parse.sh b/components/scripts/lib/build-scan-parse.sh index e3f3a047..3642e0f7 100644 --- a/components/scripts/lib/build-scan-parse.sh +++ b/components/scripts/lib/build-scan-parse.sh @@ -43,12 +43,11 @@ parse_single_build_scan() { # Parses build scan data to an array by line IFS=$'\n' read -rd '' -a build_scan_rows <<< "$build_scan_data" - parse_build_scan_row 'all_data' "${build_scan_rows[1]}" + parse_build_scan_row "${build_scan_rows[1]}" } parse_build_scans_and_build_time_metrics() { - local build_cache_metrics_only="$1" - local build_scan_data="$2" + local build_scan_data="$1" debug_build_scan_data "$build_scan_data" @@ -57,8 +56,8 @@ parse_build_scans_and_build_time_metrics() { # Parses build scan data to an array by line IFS=$'\n' read -rd '' -a build_scan_rows <<< "$build_scan_data" - parse_build_scan_row "${build_cache_metrics_only}" "${build_scan_rows[1]}" - parse_build_scan_row "${build_cache_metrics_only}" "${build_scan_rows[2]}" + parse_build_scan_row "${build_scan_rows[1]}" + parse_build_scan_row "${build_scan_rows[2]}" parse_build_time_metrics "${build_scan_rows[4]}" } @@ -74,28 +73,48 @@ debug_build_scan_data() { # shellcheck disable=SC2034 # not all scripts use all of the fetched data parse_build_scan_row() { - local build_cache_metrics_only="$1" - local build_scan_row="$2" + local build_scan_row="$1" local run_num while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20 field_21; do debug "Build Scan $field_4 is for build $run_num" - project_names[run_num]="$field_1" - build_scan_ids[run_num]="$field_4" - - if [[ "$build_cache_metrics_only" != 'build_cache_metrics_only' ]]; then - base_urls[run_num]="$field_2" - build_scan_urls[run_num]="$field_3" - git_repos[run_num]="$field_5" - git_branches[run_num]="$field_6" - git_commit_ids[run_num]="$field_7" + + project_names[run_num]="${field_1}" + + if [ -z "${base_urls[run_num]}" ]; then + base_urls[run_num]="${field_2}" + fi + + if [ -z "${build_scan_urls[run_num]}" ]; then + build_scan_urls[run_num]="${field_3}" + fi + + build_scan_ids[run_num]="${field_4}" + + if [ -z "${git_repos[run_num]}" ]; then + git_repos[run_num]="${field_5}" + fi + + if [ -z "${git_branches[run_num]}" ]; then + git_branches[run_num]="${field_6}" + fi + + if [ -z "${git_commit_ids[run_num]}" ]; then + git_commit_ids[run_num]="${field_7}" + fi + + if [ -z "${requested_tasks[run_num]}" ]; then requested_tasks[run_num]="$(remove_clean_task "${field_8}")" - build_outcomes[run_num]="$field_9" - remote_build_cache_urls[run_num]="${field_10}" - remote_build_cache_shards[run_num]="${field_11}" fi + if [ -z "${build_outcomes[run_num]}" ]; then + build_outcomes[run_num]="${field_9}" + fi + + remote_build_cache_urls[run_num]="${field_10}" + remote_build_cache_shards[run_num]="${field_11}" + # Build caching performance metrics avoided_up_to_date_num_tasks[run_num]="${field_12}" avoided_up_to_date_avoidance_savings[run_num]="${field_13}" diff --git a/components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh b/components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh index 6dbf4598..5aacad56 100755 --- a/components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh +++ b/components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh @@ -114,7 +114,7 @@ validate_required_args() { } fetch_build_scans() { - fetch_build_scans_and_build_time_metrics 'all_data' "${build_scan_urls[@]}" + fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}" } # Overrides summary.sh#print_experiment_specific_summary_info From 2428dacd36f1f86a4c9f8ca66a2f36c1efa05037 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Wed, 12 Apr 2023 16:14:17 -0500 Subject: [PATCH 2/2] Flip if condition for clarity --- components/scripts/lib/build-scan-online.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/scripts/lib/build-scan-online.sh b/components/scripts/lib/build-scan-online.sh index 7d59ae2a..12652e69 100644 --- a/components/scripts/lib/build-scan-online.sh +++ b/components/scripts/lib/build-scan-online.sh @@ -60,7 +60,7 @@ fetch_build_scans_and_build_time_metrics() { shift local build_scan_urls=("$@") - if [[ "${logging_level}" != 'brief_logging' ]]; then + if [[ "${logging_level}" == 'verbose_logging' ]]; then info "Fetching build scan data" fi