Merge pull request #396 from gradle/erichaagdev/remove-build-cache-metrics-only

Build Scan parsing is updated to read values when previously unknown

Author: erichaagdev

components/scripts/gradle/04-validate-remote-build-caching-ci-ci.sh

@@ -116,7 +116,7 @@ validate_required_args() {
 }
 
 fetch_build_scans() {
-  fetch_build_scans_and_build_time_metrics 'all_data' "${build_scan_urls[@]}"
+  fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}"
 }
 
 # Overrides summary.sh#print_experiment_specific_summary_info

components/scripts/lib/build-scan-offline.sh

@@ -40,5 +40,5 @@ read_build_scan_dumps() {
   build_scan_data="$(invoke_java "$BUILD_SCAN_SUPPORT_TOOL_JAR" extract "0,${build_scan_dumps[0]}"  "1,${build_scan_dumps[1]}")"
   echo ", done."
 
-  parse_build_scans_and_build_time_metrics 'build_cache_metrics_only' "$build_scan_data"
+  parse_build_scans_and_build_time_metrics "$build_scan_data"
 }

components/scripts/lib/build-scan-online.sh

@@ -7,7 +7,7 @@ readonly FETCH_BUILD_SCAN_DATA_JAR="${LIB_DIR}/export-api-clients/fetch-build-sc
 # Enterprise API.
 process_build_scan_data_online() {
   read_build_scan_metadata
-  fetch_build_scans_and_build_time_metrics 'build_cache_metrics_only' "${build_scan_urls[@]}"
+  fetch_build_scans_and_build_time_metrics 'brief_logging' "${build_scan_urls[@]}"
 }
 
 read_build_scan_metadata() {
@@ -53,28 +53,27 @@ fetch_single_build_scan() {
   parse_single_build_scan "${build_scan_data}"
 }
 
+# The value of logging_level should be either 'brief_logging' or
+# 'verbose_logging'
 fetch_build_scans_and_build_time_metrics() {
-  local build_cache_metrics_only="$1"
+  local logging_level="$1"
   shift
   local build_scan_urls=("$@")
 
-  local brief_logging
-  if [[ "${build_cache_metrics_only}" == 'build_cache_metrics_only' ]]; then
-    brief_logging="brief_logging"
-  else
+  if [[ "${logging_level}" == 'verbose_logging' ]]; then
     info "Fetching build scan data"
   fi
 
   local build_scan_data
-  build_scan_data="$(fetch_build_scan_data "${brief_logging}" "${build_scan_urls[@]}")"
+  build_scan_data="$(fetch_build_scan_data "${logging_level}" "${build_scan_urls[@]}")"
 
-  parse_build_scans_and_build_time_metrics "${build_cache_metrics_only}" "${build_scan_data}"
+  parse_build_scans_and_build_time_metrics "${build_scan_data}"
 }
 
 # Note: Callers of this function require stdout to be clean. No logging can be
 #       done inside this function.
 fetch_build_scan_data() {
-  local brief_logging="$1"
+  local logging_level="$1"
   shift
   local build_scan_urls=("$@")
 
@@ -90,7 +89,7 @@ fetch_build_scan_data() {
     args+=("--network-settings-file" "${SCRIPT_DIR}/network.settings")
   fi
 
-  if [[ "${brief_logging}" == "brief_logging" ]]; then
+  if [[ "${logging_level}" == "brief_logging" ]]; then
     args+=("--brief-logging")
   fi
 

components/scripts/lib/build-scan-parse.sh

@@ -43,12 +43,11 @@ parse_single_build_scan() {
   # Parses build scan data to an array by line
   IFS=$'\n' read -rd '' -a build_scan_rows <<< "$build_scan_data"
 
-  parse_build_scan_row 'all_data' "${build_scan_rows[1]}"
+  parse_build_scan_row "${build_scan_rows[1]}"
 }
 
 parse_build_scans_and_build_time_metrics() {
-  local build_cache_metrics_only="$1"
-  local build_scan_data="$2"
+  local build_scan_data="$1"
 
   debug_build_scan_data "$build_scan_data"
 
@@ -57,8 +56,8 @@ parse_build_scans_and_build_time_metrics() {
   # Parses build scan data to an array by line
   IFS=$'\n' read -rd '' -a build_scan_rows <<< "$build_scan_data"
 
-  parse_build_scan_row "${build_cache_metrics_only}" "${build_scan_rows[1]}"
-  parse_build_scan_row "${build_cache_metrics_only}" "${build_scan_rows[2]}"
+  parse_build_scan_row "${build_scan_rows[1]}"
+  parse_build_scan_row "${build_scan_rows[2]}"
 
   parse_build_time_metrics "${build_scan_rows[4]}"
 }
@@ -74,28 +73,48 @@ debug_build_scan_data() {
 
 # shellcheck disable=SC2034 # not all scripts use all of the fetched data
 parse_build_scan_row() {
-  local build_cache_metrics_only="$1"
-  local build_scan_row="$2"
+  local build_scan_row="$1"
 
   local run_num
 
   while IFS=, read -r run_num field_1 field_2 field_3 field_4 field_5 field_6 field_7 field_8 field_9 field_10 field_11 field_12 field_13 field_14 field_15 field_16 field_17 field_18 field_19 field_20 field_21; do
     debug "Build Scan $field_4 is for build $run_num"
-    project_names[run_num]="$field_1"
-    build_scan_ids[run_num]="$field_4"
-
-    if [[ "$build_cache_metrics_only" != 'build_cache_metrics_only' ]]; then
-      base_urls[run_num]="$field_2"
-      build_scan_urls[run_num]="$field_3"
-      git_repos[run_num]="$field_5"
-      git_branches[run_num]="$field_6"
-      git_commit_ids[run_num]="$field_7"
+
+    project_names[run_num]="${field_1}"
+
+    if [ -z "${base_urls[run_num]}" ]; then
+      base_urls[run_num]="${field_2}"
+    fi
+
+    if [ -z "${build_scan_urls[run_num]}" ]; then
+      build_scan_urls[run_num]="${field_3}"
+    fi
+
+    build_scan_ids[run_num]="${field_4}"
+
+    if [ -z "${git_repos[run_num]}" ]; then
+      git_repos[run_num]="${field_5}"
+    fi
+
+    if [ -z "${git_branches[run_num]}" ]; then
+      git_branches[run_num]="${field_6}"
+    fi
+
+    if [ -z "${git_commit_ids[run_num]}" ]; then
+      git_commit_ids[run_num]="${field_7}"
+    fi
+
+    if [ -z "${requested_tasks[run_num]}" ]; then
       requested_tasks[run_num]="$(remove_clean_task "${field_8}")"
-      build_outcomes[run_num]="$field_9"
-      remote_build_cache_urls[run_num]="${field_10}"
-      remote_build_cache_shards[run_num]="${field_11}"
     fi
 
+    if [ -z "${build_outcomes[run_num]}" ]; then
+      build_outcomes[run_num]="${field_9}"
+    fi
+
+    remote_build_cache_urls[run_num]="${field_10}"
+    remote_build_cache_shards[run_num]="${field_11}"
+
     # Build caching performance metrics
     avoided_up_to_date_num_tasks[run_num]="${field_12}"
     avoided_up_to_date_avoidance_savings[run_num]="${field_13}"

components/scripts/maven/03-validate-remote-build-caching-ci-ci.sh

@@ -114,7 +114,7 @@ validate_required_args() {
 }
 
 fetch_build_scans() {
-  fetch_build_scans_and_build_time_metrics 'all_data' "${build_scan_urls[@]}"
+  fetch_build_scans_and_build_time_metrics 'verbose_logging' "${build_scan_urls[@]}"
 }
 
 # Overrides summary.sh#print_experiment_specific_summary_info