From 46dcca78c5c9fbea04960bdec97f364c384c5903 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Wed, 28 Dec 2022 16:06:27 -0600 Subject: [PATCH 1/9] Remove reading project name from build-scans.csv --- components/scripts/lib/build_scan.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/scripts/lib/build_scan.sh b/components/scripts/lib/build_scan.sh index 08e57a08..d5e8401e 100644 --- a/components/scripts/lib/build_scan.sh +++ b/components/scripts/lib/build_scan.sh @@ -6,11 +6,10 @@ read_build_scan_metadata() { # This isn't the most robust way to read a CSV, # but we control the CSV so we don't have to worry about various CSV edge cases if [ -f "${BUILD_SCAN_FILE}" ]; then - while IFS=, read -r field_1 field_2 field_3 field_4; do - project_names+=("$field_1") - base_urls+=("$field_2") - build_scan_urls+=("$field_3") - build_scan_ids+=("$field_4") + while IFS=, read -r field_1 field_2 field_3; do + base_urls+=("$field_1") + build_scan_urls+=("$field_2") + build_scan_ids+=("$field_3") done < "${BUILD_SCAN_FILE}" fi } From 36826b7918edd3e62479e542676ef4aae0ae4898 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Wed, 28 Dec 2022 16:07:31 -0600 Subject: [PATCH 2/9] Remove writing project name to build-scans.csv --- .../capture-published-build-scan.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/scripts/gradle/gradle-init-scripts/capture-published-build-scan.gradle b/components/scripts/gradle/gradle-init-scripts/capture-published-build-scan.gradle index ae0caca4..e2cfe835 100644 --- a/components/scripts/gradle/gradle-init-scripts/capture-published-build-scan.gradle +++ b/components/scripts/gradle/gradle-init-scripts/capture-published-build-scan.gradle @@ -12,21 +12,21 @@ if (isTopLevelBuild) { if (!settings.pluginManager.hasPlugin("com.gradle.enterprise")) { throw new IllegalStateException("The com.gradle.enterprise plugin is missing from the project (see https://docs.gradle.com/enterprise/gradle-plugin/#gradle_6_x_and_later).") } - registerCallbacks(settings.extensions["gradleEnterprise"], settings.rootProject.name) + registerCallbacks(settings.extensions["gradleEnterprise"]) } } else if (atLeastGradle5) { projectsEvaluated { gradle -> if (!gradle.rootProject.pluginManager.hasPlugin("com.gradle.build-scan")) { throw new IllegalStateException("The com.gradle.build-scan plugin is missing from the project (see https://docs.gradle.com/enterprise/gradle-plugin/#gradle_5_x).") } - registerCallbacks(gradle.rootProject.extensions["gradleEnterprise"], gradle.rootProject.name) + registerCallbacks(gradle.rootProject.extensions["gradleEnterprise"]) } } else { throw new IllegalStateException("Build validation not supported for Gradle ${GradleVersion.current()}. Upgrade your project's build to Gradle 5 or newer.") } } -def registerCallbacks(gradleEnterprise, rootProjectName) { +void registerCallbacks(gradleEnterprise) { gradleEnterprise.with { buildScan { def scanFile = new File(experimentDir, "build-scans.csv") @@ -35,7 +35,7 @@ def registerCallbacks(gradleEnterprise, rootProjectName) { def buildScanId = buildScan.buildScanId def port = (buildScanUri.port != -1) ? ":" + buildScanUri.port : "" def baseUrl = "${buildScanUri.scheme}://${buildScanUri.host}${port}" - scanFile.append("${rootProjectName},${baseUrl},${buildScanUri},${buildScanId}\n") + scanFile.append("${baseUrl},${buildScanUri},${buildScanId}\n") } def errorFile = new File(experimentDir, "build-scan-publish-error.txt") onError { error -> From 2970e3d9f888614defb6605c5253273e76789d51 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Wed, 28 Dec 2022 16:25:16 -0600 Subject: [PATCH 3/9] Remove many redundant occurrences of read_build_scan_metadata function --- components/scripts/gradle/01-validate-incremental-building.sh | 1 - .../gradle/02-validate-local-build-caching-same-location.sh | 1 - .../03-validate-local-build-caching-different-locations.sh | 2 -- .../scripts/gradle/05-validate-remote-build-caching-ci-local.sh | 1 - components/scripts/lib/info.sh | 2 -- .../maven/01-validate-local-build-caching-same-location.sh | 1 - .../02-validate-local-build-caching-different-locations.sh | 1 - .../scripts/maven/04-validate-remote-build-caching-ci-local.sh | 1 - 8 files changed, 10 deletions(-) diff --git a/components/scripts/gradle/01-validate-incremental-building.sh b/components/scripts/gradle/01-validate-incremental-building.sh index 8c637ca1..a9cd2e91 100755 --- a/components/scripts/gradle/01-validate-incremental-building.sh +++ b/components/scripts/gradle/01-validate-incremental-building.sh @@ -300,7 +300,6 @@ EOF } explain_and_print_summary() { - read_build_scan_metadata local text if [[ "${build_scan_publishing_mode}" == "on" ]]; then IFS='' read -r -d '' text < Date: Thu, 29 Dec 2022 09:12:09 -0600 Subject: [PATCH 4/9] Add debug logging when reading Build Scan metadata --- components/scripts/lib/build_scan.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/components/scripts/lib/build_scan.sh b/components/scripts/lib/build_scan.sh index d5e8401e..2465f4d1 100644 --- a/components/scripts/lib/build_scan.sh +++ b/components/scripts/lib/build_scan.sh @@ -6,11 +6,21 @@ read_build_scan_metadata() { # This isn't the most robust way to read a CSV, # but we control the CSV so we don't have to worry about various CSV edge cases if [ -f "${BUILD_SCAN_FILE}" ]; then + local build_scan_metadata + build_scan_metadata=$(< "${BUILD_SCAN_FILE}") + + if [[ "$_arg_debug" == "on" ]]; then + debug "Raw Build Scan metadata (build-scans.csv)" + debug "---------------------------" + debug "${build_scan_metadata}" + debug "" + fi + while IFS=, read -r field_1 field_2 field_3; do base_urls+=("$field_1") build_scan_urls+=("$field_2") build_scan_ids+=("$field_3") - done < "${BUILD_SCAN_FILE}" + done <<< "${build_scan_metadata}" fi } From f1327dcf43bf0f4141f625fa1c1ac4afc3745db5 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Thu, 29 Dec 2022 09:14:24 -0600 Subject: [PATCH 5/9] Remove writing project name to build-scans.csv from Maven extension --- .../com/gradle/CaptureBuildScansListener.java | 11 +-- .../java/com/gradle/RootProjectExtractor.java | 73 ------------------- 2 files changed, 4 insertions(+), 80 deletions(-) delete mode 100644 components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/RootProjectExtractor.java diff --git a/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/CaptureBuildScansListener.java b/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/CaptureBuildScansListener.java index ee57289f..63d40f7f 100644 --- a/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/CaptureBuildScansListener.java +++ b/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/CaptureBuildScansListener.java @@ -4,7 +4,6 @@ import com.gradle.maven.extension.api.GradleEnterpriseListener; import com.gradle.maven.extension.api.scan.BuildScanApi; import org.apache.maven.execution.MavenSession; -import org.apache.maven.project.MavenProject; import org.codehaus.plexus.logging.Logger; import javax.inject.Inject; @@ -20,12 +19,10 @@ public class CaptureBuildScansListener implements GradleEnterpriseListener { private static final String EXPERIMENT_DIR = System.getProperty("com.gradle.enterprise.build_validation.experimentDir"); - private final RootProjectExtractor rootProjectExtractor; private final Logger logger; @Inject - public CaptureBuildScansListener(RootProjectExtractor rootProjectExtractor, Logger logger) { - this.rootProjectExtractor = rootProjectExtractor; + public CaptureBuildScansListener(Logger logger) { this.logger = logger; } @@ -36,7 +33,7 @@ public void configure(GradleEnterpriseApi api, MavenSession session) throws Exce BuildScanApi buildScan = api.getBuildScan(); addCustomDataOnBuildFinished(buildScan); - capturePublishedBuildScan(buildScan, rootProjectExtractor.extractRootProject(session)); + capturePublishedBuildScan(buildScan); } private static void addCustomDataOnBuildFinished(BuildScanApi buildScan) { @@ -71,7 +68,7 @@ private static String urlEncode(String str) { } } - private void capturePublishedBuildScan(BuildScanApi buildScan, MavenProject rootProject) { + private void capturePublishedBuildScan(BuildScanApi buildScan) { buildScan.buildScanPublished(scan -> { logger.debug("Saving build scan data to build-scans.csv"); String port = scan.getBuildScanUri().getPort() != -1 ? ":" + scan.getBuildScanUri().getPort() : ""; @@ -80,7 +77,7 @@ private void capturePublishedBuildScan(BuildScanApi buildScan, MavenProject root try (FileWriter fw = new FileWriter(EXPERIMENT_DIR + "/build-scans.csv", true); BufferedWriter bw = new BufferedWriter(fw); PrintWriter out = new PrintWriter(bw)) { - out.println(String.format("%s,%s,%s,%s", rootProject.getName(), baseUrl, scan.getBuildScanUri(), scan.getBuildScanId())); + out.println(String.format("%s,%s,%s", baseUrl, scan.getBuildScanUri(), scan.getBuildScanId())); } catch (IOException e) { logger.error("Unable to save scan data to build-scans.csv: " + e.getMessage(), e); throw new RuntimeException("Unable to save scan data to build-scans.csv: " + e.getMessage(), e); diff --git a/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/RootProjectExtractor.java b/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/RootProjectExtractor.java deleted file mode 100644 index c93864e2..00000000 --- a/components/capture-build-scan-url-maven-extension/src/main/java/com/gradle/RootProjectExtractor.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.gradle; - -import org.apache.maven.execution.ExecutionEvent; -import org.apache.maven.execution.MavenExecutionRequest; -import org.apache.maven.execution.MavenSession; -import org.apache.maven.model.building.ModelProcessor; -import org.apache.maven.project.MavenProject; -import org.apache.maven.project.ProjectBuilder; -import org.apache.maven.project.ProjectBuildingException; -import org.codehaus.plexus.logging.Logger; - -import javax.inject.Inject; -import java.io.File; -import java.util.*; - -import static java.util.Collections.emptyList; - -public class RootProjectExtractor { - private final ProjectBuilder projectBuilder; - private final ModelProcessor modelProcessor; - private final Logger logger; - - @Inject - public RootProjectExtractor(ProjectBuilder projectBuilder, ModelProcessor modelProcessor, Logger logger) { - this.projectBuilder = projectBuilder; - this.modelProcessor = modelProcessor; - this.logger = logger; - } - - public MavenProject extractRootProject(ExecutionEvent event) { - return extractRootProject(event.getSession()); - } - - public MavenProject extractRootProject(MavenSession session) { - List allProjects = discoverAllProjects(session.getAllProjects()); - File workspaceDirectory = getWorkspaceDirectory(session); - - if (workspaceDirectory.equals(allProjects.get(0).getBasedir())) { - return allProjects.get(0); - } - - File workspaceDirectoryPom = modelProcessor.locatePom(workspaceDirectory); - if (workspaceDirectoryPom.exists()) { - try { - return projectBuilder.build(workspaceDirectoryPom, session.getProjectBuildingRequest()).getProject(); - } catch (ProjectBuildingException e) { - logger.error("Exception locating the top level project", e); - } - } - - // We didn't successfully identify the root project, so just return the first project. - return allProjects.get(0); - } - - private File getWorkspaceDirectory(MavenSession session) { - MavenExecutionRequest request = session.getRequest(); - try { - return request.getMultiModuleProjectDirectory(); - } catch (NoSuchMethodError ignored) { - return new File(session.getExecutionRootDirectory()); - } - } - - /** - * Older Maven versions under-reported the list of discovered projects. - * This method discovers all their submodules. For newer Maven versions it is a no-op. - */ - private List discoverAllProjects(Collection sessionProjects) { - Set allProjects = new LinkedHashSet<>(sessionProjects); - sessionProjects.stream().flatMap(p -> Optional.ofNullable(p.getCollectedProjects()).orElse(emptyList()).stream()).forEach(allProjects::add); - return new ArrayList(allProjects); - } -} From 9fcdd7646ff6e68fa1fe84f9b6338cdee02b972e Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Thu, 29 Dec 2022 09:44:53 -0600 Subject: [PATCH 6/9] Introduce helper method for processing Build Scan data using Gradle Enterprise API --- .../scripts/gradle/01-validate-incremental-building.sh | 3 +-- .../02-validate-local-build-caching-same-location.sh | 3 +-- .../03-validate-local-build-caching-different-locations.sh | 3 +-- .../gradle/05-validate-remote-build-caching-ci-local.sh | 7 +------ components/scripts/lib/build_scan.sh | 7 +++++++ .../maven/01-validate-local-build-caching-same-location.sh | 3 +-- .../02-validate-local-build-caching-different-locations.sh | 3 +-- .../maven/04-validate-remote-build-caching-ci-local.sh | 7 +------ 8 files changed, 14 insertions(+), 22 deletions(-) diff --git a/components/scripts/gradle/01-validate-incremental-building.sh b/components/scripts/gradle/01-validate-incremental-building.sh index a9cd2e91..a0938574 100755 --- a/components/scripts/gradle/01-validate-incremental-building.sh +++ b/components/scripts/gradle/01-validate-incremental-building.sh @@ -155,8 +155,7 @@ print_gradle_command() { fetch_build_cache_metrics() { if [ "$build_scan_publishing_mode" == "on" ]; then - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" + process_build_scan_data_online else find_and_read_build_scan_dumps fi diff --git a/components/scripts/gradle/02-validate-local-build-caching-same-location.sh b/components/scripts/gradle/02-validate-local-build-caching-same-location.sh index 13606dfb..8a563301 100755 --- a/components/scripts/gradle/02-validate-local-build-caching-same-location.sh +++ b/components/scripts/gradle/02-validate-local-build-caching-same-location.sh @@ -158,8 +158,7 @@ print_gradle_command() { fetch_build_cache_metrics() { if [ "$build_scan_publishing_mode" == "on" ]; then - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" + process_build_scan_data_online else find_and_read_build_scan_dumps fi diff --git a/components/scripts/gradle/03-validate-local-build-caching-different-locations.sh b/components/scripts/gradle/03-validate-local-build-caching-different-locations.sh index d32f44d1..c82f75bf 100755 --- a/components/scripts/gradle/03-validate-local-build-caching-different-locations.sh +++ b/components/scripts/gradle/03-validate-local-build-caching-different-locations.sh @@ -163,8 +163,7 @@ print_gradle_command() { fetch_build_cache_metrics() { if [ "$build_scan_publishing_mode" == "on" ]; then - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" + process_build_scan_data_online else find_and_read_build_scan_dumps fi diff --git a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh index 8499ae30..7d5e186d 100755 --- a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh +++ b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh @@ -132,7 +132,7 @@ wizard_execute() { print_bl explain_measure_build_results print_bl - fetch_build_cache_metrics + process_build_scan_data_online print_bl explain_and_print_summary } @@ -213,11 +213,6 @@ print_experiment_specific_summary_info() { summary_row "Custom value mapping file:" "${mapping_file:-}" } -fetch_build_cache_metrics() { - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" -} - # Overrides info.sh#print_performance_metrics print_performance_metrics() { print_performance_characteristics diff --git a/components/scripts/lib/build_scan.sh b/components/scripts/lib/build_scan.sh index 2465f4d1..408308d5 100644 --- a/components/scripts/lib/build_scan.sh +++ b/components/scripts/lib/build_scan.sh @@ -2,6 +2,13 @@ readonly FETCH_BUILD_SCAN_DATA_JAR="${LIB_DIR}/export-api-clients/fetch-build-scan-data-cmdline-tool-${SCRIPT_VERSION}-all.jar" +# This is a helper method for the common pattern of reading Build Scan metadata from the build-scans.csv file, then +# retrieving build metrics using the Gradle Enterprise API. +process_build_scan_data_online() { + read_build_scan_metadata + fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" +} + read_build_scan_metadata() { # This isn't the most robust way to read a CSV, # but we control the CSV so we don't have to worry about various CSV edge cases diff --git a/components/scripts/maven/01-validate-local-build-caching-same-location.sh b/components/scripts/maven/01-validate-local-build-caching-same-location.sh index bc7e2857..80648673 100755 --- a/components/scripts/maven/01-validate-local-build-caching-same-location.sh +++ b/components/scripts/maven/01-validate-local-build-caching-same-location.sh @@ -159,8 +159,7 @@ print_maven_command() { fetch_build_cache_metrics() { if [ "$build_scan_publishing_mode" == "on" ]; then - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" + process_build_scan_data_online else find_and_read_build_scan_dumps fi diff --git a/components/scripts/maven/02-validate-local-build-caching-different-locations.sh b/components/scripts/maven/02-validate-local-build-caching-different-locations.sh index ce03e17e..7cdea1c1 100755 --- a/components/scripts/maven/02-validate-local-build-caching-different-locations.sh +++ b/components/scripts/maven/02-validate-local-build-caching-different-locations.sh @@ -163,8 +163,7 @@ print_maven_command() { fetch_build_cache_metrics() { if [ "$build_scan_publishing_mode" == "on" ]; then - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" + process_build_scan_data_online else find_and_read_build_scan_dumps fi diff --git a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh index 43b52946..a4b74f1a 100755 --- a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh +++ b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh @@ -130,7 +130,7 @@ wizard_execute() { print_bl explain_measure_build_results print_bl - fetch_build_cache_metrics + process_build_scan_data_online print_bl explain_and_print_summary } @@ -212,11 +212,6 @@ print_experiment_specific_summary_info() { summary_row "Custom value mapping file:" "${mapping_file:-}" } -fetch_build_cache_metrics() { - read_build_scan_metadata - fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" -} - # Overrides info.sh#print_performance_metrics print_performance_metrics() { print_performance_characteristics From 6a0fffce221ed877ef2e909c97f2855bbd050df1 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Thu, 29 Dec 2022 09:46:30 -0600 Subject: [PATCH 7/9] Reformat comment --- components/scripts/lib/build_scan.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/scripts/lib/build_scan.sh b/components/scripts/lib/build_scan.sh index 408308d5..bcc63255 100644 --- a/components/scripts/lib/build_scan.sh +++ b/components/scripts/lib/build_scan.sh @@ -2,8 +2,9 @@ readonly FETCH_BUILD_SCAN_DATA_JAR="${LIB_DIR}/export-api-clients/fetch-build-scan-data-cmdline-tool-${SCRIPT_VERSION}-all.jar" -# This is a helper method for the common pattern of reading Build Scan metadata from the build-scans.csv file, then -# retrieving build metrics using the Gradle Enterprise API. +# This is a helper method for the common pattern of reading Build Scan metadata +# from the build-scans.csv file, then retrieving build metrics using the Gradle +# Enterprise API. process_build_scan_data_online() { read_build_scan_metadata fetch_and_read_build_scan_data build_cache_metrics_only "${build_scan_urls[@]}" From fc15f8db33a25cf99abf93edf15f3afc9ecf2ce3 Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Thu, 29 Dec 2022 10:11:38 -0600 Subject: [PATCH 8/9] Add missing helper method calls --- .../scripts/gradle/05-validate-remote-build-caching-ci-local.sh | 2 +- .../scripts/maven/04-validate-remote-build-caching-ci-local.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh index 7d5e186d..0c2ae6c7 100755 --- a/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh +++ b/components/scripts/gradle/05-validate-remote-build-caching-ci-local.sh @@ -67,7 +67,7 @@ execute() { execute_build print_bl - fetch_build_cache_metrics + process_build_scan_data_online print_bl print_summary diff --git a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh index a4b74f1a..194d4803 100755 --- a/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh +++ b/components/scripts/maven/04-validate-remote-build-caching-ci-local.sh @@ -65,7 +65,7 @@ execute() { execute_build print_bl - fetch_build_cache_metrics + process_build_scan_data_online print_bl print_summary From c912797995e17924300972a2abbfad957691bd6e Mon Sep 17 00:00:00 2001 From: Eric Haag Date: Mon, 9 Jan 2023 12:17:43 -0600 Subject: [PATCH 9/9] Reword method to function Co-authored-by: Jim Hurne --- components/scripts/lib/build_scan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/scripts/lib/build_scan.sh b/components/scripts/lib/build_scan.sh index bcc63255..bcd5d75b 100644 --- a/components/scripts/lib/build_scan.sh +++ b/components/scripts/lib/build_scan.sh @@ -2,7 +2,7 @@ readonly FETCH_BUILD_SCAN_DATA_JAR="${LIB_DIR}/export-api-clients/fetch-build-scan-data-cmdline-tool-${SCRIPT_VERSION}-all.jar" -# This is a helper method for the common pattern of reading Build Scan metadata +# This is a helper function for the common pattern of reading Build Scan metadata # from the build-scans.csv file, then retrieving build metrics using the Gradle # Enterprise API. process_build_scan_data_online() {