From 70e98028609e7cb16a56f4213688332107225bac Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 20 Jun 2022 16:27:19 +0200 Subject: [PATCH 01/17] WIP snyk --- build-tools-internal/build.gradle | 4 + ...dencyMonitoringGradlePluginFuncTest.groovy | 66 +++++++++++++++ .../snyk/GenerateSnykDependencyGraph.java | 72 ++++++++++++++++ .../SnykDependencyMonitoringGradlePlugin.java | 30 +++++++ .../gradle/internal/snyk/SnykGraph.java | 82 +++++++++++++++++++ .../fixtures/AbstractGradleFuncTest.groovy | 14 ++-- ...ationCacheCompatibleAwareGradleRunner.java | 14 ++-- 7 files changed, 265 insertions(+), 17 deletions(-) create mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index c16d95c4d38f0..3aaec1a4f916f 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -135,6 +135,10 @@ gradlePlugin { id = 'elasticsearch.rest-resources' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin' } + snykDependenciesMonitoring { + id = 'elasticsearch.snyk-dependency-monitoring' + implementationClass = 'org.elasticsearch.gradle.internal.snyk.SnykDependencyMonitoringGradlePlugin' + } standaloneRestTest { id = 'elasticsearch.standalone-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.StandaloneRestTestPlugin' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy new file mode 100644 index 0000000000000..51899ac81805e --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome + +class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTest { + + def setup() { + buildFile << """ + plugins { + id 'elasticsearch.snyk-dependency-monitoring' + } + + def printGraph(Map entries, int depth = 0 ) { +// entries.each { k, v -> +// println k.indent(depth) +// println v.getClass() +//// printGraph(v, depth++) +// } + } + + tasks.named('resolveSnykDependencyGraph').configure { + doLast { + println graph + printGraph(graph.nodes) + } + } + + """ + configurationCacheCompatible = false // configuration is not cc compliant + + } + + def "can calculate snyk dependency graph"() { + given: +"".indent(3) + buildFile << """ + apply plugin:'java' + + repositories { + mavenCentral() + } + + dependencies { + implementation 'org.apache.lucene:lucene-monitor:9.2.0' + } + + tasks.named('resolveSnykDependencyGraph').configure { + configuration = configurations.runtimeClasspath + } + """ + when: + def build = gradleRunner("resolveSnykDependencyGraph").build() + + then: + build.task(":resolveSnykDependencyGraph").outcome == TaskOutcome.SUCCESS + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java new file mode 100644 index 0000000000000..cf0324456dfbe --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk; + +import org.gradle.api.artifacts.Configuration; + +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.ResolvedDependency; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.TaskAction; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class GenerateSnykDependencyGraph extends DefaultTask { + + private Configuration configuration; + + private final SnykGraph graph = new SnykGraph("root-node"); + + @Internal + public SnykGraph getGraph() { + return graph; + } + + @TaskAction + void resolveGraph() { + Set firstLevelModuleDependencies = configuration.getResolvedConfiguration().getFirstLevelModuleDependencies(); + createGraph(firstLevelModuleDependencies); + } + + private Map createGraph(Iterable deps) { + String rootId = "root-node"; + Set currentChain = new HashSet(); + loadGraph(deps, graph, rootId, currentChain); + return graph.nodes; + } + + private void loadGraph(Iterable deps, SnykGraph graph, String parentId, Set currentChain) { + System.out.println("GenerateSnykDependencyGraph.loadGraph"); + System.out.println("parentId = " + parentId); + deps.forEach( dep -> { + String childId = dep.getModuleGroup() + ":" + dep.getModuleName() + "@" + dep.getModuleVersion(); + if (graph.getNodes().get(childId) == null) { + Map childDependency = Map.of("name", dep.getModuleGroup() + + ":" + dep.getModuleName(), "version", dep.getModuleVersion() + ); + graph.setNode(childId, childDependency); + } + // In Gradle 2, there can be several instances of the same dependency present at each level, + // each for a different configuration. In this case, we need to merge the dependencies. + if (currentChain.contains(childId) == false && dep.getChildren() != null ) { + currentChain.add(childId); + loadGraph(dep.getChildren(), graph, childId, currentChain); + } + graph.setEdge(parentId, childId); + + }); + } + + public void setConfiguration(Configuration config) { + this.configuration = config; + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java new file mode 100644 index 0000000000000..18959f9b60436 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk; + +import org.gradle.api.Action; +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class SnykDependencyMonitoringGradlePlugin implements Plugin { + + @Override + public void apply(Project project) { + var task = project.getTasks().register("resolveSnykDependencyGraph", GenerateSnykDependencyGraph.class); + project.getTasks().register("uploadSnykDependencyGraph"); + var config = project.getConfigurations().create("someConfig"); + task.configure(new Action() { + @Override + public void execute(GenerateSnykDependencyGraph generateSnykDependencyGraph) { + generateSnykDependencyGraph.setConfiguration(config); + } + }); + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java new file mode 100644 index 0000000000000..f00d379e29106 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk; + +import com.google.common.collect.Maps; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class SnykGraph { + + Map nodes = Maps.newLinkedHashMap(); + + String rootId; + + public SnykGraph(String rootId) { + this.rootId = rootId; + } + + public Map setNode(String key, Map value) { + if (key == null) { + return null; + } + + if (nodes.containsKey(key)) { + return (Map) this.nodes.get(key); + } + + if (value == null) { + return null; + } + // + Map vertex = Map.of("name", value.get("name"), "version", value.get("version"), "parentIds", new HashSet<>()); + // + return (Map) nodes.put(key, vertex); + + } + + public Map getNodes() { + return nodes; + } + + public String getRootId() { + return rootId; + } + + public void setEdge(String parentId, String childId) { + if (parentId == null || childId == null || parentId == childId) { + return; + } + // root-node will be the graphlib root that first-level deps will be attached to + if (parentId != this.rootId) { + Map parentNode = this.setNode(parentId, null); + if (parentNode == null) { + return; + } + } + var childNode = this.setNode(childId, null); + if (childNode == null || ((Set) childNode.get("parentIds")).contains(parentId)) { + return; + } + ((Set) childNode.get("parentIds")).add(parentId); + } + + @Override + public String toString() { + String collect = nodes.entrySet().stream().map(stringObjectEntry -> stringObjectEntry.getKey() + ": " + stringObjectEntry.getValue()).collect(Collectors.joining("\n")); + + return "SnykGraph{" + + "nodes=\n" + collect + + ", rootId='" + rootId + '\'' + + '}'; + } +} diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 755c866dafe8c..8ed9409857faf 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -34,7 +34,7 @@ abstract class AbstractGradleFuncTest extends Specification { File propertiesFile File projectDir - boolean configurationCacheCompatible = false + public boolean configurationCacheCompatible = false def setup() { projectDir = testProjectDir.root @@ -43,7 +43,7 @@ abstract class AbstractGradleFuncTest extends Specification { buildFile = testProjectDir.newFile('build.gradle') propertiesFile = testProjectDir.newFile('gradle.properties') propertiesFile << - "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" + "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" } def cleanup() { @@ -77,15 +77,13 @@ abstract class AbstractGradleFuncTest extends Specification { new ConfigurationCacheCompatibleAwareGradleRunner( new InternalAwareGradleRunner( GradleRunner.create() - .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments() - .toString().indexOf("-agentlib:jdwp") > 0 - ) + .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0) .withProjectDir(projectDir) .withPluginClasspath() .forwardOutput() - ), configurationCacheCompatible), - projectDir - ).withArguments(arguments) + ), + configurationCacheCompatible), + projectDir).withArguments(arguments) } def assertOutputContains(String givenOutput, String expected) { diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java index 8a195fa15c62a..8f52a270af8bb 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java @@ -18,13 +18,11 @@ import java.io.File; import java.io.Writer; import java.net.URI; -import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; -/** - * A Gradle runner that delegates to another runner, optionally enabling the configuring cache parameter. - */ public class ConfigurationCacheCompatibleAwareGradleRunner extends GradleRunner { private GradleRunner delegate; private boolean ccCompatible; @@ -76,11 +74,9 @@ public List getArguments() { @Override public GradleRunner withArguments(List arguments) { - List effectiveArgs = arguments; - if (ccCompatible) { - effectiveArgs = new ArrayList<>(arguments); - effectiveArgs.add("--configuration-cache"); - } + List effectiveArgs = ccCompatible + ? Stream.concat(arguments.stream(), Stream.of("--configuration-cache")).collect(Collectors.toList()) + : arguments; delegate.withArguments(effectiveArgs); return this; } From 5f7bc3e25431754edea419edc3ed5b58af19bc2e Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 20 Jun 2022 23:52:35 +0200 Subject: [PATCH 02/17] Rework snyk dependencies monitoring Only apply snyk check for production projects --- .../internal/conventions/info/GitInfo.java | 1 - ...dencyMonitoringGradlePluginFuncTest.groovy | 138 +++++++++++++++--- .../snyk/GenerateSnykDependencyGraph.java | 133 ++++++++++++----- .../internal/snyk/SnykDependencyGraph.java | 121 +++++++++++++++ .../SnykDependencyMonitoringGradlePlugin.java | 47 +++++- .../gradle/internal/snyk/SnykGraph.java | 44 ++++-- .../internal/snyk/SnykModelBuilder.java | 64 ++++++++ .../snyk/UploadSnykDependenciesGraph.java | 82 +++++++++++ build.gradle | 8 +- 9 files changed, 565 insertions(+), 73 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java diff --git a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java index 7ada49bdbd85e..1c9db2584d03d 100644 --- a/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java +++ b/build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java @@ -168,7 +168,6 @@ private static String readFirstLine(final Path path) throws IOException { /** Find the reponame. */ public String urlFromOrigin() { - String oritin = getOrigin(); if (origin == null) { return null; // best effort, the url doesnt really matter, it is just required by maven central } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 51899ac81805e..bae1c9e76f371 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -18,25 +18,8 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes plugins { id 'elasticsearch.snyk-dependency-monitoring' } - - def printGraph(Map entries, int depth = 0 ) { -// entries.each { k, v -> -// println k.indent(depth) -// println v.getClass() -//// printGraph(v, depth++) -// } - } - - tasks.named('resolveSnykDependencyGraph').configure { - doLast { - println graph - printGraph(graph.nodes) - } - } - """ configurationCacheCompatible = false // configuration is not cc compliant - } def "can calculate snyk dependency graph"() { @@ -58,9 +41,126 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes } """ when: - def build = gradleRunner("resolveSnykDependencyGraph").build() - + def build = gradleRunner("generateSnykDependencyGraph", "-i", "--stacktrace").build() then: build.task(":resolveSnykDependencyGraph").outcome == TaskOutcome.SUCCESS + + normalized(file("build/snyk/dependencies.json").text) == """{ + "meta": { + "method": "custom gradle", + "id": "gradle", + "node": "v16.15.1", + "name": "gradle", + "plugin": "extern:gradle", + "pluginRuntime": "unknown", + "monitorGraph": true, + "version": "unspecified", + "versionBuildInfo": { + "gradleVersion": "7.4.2" + } + }, + "depGraphJSON": { + "pkgManager": { + "version": "7.4.2", + "name": "gradle" + }, + "schemaVersion": "1.2.0", + "graph": { + "rootNodeId": "root-node", + "nodes": [ + { + "nodeId": "root-node", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-monitor@9.2.0" + } + ], + "pkgId": "hello-world@unspecified" + }, + { + "nodeId": "org.apache.lucene:lucene-monitor@9.2.0", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-memory@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-analysis-common@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0" + } + ], + "pkgId": "org.apache.lucene:lucene-monitor@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-memory@9.2.0", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0" + } + ], + "pkgId": "org.apache.lucene:lucene-memory@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0", + "deps": [ + + ], + "pkgId": "org.apache.lucene:lucene-core@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-analysis-common@9.2.0", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0" + } + ], + "pkgId": "org.apache.lucene:lucene-analysis-common@9.2.0" + } + ] + }, + "pkgs": [ + { + "id": "hello-world@unspecified", + "info": { + "name": "hello-world", + "version": "unspecified" + } + }, + { + "id": "org.apache.lucene:lucene-monitor@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-monitor", + "version": "9.2.0" + } + }, + { + "id": "org.apache.lucene:lucene-memory@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-memory", + "version": "9.2.0" + } + }, + { + "id": "org.apache.lucene:lucene-core@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-core", + "version": "9.2.0" + } + }, + { + "id": "org.apache.lucene:lucene-analysis-common@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-analysis-common", + "version": "9.2.0" + } + } + ] + }, + "target": { + "remoteUrl": "http://github.com/elastic/elasticsearch.git", + "branch": "unknown" + } +}""" } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index cf0324456dfbe..731396710679b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -8,22 +8,55 @@ package org.elasticsearch.gradle.internal.snyk; -import org.gradle.api.artifacts.Configuration; +import groovy.json.JsonOutput; +import org.elasticsearch.gradle.internal.conventions.info.GitInfo; import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ResolvedDependency; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; +import org.gradle.initialization.layout.BuildLayout; -import java.util.HashSet; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; -public class GenerateSnykDependencyGraph extends DefaultTask { +import javax.inject.Inject; - private Configuration configuration; +public class GenerateSnykDependencyGraph extends DefaultTask { + private final Property configuration; + private final Property projectName; + private final Property projectPath; + private final Property version; + private final Property gradleVersion; private final SnykGraph graph = new SnykGraph("root-node"); + private final RegularFileProperty outputFile; + private BuildLayout buildLayout; + + private String jsonOutput = null; + + @Inject + public GenerateSnykDependencyGraph(ObjectFactory objectFactory, BuildLayout buildLayout) { + configuration = objectFactory.property(Configuration.class); + projectName = objectFactory.property(String.class); + projectPath = objectFactory.property(String.class); + version = objectFactory.property(String.class); + gradleVersion = objectFactory.property(String.class); + outputFile = objectFactory.fileProperty(); + this.buildLayout = buildLayout; + } @Internal public SnykGraph getGraph() { @@ -32,41 +65,75 @@ public SnykGraph getGraph() { @TaskAction void resolveGraph() { - Set firstLevelModuleDependencies = configuration.getResolvedConfiguration().getFirstLevelModuleDependencies(); - createGraph(firstLevelModuleDependencies); + Set firstLevelModuleDependencies = configuration.get() + .getResolvedConfiguration() + .getFirstLevelModuleDependencies(); + SnykModelBuilder builder = new SnykModelBuilder(gradleVersion.get()); + createGraph(firstLevelModuleDependencies, builder); + Map payload = new LinkedHashMap<>(); + payload.put("meta", generateMetaData()); + payload.put("depGraphJSON", builder.build()); + payload.put("target", buildTargetData()); + jsonOutput = JsonOutput.prettyPrint(JsonOutput.toJson(payload)); + try { + Files.writeString(getOutputFile().getAsFile().get().toPath(), jsonOutput, StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new GradleException("Cannot generate dependencies json file", e); + } + } + + private Object buildTargetData() { + Map target = new LinkedHashMap<>(); + target.put("remoteUrl", "http://github.com/elastic/elasticsearch.git"); + target.put("branch", GitInfo.gitInfo(buildLayout.getRootDirectory()).getRevision()); + return target; + } + + private Map generateMetaData() { + Map metaData = new LinkedHashMap<>(); +// metaData.put("method", "custom gradle"); +// metaData.put("id", "gradle"); +// metaData.put("node", "v16.15.1"); +// metaData.put("name", "gradle"); +// metaData.put("plugin", "extern:gradle"); +// metaData.put("pluginRuntime", "unknown"); +// metaData.put("monitorGraph", true); +// metaData.put("version", version.get()); + return metaData; + } + + private void createGraph(Set deps, SnykModelBuilder builder) { + builder.walkGraph((projectPath.equals(":") ? projectName.get() : projectPath.get()), version.get(), deps); + } + + @InputFiles + public Property getConfiguration() { + return configuration; + } + + @OutputFile + public RegularFileProperty getOutputFile() { + return outputFile; + } + + @Input + public Property getProjectPath() { + return projectPath; } - private Map createGraph(Iterable deps) { - String rootId = "root-node"; - Set currentChain = new HashSet(); - loadGraph(deps, graph, rootId, currentChain); - return graph.nodes; + @Input + public Property getVersion() { + return version; } - private void loadGraph(Iterable deps, SnykGraph graph, String parentId, Set currentChain) { - System.out.println("GenerateSnykDependencyGraph.loadGraph"); - System.out.println("parentId = " + parentId); - deps.forEach( dep -> { - String childId = dep.getModuleGroup() + ":" + dep.getModuleName() + "@" + dep.getModuleVersion(); - if (graph.getNodes().get(childId) == null) { - Map childDependency = Map.of("name", dep.getModuleGroup() + - ":" + dep.getModuleName(), "version", dep.getModuleVersion() - ); - graph.setNode(childId, childDependency); - } - // In Gradle 2, there can be several instances of the same dependency present at each level, - // each for a different configuration. In this case, we need to merge the dependencies. - if (currentChain.contains(childId) == false && dep.getChildren() != null ) { - currentChain.add(childId); - loadGraph(dep.getChildren(), graph, childId, currentChain); - } - graph.setEdge(parentId, childId); - - }); + @Input + public Property getProjectName() { + return projectName; } - public void setConfiguration(Configuration config) { - this.configuration = config; + @Input + public Property getGradleVersion() { + return gradleVersion; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java new file mode 100644 index 0000000000000..35f65fb6c4c59 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class SnykDependencyGraph { + private final String schemaVersion = "1.2.0"; + private final Map graph; + private final Set pkgs; + private final Map pkgManager; + + public SnykDependencyGraph(String gradleVersion, Set nodes, Set pkgs) { + this.pkgs = pkgs; + this.graph = new LinkedHashMap(); + graph.put("rootNodeId", "root-node"); + graph.put("nodes", nodes); + this.pkgManager = Map.of("name", "gradle", "version", gradleVersion); + } + + public String getSchemaVersion() { + return schemaVersion; + } + + public Map getPkgManager() { + return pkgManager; + } + + public Map getGraph() { + return graph; + } + + public Set getPkgs() { + return pkgs; + } + + static class SnykDependencyNode { + private String nodeId; + private String pkgId; + private Set> deps = new LinkedHashSet<>(); + + SnykDependencyNode(String nodeId, String pkgId) { + this.nodeId = nodeId; + this.pkgId = pkgId; + } + + public void addDep(String pkg) { + deps.add(Map.of("nodeId", pkg)); + } + + public String getNodeId() { + return nodeId; + } + + public String getPkgId() { + return pkgId; + } + + public Set> getDeps() { + return deps; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SnykDependencyNode that = (SnykDependencyNode) o; + return Objects.equals(nodeId, that.nodeId) && Objects.equals(pkgId, that.pkgId) && Objects.equals(deps, that.deps); + } + + @Override + public int hashCode() { + return Objects.hash(nodeId, pkgId, deps); + } + } + + static class SnykDependencyPkg { + private String id; + private Map info = new LinkedHashMap<>(); + + SnykDependencyPkg(String pkgId) { + id = pkgId; + String name = id.substring(0, id.indexOf('@')); + String version = id.substring(id.indexOf('@') + 1); + info.put("name", name); + info.put("version", version); + } + + public String getId() { + return id; + } + + public Map getInfo() { + return info; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SnykDependencyPkg that = (SnykDependencyPkg) o; + return Objects.equals(id, that.id) && Objects.equals(info, that.info); + } + + @Override + public int hashCode() { + return Objects.hash(id, info); + } + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index 18959f9b60436..2925317d94312 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -11,20 +11,51 @@ import org.gradle.api.Action; import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.ProjectLayout; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.plugins.JavaPluginExtension; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.tasks.SourceSet; + +import javax.inject.Inject; public class SnykDependencyMonitoringGradlePlugin implements Plugin { + private ProjectLayout projectLayout; + private ProviderFactory providerFactory; + + @Inject + public SnykDependencyMonitoringGradlePlugin(ProjectLayout projectLayout, ProviderFactory providerFactory) { + this.projectLayout = projectLayout; + this.providerFactory = providerFactory; + } + @Override public void apply(Project project) { - var task = project.getTasks().register("resolveSnykDependencyGraph", GenerateSnykDependencyGraph.class); - project.getTasks().register("uploadSnykDependencyGraph"); - var config = project.getConfigurations().create("someConfig"); - task.configure(new Action() { - @Override - public void execute(GenerateSnykDependencyGraph generateSnykDependencyGraph) { - generateSnykDependencyGraph.setConfiguration(config); - } + var generateTaskProvider = project.getTasks() + .register("generateSnykDependencyGraph", GenerateSnykDependencyGraph.class, generateSnykDependencyGraph -> { + generateSnykDependencyGraph.getProjectPath().set(project.getPath()); + generateSnykDependencyGraph.getProjectName().set(project.getName()); + generateSnykDependencyGraph.getVersion().set(project.getVersion().toString()); + generateSnykDependencyGraph.getGradleVersion().set(project.getGradle().getGradleVersion()); + generateSnykDependencyGraph.getOutputFile().set(projectLayout.getBuildDirectory().file("snyk/dependencies.json")); + }); + + project.getTasks().register("uploadSnykDependencyGraph", UploadSnykDependenciesGraph.class, t -> { + t.getInputFile().set(generateTaskProvider.get().getOutputFile()); + t.getUrl().set("https://snyk.io/api/v1/monitor/gradle/graph"); + t.getToken().set(providerFactory.gradleProperty("snykToken")); +// the elasticsearch snyk project id +// t.getProjectId().set("f27934bf-9ad1-4d91-901c-cb77168a34db"); }); + + project.getPlugins().withType(JavaPlugin.class, javaPlugin -> generateTaskProvider.configure(generateSnykDependencyGraph -> { + SourceSet main = project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME); + Configuration runtimeConfiguration = project.getConfigurations().getByName(main.getRuntimeClasspathConfigurationName()); + generateSnykDependencyGraph.getConfiguration().set(runtimeConfiguration); + })); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java index f00d379e29106..e3545020bb336 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java @@ -10,7 +10,8 @@ import com.google.common.collect.Maps; -import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -29,17 +30,16 @@ public Map setNode(String key, Map value) { if (key == null) { return null; } - if (nodes.containsKey(key)) { return (Map) this.nodes.get(key); } - if (value == null) { return null; } - // - Map vertex = Map.of("name", value.get("name"), "version", value.get("version"), "parentIds", new HashSet<>()); - // + Map vertex = Maps.newLinkedHashMapWithExpectedSize(3); + vertex.put("name", value.get("name")); + vertex.put("version", value.get("version")); + vertex.put("parentIds", new LinkedHashSet<>()); return (Map) nodes.put(key, vertex); } @@ -72,11 +72,33 @@ public void setEdge(String parentId, String childId) { @Override public String toString() { - String collect = nodes.entrySet().stream().map(stringObjectEntry -> stringObjectEntry.getKey() + ": " + stringObjectEntry.getValue()).collect(Collectors.joining("\n")); + String collect = nodes.entrySet() + .stream() + .map(stringObjectEntry -> stringObjectEntry.getKey() + ": " + stringObjectEntry.getValue()) + .collect(Collectors.joining("\n")); + + return "SnykGraph{" + "nodes=\n" + collect + ", rootId='" + rootId + '\'' + '}'; + } - return "SnykGraph{" + - "nodes=\n" + collect + - ", rootId='" + rootId + '\'' + - '}'; + public void toSnykApiGraph() { + Set pkgs = new LinkedHashSet(); + Set nodes = new LinkedHashSet(); + getNodes().forEach((nodeId, nodeData) -> { + LinkedHashMap pkg = new LinkedHashMap<>(); + LinkedHashMap pkgInfo = new LinkedHashMap<>(); + pkgInfo.put("name", ((Map) nodeData).get("name")); + pkgInfo.put("version", ((Map) nodeData).get("version")); + pkg.put("id", nodeId); + pkg.put("info", pkgInfo); + pkgs.add(pkg); + + LinkedHashMap node = new LinkedHashMap<>(); + node.put("nodeId", nodeId); + node.put("pkgId", nodeId); + + LinkedHashMap deps = new LinkedHashMap<>(); + Object parentIds = ((Map) nodeData).get("parentIds"); + + }); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java new file mode 100644 index 0000000000000..80e04722bb73b --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk; + +import org.elasticsearch.gradle.internal.snyk.SnykDependencyGraph.SnykDependencyNode; +import org.gradle.api.artifacts.ResolvedDependency; + +import java.util.LinkedHashSet; +import java.util.Set; + +public class SnykModelBuilder { + + private Set nodes = new LinkedHashSet<>(); + private Set pkgs = new LinkedHashSet<>(); + + private SnykDependencyNode currentNode; + private String gradleVersion; + + public SnykModelBuilder(String gradleVersion) { + this.gradleVersion = gradleVersion; + } + + public SnykDependencyNode addNode(String nodeId, String pkgIdPrefix, String version) { + String pkgId = pkgIdPrefix + "@" + version; + SnykDependencyNode node = new SnykDependencyNode(nodeId, pkgId); + SnykDependencyGraph.SnykDependencyPkg pkg = new SnykDependencyGraph.SnykDependencyPkg(pkgId); + nodes.add(node); + if (currentNode != null) { + currentNode.addDep(pkgId); + } + pkgs.add(pkg); + return node; + } + + public SnykDependencyNode addDependency(ResolvedDependency dep) { + String pkgPrefix = dep.getModuleGroup() + ":" + dep.getModuleName(); + String nodeId = pkgPrefix + "@" + dep.getModuleVersion(); + return addNode(nodeId, pkgPrefix, dep.getModuleVersion()); + } + + private void loadGraph(SnykDependencyNode parent, Set deps) { + this.currentNode = parent; + deps.forEach(dep -> { + SnykDependencyGraph.SnykDependencyNode snykDependencyNode = addDependency(dep); + loadGraph(snykDependencyNode, dep.getChildren()); + this.currentNode = parent; + }); + } + + public SnykDependencyGraph build() { + return new SnykDependencyGraph(gradleVersion, nodes, pkgs); + } + + public void walkGraph(String rootPkgId, String version, Set deps) { + SnykDependencyNode root = addNode("root-node", rootPkgId, version); + loadGraph(root, deps); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java new file mode 100644 index 0000000000000..c56d1cda3e09c --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.snyk; + +import org.apache.commons.io.FileUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.TaskAction; + +import javax.inject.Inject; + +public class UploadSnykDependenciesGraph extends DefaultTask { + + private final RegularFileProperty inputFile; + private final Property token; + private final Property url; + private final Property projectId; + + @Inject + public UploadSnykDependenciesGraph(ObjectFactory objectFactory) { + url = objectFactory.property(String.class); + projectId = objectFactory.property(String.class); + token = objectFactory.property(String.class); + inputFile = objectFactory.fileProperty(); + } + + @TaskAction + void upload() { + String endpoint = url.map(u -> u + projectId.map(id -> "?org=" + id).getOrElse("")).get(); + try (CloseableHttpClient client = HttpClients.createDefault()) { + String content = FileUtils.readFileToString(inputFile.getAsFile().get()); + HttpPut putRequest = new HttpPut(endpoint); + putRequest.addHeader("Authorization", "token " + token.get()); + putRequest.addHeader("Content-Type", "application/json"); + putRequest.setEntity(new StringEntity(content)); + CloseableHttpResponse response = client.execute(putRequest); + getLogger().info("API call response status: " + response.getStatusLine().getStatusCode()); + getLogger().info(EntityUtils.toString(response.getEntity())); + } catch (Exception e) { + throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e); + } + } + + @Input + public Property getToken() { + return token; + } + + @Input + public Property getUrl() { + return url; + } + + @Input + @Optional + public Property getProjectId() { + return projectId; + } + + @InputFile + public RegularFileProperty getInputFile() { + return inputFile; + } +} diff --git a/build.gradle b/build.gradle index 6ccd88e443b01..194a641d8ed03 100644 --- a/build.gradle +++ b/build.gradle @@ -155,8 +155,14 @@ if (project.gradle.startParameter.taskNames.find { it.startsWith("checkPart") } bwc_tests_enabled = false } -subprojects { +subprojects { proj -> apply plugin: 'elasticsearch.base' + + plugins.withType(JavaPlugin) { + if(proj.getPath().contains(":qa:") == false) { + project.apply plugin: 'elasticsearch.snyk-dependency-monitoring' + } + } } allprojects { From 163d740bcdaa63233171b98ff51d3baad1af3788 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 23 Jun 2022 13:40:44 +0200 Subject: [PATCH 03/17] Fix issue with duplicate nodes in snyk graph --- ...dencyMonitoringGradlePluginFuncTest.groovy | 2 +- .../snyk/GenerateSnykDependencyGraph.java | 44 +++++++++++-------- .../internal/snyk/SnykDependencyGraph.java | 9 +++- .../SnykDependencyMonitoringGradlePlugin.java | 12 ++--- .../internal/snyk/SnykModelBuilder.java | 13 ++++++ .../snyk/UploadSnykDependenciesGraph.java | 15 +++++-- build.gradle | 2 +- 7 files changed, 65 insertions(+), 32 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index bae1c9e76f371..2cceddcf4538c 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -41,7 +41,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes } """ when: - def build = gradleRunner("generateSnykDependencyGraph", "-i", "--stacktrace").build() + def build = gradleRunner("generateSnykDependencyGraph").build() then: build.task(":resolveSnykDependencyGraph").outcome == TaskOutcome.SUCCESS diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index 731396710679b..45034c87ca33f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -65,21 +65,31 @@ public SnykGraph getGraph() { @TaskAction void resolveGraph() { + Map payload = generateGradleGraphPayload(); + jsonOutput = JsonOutput.prettyPrint(JsonOutput.toJson(payload)); + try { + Files.writeString( + getOutputFile().getAsFile().get().toPath(), + jsonOutput, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + } catch (IOException e) { + throw new GradleException("Cannot generate dependencies json file", e); + } + } + + private Map generateGradleGraphPayload() { Set firstLevelModuleDependencies = configuration.get() .getResolvedConfiguration() .getFirstLevelModuleDependencies(); SnykModelBuilder builder = new SnykModelBuilder(gradleVersion.get()); - createGraph(firstLevelModuleDependencies, builder); + builder.walkGraph((projectPath.equals(":") ? projectName.get() : projectPath.get()), version.get(), firstLevelModuleDependencies); Map payload = new LinkedHashMap<>(); payload.put("meta", generateMetaData()); payload.put("depGraphJSON", builder.build()); payload.put("target", buildTargetData()); - jsonOutput = JsonOutput.prettyPrint(JsonOutput.toJson(payload)); - try { - Files.writeString(getOutputFile().getAsFile().get().toPath(), jsonOutput, StandardOpenOption.TRUNCATE_EXISTING); - } catch (IOException e) { - throw new GradleException("Cannot generate dependencies json file", e); - } + return payload; } private Object buildTargetData() { @@ -91,21 +101,17 @@ private Object buildTargetData() { private Map generateMetaData() { Map metaData = new LinkedHashMap<>(); -// metaData.put("method", "custom gradle"); -// metaData.put("id", "gradle"); -// metaData.put("node", "v16.15.1"); -// metaData.put("name", "gradle"); -// metaData.put("plugin", "extern:gradle"); -// metaData.put("pluginRuntime", "unknown"); -// metaData.put("monitorGraph", true); -// metaData.put("version", version.get()); + metaData.put("method", "custom gradle"); + metaData.put("id", "gradle"); + metaData.put("node", "v16.15.1"); + metaData.put("name", "gradle"); + metaData.put("plugin", "extern:gradle"); + metaData.put("pluginRuntime", "unknown"); + metaData.put("monitorGraph", true); + // metaData.put("version", version.get()); return metaData; } - private void createGraph(Set deps, SnykModelBuilder builder) { - builder.walkGraph((projectPath.equals(":") ? projectName.get() : projectPath.get()), version.get(), deps); - } - @InputFiles public Property getConfiguration() { return configuration; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java index 35f65fb6c4c59..1b590b788890c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java @@ -75,12 +75,17 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; SnykDependencyNode that = (SnykDependencyNode) o; - return Objects.equals(nodeId, that.nodeId) && Objects.equals(pkgId, that.pkgId) && Objects.equals(deps, that.deps); + return Objects.equals(nodeId, that.nodeId) && Objects.equals(pkgId, that.pkgId); } @Override public int hashCode() { - return Objects.hash(nodeId, pkgId, deps); + return Objects.hash(nodeId, pkgId); + } + + @Override + public String toString() { + return "SnykDependencyNode{" + "nodeId='" + nodeId + '\'' + ", pkgId='" + pkgId + '\'' + '}'; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index 2925317d94312..63e3c05e05d8e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -8,12 +8,10 @@ package org.elasticsearch.gradle.internal.snyk; -import org.gradle.api.Action; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.file.ProjectLayout; -import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.plugins.JavaPluginExtension; import org.gradle.api.provider.ProviderFactory; @@ -45,14 +43,16 @@ public void apply(Project project) { project.getTasks().register("uploadSnykDependencyGraph", UploadSnykDependenciesGraph.class, t -> { t.getInputFile().set(generateTaskProvider.get().getOutputFile()); - t.getUrl().set("https://snyk.io/api/v1/monitor/gradle/graph"); t.getToken().set(providerFactory.gradleProperty("snykToken")); -// the elasticsearch snyk project id -// t.getProjectId().set("f27934bf-9ad1-4d91-901c-cb77168a34db"); + // the elasticsearch snyk project id + t.getProjectId().set("f27934bf-9ad1-4d91-901c-cb77168a34db"); }); project.getPlugins().withType(JavaPlugin.class, javaPlugin -> generateTaskProvider.configure(generateSnykDependencyGraph -> { - SourceSet main = project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME); + SourceSet main = project.getExtensions() + .getByType(JavaPluginExtension.class) + .getSourceSets() + .getByName(SourceSet.MAIN_SOURCE_SET_NAME); Configuration runtimeConfiguration = project.getConfigurations().getByName(main.getRuntimeClasspathConfigurationName()); generateSnykDependencyGraph.getConfiguration().set(runtimeConfiguration); })); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java index 80e04722bb73b..64792ff054029 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java @@ -11,8 +11,10 @@ import org.elasticsearch.gradle.internal.snyk.SnykDependencyGraph.SnykDependencyNode; import org.gradle.api.artifacts.ResolvedDependency; +import java.util.Comparator; import java.util.LinkedHashSet; import java.util.Set; +import java.util.function.Consumer; public class SnykModelBuilder { @@ -54,6 +56,17 @@ private void loadGraph(SnykDependencyNode parent, Set deps) } public SnykDependencyGraph build() { + nodes.stream().sorted(new Comparator() { + @Override + public int compare(SnykDependencyNode o1, SnykDependencyNode o2) { + return o1.getNodeId().compareTo(o2.getNodeId()); + } + }).forEach(new Consumer() { + @Override + public void accept(SnykDependencyNode snykDependencyNode) { + System.out.println("snykDependencyNode = " + snykDependencyNode); + } + }); return new SnykDependencyGraph(gradleVersion, nodes, pkgs); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index c56d1cda3e09c..7b61e013e18ff 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -29,6 +29,9 @@ public class UploadSnykDependenciesGraph extends DefaultTask { + public static final String GRADLE_GRAPH_ENDPOINT_URL = "https://snyk.io/api/v1/monitor/gradle/graph"; + public static final String SNYK_DEP_GRAPH_API_ENDPOINT_URL = "https://snyk.io/api/v1/monitor/dep-graph"; + private final RegularFileProperty inputFile; private final Property token; private final Property url; @@ -36,7 +39,7 @@ public class UploadSnykDependenciesGraph extends DefaultTask { @Inject public UploadSnykDependenciesGraph(ObjectFactory objectFactory) { - url = objectFactory.property(String.class); + url = objectFactory.property(String.class).convention(GRADLE_GRAPH_ENDPOINT_URL); projectId = objectFactory.property(String.class); token = objectFactory.property(String.class); inputFile = objectFactory.fileProperty(); @@ -44,7 +47,7 @@ public UploadSnykDependenciesGraph(ObjectFactory objectFactory) { @TaskAction void upload() { - String endpoint = url.map(u -> u + projectId.map(id -> "?org=" + id).getOrElse("")).get(); + String endpoint = calculateEffectiveEndpoint(); try (CloseableHttpClient client = HttpClients.createDefault()) { String content = FileUtils.readFileToString(inputFile.getAsFile().get()); HttpPut putRequest = new HttpPut(endpoint); @@ -52,13 +55,19 @@ void upload() { putRequest.addHeader("Content-Type", "application/json"); putRequest.setEntity(new StringEntity(content)); CloseableHttpResponse response = client.execute(putRequest); - getLogger().info("API call response status: " + response.getStatusLine().getStatusCode()); + int statusCode = response.getStatusLine().getStatusCode(); + getLogger().info("API call response status: " + statusCode); getLogger().info(EntityUtils.toString(response.getEntity())); } catch (Exception e) { throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e); } } + private String calculateEffectiveEndpoint() { + String url = this.url.get(); + return url.equals(GRADLE_GRAPH_ENDPOINT_URL) ? url : projectId.map(id -> url + "?org=" + id).getOrElse(url); + } + @Input public Property getToken() { return token; diff --git a/build.gradle b/build.gradle index 194a641d8ed03..1596e35110b7b 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ subprojects { proj -> apply plugin: 'elasticsearch.base' plugins.withType(JavaPlugin) { - if(proj.getPath().contains(":qa:") == false) { + if(proj.name != 'qa' && proj.getPath().contains(":qa:") == false) { project.apply plugin: 'elasticsearch.snyk-dependency-monitoring' } } From 42d6473c39a118e0445cac138b2cfce0ae2d45c2 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 23 Jun 2022 23:44:42 +0200 Subject: [PATCH 04/17] Simplify testing using json assert --- build-tools-internal/build.gradle | 1 + ...dencyMonitoringGradlePluginFuncTest.groovy | 213 +++++++++--------- .../snyk/GenerateSnykDependencyGraph.java | 65 +++--- .../internal/snyk/SnykDependencyGraph.java | 11 +- ...r.java => SnykDependencyGraphBuilder.java} | 17 +- .../gradle/internal/snyk/SnykGraph.java | 104 --------- .../snyk/UploadSnykDependenciesGraph.java | 6 +- gradle/build.versions.toml | 1 + 8 files changed, 151 insertions(+), 267 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/{SnykModelBuilder.java => SnykDependencyGraphBuilder.java} (78%) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 3aaec1a4f916f..1e4dcfeb7fffc 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -295,6 +295,7 @@ dependencies { exclude module: "groovy" } testImplementation buildLibs.spock.junit4 + testImplementation buildLibs.json.assert integTestImplementation buildLibs.xmlunit.core } diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 2cceddcf4538c..6ee46535abe83 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.snyk import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest import org.gradle.testkit.runner.TaskOutcome +import org.skyscreamer.jsonassert.JSONAssert class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTest { @@ -18,6 +19,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes plugins { id 'elasticsearch.snyk-dependency-monitoring' } + version = "1.0-SNAPSHOT" """ configurationCacheCompatible = false // configuration is not cc compliant } @@ -36,131 +38,126 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes implementation 'org.apache.lucene:lucene-monitor:9.2.0' } - tasks.named('resolveSnykDependencyGraph').configure { + tasks.named('generateSnykDependencyGraph').configure { configuration = configurations.runtimeClasspath } """ when: def build = gradleRunner("generateSnykDependencyGraph").build() then: - build.task(":resolveSnykDependencyGraph").outcome == TaskOutcome.SUCCESS - - normalized(file("build/snyk/dependencies.json").text) == """{ - "meta": { - "method": "custom gradle", - "id": "gradle", - "node": "v16.15.1", - "name": "gradle", - "plugin": "extern:gradle", - "pluginRuntime": "unknown", - "monitorGraph": true, - "version": "unspecified", - "versionBuildInfo": { - "gradleVersion": "7.4.2" - } - }, - "depGraphJSON": { - "pkgManager": { - "version": "7.4.2", - "name": "gradle" - }, - "schemaVersion": "1.2.0", - "graph": { - "rootNodeId": "root-node", - "nodes": [ - { - "nodeId": "root-node", - "deps": [ - { - "nodeId": "org.apache.lucene:lucene-monitor@9.2.0" - } - ], - "pkgId": "hello-world@unspecified" + build.task(":generateSnykDependencyGraph").outcome == TaskOutcome.SUCCESS + JSONAssert.assertEquals(file( "build/snyk/dependencies.json").text, """{ + "meta": { + "method": "custom gradle", + "id": "gradle", + "node": "v16.15.1", + "name": "gradle", + "plugin": "extern:gradle", + "pluginRuntime": "unknown", + "monitorGraph": true + }, + "depGraphJSON": { + "pkgManager": { + "version": "7.4.2", + "name": "gradle" }, - { - "nodeId": "org.apache.lucene:lucene-monitor@9.2.0", - "deps": [ + "schemaVersion": "1.2.0", + "graph": { + "rootNodeId": "root-node", + "nodes": [ { - "nodeId": "org.apache.lucene:lucene-memory@9.2.0" + "nodeId": "root-node", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-monitor@9.2.0" + } + ], + "pkgId": "hello-world@1.0-SNAPSHOT" }, { - "nodeId": "org.apache.lucene:lucene-analysis-common@9.2.0" + "nodeId": "org.apache.lucene:lucene-monitor@9.2.0", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-memory@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-analysis-common@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0" + } + ], + "pkgId": "org.apache.lucene:lucene-monitor@9.2.0" }, { - "nodeId": "org.apache.lucene:lucene-core@9.2.0" - } - ], - "pkgId": "org.apache.lucene:lucene-monitor@9.2.0" - }, - { - "nodeId": "org.apache.lucene:lucene-memory@9.2.0", - "deps": [ + "nodeId": "org.apache.lucene:lucene-memory@9.2.0", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0" + } + ], + "pkgId": "org.apache.lucene:lucene-memory@9.2.0" + }, + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0", + "deps": [ + + ], + "pkgId": "org.apache.lucene:lucene-core@9.2.0" + }, { - "nodeId": "org.apache.lucene:lucene-core@9.2.0" + "nodeId": "org.apache.lucene:lucene-analysis-common@9.2.0", + "deps": [ + { + "nodeId": "org.apache.lucene:lucene-core@9.2.0" + } + ], + "pkgId": "org.apache.lucene:lucene-analysis-common@9.2.0" } - ], - "pkgId": "org.apache.lucene:lucene-memory@9.2.0" + ] }, - { - "nodeId": "org.apache.lucene:lucene-core@9.2.0", - "deps": [ - - ], - "pkgId": "org.apache.lucene:lucene-core@9.2.0" - }, - { - "nodeId": "org.apache.lucene:lucene-analysis-common@9.2.0", - "deps": [ - { - "nodeId": "org.apache.lucene:lucene-core@9.2.0" + "pkgs": [ + { + "id": "hello-world@1.0-SNAPSHOT", + "info": { + "name": "hello-world", + "version": "1.0-SNAPSHOT" } - ], - "pkgId": "org.apache.lucene:lucene-analysis-common@9.2.0" - } - ] - }, - "pkgs": [ - { - "id": "hello-world@unspecified", - "info": { - "name": "hello-world", - "version": "unspecified" - } - }, - { - "id": "org.apache.lucene:lucene-monitor@9.2.0", - "info": { - "name": "org.apache.lucene:lucene-monitor", - "version": "9.2.0" - } - }, - { - "id": "org.apache.lucene:lucene-memory@9.2.0", - "info": { - "name": "org.apache.lucene:lucene-memory", - "version": "9.2.0" - } - }, - { - "id": "org.apache.lucene:lucene-core@9.2.0", - "info": { - "name": "org.apache.lucene:lucene-core", - "version": "9.2.0" - } + }, + { + "id": "org.apache.lucene:lucene-monitor@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-monitor", + "version": "9.2.0" + } + }, + { + "id": "org.apache.lucene:lucene-memory@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-memory", + "version": "9.2.0" + } + }, + { + "id": "org.apache.lucene:lucene-core@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-core", + "version": "9.2.0" + } + }, + { + "id": "org.apache.lucene:lucene-analysis-common@9.2.0", + "info": { + "name": "org.apache.lucene:lucene-analysis-common", + "version": "9.2.0" + } + } + ] }, - { - "id": "org.apache.lucene:lucene-analysis-common@9.2.0", - "info": { - "name": "org.apache.lucene:lucene-analysis-common", - "version": "9.2.0" - } + "target": { + "remoteUrl": "http://github.com/elastic/elasticsearch.git", + "branch": "unknown" } - ] - }, - "target": { - "remoteUrl": "http://github.com/elastic/elasticsearch.git", - "branch": "unknown" - } -}""" + }""", true) } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index 45034c87ca33f..9444e065e6584 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -20,7 +20,6 @@ import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; -import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.gradle.initialization.layout.BuildLayout; @@ -28,7 +27,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardOpenOption; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -41,11 +39,8 @@ public class GenerateSnykDependencyGraph extends DefaultTask { private final Property projectPath; private final Property version; private final Property gradleVersion; - private final SnykGraph graph = new SnykGraph("root-node"); private final RegularFileProperty outputFile; - private BuildLayout buildLayout; - - private String jsonOutput = null; + private final BuildLayout buildLayout; @Inject public GenerateSnykDependencyGraph(ObjectFactory objectFactory, BuildLayout buildLayout) { @@ -58,15 +53,10 @@ public GenerateSnykDependencyGraph(ObjectFactory objectFactory, BuildLayout buil this.buildLayout = buildLayout; } - @Internal - public SnykGraph getGraph() { - return graph; - } - @TaskAction void resolveGraph() { Map payload = generateGradleGraphPayload(); - jsonOutput = JsonOutput.prettyPrint(JsonOutput.toJson(payload)); + String jsonOutput = JsonOutput.prettyPrint(JsonOutput.toJson(payload)); try { Files.writeString( getOutputFile().getAsFile().get().toPath(), @@ -83,33 +73,42 @@ private Map generateGradleGraphPayload() { Set firstLevelModuleDependencies = configuration.get() .getResolvedConfiguration() .getFirstLevelModuleDependencies(); - SnykModelBuilder builder = new SnykModelBuilder(gradleVersion.get()); - builder.walkGraph((projectPath.equals(":") ? projectName.get() : projectPath.get()), version.get(), firstLevelModuleDependencies); - Map payload = new LinkedHashMap<>(); - payload.put("meta", generateMetaData()); - payload.put("depGraphJSON", builder.build()); - payload.put("target", buildTargetData()); - return payload; + SnykDependencyGraphBuilder builder = new SnykDependencyGraphBuilder(gradleVersion.get()); + String effectiveProjectPath = projectPath.get(); + builder.walkGraph( + (effectiveProjectPath.equals(":") ? projectName.get() : effectiveProjectPath), + version.get(), + firstLevelModuleDependencies + ); + return Map.of("meta", generateMetaData(), "depGraphJSON", builder.build(), "target", buildTargetData()); } private Object buildTargetData() { - Map target = new LinkedHashMap<>(); - target.put("remoteUrl", "http://github.com/elastic/elasticsearch.git"); - target.put("branch", GitInfo.gitInfo(buildLayout.getRootDirectory()).getRevision()); - return target; + return Map.of( + "remoteUrl", + "http://github.com/elastic/elasticsearch.git", + "branch", + GitInfo.gitInfo(buildLayout.getRootDirectory()).getRevision() + ); } private Map generateMetaData() { - Map metaData = new LinkedHashMap<>(); - metaData.put("method", "custom gradle"); - metaData.put("id", "gradle"); - metaData.put("node", "v16.15.1"); - metaData.put("name", "gradle"); - metaData.put("plugin", "extern:gradle"); - metaData.put("pluginRuntime", "unknown"); - metaData.put("monitorGraph", true); - // metaData.put("version", version.get()); - return metaData; + return Map.of( + "method", + "custom gradle", + "id", + "gradle", + "node", + "v16.15.1", + "name", + "gradle", + "plugin", + "extern:gradle", + "pluginRuntime", + "unknown", + "monitorGraph", + true + ); } @InputFiles diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java index 1b590b788890c..02f179cb995fb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java @@ -8,7 +8,7 @@ package org.elasticsearch.gradle.internal.snyk; -import java.util.LinkedHashMap; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Objects; @@ -22,7 +22,7 @@ public class SnykDependencyGraph { public SnykDependencyGraph(String gradleVersion, Set nodes, Set pkgs) { this.pkgs = pkgs; - this.graph = new LinkedHashMap(); + this.graph = new HashMap(); graph.put("rootNodeId", "root-node"); graph.put("nodes", nodes); this.pkgManager = Map.of("name", "gradle", "version", gradleVersion); @@ -90,15 +90,14 @@ public String toString() { } static class SnykDependencyPkg { - private String id; - private Map info = new LinkedHashMap<>(); + private final String id; + private final Map info; SnykDependencyPkg(String pkgId) { id = pkgId; String name = id.substring(0, id.indexOf('@')); String version = id.substring(id.indexOf('@') + 1); - info.put("name", name); - info.put("version", version); + info = Map.of("name", name, "version", version); } public String getId() { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java similarity index 78% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java index 64792ff054029..4210c375921ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykModelBuilder.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraphBuilder.java @@ -11,12 +11,10 @@ import org.elasticsearch.gradle.internal.snyk.SnykDependencyGraph.SnykDependencyNode; import org.gradle.api.artifacts.ResolvedDependency; -import java.util.Comparator; import java.util.LinkedHashSet; import java.util.Set; -import java.util.function.Consumer; -public class SnykModelBuilder { +public class SnykDependencyGraphBuilder { private Set nodes = new LinkedHashSet<>(); private Set pkgs = new LinkedHashSet<>(); @@ -24,7 +22,7 @@ public class SnykModelBuilder { private SnykDependencyNode currentNode; private String gradleVersion; - public SnykModelBuilder(String gradleVersion) { + public SnykDependencyGraphBuilder(String gradleVersion) { this.gradleVersion = gradleVersion; } @@ -56,17 +54,6 @@ private void loadGraph(SnykDependencyNode parent, Set deps) } public SnykDependencyGraph build() { - nodes.stream().sorted(new Comparator() { - @Override - public int compare(SnykDependencyNode o1, SnykDependencyNode o2) { - return o1.getNodeId().compareTo(o2.getNodeId()); - } - }).forEach(new Consumer() { - @Override - public void accept(SnykDependencyNode snykDependencyNode) { - System.out.println("snykDependencyNode = " + snykDependencyNode); - } - }); return new SnykDependencyGraph(gradleVersion, nodes, pkgs); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java deleted file mode 100644 index e3545020bb336..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykGraph.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.internal.snyk; - -import com.google.common.collect.Maps; - -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -public class SnykGraph { - - Map nodes = Maps.newLinkedHashMap(); - - String rootId; - - public SnykGraph(String rootId) { - this.rootId = rootId; - } - - public Map setNode(String key, Map value) { - if (key == null) { - return null; - } - if (nodes.containsKey(key)) { - return (Map) this.nodes.get(key); - } - if (value == null) { - return null; - } - Map vertex = Maps.newLinkedHashMapWithExpectedSize(3); - vertex.put("name", value.get("name")); - vertex.put("version", value.get("version")); - vertex.put("parentIds", new LinkedHashSet<>()); - return (Map) nodes.put(key, vertex); - - } - - public Map getNodes() { - return nodes; - } - - public String getRootId() { - return rootId; - } - - public void setEdge(String parentId, String childId) { - if (parentId == null || childId == null || parentId == childId) { - return; - } - // root-node will be the graphlib root that first-level deps will be attached to - if (parentId != this.rootId) { - Map parentNode = this.setNode(parentId, null); - if (parentNode == null) { - return; - } - } - var childNode = this.setNode(childId, null); - if (childNode == null || ((Set) childNode.get("parentIds")).contains(parentId)) { - return; - } - ((Set) childNode.get("parentIds")).add(parentId); - } - - @Override - public String toString() { - String collect = nodes.entrySet() - .stream() - .map(stringObjectEntry -> stringObjectEntry.getKey() + ": " + stringObjectEntry.getValue()) - .collect(Collectors.joining("\n")); - - return "SnykGraph{" + "nodes=\n" + collect + ", rootId='" + rootId + '\'' + '}'; - } - - public void toSnykApiGraph() { - Set pkgs = new LinkedHashSet(); - Set nodes = new LinkedHashSet(); - getNodes().forEach((nodeId, nodeData) -> { - LinkedHashMap pkg = new LinkedHashMap<>(); - LinkedHashMap pkgInfo = new LinkedHashMap<>(); - pkgInfo.put("name", ((Map) nodeData).get("name")); - pkgInfo.put("version", ((Map) nodeData).get("version")); - pkg.put("id", nodeId); - pkg.put("info", pkgInfo); - pkgs.add(pkg); - - LinkedHashMap node = new LinkedHashMap<>(); - node.put("nodeId", nodeId); - node.put("pkgId", nodeId); - - LinkedHashMap deps = new LinkedHashMap<>(); - Object parentIds = ((Map) nodeData).get("parentIds"); - - }); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index 7b61e013e18ff..509cbfee8a387 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -57,7 +57,11 @@ void upload() { CloseableHttpResponse response = client.execute(putRequest); int statusCode = response.getStatusLine().getStatusCode(); getLogger().info("API call response status: " + statusCode); - getLogger().info(EntityUtils.toString(response.getEntity())); + String responseString = EntityUtils.toString(response.getEntity()); + getLogger().info(responseString); + if (statusCode != 201) { + throw new GradleException("Uploading Snyk Graph failed with http code " + statusCode + ":" + responseString); + } } catch (Exception e) { throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e); } diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 8fcc2f069ff01..40ceeb034cf42 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -21,6 +21,7 @@ httpcore = "org.apache.httpcomponents:httpcore:4.4.12" httpclient = "org.apache.httpcomponents:httpclient:4.5.10" idea-ext = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.4" json-schema-validator = "com.networknt:json-schema-validator:1.0.49" +json-assert = "org.skyscreamer:jsonassert:1.5.0" jackson-dataformat-yaml = { group = "com.fasterxml.jackson.dataformat", name="jackson-dataformat-yaml", version.ref="jackson" } jackson-platform = { group = "com.fasterxml.jackson", name="jackson-bom", version.ref="jackson" } jna = "net.java.dev.jna:jna:5.10.0" From c329367a51798755e50d34163c6fd720b41e583b Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 08:44:32 +0200 Subject: [PATCH 05/17] Add test coverage for upload snyk graph --- .../fixtures/http/HttpServerRule.groovy | 92 +++++++++++++++++++ ...dencyMonitoringGradlePluginFuncTest.groovy | 28 +++++- .../snyk/UploadSnykDependenciesGraph.java | 16 ++-- 3 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy new file mode 100644 index 0000000000000..fba7124c5c0b8 --- /dev/null +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.fixtures.http + +import com.sun.net.httpserver.HttpContext +import com.sun.net.httpserver.HttpExchange +import com.sun.net.httpserver.HttpHandler +import com.sun.net.httpserver.HttpServer +import org.apache.commons.io.IOUtils +import org.junit.rules.ExternalResource + +import java.net.InetSocketAddress +import java.util.ArrayList +import java.util.List +import java.util.concurrent.Callable +import java.util.function.Consumer + +class HttpServerRule extends ExternalResource { + private static final int PORT = 6991 + + private HttpServer server + private List contexts = new ArrayList<>() + + @Override + protected void before() throws Throwable { + server = HttpServer.create(new InetSocketAddress(PORT), 0) // localhost:6991 + server.setExecutor(null) // creates a default executor + server.start() + } + + @Override + protected void after() { + if (server != null) { + server.stop(0) // doesn't wait all current exchange handlers complete + } + } + + String getUriFor(String path) { + if (path.startsWith("/") == false) { + path = "/" + path + } + String host = "http://localhost:" + PORT + return host + path + } + + void registerHandler(String uriToHandle, Closure configuration) { + registerHandler(uriToHandle, new Consumer() { + @Override + void accept(SimpleHttpHandler httpHandler) { + configuration.call(httpHandler) + } + }) + } + + void registerHandler(String uriToHandle, Consumer configuration) { + if(contexts.contains(uriToHandle)) { + server.removeContext(uriToHandle) + } + + def handler = new SimpleHttpHandler() + configuration.accept(handler) + server.createContext(uriToHandle, handler) + } + + private static class SimpleHttpHandler implements HttpHandler { + private String responseBody = ""; + private String contentType = "text/plain"; + private int expectedHttpResponseCode + + @Override + void handle(HttpExchange exchange) throws IOException { + exchange.getResponseHeaders().add("Content-Type", contentType) + exchange.sendResponseHeaders(expectedHttpResponseCode, responseBody.length()) + IOUtils.write(responseBody, exchange.getResponseBody()) + exchange.close() + } + + void setResponseBody(String responseBody) { + this.responseBody = responseBody + } + + void setExpectedHttpResponseCode(int expectedHttpResponseCode) { + this.expectedHttpResponseCode = expectedHttpResponseCode + } + } +} \ No newline at end of file diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 6ee46535abe83..b0beeaa17310e 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -9,11 +9,16 @@ package org.elasticsearch.gradle.internal.snyk import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.http.HttpServerRule import org.gradle.testkit.runner.TaskOutcome +import org.junit.Rule import org.skyscreamer.jsonassert.JSONAssert class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTest { + @Rule + public HttpServerRule httpServer = new HttpServerRule(); + def setup() { buildFile << """ plugins { @@ -26,7 +31,6 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes def "can calculate snyk dependency graph"() { given: -"".indent(3) buildFile << """ apply plugin:'java' @@ -160,4 +164,26 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes } }""", true) } + + def "upload snyk graph fails if new entry could not be created"() { + given: + httpServer.registerHandler("/api/v1/monitor/gradle/graph") { handler -> + handler.responseBody = "Success!" + handler.expectedHttpResponseCode = HttpURLConnection.HTTP_CREATED + } + + buildFile << """ + apply plugin:'java' + + tasks.named('uploadSnykDependencyGraph').configure { + getUrl().set("${httpServer.getUriFor('/api/v1/monitor/gradle/graph')}") + getToken().set("myToken") + } + """ + when: + def build = gradleRunner("uploadSnykDependencyGraph", '-i').build() + then: + build.task(":uploadSnykDependencyGraph").outcome == TaskOutcome.SUCCESS + build.output.contains("Snyk API call response status: \" + statusCode") + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index 509cbfee8a387..fc8fcaa0fd09f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -25,6 +25,9 @@ import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.TaskAction; +import java.io.IOException; +import java.net.HttpURLConnection; + import javax.inject.Inject; public class UploadSnykDependenciesGraph extends DefaultTask { @@ -48,21 +51,22 @@ public UploadSnykDependenciesGraph(ObjectFactory objectFactory) { @TaskAction void upload() { String endpoint = calculateEffectiveEndpoint(); + CloseableHttpResponse response; try (CloseableHttpClient client = HttpClients.createDefault()) { String content = FileUtils.readFileToString(inputFile.getAsFile().get()); HttpPut putRequest = new HttpPut(endpoint); putRequest.addHeader("Authorization", "token " + token.get()); putRequest.addHeader("Content-Type", "application/json"); putRequest.setEntity(new StringEntity(content)); - CloseableHttpResponse response = client.execute(putRequest); + response = client.execute(putRequest); int statusCode = response.getStatusLine().getStatusCode(); - getLogger().info("API call response status: " + statusCode); String responseString = EntityUtils.toString(response.getEntity()); - getLogger().info(responseString); - if (statusCode != 201) { - throw new GradleException("Uploading Snyk Graph failed with http code " + statusCode + ":" + responseString); + getLogger().info("Snyk API call response status: " + statusCode); + if (statusCode != HttpURLConnection.HTTP_CREATED) { + throw new GradleException("Uploading Snyk Graph failed with http code " + statusCode + ": " + responseString); } - } catch (Exception e) { + getLogger().info(responseString); + } catch (IOException e) { throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e); } } From 7ce924e0a29ded51ab1f2d9204f2389343a21085 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 11:26:13 +0200 Subject: [PATCH 06/17] Tweak test coverage and dynamic port allocation --- .../fixtures/http/HttpServerRule.groovy | 17 ++++++++-------- ...dencyMonitoringGradlePluginFuncTest.groovy | 20 +++++++++++++++---- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy index fba7124c5c0b8..a2d51522dbb54 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy @@ -8,28 +8,28 @@ package org.elasticsearch.gradle.fixtures.http -import com.sun.net.httpserver.HttpContext import com.sun.net.httpserver.HttpExchange import com.sun.net.httpserver.HttpHandler import com.sun.net.httpserver.HttpServer import org.apache.commons.io.IOUtils +import org.elasticsearch.gradle.internal.util.ports.AvailablePortAllocator import org.junit.rules.ExternalResource -import java.net.InetSocketAddress -import java.util.ArrayList -import java.util.List -import java.util.concurrent.Callable import java.util.function.Consumer class HttpServerRule extends ExternalResource { - private static final int PORT = 6991 private HttpServer server private List contexts = new ArrayList<>() + private int port @Override protected void before() throws Throwable { - server = HttpServer.create(new InetSocketAddress(PORT), 0) // localhost:6991 + // TODO revisit port range allocation + def socket = new ServerSocket(0) + port = socket.getLocalPort() + socket.close() + server = HttpServer.create(new InetSocketAddress(port), 0) // localhost:6991 server.setExecutor(null) // creates a default executor server.start() } @@ -45,7 +45,7 @@ class HttpServerRule extends ExternalResource { if (path.startsWith("/") == false) { path = "/" + path } - String host = "http://localhost:" + PORT + String host = "http://localhost:" + port return host + path } @@ -66,6 +66,7 @@ class HttpServerRule extends ExternalResource { def handler = new SimpleHttpHandler() configuration.accept(handler) server.createContext(uriToHandle, handler) + contexts.add(uriToHandle) } private static class SimpleHttpHandler implements HttpHandler { diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index b0beeaa17310e..3dc9d3e29375d 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -165,7 +165,7 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes }""", true) } - def "upload snyk graph fails if new entry could not be created"() { + def "upload fails with reasonable error message"() { given: httpServer.registerHandler("/api/v1/monitor/gradle/graph") { handler -> handler.responseBody = "Success!" @@ -181,9 +181,21 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes } """ when: - def build = gradleRunner("uploadSnykDependencyGraph", '-i').build() + def result = gradleRunner("uploadSnykDependencyGraph", '-i').build() then: - build.task(":uploadSnykDependencyGraph").outcome == TaskOutcome.SUCCESS - build.output.contains("Snyk API call response status: \" + statusCode") + result.task(":uploadSnykDependencyGraph").outcome == TaskOutcome.SUCCESS + result.output.contains("Snyk API call response status: 201") + + when: + httpServer.registerHandler("/api/v1/monitor/gradle/graph") { handler -> + handler.responseBody = "Internal Error" + handler.expectedHttpResponseCode = HttpURLConnection.HTTP_INTERNAL_ERROR + } + + result = gradleRunner("uploadSnykDependencyGraph").buildAndFail() + + then: + result.task(":uploadSnykDependencyGraph").outcome == TaskOutcome.FAILED + result.output.contains("Uploading Snyk Graph failed with http code 500: Internal Error") } } From 616062e21e3461a4315e47b23f5ecccf0c20ce07 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 12:31:21 +0200 Subject: [PATCH 07/17] Remove dependencygraph plugin --- .../fixtures/http/HttpServerRule.groovy | 5 +- ...gConventionsPrecommitPluginFuncTest.groovy | 2 - ...dencyMonitoringGradlePluginFuncTest.groovy | 19 ++- .../gradle/internal/BuildPlugin.java | 3 +- .../internal/DependenciesGraphPlugin.java | 67 -------- .../internal/DependenciesGraphTask.java | 152 ------------------ build.gradle | 9 -- client/test/build.gradle | 1 - test/framework/build.gradle | 1 - test/x-content/build.gradle | 1 - 10 files changed, 13 insertions(+), 247 deletions(-) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy index a2d51522dbb54..553a3e30d4664 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy @@ -12,7 +12,6 @@ import com.sun.net.httpserver.HttpExchange import com.sun.net.httpserver.HttpHandler import com.sun.net.httpserver.HttpServer import org.apache.commons.io.IOUtils -import org.elasticsearch.gradle.internal.util.ports.AvailablePortAllocator import org.junit.rules.ExternalResource import java.util.function.Consumer @@ -70,8 +69,8 @@ class HttpServerRule extends ExternalResource { } private static class SimpleHttpHandler implements HttpHandler { - private String responseBody = ""; - private String contentType = "text/plain"; + private String responseBody = "" + private String contentType = "text/plain" private int expectedHttpResponseCode @Override diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy index 19d5296b8defe..2d835114ed6d7 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy @@ -18,8 +18,6 @@ import spock.lang.IgnoreIf import spock.lang.Shared import spock.lang.Unroll -// see https://github.com/elastic/elasticsearch/issues/87913 -@IgnoreIf({ os.windows }) class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPluginFuncTest { Class pluginClassUnderTest = TestingConventionsPrecommitPlugin.class diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 3dc9d3e29375d..506e9df3efe48 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -8,24 +8,22 @@ package org.elasticsearch.gradle.internal.snyk -import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest import org.elasticsearch.gradle.fixtures.http.HttpServerRule +import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersPrecommitPlugin +import org.gradle.api.Plugin import org.gradle.testkit.runner.TaskOutcome import org.junit.Rule import org.skyscreamer.jsonassert.JSONAssert -class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTest { +class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInternalPluginFuncTest { @Rule - public HttpServerRule httpServer = new HttpServerRule(); + public HttpServerRule httpServer = new HttpServerRule() + + Class pluginClassUnderTest = SnykDependencyMonitoringGradlePlugin.class def setup() { - buildFile << """ - plugins { - id 'elasticsearch.snyk-dependency-monitoring' - } - version = "1.0-SNAPSHOT" - """ configurationCacheCompatible = false // configuration is not cc compliant } @@ -33,7 +31,8 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleFuncTes given: buildFile << """ apply plugin:'java' - + version = "1.0-SNAPSHOT" + repositories { mavenCentral() } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java index e5b932bc408e9..6849796579ad9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java @@ -10,6 +10,7 @@ import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks; +import org.elasticsearch.gradle.internal.snyk.SnykDependencyMonitoringGradlePlugin; import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -59,7 +60,7 @@ public void apply(final Project project) { project.getPluginManager().apply("elasticsearch.publish"); project.getPluginManager().apply(ElasticsearchJavadocPlugin.class); project.getPluginManager().apply(DependenciesInfoPlugin.class); - project.getPluginManager().apply(DependenciesGraphPlugin.class); + project.getPluginManager().apply(SnykDependencyMonitoringGradlePlugin.class); InternalPrecommitTasks.create(project, true); configureLicenseAndNotice(project); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java deleted file mode 100644 index 8dee4d27f6a53..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.internal; - -import org.gradle.api.GradleException; -import org.gradle.api.Plugin; -import org.gradle.api.Project; -import org.gradle.api.Task; -import org.gradle.api.plugins.JavaPlugin; -import org.gradle.api.tasks.TaskProvider; - -import java.util.List; -import java.util.stream.Collectors; - -public class DependenciesGraphPlugin implements Plugin { - - public void apply(Project project) { - project.getRootProject().getPluginManager().apply(DependenciesGraphHookPlugin.class); - final String url = System.getenv("SCA_URL"); - final String token = System.getenv("SCA_TOKEN"); - TaskProvider depsGraph = project.getTasks().register("dependenciesGraph", DependenciesGraphTask.class); - depsGraph.configure(t -> { - project.getPlugins().withType(JavaPlugin.class, p -> { - t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME)); - t.setToken(token); - t.setUrl(url); - }); - }); - } - - static class DependenciesGraphHookPlugin implements Plugin { - - @Override - public void apply(Project project) { - if (project != project.getRootProject()) { - throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project."); - } - final String url = System.getenv("SCA_URL"); - final String token = System.getenv("SCA_TOKEN"); - project.getGradle().getTaskGraph().whenReady(graph -> { - List depGraphTasks = graph.getAllTasks() - .stream() - .filter(t -> t instanceof DependenciesGraphTask) - .map(Task::getPath) - .collect(Collectors.toList()); - if (depGraphTasks.size() > 0) { - if (url == null || token == null) { - // If there are more than one DependenciesGraphTasks to run, print the message only for one of - // them as the resolving action is the same for all - throw new GradleException( - "The environment variables SCA_URL and SCA_TOKEN need to be set before task " - + depGraphTasks.get(0) - + " can run" - ); - } - } - }); - - } - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java deleted file mode 100644 index 2863aec90b791..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.internal; - -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; -import org.gradle.StartParameter; -import org.gradle.api.DefaultTask; -import org.gradle.api.GradleException; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.Dependency; -import org.gradle.api.artifacts.DependencySet; -import org.gradle.api.artifacts.ProjectDependency; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.InputFiles; -import org.gradle.api.tasks.TaskAction; - -import java.util.HashSet; -import java.util.Set; - -import javax.inject.Inject; - -import static org.elasticsearch.gradle.util.GradleUtils.projectPath; - -/** - * A task to generate a dependency graph of our runtime dependencies and push that via - * an API call to a given endpoint of a SCA tool/service. - * The graph is built according to the specification in https://github.com/snyk/dep-graph#depgraphdata - * - * Due to the nature of our dependency resolution in gradle, we are abusing the aforementioned graph definition as - * the graph we construct has a single root ( the subproject ) and all dependencies are children of that root, - * irrespective of if they are direct dependencies or transitive ones ( that should be children of other children ). - * Although we end up lacking some contextual information, this allows us to scan and monitor only the dependencies - * that are bundled and used in runtime. - */ -public class DependenciesGraphTask extends DefaultTask { - - private Configuration runtimeConfiguration; - private String token; - private String url; - private StartParameter startParameter; - - @Input - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - @Input - public String getToken() { - return token; - } - - public void setToken(String token) { - this.token = token; - } - - @InputFiles - public Configuration getRuntimeConfiguration() { - return runtimeConfiguration; - } - - public void setRuntimeConfiguration(Configuration runtimeConfiguration) { - this.runtimeConfiguration = runtimeConfiguration; - } - - @Inject - public DependenciesGraphTask(StartParameter startParameter) { - this.startParameter = startParameter; - } - - @TaskAction - void generateDependenciesGraph() { - if (startParameter.isOffline()) { - throw new GradleException("Must run in online mode in order to submit the dependency graph to the SCA service"); - } - - final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies(); - final Set packages = new HashSet<>(); - final Set nodes = new HashSet<>(); - final Set nodeIds = new HashSet<>(); - for (final Dependency dependency : runtimeDependencies) { - final String id = dependency.getGroup() + ":" + dependency.getName(); - final String versionedId = id + "@" + dependency.getVersion(); - final StringBuilder nodeString = new StringBuilder(); - if (dependency instanceof ProjectDependency) { - continue; - } - packages.add(""" - {"id": "%s","info": {"name": "%s","version": "%s"}}\ - """.formatted(versionedId, id, dependency.getVersion())); - nodeString.append(""" - {"nodeId": "%s","pkgId": "%s","deps": []}\ - """.formatted(versionedId, versionedId)); - nodes.add(nodeString.toString()); - nodeIds.add(""" - {"nodeId": "%s"}\ - """.formatted(versionedId)); - } - // We add one package and one node for each dependency, it suffices to check packages. - if (packages.size() > 0) { - final String projectName = "elastic/elasticsearch" + projectPath(getPath()); - final String output = """ - { - "depGraph": { - "schemaVersion": "1.2.0", - "pkgManager": {"name": "gradle"}, - "pkgs": [ - { - "id": "%s@0.0.0", - "info": {"name": "%1$s", "version": "0.0.0"} - }, - %s - ], - "graph": { - "rootNodeId": "%1$s@0.0.0", - "nodes": [ - { "nodeId": "%1$s@0.0.0","pkgId": "%1$s@0.0.0","deps": [%s] }, - %s - ] - } - } - }""".formatted(projectName, String.join(",", packages), String.join(",", nodeIds), String.join(",", nodes)); - getLogger().debug("Dependency Graph: " + output); - try (CloseableHttpClient client = HttpClients.createDefault()) { - HttpPost postRequest = new HttpPost(url); - postRequest.addHeader("Authorization", "token " + token); - postRequest.addHeader("Content-Type", "application/json"); - postRequest.setEntity(new StringEntity(output.toString())); - CloseableHttpResponse response = client.execute(postRequest); - getLogger().info("API call response status: " + response.getStatusLine().getStatusCode()); - getLogger().debug(EntityUtils.toString(response.getEntity())); - } catch (Exception e) { - throw new GradleException("Failed to call API endpoint to submit updated dependency graph", e); - } - } - } - -} diff --git a/build.gradle b/build.gradle index 1596e35110b7b..adcc3547827b1 100644 --- a/build.gradle +++ b/build.gradle @@ -157,12 +157,6 @@ if (project.gradle.startParameter.taskNames.find { it.startsWith("checkPart") } subprojects { proj -> apply plugin: 'elasticsearch.base' - - plugins.withType(JavaPlugin) { - if(proj.name != 'qa' && proj.getPath().contains(":qa:") == false) { - project.apply plugin: 'elasticsearch.snyk-dependency-monitoring' - } - } } allprojects { @@ -235,9 +229,6 @@ allprojects { maybeConfigure(project.tasks, 'dependenciesInfo') { it.enabled = false } - maybeConfigure(project.tasks, 'dependenciesGraph') { - it.enabled = false - } } project.afterEvaluate { diff --git a/client/test/build.gradle b/client/test/build.gradle index 3b2fea0b56e25..4a681a94657e9 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -53,7 +53,6 @@ tasks.named("jarHell").configure { enabled = false } // TODO: should we have licenses for our test deps? tasks.named("dependencyLicenses").configure { enabled = false } tasks.named("dependenciesInfo").configure { enabled = false } -tasks.named("dependenciesGraph").configure { it.enabled = false } //we aren't releasing this jar tasks.named("test").configure { enabled = false } diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 323f64a475085..5b5674181d3d1 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -47,7 +47,6 @@ tasks.named('forbiddenApisMain').configure { // TODO: should we have licenses for our test deps? tasks.named("dependencyLicenses").configure { enabled = false } tasks.named("dependenciesInfo").configure { enabled = false } -tasks.named("dependenciesGraph").configure { enabled = false } tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( // classes are missing diff --git a/test/x-content/build.gradle b/test/x-content/build.gradle index 147bdecc7aa6d..5653fad724fa5 100644 --- a/test/x-content/build.gradle +++ b/test/x-content/build.gradle @@ -30,7 +30,6 @@ tasks.named('forbiddenApisMain').configure { // TODO: should we have licenses for our test deps? tasks.named("dependencyLicenses").configure { enabled = false } tasks.named("dependenciesInfo").configure { enabled = false } -tasks.named("dependenciesGraph").configure { enabled = false } tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( From 6cc5f8276f2047dd16d3d256cf5c22a6472687c4 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 18:50:21 +0200 Subject: [PATCH 08/17] minor cleanup on client and tools projects --- .../internal/snyk/SnykDependencyMonitoringGradlePlugin.java | 3 ++- client/benchmark/build.gradle | 3 +-- client/client-benchmark-noop-api-plugin/build.gradle | 3 +-- client/test/build.gradle | 4 +--- distribution/tools/cli-launcher/build.gradle | 2 +- rest-api-spec/build.gradle | 2 -- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index 63e3c05e05d8e..f402a51fe0e78 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -21,6 +21,7 @@ public class SnykDependencyMonitoringGradlePlugin implements Plugin { + public static final String UPLOAD_TASK_NAME = "uploadSnykDependencyGraph"; private ProjectLayout projectLayout; private ProviderFactory providerFactory; @@ -41,7 +42,7 @@ public void apply(Project project) { generateSnykDependencyGraph.getOutputFile().set(projectLayout.getBuildDirectory().file("snyk/dependencies.json")); }); - project.getTasks().register("uploadSnykDependencyGraph", UploadSnykDependenciesGraph.class, t -> { + project.getTasks().register(UPLOAD_TASK_NAME, UploadSnykDependenciesGraph.class, t -> { t.getInputFile().set(generateTaskProvider.get().getOutputFile()); t.getToken().set(providerFactory.gradleProperty("snykToken")); // the elasticsearch snyk project id diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle index ede0b6961c56e..f0ce2499da10b 100644 --- a/client/benchmark/build.gradle +++ b/client/benchmark/build.gradle @@ -17,8 +17,7 @@ tasks.named("assemble").configure { enabled = true } archivesBaseName = 'client-benchmarks' mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain' -// never try to invoke tests on the benchmark project - there aren't any -tasks.named("test").configure {enabled = false } +tasks.named("uploadSnykDependencyGraph").configure {enabled = false } dependencies { api 'org.apache.commons:commons-math3:3.2' diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle index 198775e61b8a2..839cd9052b4f5 100644 --- a/client/client-benchmark-noop-api-plugin/build.gradle +++ b/client/client-benchmark-noop-api-plugin/build.gradle @@ -20,5 +20,4 @@ esplugin { tasks.named("assemble").configure { enabled = false } tasks.named("dependencyLicenses").configure { enabled = false } tasks.named("dependenciesInfo").configure { enabled = false } -// no unit tests -tasks.named("test").configure { enabled = false } +tasks.named("uploadSnykDependencyGraph").configure { enabled = false } \ No newline at end of file diff --git a/client/test/build.gradle b/client/test/build.gradle index 4a681a94657e9..c8ba708140866 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -53,9 +53,7 @@ tasks.named("jarHell").configure { enabled = false } // TODO: should we have licenses for our test deps? tasks.named("dependencyLicenses").configure { enabled = false } tasks.named("dependenciesInfo").configure { enabled = false } - -//we aren't releasing this jar -tasks.named("test").configure { enabled = false } +tasks.named("uploadSnykDependencyGraph").configure { enabled = false } tasks.withType(LicenseHeadersTask.class).configureEach { approvedLicenses = ['Apache', 'Generated', 'Vendored'] diff --git a/distribution/tools/cli-launcher/build.gradle b/distribution/tools/cli-launcher/build.gradle index 17eaf3e018a09..af7530e92bdf9 100644 --- a/distribution/tools/cli-launcher/build.gradle +++ b/distribution/tools/cli-launcher/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'elasticsearch.java' +apply plugin: 'elasticsearch.build' dependencies { compileOnly project(':server') diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a74f25dc8a551..b0b327a9f4ff2 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -38,8 +38,6 @@ testClusters.configureEach { requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0") } -tasks.named("test").configure { enabled = false } - tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTestsByFilePattern("**/cat*/*.yml", "Cat API are meant to be consumed by humans, so will not be supported by Compatible REST API") From cc2fe7d45c9faf59c3f4b2029e231739bc44477b Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 22:45:39 +0200 Subject: [PATCH 09/17] Apply elasticsearch.build to missing projects --- .../tools/java-version-checker/build.gradle | 5 +- .../tools/windows-service-cli/build.gradle | 6 +- .../hadoop-client-api/build.gradle | 8 +- .../licenses/hadoop-client-api-3.3.1.jar.sha1 | 1 + .../licenses/hadoop-client-api-LICENSE.txt | 202 ++++++++++++++++++ .../licenses/hadoop-client-api-NOTICE.txt | 2 + 6 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 create mode 100644 plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt create mode 100644 plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/java-version-checker/build.gradle index 576baae3d040e..7d5298c199b58 100644 --- a/distribution/tools/java-version-checker/build.gradle +++ b/distribution/tools/java-version-checker/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'elasticsearch.java' +apply plugin: 'elasticsearch.build' sourceSets { unsupportedJdkVersionEntrypoint @@ -22,6 +22,7 @@ tasks.named("jar") { } } -["test", "javadoc"].each { +// TODO revisit forbiddenApis issues +["javadoc", "forbiddenApisMain", "forbiddenApisUnsupportedJdkVersionEntrypoint"].each { tasks.named(it).configure { enabled = false } } diff --git a/distribution/tools/windows-service-cli/build.gradle b/distribution/tools/windows-service-cli/build.gradle index 103e5322913b9..77da0d407a40d 100644 --- a/distribution/tools/windows-service-cli/build.gradle +++ b/distribution/tools/windows-service-cli/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'elasticsearch.java' +apply plugin: 'elasticsearch.build' dependencies { compileOnly project(":server") @@ -7,3 +7,7 @@ dependencies { testImplementation project(":test:framework") } + +tasks.named('forbiddenApisMain').configure { + enabled = false +} diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle index 345f8605eac90..1b22965d02c0f 100644 --- a/plugins/repository-hdfs/hadoop-client-api/build.gradle +++ b/plugins/repository-hdfs/hadoop-client-api/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'elasticsearch.java' +apply plugin: 'elasticsearch.build' apply plugin: 'com.github.johnrengelman.shadow' dependencies { @@ -8,3 +8,9 @@ dependencies { tasks.named('shadowJar').configure { exclude 'org/apache/hadoop/util/ShutdownHookManager$*.class' } + +['jarHell', 'forbiddenApisMain', 'splitPackagesAudit'].each { + tasks.named(it).configure { + enabled = false + } +} diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 new file mode 100644 index 0000000000000..dc2f20e310d30 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 @@ -0,0 +1 @@ +4b9c9cdd9967495838fb521001699c4c9dddf183 \ No newline at end of file diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt new file mode 100644 index 0000000000000..62fc5816c996b --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-NOTICE.txt @@ -0,0 +1,2 @@ +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). From b6da832a74305345b6327e0ee624f4fed3be8696 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 22:56:01 +0200 Subject: [PATCH 10/17] disable thirdparty audit for now for hadoop-client-api project as it fails with oom --- plugins/repository-hdfs/hadoop-client-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-hdfs/hadoop-client-api/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle index 1b22965d02c0f..4ac6f79530fcb 100644 --- a/plugins/repository-hdfs/hadoop-client-api/build.gradle +++ b/plugins/repository-hdfs/hadoop-client-api/build.gradle @@ -9,7 +9,7 @@ tasks.named('shadowJar').configure { exclude 'org/apache/hadoop/util/ShutdownHookManager$*.class' } -['jarHell', 'forbiddenApisMain', 'splitPackagesAudit'].each { +['jarHell', 'thirdPartyAudit', 'forbiddenApisMain', 'splitPackagesAudit'].each { tasks.named(it).configure { enabled = false } From 952d26fe09657e9d2ace5e47243b8169c2a218af Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Jun 2022 23:45:33 +0200 Subject: [PATCH 11/17] Some cleanup before initial review --- build-tools-internal/build.gradle | 4 ---- ...pendencyMonitoringGradlePluginFuncTest.groovy | 5 ----- .../snyk/UploadSnykDependenciesGraph.java | 4 ++++ .../fixtures/AbstractGradleFuncTest.groovy | 16 ++++++++-------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 1e4dcfeb7fffc..7d0beba671eb8 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -135,10 +135,6 @@ gradlePlugin { id = 'elasticsearch.rest-resources' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin' } - snykDependenciesMonitoring { - id = 'elasticsearch.snyk-dependency-monitoring' - implementationClass = 'org.elasticsearch.gradle.internal.snyk.SnykDependencyMonitoringGradlePlugin' - } standaloneRestTest { id = 'elasticsearch.standalone-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.StandaloneRestTestPlugin' diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 506e9df3efe48..619b758ca24d7 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -10,7 +10,6 @@ package org.elasticsearch.gradle.internal.snyk import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest import org.elasticsearch.gradle.fixtures.http.HttpServerRule -import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersPrecommitPlugin import org.gradle.api.Plugin import org.gradle.testkit.runner.TaskOutcome import org.junit.Rule @@ -40,10 +39,6 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna dependencies { implementation 'org.apache.lucene:lucene-monitor:9.2.0' } - - tasks.named('generateSnykDependencyGraph').configure { - configuration = configurations.runtimeClasspath - } """ when: def build = gradleRunner("generateSnykDependencyGraph").build() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index fc8fcaa0fd09f..e4408afaa62c7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -33,6 +33,10 @@ public class UploadSnykDependenciesGraph extends DefaultTask { public static final String GRADLE_GRAPH_ENDPOINT_URL = "https://snyk.io/api/v1/monitor/gradle/graph"; + + // This is new `experimental` api endpoint we might want to support in the future. For now it + // does not allow grouping projects and adding custom metadata to the graph data. Therefore + // we do not support this yet but keep it here for documentation purposes public static final String SNYK_DEP_GRAPH_API_ENDPOINT_URL = "https://snyk.io/api/v1/monitor/dep-graph"; private final RegularFileProperty inputFile; diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 8ed9409857faf..f4d7399c77510 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -43,7 +43,7 @@ abstract class AbstractGradleFuncTest extends Specification { buildFile = testProjectDir.newFile('build.gradle') propertiesFile = testProjectDir.newFile('gradle.properties') propertiesFile << - "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" + "org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME" } def cleanup() { @@ -131,13 +131,13 @@ abstract class AbstractGradleFuncTest extends Specification { } File internalBuild( - List extraPlugins = [], - String bugfix = "7.15.2", - String bugfixLucene = "8.9.0", - String staged = "7.16.0", - String stagedLucene = "8.10.0", - String minor = "8.0.0", - String minorLucene = "9.0.0" + List extraPlugins = [], + String bugfix = "7.15.2", + String bugfixLucene = "8.9.0", + String staged = "7.16.0", + String stagedLucene = "8.10.0", + String minor = "8.0.0", + String minorLucene = "9.0.0" ) { buildFile << """plugins { id 'elasticsearch.global-build-info' From 74fc1c2e69132db317d0b574d688b60687ea24bf Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 27 Jun 2022 09:12:47 +0200 Subject: [PATCH 12/17] Reuse wiremock for mocking http requests --- .../fixtures/http/HttpServerRule.groovy | 92 ------------------- ...dencyMonitoringGradlePluginFuncTest.groovy | 41 +++++---- .../snyk/UploadSnykDependenciesGraph.java | 9 +- .../gradle/fixtures/WiremockFixture.groovy | 27 +++++- 4 files changed, 49 insertions(+), 120 deletions(-) delete mode 100644 build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy deleted file mode 100644 index 553a3e30d4664..0000000000000 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/http/HttpServerRule.groovy +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.fixtures.http - -import com.sun.net.httpserver.HttpExchange -import com.sun.net.httpserver.HttpHandler -import com.sun.net.httpserver.HttpServer -import org.apache.commons.io.IOUtils -import org.junit.rules.ExternalResource - -import java.util.function.Consumer - -class HttpServerRule extends ExternalResource { - - private HttpServer server - private List contexts = new ArrayList<>() - private int port - - @Override - protected void before() throws Throwable { - // TODO revisit port range allocation - def socket = new ServerSocket(0) - port = socket.getLocalPort() - socket.close() - server = HttpServer.create(new InetSocketAddress(port), 0) // localhost:6991 - server.setExecutor(null) // creates a default executor - server.start() - } - - @Override - protected void after() { - if (server != null) { - server.stop(0) // doesn't wait all current exchange handlers complete - } - } - - String getUriFor(String path) { - if (path.startsWith("/") == false) { - path = "/" + path - } - String host = "http://localhost:" + port - return host + path - } - - void registerHandler(String uriToHandle, Closure configuration) { - registerHandler(uriToHandle, new Consumer() { - @Override - void accept(SimpleHttpHandler httpHandler) { - configuration.call(httpHandler) - } - }) - } - - void registerHandler(String uriToHandle, Consumer configuration) { - if(contexts.contains(uriToHandle)) { - server.removeContext(uriToHandle) - } - - def handler = new SimpleHttpHandler() - configuration.accept(handler) - server.createContext(uriToHandle, handler) - contexts.add(uriToHandle) - } - - private static class SimpleHttpHandler implements HttpHandler { - private String responseBody = "" - private String contentType = "text/plain" - private int expectedHttpResponseCode - - @Override - void handle(HttpExchange exchange) throws IOException { - exchange.getResponseHeaders().add("Content-Type", contentType) - exchange.sendResponseHeaders(expectedHttpResponseCode, responseBody.length()) - IOUtils.write(responseBody, exchange.getResponseBody()) - exchange.close() - } - - void setResponseBody(String responseBody) { - this.responseBody = responseBody - } - - void setExpectedHttpResponseCode(int expectedHttpResponseCode) { - this.expectedHttpResponseCode = expectedHttpResponseCode - } - } -} \ No newline at end of file diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy index 619b758ca24d7..1da3f063285e4 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePluginFuncTest.groovy @@ -9,16 +9,17 @@ package org.elasticsearch.gradle.internal.snyk import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest -import org.elasticsearch.gradle.fixtures.http.HttpServerRule import org.gradle.api.Plugin import org.gradle.testkit.runner.TaskOutcome -import org.junit.Rule import org.skyscreamer.jsonassert.JSONAssert -class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInternalPluginFuncTest { +import static java.net.HttpURLConnection.HTTP_CREATED +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR +import static org.elasticsearch.gradle.fixtures.WiremockFixture.PUT +import static org.elasticsearch.gradle.fixtures.WiremockFixture.withWireMock +import static org.elasticsearch.gradle.internal.snyk.UploadSnykDependenciesGraph.GRADLE_GRAPH_ENDPOINT - @Rule - public HttpServerRule httpServer = new HttpServerRule() +class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInternalPluginFuncTest { Class pluginClassUnderTest = SnykDependencyMonitoringGradlePlugin.class @@ -161,33 +162,33 @@ class SnykDependencyMonitoringGradlePluginFuncTest extends AbstractGradleInterna def "upload fails with reasonable error message"() { given: - httpServer.registerHandler("/api/v1/monitor/gradle/graph") { handler -> - handler.responseBody = "Success!" - handler.expectedHttpResponseCode = HttpURLConnection.HTTP_CREATED - } - buildFile << """ apply plugin:'java' - + """ + when: + def result = withWireMock(PUT, "/api/v1/monitor/gradle/graph", "OK", HTTP_CREATED) { server -> + buildFile << """ tasks.named('uploadSnykDependencyGraph').configure { - getUrl().set("${httpServer.getUriFor('/api/v1/monitor/gradle/graph')}") + getUrl().set('${server.baseUrl()}/api/v1/monitor/gradle/graph') getToken().set("myToken") } - """ - when: - def result = gradleRunner("uploadSnykDependencyGraph", '-i').build() + """ + gradleRunner("uploadSnykDependencyGraph", '-i', '--stacktrace').build() + } then: result.task(":uploadSnykDependencyGraph").outcome == TaskOutcome.SUCCESS result.output.contains("Snyk API call response status: 201") when: - httpServer.registerHandler("/api/v1/monitor/gradle/graph") { handler -> - handler.responseBody = "Internal Error" - handler.expectedHttpResponseCode = HttpURLConnection.HTTP_INTERNAL_ERROR + result = withWireMock(PUT, GRADLE_GRAPH_ENDPOINT, "Internal Error", HTTP_INTERNAL_ERROR) { server -> + buildFile << """ + tasks.named('uploadSnykDependencyGraph').configure { + getUrl().set('${server.baseUrl()}/api/v1/monitor/gradle/graph') + } + """ + gradleRunner("uploadSnykDependencyGraph", '-i').buildAndFail() } - result = gradleRunner("uploadSnykDependencyGraph").buildAndFail() - then: result.task(":uploadSnykDependencyGraph").outcome == TaskOutcome.FAILED result.output.contains("Uploading Snyk Graph failed with http code 500: Internal Error") diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index e4408afaa62c7..a2a2a35bd6b91 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -32,12 +32,13 @@ public class UploadSnykDependenciesGraph extends DefaultTask { - public static final String GRADLE_GRAPH_ENDPOINT_URL = "https://snyk.io/api/v1/monitor/gradle/graph"; + public static final String DEFAULT_SERVER = "https://snyk.io"; + public static final String GRADLE_GRAPH_ENDPOINT = "/api/v1/monitor/gradle/graph"; // This is new `experimental` api endpoint we might want to support in the future. For now it // does not allow grouping projects and adding custom metadata to the graph data. Therefore // we do not support this yet but keep it here for documentation purposes - public static final String SNYK_DEP_GRAPH_API_ENDPOINT_URL = "https://snyk.io/api/v1/monitor/dep-graph"; + public static final String SNYK_DEP_GRAPH_API_ENDPOINT = "/api/v1/monitor/dep-graph"; private final RegularFileProperty inputFile; private final Property token; @@ -46,7 +47,7 @@ public class UploadSnykDependenciesGraph extends DefaultTask { @Inject public UploadSnykDependenciesGraph(ObjectFactory objectFactory) { - url = objectFactory.property(String.class).convention(GRADLE_GRAPH_ENDPOINT_URL); + url = objectFactory.property(String.class).convention(DEFAULT_SERVER + GRADLE_GRAPH_ENDPOINT); projectId = objectFactory.property(String.class); token = objectFactory.property(String.class); inputFile = objectFactory.fileProperty(); @@ -77,7 +78,7 @@ void upload() { private String calculateEffectiveEndpoint() { String url = this.url.get(); - return url.equals(GRADLE_GRAPH_ENDPOINT_URL) ? url : projectId.map(id -> url + "?org=" + id).getOrElse(url); + return url.endsWith(GRADLE_GRAPH_ENDPOINT) ? url : projectId.map(id -> url + "?org=" + id).getOrElse(url); } @Input diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy index 7c0450760359c..cecdda15c5d8c 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy @@ -14,6 +14,7 @@ import org.gradle.testkit.runner.BuildResult import static com.github.tomakehurst.wiremock.client.WireMock.aResponse import static com.github.tomakehurst.wiremock.client.WireMock.get import static com.github.tomakehurst.wiremock.client.WireMock.head +import static com.github.tomakehurst.wiremock.client.WireMock.put import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo /** @@ -21,6 +22,9 @@ import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo * */ class WiremockFixture { + static String GET = "GET" + static String PUT = "PUT" + /** * the buildRunClosure has passed an instance of WireMockServer that can be used to access e.g. the baseUrl of * the configured server: @@ -38,13 +42,28 @@ class WiremockFixture { * gadleRunner('myTask').build() * * */ + static BuildResult withWireMock(String expectedUrl, byte[] expectedContent, Closure buildRunClosure) { + withWireMock(GET, expectedUrl, expectedContent, HttpURLConnection.HTTP_OK, buildRunClosure) + } + + static BuildResult withWireMock(String httpMethod, String expectedUrl, String expectedContent, int expectedHttpCode, Closure buildRunClosure) { + withWireMock(httpMethod, expectedUrl, expectedContent.getBytes(), expectedHttpCode, buildRunClosure) + } + + static BuildResult withWireMock(String httpMethod, String expectedUrl, byte[] expectedContent, int expectedHttpCode, Closure buildRunClosure) { WireMockServer wireMock = new WireMockServer(0); try { - wireMock.stubFor(head(urlEqualTo(expectedUrl)).willReturn(aResponse().withStatus(200))); - wireMock.stubFor( - get(urlEqualTo(expectedUrl)).willReturn(aResponse().withStatus(200).withBody(expectedContent)) - ) + wireMock.stubFor(head(urlEqualTo(expectedUrl)).willReturn(aResponse().withStatus(expectedHttpCode))); + if(httpMethod == GET) { + wireMock.stubFor( + get(urlEqualTo(expectedUrl)).willReturn(aResponse().withStatus(expectedHttpCode).withBody(expectedContent)) + ) + } else if(httpMethod == PUT) { + wireMock.stubFor( + put(urlEqualTo(expectedUrl)).willReturn(aResponse().withStatus(expectedHttpCode).withBody(expectedContent)) + ) + } wireMock.start(); return buildRunClosure.call(wireMock); } catch (Exception e) { From 495188fd406af635b609612bb6320558efbacebb Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 27 Jun 2022 14:36:22 +0200 Subject: [PATCH 13/17] Cleanup after merging --- ...gConventionsPrecommitPluginFuncTest.groovy | 2 ++ .../fixtures/AbstractGradleFuncTest.groovy | 26 ++++++++++--------- ...ationCacheCompatibleAwareGradleRunner.java | 14 ++++++---- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy index 2d835114ed6d7..19d5296b8defe 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy @@ -18,6 +18,8 @@ import spock.lang.IgnoreIf import spock.lang.Shared import spock.lang.Unroll +// see https://github.com/elastic/elasticsearch/issues/87913 +@IgnoreIf({ os.windows }) class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPluginFuncTest { Class pluginClassUnderTest = TestingConventionsPrecommitPlugin.class diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index f4d7399c77510..755c866dafe8c 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -34,7 +34,7 @@ abstract class AbstractGradleFuncTest extends Specification { File propertiesFile File projectDir - public boolean configurationCacheCompatible = false + boolean configurationCacheCompatible = false def setup() { projectDir = testProjectDir.root @@ -77,13 +77,15 @@ abstract class AbstractGradleFuncTest extends Specification { new ConfigurationCacheCompatibleAwareGradleRunner( new InternalAwareGradleRunner( GradleRunner.create() - .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0) + .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments() + .toString().indexOf("-agentlib:jdwp") > 0 + ) .withProjectDir(projectDir) .withPluginClasspath() .forwardOutput() - ), - configurationCacheCompatible), - projectDir).withArguments(arguments) + ), configurationCacheCompatible), + projectDir + ).withArguments(arguments) } def assertOutputContains(String givenOutput, String expected) { @@ -131,13 +133,13 @@ abstract class AbstractGradleFuncTest extends Specification { } File internalBuild( - List extraPlugins = [], - String bugfix = "7.15.2", - String bugfixLucene = "8.9.0", - String staged = "7.16.0", - String stagedLucene = "8.10.0", - String minor = "8.0.0", - String minorLucene = "9.0.0" + List extraPlugins = [], + String bugfix = "7.15.2", + String bugfixLucene = "8.9.0", + String staged = "7.16.0", + String stagedLucene = "8.10.0", + String minor = "8.0.0", + String minorLucene = "9.0.0" ) { buildFile << """plugins { id 'elasticsearch.global-build-info' diff --git a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java index 8f52a270af8bb..8a195fa15c62a 100644 --- a/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java +++ b/build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java @@ -18,11 +18,13 @@ import java.io.File; import java.io.Writer; import java.net.URI; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; +/** + * A Gradle runner that delegates to another runner, optionally enabling the configuring cache parameter. + */ public class ConfigurationCacheCompatibleAwareGradleRunner extends GradleRunner { private GradleRunner delegate; private boolean ccCompatible; @@ -74,9 +76,11 @@ public List getArguments() { @Override public GradleRunner withArguments(List arguments) { - List effectiveArgs = ccCompatible - ? Stream.concat(arguments.stream(), Stream.of("--configuration-cache")).collect(Collectors.toList()) - : arguments; + List effectiveArgs = arguments; + if (ccCompatible) { + effectiveArgs = new ArrayList<>(arguments); + effectiveArgs.add("--configuration-cache"); + } delegate.withArguments(effectiveArgs); return this; } From f36b640edae2cb28430cd9424746f2c886a9e2b8 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 27 Jun 2022 14:40:03 +0200 Subject: [PATCH 14/17] Minor formatting fix --- .../gradle/fixtures/WiremockFixture.groovy | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy index cecdda15c5d8c..a3281780e16e0 100644 --- a/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy +++ b/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/WiremockFixture.groovy @@ -25,6 +25,14 @@ class WiremockFixture { static String GET = "GET" static String PUT = "PUT" + static BuildResult withWireMock(String expectedUrl, byte[] expectedContent, Closure buildRunClosure) { + withWireMock(GET, expectedUrl, expectedContent, HttpURLConnection.HTTP_OK, buildRunClosure) + } + + static BuildResult withWireMock(String httpMethod, String expectedUrl, String expectedContent, int expectedHttpCode, Closure buildRunClosure) { + withWireMock(httpMethod, expectedUrl, expectedContent.getBytes(), expectedHttpCode, buildRunClosure) + } + /** * the buildRunClosure has passed an instance of WireMockServer that can be used to access e.g. the baseUrl of * the configured server: @@ -33,7 +41,7 @@ class WiremockFixture { * WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server -> * buildFile << """ * // wire a gradle repository with wiremock -* repositories { + * repositories { * maven { * url = '${server.baseUrl()}' * } @@ -42,15 +50,6 @@ class WiremockFixture { * gadleRunner('myTask').build() * * */ - - static BuildResult withWireMock(String expectedUrl, byte[] expectedContent, Closure buildRunClosure) { - withWireMock(GET, expectedUrl, expectedContent, HttpURLConnection.HTTP_OK, buildRunClosure) - } - - static BuildResult withWireMock(String httpMethod, String expectedUrl, String expectedContent, int expectedHttpCode, Closure buildRunClosure) { - withWireMock(httpMethod, expectedUrl, expectedContent.getBytes(), expectedHttpCode, buildRunClosure) - } - static BuildResult withWireMock(String httpMethod, String expectedUrl, byte[] expectedContent, int expectedHttpCode, Closure buildRunClosure) { WireMockServer wireMock = new WireMockServer(0); try { From 670362ec2b2e18649b3c056e0da284a37c70a352 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 28 Jun 2022 08:36:34 +0200 Subject: [PATCH 15/17] Apply review feedback --- .../snyk/GenerateSnykDependencyGraph.java | 46 ++++++++----------- .../internal/snyk/SnykDependencyGraph.java | 2 +- .../SnykDependencyMonitoringGradlePlugin.java | 2 +- .../snyk/UploadSnykDependenciesGraph.java | 4 +- client/benchmark/build.gradle | 10 +--- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java index 9444e065e6584..079b51cd05f3d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/GenerateSnykDependencyGraph.java @@ -10,7 +10,7 @@ import groovy.json.JsonOutput; -import org.elasticsearch.gradle.internal.conventions.info.GitInfo; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.artifacts.Configuration; @@ -34,6 +34,22 @@ public class GenerateSnykDependencyGraph extends DefaultTask { + private static final Map FIXED_META_DATA = Map.of( + "method", + "custom gradle", + "id", + "gradle", + "node", + "v16.15.1", + "name", + "gradle", + "plugin", + "extern:gradle", + "pluginRuntime", + "unknown", + "monitorGraph", + true + ); private final Property configuration; private final Property projectName; private final Property projectPath; @@ -80,35 +96,11 @@ private Map generateGradleGraphPayload() { version.get(), firstLevelModuleDependencies ); - return Map.of("meta", generateMetaData(), "depGraphJSON", builder.build(), "target", buildTargetData()); + return Map.of("meta", FIXED_META_DATA, "depGraphJSON", builder.build(), "target", buildTargetData()); } private Object buildTargetData() { - return Map.of( - "remoteUrl", - "http://github.com/elastic/elasticsearch.git", - "branch", - GitInfo.gitInfo(buildLayout.getRootDirectory()).getRevision() - ); - } - - private Map generateMetaData() { - return Map.of( - "method", - "custom gradle", - "id", - "gradle", - "node", - "v16.15.1", - "name", - "gradle", - "plugin", - "extern:gradle", - "pluginRuntime", - "unknown", - "monitorGraph", - true - ); + return Map.of("remoteUrl", "http://github.com/elastic/elasticsearch.git", "branch", BuildParams.getGitRevision()); } @InputFiles diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java index 02f179cb995fb..fe62eb4e649db 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyGraph.java @@ -15,7 +15,7 @@ import java.util.Set; public class SnykDependencyGraph { - private final String schemaVersion = "1.2.0"; + static private final String schemaVersion = "1.2.0"; private final Map graph; private final Set pkgs; private final Map pkgManager; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java index f402a51fe0e78..4df33d80ee275 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/SnykDependencyMonitoringGradlePlugin.java @@ -46,7 +46,7 @@ public void apply(Project project) { t.getInputFile().set(generateTaskProvider.get().getOutputFile()); t.getToken().set(providerFactory.gradleProperty("snykToken")); // the elasticsearch snyk project id - t.getProjectId().set("f27934bf-9ad1-4d91-901c-cb77168a34db"); + t.getProjectId().set(providerFactory.gradleProperty("snykProjectId")); }); project.getPlugins().withType(JavaPlugin.class, javaPlugin -> generateTaskProvider.configure(generateSnykDependencyGraph -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index a2a2a35bd6b91..f198e13d38bc5 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -11,6 +11,7 @@ import org.apache.commons.io.FileUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.FileEntity; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; @@ -58,11 +59,10 @@ void upload() { String endpoint = calculateEffectiveEndpoint(); CloseableHttpResponse response; try (CloseableHttpClient client = HttpClients.createDefault()) { - String content = FileUtils.readFileToString(inputFile.getAsFile().get()); HttpPut putRequest = new HttpPut(endpoint); putRequest.addHeader("Authorization", "token " + token.get()); putRequest.addHeader("Content-Type", "application/json"); - putRequest.setEntity(new StringEntity(content)); + putRequest.setEntity(new FileEntity(inputFile.getAsFile().get())); response = client.execute(putRequest); int statusCode = response.getStatusLine().getStatusCode(); String responseString = EntityUtils.toString(response.getEntity()); diff --git a/client/benchmark/build.gradle b/client/benchmark/build.gradle index f0ce2499da10b..40c277067fe6c 100644 --- a/client/benchmark/build.gradle +++ b/client/benchmark/build.gradle @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.java' apply plugin: 'application' group = 'org.elasticsearch.client' @@ -17,8 +17,6 @@ tasks.named("assemble").configure { enabled = true } archivesBaseName = 'client-benchmarks' mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain' -tasks.named("uploadSnykDependencyGraph").configure {enabled = false } - dependencies { api 'org.apache.commons:commons-math3:3.2' @@ -26,8 +24,4 @@ dependencies { api project(':client:rest') // bottleneck should be the client, not Elasticsearch api project(path: ':client:client-benchmark-noop-api-plugin') -} - -// No licenses for our benchmark deps (we don't ship benchmarks) -tasks.named("dependencyLicenses").configure { enabled = false } -tasks.named("dependenciesInfo").configure{enabled = false } +} \ No newline at end of file From 47206c48589ce5bf2a1113aa57ff9b787309069b Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 28 Jun 2022 11:06:13 +0200 Subject: [PATCH 16/17] Remove unused imports --- .../gradle/internal/snyk/UploadSnykDependenciesGraph.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java index f198e13d38bc5..ecfd9afb6b02d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/snyk/UploadSnykDependenciesGraph.java @@ -8,11 +8,9 @@ package org.elasticsearch.gradle.internal.snyk; -import org.apache.commons.io.FileUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPut; import org.apache.http.entity.FileEntity; -import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; From 1112c574401e1c25c3fdf449c0da02bcf9027cbc Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 29 Jun 2022 11:25:16 +0200 Subject: [PATCH 17/17] Update hadoop client api checksum --- .../hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 | 1 - .../hadoop-client-api/licenses/hadoop-client-api-3.3.3.jar.sha1 | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 create mode 100644 plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.3.jar.sha1 diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 deleted file mode 100644 index dc2f20e310d30..0000000000000 --- a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4b9c9cdd9967495838fb521001699c4c9dddf183 \ No newline at end of file diff --git a/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.3.jar.sha1 b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.3.jar.sha1 new file mode 100644 index 0000000000000..8df133d0bd106 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/licenses/hadoop-client-api-3.3.3.jar.sha1 @@ -0,0 +1 @@ +d0593aed2d4df9bcee507550913d29d589ebd84a \ No newline at end of file