From d76a418792322fe63ebe1feceb87fa6f954be58b Mon Sep 17 00:00:00 2001 From: David Nestorovic Date: Fri, 8 Nov 2024 13:48:20 +0100 Subject: [PATCH] Extract gradle task for fetching newer versions --- .../workflows/check-new-library-versions.yml | 2 +- .../org.graalvm.internal.tck-harness.gradle | 63 ++------- ...hExistingLibrariesWithNewerVersions.groovy | 130 ++++++++++++++++++ .../internal/tck/harness/TckExtension.java | 63 +-------- 4 files changed, 142 insertions(+), 116 deletions(-) create mode 100644 tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tasks/FetchExistingLibrariesWithNewerVersions.groovy diff --git a/.github/workflows/check-new-library-versions.yml b/.github/workflows/check-new-library-versions.yml index b8085016a..f94046584 100644 --- a/.github/workflows/check-new-library-versions.yml +++ b/.github/workflows/check-new-library-versions.yml @@ -38,7 +38,7 @@ jobs: - name: "🕸️ Populate matrix" id: set-matrix run: | - ./gradlew getExistingLibrariesWithNewVersions -PlibraryMatrixLimit=200 + ./gradlew fetchExistingLibrariesWithNewerVersions --matrixLimit=200 - name: "🔨 Create branch" run: | git config --local user.email "actions@github.com" diff --git a/tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle b/tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle index d1271fad2..5d7dec14d 100644 --- a/tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle +++ b/tests/tck-build-logic/src/main/groovy/org.graalvm.internal.tck-harness.gradle @@ -10,6 +10,7 @@ plugins { } import groovy.json.JsonOutput +import org.graalvm.internal.tasks.FetchExistingLibrariesWithNewerVersions import org.graalvm.internal.tck.DockerTask import org.graalvm.internal.tck.ConfigFilesChecker import org.graalvm.internal.tck.ScaffoldTask @@ -103,60 +104,6 @@ def matrixDefault = [ final String METADATA_GROUP = "Metadata" -List getNewerVersionsFor(String library, String startingVersion) { - def baseUrl = "https://repo1.maven.org/maven2" - String[] libraryParts = library.split(":") - String group = libraryParts[0].replace(".", "/") - String artifact = libraryParts[1] - def data = new URL(baseUrl + "/" + group + "/" + artifact + "/" + "maven-metadata.xml").getText() - - return tck.getNewerVersionsFromLibraryIndex(data, startingVersion, library) -} - -// gradle getExistingLibrariesWithNewVersions -Provider getExistingLibrariesWithNewVersions = tasks.register("getExistingLibrariesWithNewVersions", DefaultTask) { task -> - task.setDescription("Returns list of all libraries coordinates") - task.setGroup(METADATA_GROUP) - task.doFirst { - if (!project.hasProperty("libraryMatrixLimit")) { - throw new IllegalArgumentException("Please provide the maximum number of libraries versions!") - } - - // get all existing libraries - Set libraries = [] - matchingCoordinates.forEach { - libraries.add(it.substring(0, it.lastIndexOf(":"))) - } - - // foreach existing library find newer versions than the latest one tested except for infrastructure tests - List infrastructureTests = List.of("samples", "org.example") - List newerVersions = new ArrayList<>() - libraries.forEach { - String libraryName = it - if (infrastructureTests.stream().noneMatch(testName -> libraryName.startsWith(testName))) { - List versions = getNewerVersionsFor(libraryName, tck.getLatestLibraryVersion(libraryName)) - versions.forEach { - newerVersions.add(libraryName.concat(":").concat(it)) - } - } - } - - // number of discovered versions is potentially huge and we want to limit it - int limit = Integer.parseInt((String) project.findProperty("libraryMatrixLimit")) - if (newerVersions.size() > limit) { - newerVersions = newerVersions.subList(0, limit) - } - - def matrix = [ - "coordinates" : newerVersions, - "version" : ["17"], - "os" : ["ubuntu-latest"] - ] - - println "::set-output name=matrix::${JsonOutput.toJson(matrix)}" - } -} - // gradle generateMatrixMatchingCoordinates -Pcoordinates= Provider generateMatrixMatchingCoordinates = tasks.register("generateMatrixMatchingCoordinates", DefaultTask) { task -> task.setDescription("Returns matrix definition populated with all matching coordinates") @@ -216,6 +163,14 @@ Provider generateMatrixDiffCoordinates = tasks.register("generateMatrixDif } } +// groovy tasks +tasks.register("fetchExistingLibrariesWithNewerVersions", FetchExistingLibrariesWithNewerVersions.class) { task -> + task.setGroup(METADATA_GROUP) + task.setDescription("Returns list of all libraries coordinates") + task.setAllLibraryCoordinates(matchingCoordinates) +} + +// java tasks tasks.register("checkAllowedDockerImages", GrypeTask.class) { task -> task.setDescription("Returns list of allowed docker images") task.setGroup(METADATA_GROUP) diff --git a/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tasks/FetchExistingLibrariesWithNewerVersions.groovy b/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tasks/FetchExistingLibrariesWithNewerVersions.groovy new file mode 100644 index 000000000..455f23223 --- /dev/null +++ b/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tasks/FetchExistingLibrariesWithNewerVersions.groovy @@ -0,0 +1,130 @@ +package org.graalvm.internal.tasks + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.SerializationFeature +import groovy.json.JsonOutput +import org.graalvm.internal.tck.model.MetadataVersionsIndexEntry +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.options.Option +import org.gradle.util.internal.VersionNumber + +import java.util.regex.Matcher +import java.util.regex.Pattern + + +abstract class FetchExistingLibrariesWithNewerVersions extends DefaultTask { + + @Input + List allLibraryCoordinates + + @Input + @Option(option = "matrixLimit", description = "Sets the maximum number of coordinates in the final matrix") + Integer matrixLimit + + private static final List INFRASTRUCTURE_TESTS = List.of("samples", "org.example") + + @TaskAction + void action() { + // get all existing libraries + Set libraries = [] + allLibraryCoordinates.forEach { + libraries.add(it.substring(0, it.lastIndexOf(":"))) + } + + // foreach existing library find newer versions than the latest one tested except for infrastructure tests + List newerVersions = new ArrayList<>() + libraries.forEach { + String libraryName = it + if (INFRASTRUCTURE_TESTS.stream().noneMatch(testName -> libraryName.startsWith(testName))) { + List versions = getNewerVersionsFor(libraryName, getLatestLibraryVersion(libraryName)) + versions.forEach { + newerVersions.add(libraryName.concat(":").concat(it)) + } + } + } + + if (newerVersions.size() > getMatrixLimit()) { + newerVersions = newerVersions.subList(0, getMatrixLimit()) + } + + def matrix = [ + "coordinates": newerVersions, + "version" : ["17"], + "os" : ["ubuntu-latest"] + ] + + println "::set-output name=matrix::${JsonOutput.toJson(matrix)}" + } + + static List getNewerVersionsFor(String library, String startingVersion) { + def baseUrl = "https://repo1.maven.org/maven2" + String[] libraryParts = library.split(":") + String group = libraryParts[0].replace(".", "/") + String artifact = libraryParts[1] + def data = new URL(baseUrl + "/" + group + "/" + artifact + "/" + "maven-metadata.xml").getText() + + return getNewerVersionsFromLibraryIndex(data, startingVersion, library) + } + + static List getNewerVersionsFromLibraryIndex(String index, String startingVersion, String libraryName) { + Pattern pattern = Pattern.compile("(.*)"); + Matcher matcher = pattern.matcher(index); + List allVersions = new ArrayList<>(); + + if (matcher.groupCount() < 1) { + throw new RuntimeException("Cannot find versions in the given index file: " + libraryName); + } + + while (matcher.find()) { + allVersions.add(matcher.group(1)); + } + + int indexOfStartingVersion = allVersions.indexOf(startingVersion); + if (indexOfStartingVersion < 0) { + System.out.println("Cannot find starting version in index file: " + libraryName + " for version " + startingVersion); + return new ArrayList<>(); + } + + allVersions = allVersions.subList(indexOfStartingVersion, allVersions.size()); + if (allVersions.size() <= 1) { + System.out.println("Cannot find newer versions for " + libraryName + " after the version " + startingVersion); + } + + return allVersions.subList(1, allVersions.size()); + } + + static String getLatestLibraryVersion(String libraryModule) { + try { + String[] coordinates = libraryModule.split(":"); + String group = coordinates[0]; + String artifact = coordinates[1]; + + File coordinatesMetadataIndex = new File("metadata/" + group + "/" + artifact +"/index.json"); + ObjectMapper objectMapper = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + + List entries = objectMapper.readValue(coordinatesMetadataIndex, new TypeReference>() { + }); + + List allTested = new ArrayList<>(); + for (MetadataVersionsIndexEntry entry : entries) { + allTested.addAll(entry.testedVersions()); + } + + if (allTested.isEmpty()) { + throw new IllegalStateException("Cannot find any tested version for: " + libraryModule); + } + + allTested.sort(Comparator.comparing(VersionNumber::parse)); + return allTested.get(allTested.size() - 1); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/TckExtension.java b/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/TckExtension.java index d6ae9edcc..bf2c4fc38 100644 --- a/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/TckExtension.java +++ b/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/harness/TckExtension.java @@ -301,9 +301,7 @@ List getMatchingCoordinates(String coordinateFilter) { String artifactId = strings.get(1); String version = strings.get(2); - Set matchingCoordinates = new HashSet<>(); - for (String directory : getMatchingMetadataDirs(groupId, artifactId)) { Path index = metadataRoot().resolve(directory).resolve("index.json"); List> metadataIndex = (List>) extractJsonFile(index); @@ -324,7 +322,8 @@ List getMatchingCoordinates(String coordinateFilter) { } } } - return matchingCoordinates.stream().collect(Collectors.toList()); + + return new ArrayList<>(matchingCoordinates); } /** @@ -353,62 +352,4 @@ List getMetadataFileList(Path directory) throws IOException { return foundFiles; } } - - String getLatestLibraryVersion(String libraryModule) { - try { - String[] coordinates = libraryModule.split(":"); - String group = coordinates[0]; - String artifact = coordinates[1]; - - File coordinatesMetadataIndex = new File("metadata/" + group + "/" + artifact +"/index.json"); - ObjectMapper objectMapper = new ObjectMapper() - .enable(SerializationFeature.INDENT_OUTPUT) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - - List entries = objectMapper.readValue(coordinatesMetadataIndex, new TypeReference<>() { - }); - - List allTested = new ArrayList<>(); - for (MetadataVersionsIndexEntry entry : entries) { - allTested.addAll(entry.testedVersions()); - } - - if (allTested.isEmpty()) { - throw new IllegalStateException("Cannot find any tested version for: " + libraryModule); - } - - allTested.sort(Comparator.comparing(VersionNumber::parse)); - return allTested.get(allTested.size() - 1); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - List getNewerVersionsFromLibraryIndex(String index, String startingVersion, String libraryName) { - Pattern pattern = Pattern.compile("(.*)"); - Matcher matcher = pattern.matcher(index); - List allVersions = new ArrayList<>(); - - if (matcher.groupCount() < 1) { - throw new RuntimeException("Cannot find versions in the given index file: " + libraryName); - } - - while (matcher.find()) { - allVersions.add(matcher.group(1)); - } - - int indexOfStartingVersion = allVersions.indexOf(startingVersion); - if (indexOfStartingVersion < 0) { - System.out.println("Cannot find starting version in index file: " + libraryName + " for version " + startingVersion); - return new ArrayList<>(); - } - - allVersions = allVersions.subList(indexOfStartingVersion, allVersions.size()); - if (allVersions.size() <= 1) { - System.out.println("Cannot find newer versions for " + libraryName + " after the version " + startingVersion); - } - - return allVersions.subList(1, allVersions.size()); - } - }