From af1d614b2d415c553158811e7bcc6dbb4681af1a Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Wed, 24 May 2023 16:02:11 -0700 Subject: [PATCH 1/5] Add compatibility check task Signed-off-by: Sayali Gaikawad --- buildSrc/build.gradle | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index b58652e599ec8..11259bea1aff7 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -35,6 +35,7 @@ plugins { id 'java-gradle-plugin' id 'groovy' id 'java-test-fixtures' + id 'org.ajoberstar.grgit' version '5.2.0' } group = 'org.opensearch.gradle' @@ -323,3 +324,42 @@ class VersionPropertiesLoader { loadedProps.put("opensearch", opensearch) } } + + +task cloneGitRepoAndCheckCompatibility { + def repositoryUrls = project.hasProperty('repositoryUrls') ? project.property('repositoryUrls').split(',') : []; + String refToCheckout = project.hasProperty('refToCheckout') ?: 'main' + def failedComponents = [] + dependsOn 'publishToMavenLocal' + doLast { + if (repositoryUrls.size() == 0) { + throw new IllegalArgumentException("repositoryUrls must be provided as arguments.") + } + def tempDir = File.createTempDir() + repositoryUrls.each { repositoryUrl -> + try { + exec { + commandLine 'git', 'clone', repositoryUrl, tempDir + } + exec { + def stdout = new ByteArrayOutputStream() + workingDir = tempDir + commandLine 'git', 'checkout', refToCheckout + commandLine './gradlew', 'assemble' + standardOutput stdout + } + } catch (ignored) { + failedComponents.add(repositoryUrl) + logger.info("Failed for component: ", repositoryUrl) + } finally { + tempDir.deleteDir() + } + } + if (!failedComponents.isEmpty()) { + println("Components incompatible with this change: $failedComponents") + } else { + println('Everything looks good!') + } + } +} + From 9b8439bbde48d4821b531fb0649c6278409b36f0 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Wed, 24 May 2023 16:02:46 -0700 Subject: [PATCH 2/5] Add compatibility check task Signed-off-by: Sayali Gaikawad --- buildSrc/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 11259bea1aff7..6680fd43e6281 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -326,7 +326,7 @@ class VersionPropertiesLoader { } -task cloneGitRepoAndCheckCompatibility { +task checkCompatibility { def repositoryUrls = project.hasProperty('repositoryUrls') ? project.property('repositoryUrls').split(',') : []; String refToCheckout = project.hasProperty('refToCheckout') ?: 'main' def failedComponents = [] From 9aa033412ace1083d256427e6fe73c940975b734 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Thu, 25 May 2023 12:39:44 -0700 Subject: [PATCH 3/5] Add compatibility check task Signed-off-by: Sayali Gaikawad --- build.gradle | 46 +++++++++++++++++++++++++++++++++++++++++++ buildSrc/build.gradle | 37 ---------------------------------- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/build.gradle b/build.gradle index aa0a3ecd96d89..9527421264eb7 100644 --- a/build.gradle +++ b/build.gradle @@ -57,6 +57,7 @@ plugins { id "org.gradle.test-retry" version "1.5.3" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' + id "com.dorongold.task-tree" version "2.1.1" } apply from: 'gradle/build-complete.gradle' @@ -643,3 +644,48 @@ tasks.withType(TestTaskReports).configureEach { tasks.named(JavaBasePlugin.CHECK_TASK_NAME) { dependsOn tasks.named('testAggregateTestReport', TestReport) } + +task checkCompatibility { + def repositoryUrls = project.hasProperty('repositoryUrls') ? project.property('repositoryUrls').split(',') : []; + String refToCheckout = project.hasProperty('refToCheckout') ?: 'main' + def failedComponents = [] + doLast { + if (repositoryUrls.size() == 0) { + throw new IllegalArgumentException("repositoryUrls must be provided as arguments.") + } + def tempDir = File.createTempDir() + repositoryUrls.each { repositoryUrl -> + try { + exec { + commandLine 'git', 'clone', repositoryUrl, tempDir + } + exec { + def stdout = new ByteArrayOutputStream() + workingDir = tempDir + commandLine 'git', 'checkout', refToCheckout + commandLine './gradlew', 'assemble' + standardOutput stdout + } + } catch (ignored) { + failedComponents.add(repositoryUrl) + logger.info("Failed for component: ", repositoryUrl) + } finally { + tempDir.deleteDir() + } + } + if (!failedComponents.isEmpty()) { + println("Components incompatible with this change: $failedComponents") + } else { + println('Everything looks good!') + } + } +} + +allprojects {project -> + project.afterEvaluate { + if (project.tasks.findByName('publishToMavenLocal')){ + checkCompatibility.dependsOn(project.tasks.publishToMavenLocal) + } + } +} + diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 6680fd43e6281..43b9e7ca05498 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -326,40 +326,3 @@ class VersionPropertiesLoader { } -task checkCompatibility { - def repositoryUrls = project.hasProperty('repositoryUrls') ? project.property('repositoryUrls').split(',') : []; - String refToCheckout = project.hasProperty('refToCheckout') ?: 'main' - def failedComponents = [] - dependsOn 'publishToMavenLocal' - doLast { - if (repositoryUrls.size() == 0) { - throw new IllegalArgumentException("repositoryUrls must be provided as arguments.") - } - def tempDir = File.createTempDir() - repositoryUrls.each { repositoryUrl -> - try { - exec { - commandLine 'git', 'clone', repositoryUrl, tempDir - } - exec { - def stdout = new ByteArrayOutputStream() - workingDir = tempDir - commandLine 'git', 'checkout', refToCheckout - commandLine './gradlew', 'assemble' - standardOutput stdout - } - } catch (ignored) { - failedComponents.add(repositoryUrl) - logger.info("Failed for component: ", repositoryUrl) - } finally { - tempDir.deleteDir() - } - } - if (!failedComponents.isEmpty()) { - println("Components incompatible with this change: $failedComponents") - } else { - println('Everything looks good!') - } - } -} - From 58f1589ed363fbc4181af4762005d6cbe423fef3 Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Thu, 1 Jun 2023 11:14:04 -0700 Subject: [PATCH 4/5] Comp check Signed-off-by: Sayali Gaikawad --- build.gradle | 42 ++++++++++++++++++++++++++++-------------- buildSrc/build.gradle | 3 --- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index 9527421264eb7..8773ad5c0a5d9 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,7 @@ import org.gradle.plugins.ide.eclipse.model.EclipseJdt import org.gradle.plugins.ide.eclipse.model.SourceFolder import org.gradle.api.Project; import org.gradle.process.ExecResult; +import groovy.json.JsonSlurper import static org.opensearch.gradle.util.GradleUtils.maybeConfigure @@ -646,29 +647,43 @@ tasks.named(JavaBasePlugin.CHECK_TASK_NAME) { } task checkCompatibility { - def repositoryUrls = project.hasProperty('repositoryUrls') ? project.property('repositoryUrls').split(',') : []; - String refToCheckout = project.hasProperty('refToCheckout') ?: 'main' + ext.getPluginRepoURL = { jsonUrl-> + def url = new URL(jsonUrl) + def jsonSlurper = new JsonSlurper() + def json = jsonSlurper.parseText(url.text) + def labels = json.projects.values() + return labels + } + ArrayList repositoryUrls = new ArrayList() + if (project.hasProperty('repositoryUrls')) { + repositoryUrls = project.property('repositoryUrls').split(',') + } else { + println('repositoryUrls property was not provided, using components list from https://raw.githubusercontent.com/opensearch-project/opensearch-plugins/main/plugins/.meta ') + repositoryUrls = getPluginRepoURL('https://raw.githubusercontent.com/opensearch-project/opensearch-plugins/main/plugins/.meta') + }; + String refToCheckout = project.hasProperty('refToCheckout') ? project.property('refToCheckout'): 'main' def failedComponents = [] doLast { - if (repositoryUrls.size() == 0) { - throw new IllegalArgumentException("repositoryUrls must be provided as arguments.") - } - def tempDir = File.createTempDir() - repositoryUrls.each { repositoryUrl -> + println("Repository URLs: $repositoryUrls") + println("Reference that is being checked out: $refToCheckout") + repositoryUrls.parallelStream().forEach { repositoryUrl -> + def tempDir = File.createTempDir() + exec { + logger.info('Cloning repo:', repositoryUrl) + commandLine 'git', 'clone', repositoryUrl, tempDir + } try { exec { - commandLine 'git', 'clone', repositoryUrl, tempDir - } - exec { - def stdout = new ByteArrayOutputStream() workingDir = tempDir + def stdout = new ByteArrayOutputStream() + logger.info('Checking out:', refToCheckout) commandLine 'git', 'checkout', refToCheckout commandLine './gradlew', 'assemble' standardOutput stdout } } catch (ignored) { failedComponents.add(repositoryUrl) - logger.info("Failed for component: ", repositoryUrl) + logger.info("Gradle assemble failed for component: ", repositoryUrl) } finally { tempDir.deleteDir() } @@ -676,7 +691,7 @@ task checkCompatibility { if (!failedComponents.isEmpty()) { println("Components incompatible with this change: $failedComponents") } else { - println('Everything looks good!') + println('All components are compatible with this change') } } } @@ -688,4 +703,3 @@ allprojects {project -> } } } - diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 43b9e7ca05498..b58652e599ec8 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -35,7 +35,6 @@ plugins { id 'java-gradle-plugin' id 'groovy' id 'java-test-fixtures' - id 'org.ajoberstar.grgit' version '5.2.0' } group = 'org.opensearch.gradle' @@ -324,5 +323,3 @@ class VersionPropertiesLoader { loadedProps.put("opensearch", opensearch) } } - - From 01748c4c77d39b006a53ea7efae24370c68c3f8d Mon Sep 17 00:00:00 2001 From: Sayali Gaikawad Date: Fri, 2 Jun 2023 13:03:49 -0700 Subject: [PATCH 5/5] Comp check Signed-off-by: Sayali Gaikawad --- build.gradle | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 8773ad5c0a5d9..2e4d5123f6d55 100644 --- a/build.gradle +++ b/build.gradle @@ -647,25 +647,30 @@ tasks.named(JavaBasePlugin.CHECK_TASK_NAME) { } task checkCompatibility { + description('Checks code compatibility with remote component') ext.getPluginRepoURL = { jsonUrl-> def url = new URL(jsonUrl) - def jsonSlurper = new JsonSlurper() - def json = jsonSlurper.parseText(url.text) + def json = new JsonSlurper().parseText(url.text) def labels = json.projects.values() return labels } + ext.getGradleExec = { + if(System.properties['os.name'].toLowerCase().contains('windows')){ + return 'gradlew.bat' + } + return './gradlew' + } ArrayList repositoryUrls = new ArrayList() if (project.hasProperty('repositoryUrls')) { repositoryUrls = project.property('repositoryUrls').split(',') } else { - println('repositoryUrls property was not provided, using components list from https://raw.githubusercontent.com/opensearch-project/opensearch-plugins/main/plugins/.meta ') + logger.info('repositoryUrls property was not provided, using components list from https://raw.githubusercontent.com/opensearch-project/opensearch-plugins/main/plugins/.meta ') repositoryUrls = getPluginRepoURL('https://raw.githubusercontent.com/opensearch-project/opensearch-plugins/main/plugins/.meta') }; - String refToCheckout = project.hasProperty('refToCheckout') ? project.property('refToCheckout'): 'main' + String ref = project.hasProperty('ref') ? project.property('ref'): 'main' def failedComponents = [] doLast { - println("Repository URLs: $repositoryUrls") - println("Reference that is being checked out: $refToCheckout") + println("Checking compability for: $repositoryUrls for $ref") repositoryUrls.parallelStream().forEach { repositoryUrl -> def tempDir = File.createTempDir() exec { @@ -676,14 +681,15 @@ task checkCompatibility { exec { workingDir = tempDir def stdout = new ByteArrayOutputStream() - logger.info('Checking out:', refToCheckout) - commandLine 'git', 'checkout', refToCheckout - commandLine './gradlew', 'assemble' + def gradleScript = getGradleExec() + logger.info('Checking out:', ref) + commandLine 'git', 'checkout', ref + commandLine gradleScript, 'assemble' standardOutput stdout } - } catch (ignored) { + } catch (ex) { failedComponents.add(repositoryUrl) - logger.info("Gradle assemble failed for component: ", repositoryUrl) + logger.info("Gradle assemble failed for component: ", repositoryUrl, "Error logs:", ex) } finally { tempDir.deleteDir() }