From 0c24b03ed595d696611a438fda86a86c683412f7 Mon Sep 17 00:00:00 2001 From: Oliver Heger Date: Tue, 30 Mar 2021 06:56:25 +0200 Subject: [PATCH 1/2] Add a command to apply a JSLT transformation to an ORT result JSLT [1] supports powerful transformations on JSON data. This command is useful for instance if there are breaking changes in ORT's data format; by applying a corresponding transformation, stored results can be upgraded to the new format. [1] https://github.com/schibsted/jslt Signed-off-by: Oliver Heger --- gradle.properties | 1 + helper-cli/build.gradle.kts | 2 + helper-cli/src/main/kotlin/HelperMain.kt | 2 + .../kotlin/commands/TransformResultCommand.kt | 75 +++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 helper-cli/src/main/kotlin/commands/TransformResultCommand.kt diff --git a/gradle.properties b/gradle.properties index 7b074d5e5fba6..569f0c06c308f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -43,6 +43,7 @@ hopliteVersion = 1.4.0 jacksonVersion = 2.12.2 jgitVersion = 5.11.0.202103091610-r jSchAgentProxyVersion = 0.0.7 +jsltVersion = 0.1.11 kotestVersion = 4.4.3 kotlinxCoroutinesVersion = 1.4.3 kotlinxHtmlVersion = 0.7.2 diff --git a/helper-cli/build.gradle.kts b/helper-cli/build.gradle.kts index e3088ed065bee..0f02723471b05 100644 --- a/helper-cli/build.gradle.kts +++ b/helper-cli/build.gradle.kts @@ -19,6 +19,7 @@ */ val cliktVersion: String by project +val jsltVersion: String by project val log4jCoreVersion: String by project plugins { @@ -71,6 +72,7 @@ dependencies { implementation(project(":utils")) implementation("com.github.ajalt.clikt:clikt:$cliktVersion") + implementation("com.schibsted.spt.data:jslt:$jsltVersion") implementation("org.apache.logging.log4j:log4j-core:$log4jCoreVersion") implementation("org.apache.logging.log4j:log4j-slf4j-impl:$log4jCoreVersion") } diff --git a/helper-cli/src/main/kotlin/HelperMain.kt b/helper-cli/src/main/kotlin/HelperMain.kt index 998f39e15d226..8da26502ad2de 100644 --- a/helper-cli/src/main/kotlin/HelperMain.kt +++ b/helper-cli/src/main/kotlin/HelperMain.kt @@ -53,6 +53,7 @@ import org.ossreviewtoolkit.helper.commands.MergeRepositoryConfigurationsCommand import org.ossreviewtoolkit.helper.commands.RemoveConfigurationEntriesCommand import org.ossreviewtoolkit.helper.commands.SortRepositoryConfigurationCommand import org.ossreviewtoolkit.helper.commands.SubtractScanResultsCommand +import org.ossreviewtoolkit.helper.commands.TransformResultCommand import org.ossreviewtoolkit.helper.commands.VerifySourceArtifactCurationsCommand import org.ossreviewtoolkit.helper.commands.packageconfig.PackageConfigurationCommand import org.ossreviewtoolkit.helper.common.ORTH_NAME @@ -103,6 +104,7 @@ internal class HelperMain : CliktCommand(name = ORTH_NAME, epilog = "* denotes r RemoveConfigurationEntriesCommand(), SortRepositoryConfigurationCommand(), SubtractScanResultsCommand(), + TransformResultCommand(), VerifySourceArtifactCurationsCommand() ) } diff --git a/helper-cli/src/main/kotlin/commands/TransformResultCommand.kt b/helper-cli/src/main/kotlin/commands/TransformResultCommand.kt new file mode 100644 index 0000000000000..727eddb1eb85c --- /dev/null +++ b/helper-cli/src/main/kotlin/commands/TransformResultCommand.kt @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2021 Bosch.IO GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.helper.commands + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.convert +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.file + +import com.schibsted.spt.data.jslt.Parser + +import org.ossreviewtoolkit.model.yamlMapper +import org.ossreviewtoolkit.utils.expandTilde + +/** + * A command to transform an ORT result using a JSLT expression. + * + * See https://github.com/schibsted/jslt. + */ +class TransformResultCommand : CliktCommand( + name = "transform", help = "Implements a JSLT transformation on the given ORT result file." +) { + private val ortResultFile by option( + "--ort-result-file", "-i", + help = "The ORT result file to read as input." + ).convert { it.expandTilde() } + .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) + .convert { it.absoluteFile.normalize() } + .required() + + private val transformationFile by option( + "--transformation-file", + help = "The file defining the JSLT transformation to apply." + ).convert { it.expandTilde() } + .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) + .convert { it.absoluteFile.normalize() } + .required() + + private val outputFile by option( + "--output-file", "-o", + help = "The file in which to write the result of the transformation." + ).convert { it.expandTilde() } + .file(mustExist = false, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) + .convert { it.absoluteFile.normalize() } + .required() + + override fun run() { + val node = yamlMapper.readTree(ortResultFile) + + val expression = Parser.compile(transformationFile) + + val result = expression.apply(node) + yamlMapper.writeValue(outputFile, result) + + println("Wrote result to $outputFile.") + } +} From 7ea374c8817d574a38b4e67af3aeb83bb2daff44 Mon Sep 17 00:00:00 2001 From: Oliver Heger Date: Thu, 8 Apr 2021 09:01:22 +0200 Subject: [PATCH 2/2] helper-cli: Add -i alias for the input to commands where applicable Align the helper commands to the convention used by the standard commands where it makes sense. Signed-off-by: Oliver Heger --- .../kotlin/commands/ExportLicenseFindingCurationsCommand.kt | 2 +- .../src/main/kotlin/commands/ExportPathExcludesCommand.kt | 2 +- .../kotlin/commands/ExtractRepositoryConfigurationCommand.kt | 2 +- .../src/main/kotlin/commands/GenerateProjectExcludesCommand.kt | 2 +- .../kotlin/commands/GenerateRuleViolationResolutionsCommand.kt | 2 +- .../src/main/kotlin/commands/GenerateScopeExcludesCommand.kt | 2 +- .../kotlin/commands/GenerateTimeoutErrorResolutionsCommand.kt | 2 +- .../kotlin/commands/ImportLicenseFindingCurationsCommand.kt | 2 +- .../src/main/kotlin/commands/ImportPathExcludesCommand.kt | 2 +- helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt | 2 +- helper-cli/src/main/kotlin/commands/ListCopyrightsCommand.kt | 2 +- .../src/main/kotlin/commands/ListLicenseCategoriesCommand.kt | 2 +- helper-cli/src/main/kotlin/commands/ListLicensesCommand.kt | 2 +- helper-cli/src/main/kotlin/commands/ListPackagesCommand.kt | 2 +- .../main/kotlin/commands/RemoveConfigurationEntriesCommand.kt | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/helper-cli/src/main/kotlin/commands/ExportLicenseFindingCurationsCommand.kt b/helper-cli/src/main/kotlin/commands/ExportLicenseFindingCurationsCommand.kt index d2d258d01db38..aa224519ba619 100644 --- a/helper-cli/src/main/kotlin/commands/ExportLicenseFindingCurationsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ExportLicenseFindingCurationsCommand.kt @@ -51,7 +51,7 @@ internal class ExportLicenseFindingCurationsCommand : CliktCommand( .required() private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which the license finding curations are to be read." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ExportPathExcludesCommand.kt b/helper-cli/src/main/kotlin/commands/ExportPathExcludesCommand.kt index bd21d1575cc8a..648358bd5337f 100644 --- a/helper-cli/src/main/kotlin/commands/ExportPathExcludesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ExportPathExcludesCommand.kt @@ -51,7 +51,7 @@ internal class ExportPathExcludesCommand : CliktCommand( .required() private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which the path excludes are to be read." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ExtractRepositoryConfigurationCommand.kt b/helper-cli/src/main/kotlin/commands/ExtractRepositoryConfigurationCommand.kt index 3091dfe3ef630..c080337f16a78 100644 --- a/helper-cli/src/main/kotlin/commands/ExtractRepositoryConfigurationCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ExtractRepositoryConfigurationCommand.kt @@ -34,7 +34,7 @@ class ExtractRepositoryConfigurationCommand : CliktCommand( help = "Extract the repository configuration from the given ORT result file." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which repository configuration shall be extracted." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) diff --git a/helper-cli/src/main/kotlin/commands/GenerateProjectExcludesCommand.kt b/helper-cli/src/main/kotlin/commands/GenerateProjectExcludesCommand.kt index 926dd0f01c78a..38979fc49b2e3 100644 --- a/helper-cli/src/main/kotlin/commands/GenerateProjectExcludesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/GenerateProjectExcludesCommand.kt @@ -40,7 +40,7 @@ internal class GenerateProjectExcludesCommand : CliktCommand( "the given repository configuration file." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which the rule violations are read." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) diff --git a/helper-cli/src/main/kotlin/commands/GenerateRuleViolationResolutionsCommand.kt b/helper-cli/src/main/kotlin/commands/GenerateRuleViolationResolutionsCommand.kt index 728c06d087a55..dcde61232f2c7 100644 --- a/helper-cli/src/main/kotlin/commands/GenerateRuleViolationResolutionsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/GenerateRuleViolationResolutionsCommand.kt @@ -44,7 +44,7 @@ internal class GenerateRuleViolationResolutionsCommand : CliktCommand( "configuration file." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which the rule violations are read." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) diff --git a/helper-cli/src/main/kotlin/commands/GenerateScopeExcludesCommand.kt b/helper-cli/src/main/kotlin/commands/GenerateScopeExcludesCommand.kt index 945b9a24e5f9e..3b4d1df316e5f 100644 --- a/helper-cli/src/main/kotlin/commands/GenerateScopeExcludesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/GenerateScopeExcludesCommand.kt @@ -41,7 +41,7 @@ internal class GenerateScopeExcludesCommand : CliktCommand( "given repository configuration file." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which the rule violations are read." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) diff --git a/helper-cli/src/main/kotlin/commands/GenerateTimeoutErrorResolutionsCommand.kt b/helper-cli/src/main/kotlin/commands/GenerateTimeoutErrorResolutionsCommand.kt index 90b8e0f8e6cf0..d090d55c7c2b1 100644 --- a/helper-cli/src/main/kotlin/commands/GenerateTimeoutErrorResolutionsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/GenerateTimeoutErrorResolutionsCommand.kt @@ -41,7 +41,7 @@ internal class GenerateTimeoutErrorResolutionsCommand : CliktCommand( help = "Generates resolutions for scanner timeout errors. The result is written to the standard output." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file containing the scan timeout errors." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ImportLicenseFindingCurationsCommand.kt b/helper-cli/src/main/kotlin/commands/ImportLicenseFindingCurationsCommand.kt index 88d1fa29227af..1b20f4cc7927c 100644 --- a/helper-cli/src/main/kotlin/commands/ImportLicenseFindingCurationsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ImportLicenseFindingCurationsCommand.kt @@ -52,7 +52,7 @@ internal class ImportLicenseFindingCurationsCommand : CliktCommand( .required() private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The ORT file containing the findings the imported curations need to match against." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ImportPathExcludesCommand.kt b/helper-cli/src/main/kotlin/commands/ImportPathExcludesCommand.kt index 7d6628a1bce29..55310090acfb6 100644 --- a/helper-cli/src/main/kotlin/commands/ImportPathExcludesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ImportPathExcludesCommand.kt @@ -40,7 +40,7 @@ internal class ImportPathExcludesCommand : CliktCommand( help = "Import path excludes by repository from a file into the given repository configuration." ) { private val pathExcludesFile by option( - "--path-excludes-file", + "--path-excludes-file", "-i", help = "The input path excludes file." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt b/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt index b85733e9310d9..68ab8e9ceb450 100644 --- a/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ImportScanResultsCommand.kt @@ -35,7 +35,7 @@ internal class ImportScanResultsCommand : CliktCommand( help = "Import all scan results from the given ORT result file to the file based scan results storage directory." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The input ORT file from which repository configuration shall be extracted." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = false) diff --git a/helper-cli/src/main/kotlin/commands/ListCopyrightsCommand.kt b/helper-cli/src/main/kotlin/commands/ListCopyrightsCommand.kt index 7ad8587e3c47d..538158220024f 100644 --- a/helper-cli/src/main/kotlin/commands/ListCopyrightsCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ListCopyrightsCommand.kt @@ -42,7 +42,7 @@ internal class ListCopyrightsCommand : CliktCommand( help = "Lists the copyright findings." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The ORT result file to read as input." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ListLicenseCategoriesCommand.kt b/helper-cli/src/main/kotlin/commands/ListLicenseCategoriesCommand.kt index b852ed3c930cd..cfe3446abd558 100644 --- a/helper-cli/src/main/kotlin/commands/ListLicenseCategoriesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ListLicenseCategoriesCommand.kt @@ -35,7 +35,7 @@ class ListLicenseCategoriesCommand : CliktCommand( help = "Lists the license categories." ) { private val licenseClassificationsFile by option( - "--license-classifications-file", + "--license-classifications-file", "-i", help = "The license classifications file." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ListLicensesCommand.kt b/helper-cli/src/main/kotlin/commands/ListLicensesCommand.kt index b935fb25704c8..b4e5d490a0659 100644 --- a/helper-cli/src/main/kotlin/commands/ListLicensesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ListLicensesCommand.kt @@ -59,7 +59,7 @@ internal class ListLicensesCommand : CliktCommand( help = "Lists the license findings for a given package as distinct text locations." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The ORT result file to read as input." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/ListPackagesCommand.kt b/helper-cli/src/main/kotlin/commands/ListPackagesCommand.kt index 0bab553d66484..ca33404176334 100644 --- a/helper-cli/src/main/kotlin/commands/ListPackagesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/ListPackagesCommand.kt @@ -38,7 +38,7 @@ class ListPackagesCommand : CliktCommand( help = "Lists the packages and projects contained in the given ORT result file." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The ORT result file to read as input." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true) diff --git a/helper-cli/src/main/kotlin/commands/RemoveConfigurationEntriesCommand.kt b/helper-cli/src/main/kotlin/commands/RemoveConfigurationEntriesCommand.kt index b4ad69d37d204..11f6012a65164 100644 --- a/helper-cli/src/main/kotlin/commands/RemoveConfigurationEntriesCommand.kt +++ b/helper-cli/src/main/kotlin/commands/RemoveConfigurationEntriesCommand.kt @@ -44,7 +44,7 @@ internal class RemoveConfigurationEntriesCommand : CliktCommand( "written to the given repository configuration file." ) { private val ortResultFile by option( - "--ort-result-file", + "--ort-result-file", "-i", help = "The ORT result file to read as input which should contain an evaluator result." ).convert { it.expandTilde() } .file(mustExist = true, canBeFile = true, canBeDir = false, mustBeWritable = false, mustBeReadable = true)