From c2c6f1b34c91e72310e992def650211d2172aa99 Mon Sep 17 00:00:00 2001 From: Albert Tregnaghi Date: Mon, 18 Nov 2024 14:37:55 +0100 Subject: [PATCH] Implemented PDS asset handling #3524 - introduced new usecase for PDS job execution (makes it easier for developers to find the relevant parts) --- .../commons/archive/ArchiveSupport.java | 35 +++- .../pds/PDSDefaultRuntimeKeyConstants.java | 2 + ...PDSLauncherScriptEnvironmentConstants.java | 1 + .../commons/pds/data/PDSTemplateMetaData.java | 54 +++--- .../pds/data/PDSTemplateMetaDataTest.java | 16 +- .../asciidoc/documents/pds/pds_config.adoc | 19 +- .../pds-param-template-metadata-example1.json | 8 +- .../pds-param-template-metadata-syntax.json | 8 +- .../integrationtest-webscan.sh | 5 + .../PDSWebScanJobScenario12IntTest.java | 20 +- .../pds/job/PDSJobConfigurationSupport.java | 11 ++ .../pds/usecase/PDSUseCaseIdentifier.java | 2 + .../pds/usecase/UseCaseSystemExecutesJob.java | 19 ++ .../pds/batch/PDSBatchTriggerService.java | 6 +- .../pds/execution/PDSExecutionCallable.java | 2 + .../PDSExecutionEnvironmentService.java | 4 + .../pds/execution/PDSExecutionService.java | 2 + .../sechub/pds/job/PDSWorkspaceService.java | 181 ++++++++++++++++-- .../sechub/pds/job/WorkspaceLocationData.java | 5 + .../pds/job/PDSWorkspaceServiceTest.java | 117 +++++++++-- .../pds/PDSTemplateMetaDataService.java | 12 +- .../pds/PDSExecutorConfigSupportTest.java | 4 +- .../pds/PDSTemplateMetaDataServiceTest.java | 16 +- 23 files changed, 436 insertions(+), 113 deletions(-) create mode 100644 sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/UseCaseSystemExecutesJob.java diff --git a/sechub-commons-archive/src/main/java/com/mercedesbenz/sechub/commons/archive/ArchiveSupport.java b/sechub-commons-archive/src/main/java/com/mercedesbenz/sechub/commons/archive/ArchiveSupport.java index e03c99e287..a57352dc9f 100644 --- a/sechub-commons-archive/src/main/java/com/mercedesbenz/sechub/commons/archive/ArchiveSupport.java +++ b/sechub-commons-archive/src/main/java/com/mercedesbenz/sechub/commons/archive/ArchiveSupport.java @@ -4,6 +4,7 @@ import static java.util.Objects.requireNonNull; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -329,6 +330,28 @@ public void compressFolder(ArchiveType type, File folder, File targetArchiveFile } + /** + * Just extracts the given archive file to target folder, without any + * adjustments. + * + * @param archiveType archive type to use + * @param archiveFile file which shall be extracted + * @param targetFolder target folder where extraction shall be done + * @throws IOException + */ + public ArchiveExtractionResult extractFileAsIsToFolder(ArchiveType archiveType, File archiveFile, File targetFolder, + ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException { + try (FileInputStream sourceInputStream = new FileInputStream(archiveFile)) { + String sourceLocation = archiveFile.getAbsolutePath(); + var result = switch (archiveType) { + case TAR -> extractTar(sourceInputStream, sourceLocation, targetFolder, null, archiveExtractionConstraints); + case ZIP -> extractZip(sourceInputStream, sourceLocation, targetFolder, null, archiveExtractionConstraints); + default -> throw new IllegalArgumentException("Archive type: " + archiveType + " is not supported!"); + }; + return result; + } + } + private void compressRecursively(String basePath, ArchiveOutputStream outputStream, File file, ArchiveType type, String pathAddition, CreationPathContext creationPathContext) throws IOException { @@ -407,13 +430,13 @@ private void compressRecursively(String basePath, ArchiveOutputStream outputStre } private ArchiveExtractionResult extractTar(InputStream sourceInputStream, String sourceLocation, File outputDir, - SecHubFileStructureDataProvider fileStructureProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException { + SecHubFileStructureDataProvider fileStructurDataProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException { try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveType.TAR.getType(), sourceInputStream)) { if (!(archiveInputStream instanceof TarArchiveInputStream)) { throw new IOException("Cannot extract: " + sourceLocation + " because it is not a tar tar"); } try (SafeArchiveInputStream safeArchiveInputStream = new SafeArchiveInputStream(archiveInputStream, archiveExtractionConstraints)) { - return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructureProvider); + return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructurDataProvider); } } catch (ArchiveException e) { throw new IOException("Was not able to extract tar:" + sourceLocation + " at " + outputDir, e); @@ -422,11 +445,11 @@ private ArchiveExtractionResult extractTar(InputStream sourceInputStream, String } private ArchiveExtractionResult extractZip(InputStream sourceInputStream, String sourceLocation, File outputDir, - SecHubFileStructureDataProvider configuration, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException { + SecHubFileStructureDataProvider fileStructurDataProvider, ArchiveExtractionConstraints archiveExtractionConstraints) throws IOException { try (ArchiveInputStream archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveType.ZIP.getType(), sourceInputStream); SafeArchiveInputStream safeArchiveInputStream = new SafeArchiveInputStream(archiveInputStream, archiveExtractionConstraints)) { - return extract(safeArchiveInputStream, sourceLocation, outputDir, configuration); + return extract(safeArchiveInputStream, sourceLocation, outputDir, fileStructurDataProvider); } catch (ArchiveException e) { throw new IOException("Was not able to extract tar:" + sourceLocation + " at " + outputDir, e); @@ -460,7 +483,7 @@ public boolean isZipFileStream(InputStream inputStream) { } private ArchiveExtractionResult extract(SafeArchiveInputStream safeArchiveInputStream, String sourceLocation, File outputDir, - SecHubFileStructureDataProvider fileStructureProvider) throws ArchiveException, IOException { + SecHubFileStructureDataProvider fileStructurDataProvider) throws ArchiveException, IOException { ArchiveExtractionResult result = new ArchiveExtractionResult(); result.targetLocation = outputDir.getAbsolutePath(); @@ -473,7 +496,7 @@ private ArchiveExtractionResult extract(SafeArchiveInputStream safeArchiveInputS throw new IllegalStateException("Entry path is null - cannot be handled!"); } - ArchiveTransformationData data = createTransformationData(fileStructureProvider, name); + ArchiveTransformationData data = createTransformationData(fileStructurDataProvider, name); if (data == null) { continue; } diff --git a/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSDefaultRuntimeKeyConstants.java b/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSDefaultRuntimeKeyConstants.java index f45b60e0e1..44fcd89635 100644 --- a/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSDefaultRuntimeKeyConstants.java +++ b/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSDefaultRuntimeKeyConstants.java @@ -30,6 +30,8 @@ public class PDSDefaultRuntimeKeyConstants { public static final String RT_KEY_PDS_JOB_BINARIES_TAR_FILE = "pds.job.binaries.tar.file"; + public static final String RT_KEY_PDS_JOB_EXTRACTED_ASSETS_FOLDER = "pds.job.extracted.assets.folder"; + public static final String RT_KEY_PDS_JOB_EXTRACTED_SOURCES_FOLDER = "pds.job.extracted.sources.folder"; public static final String RT_KEY_PDS_JOB_HAS_EXTRACTED_BINARIES = "pds.job.has.extracted.binaries"; diff --git a/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSLauncherScriptEnvironmentConstants.java b/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSLauncherScriptEnvironmentConstants.java index cb8b2aa650..cf2232f0ea 100644 --- a/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSLauncherScriptEnvironmentConstants.java +++ b/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/PDSLauncherScriptEnvironmentConstants.java @@ -27,6 +27,7 @@ public class PDSLauncherScriptEnvironmentConstants { @Deprecated public static final String PDS_JOB_SOURCECODE_UNZIPPED_FOLDER = "PDS_JOB_SOURCECODE_UNZIPPED_FOLDER"; + public static final String PDS_JOB_EXTRACTED_ASSETS_FOLDER = "PDS_JOB_EXTRACTED_ASSETS_FOLDER"; public static final String PDS_JOB_EXTRACTED_SOURCES_FOLDER = "PDS_JOB_EXTRACTED_SOURCES_FOLDER"; public static final String PDS_JOB_HAS_EXTRACTED_SOURCES = "PDS_JOB_HAS_EXTRACTED_SOURCES"; public static final String PDS_JOB_SOURCECODE_ZIP_FILE = "PDS_JOB_SOURCECODE_ZIP_FILE"; diff --git a/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaData.java b/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaData.java index 78acd6b804..8d1712ff90 100644 --- a/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaData.java +++ b/sechub-commons-pds/src/main/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaData.java @@ -7,28 +7,28 @@ public class PDSTemplateMetaData { - private String template; - private TemplateType type; + private String templateId; + private TemplateType templateType; private PDSAssetData assetData; public PDSAssetData getAssetData() { return assetData; } - public String getTemplate() { - return template; + public String getTemplateId() { + return templateId; } - public TemplateType getType() { - return type; + public TemplateType getTemplateType() { + return templateType; } - public void setType(TemplateType type) { - this.type = type; + public void setTemplateType(TemplateType type) { + this.templateType = type; } - public void setTemplate(String templateId) { - this.template = templateId; + public void setTemplateId(String templateId) { + this.templateId = templateId; } public void setAssetData(PDSAssetData assetData) { @@ -37,7 +37,7 @@ public void setAssetData(PDSAssetData assetData) { @Override public int hashCode() { - return Objects.hash(assetData, template, type); + return Objects.hash(assetData, templateId, templateType); } @Override @@ -49,34 +49,34 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; PDSTemplateMetaData other = (PDSTemplateMetaData) obj; - return Objects.equals(assetData, other.assetData) && Objects.equals(template, other.template) && type == other.type; + return Objects.equals(assetData, other.assetData) && Objects.equals(templateId, other.templateId) && templateType == other.templateType; } @Override public String toString() { - return "PDSTemplateMetaData [" + (template != null ? "template=" + template + ", " : "") + (type != null ? "type=" + type + ", " : "") - + (assetData != null ? "assetData=" + assetData : "") + "]"; + return "PDSTemplateMetaData [" + (templateId != null ? "template=" + templateId + ", " : "") + + (templateType != null ? "type=" + templateType + ", " : "") + (assetData != null ? "assetData=" + assetData : "") + "]"; } public static class PDSAssetData { - private String asset; - private String file; + private String assetId; + private String fileName; private String checksum; - public String getAsset() { - return asset; + public String getAssetId() { + return assetId; } - public void setAsset(String asset) { - this.asset = asset; + public void setAssetId(String assetId) { + this.assetId = assetId; } - public String getFile() { - return file; + public String getFileName() { + return fileName; } - public void setFile(String file) { - this.file = file; + public void setFileName(String fileName) { + this.fileName = fileName; } public String getChecksum() { @@ -89,7 +89,7 @@ public void setChecksum(String checksum) { @Override public int hashCode() { - return Objects.hash(asset, checksum, file); + return Objects.hash(assetId, checksum, fileName); } @Override @@ -101,12 +101,12 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; PDSAssetData other = (PDSAssetData) obj; - return Objects.equals(asset, other.asset) && Objects.equals(checksum, other.checksum) && Objects.equals(file, other.file); + return Objects.equals(assetId, other.assetId) && Objects.equals(checksum, other.checksum) && Objects.equals(fileName, other.fileName); } @Override public String toString() { - return "PDSAssetData [" + (asset != null ? "asset=" + asset + ", " : "") + (file != null ? "file=" + file + ", " : "") + return "PDSAssetData [" + (assetId != null ? "assetId=" + assetId + ", " : "") + (fileName != null ? "fileName=" + fileName + ", " : "") + (checksum != null ? "checksum=" + checksum : "") + "]"; } diff --git a/sechub-commons-pds/src/test/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaDataTest.java b/sechub-commons-pds/src/test/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaDataTest.java index ddf3aeae8a..c752a997f7 100644 --- a/sechub-commons-pds/src/test/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaDataTest.java +++ b/sechub-commons-pds/src/test/java/com/mercedesbenz/sechub/commons/pds/data/PDSTemplateMetaDataTest.java @@ -1,10 +1,15 @@ // SPDX-License-Identifier: MIT package com.mercedesbenz.sechub.commons.pds.data; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; + import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import com.mercedesbenz.sechub.commons.model.JSONConverter; +import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData.PDSAssetData; import com.mercedesbenz.sechub.test.TestFileReader; class PDSTemplateMetaDataTest { @@ -16,8 +21,17 @@ void examples_in_doc_are_valid(String fileName) { String json = TestFileReader.readTextFromFile("./../sechub-doc/src/docs/asciidoc/documents/shared/snippet/" + fileName); /* execute */ - JSONConverter.get().fromJSONtoListOf(PDSTemplateMetaData.class, json); + List result = JSONConverter.get().fromJSONtoListOf(PDSTemplateMetaData.class, json); + /* test */ + assertEquals(1, result.size()); + PDSTemplateMetaData entry = result.iterator().next(); + assertNotNull(entry.getTemplateId()); + PDSAssetData assetData = entry.getAssetData(); + assertNotNull(assetData); + assertNotNull(assetData.getAssetId()); + assertNotNull(assetData.getChecksum()); + assertNotNull(assetData.getFileName()); } } diff --git a/sechub-doc/src/docs/asciidoc/documents/pds/pds_config.adoc b/sechub-doc/src/docs/asciidoc/documents/pds/pds_config.adoc index d4bc584562..9e798d1065 100644 --- a/sechub-doc/src/docs/asciidoc/documents/pds/pds_config.adoc +++ b/sechub-doc/src/docs/asciidoc/documents/pds/pds_config.adoc @@ -48,7 +48,7 @@ Either use system properties pds.techuser.userid pds.techuser.apitoken ---- -or env entries +or environment variables ---- PDS_TECHUSER_USERID PDS_TECHUSER_APITOKEN @@ -187,6 +187,12 @@ how the data is gathered. |PDS_JOB_SOURCECODE_ZIP_FILE | The absolute path to the uploaded "sourcecode.zip" file |PDS_JOB_EXTRACTED_SOURCES_FOLDER | When auto extracting is enabled (default) the uploaded source code is extracted to this folder |PDS_JOB_EXTRACTED_BINARIES_FOLDER | When auto extracting is enabled (default) the uploaded binaries are extracted to this folder +|PDS_JOB_EXTRACTED_ASSETS_FOLDER | The absolute path to the extracted assets. + + + + Files for template types are located in dedicated sub folders which are named by the id of the template type. + + + + + For example: WEBSCAN_LOGIN template type has id: `webscan-login`. To access the file `custom-login.groovy` a script + or a wrapper application can simply fetch the file via `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/custom-login.groovy`. |PDS_SCAN_TARGET_URL | Target URL for current scan (e.g webscan). Will not be set in all scan types. E.g. for a code scan this environemnt variable will not be available |PDS_SCAN_CONFIGURATION | Contains the SecHub configuration as JSON _(but reduced to current scan type, so e.g. a web scan will have no code scan configuration data available)_ + + @@ -216,10 +222,15 @@ include::../gen/gen_pds_executor_config_parameters.adoc[] ==== File locations ===== Upload -`$PDS_JOB_WORKSPACE_LOCATION/upload/` +Content from uploaded user archives is extracted to: + +`PDS_JOB_EXTRACTED_SOURCES_FOLDER`, + +`PDS_JOB_EXTRACTED_BINARIES_FOLDER` + +Content from uploaded asset files is extracted to: + `PDS_JOB_EXTRACTED_ASSETS_FOLDER`, -Automatically unzipped content is available inside + -`$PDS_JOB_WORKSPACE_LOCATION/upload/unzipped` ===== Output diff --git a/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-example1.json b/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-example1.json index 28d91edf51..9f67da8c86 100644 --- a/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-example1.json +++ b/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-example1.json @@ -1,9 +1,9 @@ [ { - "template" : "single-singon", - "type" : "webscan-login", + "templateId" : "single-singon", + "templateType" : "webscan-login", "assetData" : { - "asset" : "custom-webscan-setup", - "file" : "WEBSCAN_PRODUCT_ID.zip", + "assetId" : "custom-webscan-setup", + "fileName" : "WEBSCAN_PRODUCT_ID.zip", "checksum" : "434c6c6ec1b0ed9844149069d7d45ac18e72505b" } } ] \ No newline at end of file diff --git a/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-syntax.json b/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-syntax.json index b45a65ff72..13346713f2 100644 --- a/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-syntax.json +++ b/sechub-doc/src/docs/asciidoc/documents/shared/snippet/pds-param-template-metadata-syntax.json @@ -1,11 +1,11 @@ [ //<1> { - "template" : "templateId", //<2> - "type": "WEBSCAN_LOGIN", //<3> + "templateId" : "templateId", //<2> + "tempplateType": "WEBSCAN_LOGIN", //<3> "assetData" : { //<4> - "asset" : "assetId", //<5> - "file" : "fileName", //<6> + "assetId" : "assetId", //<5> + "fileName" : "fileName", //<6> "checksum" : "fileChecksum" //<7> } diff --git a/sechub-integrationtest/pds/product-scripts/integrationtest-webscan.sh b/sechub-integrationtest/pds/product-scripts/integrationtest-webscan.sh index 7fd103086f..c89235bf74 100755 --- a/sechub-integrationtest/pds/product-scripts/integrationtest-webscan.sh +++ b/sechub-integrationtest/pds/product-scripts/integrationtest-webscan.sh @@ -20,6 +20,11 @@ dumpPDSVariables # We added pds.config.templates.metadata.list as optional parameter here for testing # So we can dump the variable here - used in scenario12 integration test dumpVariable "PDS_CONFIG_TEMPLATE_METADATA_LIST" + +ASSET_FILE1="$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/testfile1.txt" +TEST_CONTENT_FROM_ASSETFILE=$(cat $ASSET_FILE1) +# Afterwards TEST_CONTENT_FROM_ASSETFILE=i am "testfile1.txt" for scenario12 integration tests +dumpVariable "TEST_CONTENT_FROM_ASSETFILE" if [[ "$PDS_TEST_KEY_VARIANTNAME" = "a" ]]; then diff --git a/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario12/PDSWebScanJobScenario12IntTest.java b/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario12/PDSWebScanJobScenario12IntTest.java index 3a3504587b..f813483f04 100644 --- a/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario12/PDSWebScanJobScenario12IntTest.java +++ b/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario12/PDSWebScanJobScenario12IntTest.java @@ -62,17 +62,20 @@ public class PDSWebScanJobScenario12IntTest { * * The tests checks following: * - * - pds_web_scan_has_expected_info_finding_with_given_target_url_and_product2_level_information_and_sechub_web_config_parts - * - pds web scan has expected info finding, with - * - given target url + * - PDS web scan has expected info finding, with + * - given target URL * - product level information - * - sechub web configuration parts + * - SecHub web configuration parts * * - PDS parameter for template meta data configuration is correct and transmitted to PDS * The parameter "pds.config.template.metadata.list" is normally not available inside * the scripts, but for testing we added the parameter inside server configuration so it * will be added to script level and can be checked by TestAPI * + * - PDS will download and extract the uploaded asset file automatically and the + * extracted content is available inside the test bash script (executed by PDS) + * + * * @formatter:on */ @Test @@ -163,13 +166,13 @@ public void pds_web_scan_can_be_executed_and_works() throws Exception { assertTrue(includes.contains("/customer/<*>")); assertTrue(excludes.contains("<*>/admin/<*>")); - // config must contain the expected headers + // web configuration must contain the expected headers assertExpectedHeaders(webConfiguration); - // config must contain the expected client certificate + // web configuration must contain the expected client certificate assertExpectedClientCertificate(webConfiguration); - // config must contain the expected openApi definition + // web configuration must contain the expected openApi definition assertExpectedOpenApiDefinition(webConfiguration); /* additional testing : messages*/ @@ -185,9 +188,10 @@ public void pds_web_scan_can_be_executed_and_works() throws Exception { Map variables = fetchPDSVariableTestOutputMap(pdsJobUUID); String expectedMetaDataListJson = """ - [{"template":"template-scenario12-1","type":"WEBSCAN_LOGIN","assetData":{"asset":"asset-s12-pds-inttest-webscan","file":"PDS_INTTEST_PRODUCT_WEBSCAN.zip","checksum":"ff06430bfc2d8c698ab8effa41b914525b8cca1c1eecefa76d248b25cc598fba"}}] + [{"templateId":"template-scenario12-1","templateType":"WEBSCAN_LOGIN","assetData":{"assetId":"asset-s12-pds-inttest-webscan","fileName":"PDS_INTTEST_PRODUCT_WEBSCAN.zip","checksum":"ff06430bfc2d8c698ab8effa41b914525b8cca1c1eecefa76d248b25cc598fba"}}] """.trim(); assertThat(variables.get("PDS_CONFIG_TEMPLATE_METADATA_LIST")).isEqualTo(expectedMetaDataListJson); + assertThat(variables.get("TEST_CONTENT_FROM_ASSETFILE")).isEqualTo("i am \"testfile1.txt\" for scenario12 integration tests"); /* @formatter:on */ } diff --git a/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/job/PDSJobConfigurationSupport.java b/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/job/PDSJobConfigurationSupport.java index 8482876303..414831af46 100644 --- a/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/job/PDSJobConfigurationSupport.java +++ b/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/job/PDSJobConfigurationSupport.java @@ -13,12 +13,14 @@ import org.slf4j.LoggerFactory; import com.mercedesbenz.sechub.commons.core.util.SimpleStringUtils; +import com.mercedesbenz.sechub.commons.model.JSONConverter; import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModel; import com.mercedesbenz.sechub.commons.model.SecHubDataConfigurationType; import com.mercedesbenz.sechub.commons.model.SecHubDataConfigurationTypeListParser; import com.mercedesbenz.sechub.commons.model.SecHubScanConfiguration; import com.mercedesbenz.sechub.commons.pds.PDSDefaultParameterKeyConstants; import com.mercedesbenz.sechub.commons.pds.PDSDefaultParameterValueConstants; +import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData; import com.mercedesbenz.sechub.pds.execution.PDSExecutionParameterEntry; public class PDSJobConfigurationSupport { @@ -200,4 +202,13 @@ public int getJobStorageReadResiliencRetryWaitSeconds(int defaultValue) { return result; } + public List getTemplateMetaData() { + + String json = getStringParameterOrNull(PARAM_KEY_PDS_CONFIG_TEMPLATE_META_DATA_LIST); + if (json == null || json.isBlank()) { + return Collections.emptyList(); + } + return JSONConverter.get().fromJSONtoListOf(PDSTemplateMetaData.class, json); + } + } diff --git a/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/PDSUseCaseIdentifier.java b/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/PDSUseCaseIdentifier.java index 70de74439b..f59264d4dc 100644 --- a/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/PDSUseCaseIdentifier.java +++ b/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/PDSUseCaseIdentifier.java @@ -47,6 +47,8 @@ public enum PDSUseCaseIdentifier { UC_SYSTEM_SIGTERM_HANDLING(19, false), + UC_SYSTEM_JOB_EXECUTION(20, false), + ; /* +---------------------------------------------------------------------+ */ diff --git a/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/UseCaseSystemExecutesJob.java b/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/UseCaseSystemExecutesJob.java new file mode 100644 index 0000000000..d3be4feb73 --- /dev/null +++ b/sechub-pds-core/src/main/java/com/mercedesbenz/sechub/pds/usecase/UseCaseSystemExecutesJob.java @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT +package com.mercedesbenz.sechub.pds.usecase; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/* @formatter:off */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@PDSUseCaseDefinition( + id=PDSUseCaseIdentifier.UC_SYSTEM_JOB_EXECUTION, + group=PDSUseCaseGroup.JOB_EXECUTION, + title="System executes job", + description="The PDS does execute a PDS job.") +public @interface UseCaseSystemExecutesJob { + PDSStep value(); +} diff --git a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/batch/PDSBatchTriggerService.java b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/batch/PDSBatchTriggerService.java index 5fa5765a36..6485696c34 100644 --- a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/batch/PDSBatchTriggerService.java +++ b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/batch/PDSBatchTriggerService.java @@ -16,6 +16,8 @@ import com.mercedesbenz.sechub.pds.execution.PDSExecutionService; import com.mercedesbenz.sechub.pds.job.PDSJobRepository; import com.mercedesbenz.sechub.pds.job.PDSJobTransactionService; +import com.mercedesbenz.sechub.pds.usecase.PDSStep; +import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemExecutesJob; import jakarta.annotation.PostConstruct; @@ -59,6 +61,7 @@ protected void postConstruct() { // default 10 seconds delay and 5 seconds initial @Scheduled(initialDelayString = "${pds.config.trigger.nextjob.initialdelay:" + DEFAULT_INITIAL_DELAY_MILLIS + "}", fixedDelayString = "${pds.config.trigger.nextjob.delay:" + DEFAULT_FIXED_DELAY_MILLIS + "}") + @UseCaseSystemExecutesJob(@PDSStep(number = 1, name = "PDS trigger service fills queue", description = "Trigger service adds jobs to queue (when queue not full)")) public void triggerExecutionOfNextJob() { if (!schedulingEnabled) { LOG.trace("Trigger execution of next job canceled, because scheduling disabled."); @@ -86,8 +89,7 @@ public void triggerExecutionOfNextJob() { } catch (ObjectOptimisticLockingFailureException e) { /* * This can happen when PDS instances are started at same time, so the check for - * next jobs can lead to race condiitons - and optmistic locks will occurre - * here. + * next jobs can lead to race conditions - and optimistic locks will occur here. * * To avoid this to happen again, we wait a random time here. So next call on * this machine should normally not collide again. diff --git a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionCallable.java b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionCallable.java index 6dfb1085f2..61bac36ec3 100644 --- a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionCallable.java +++ b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionCallable.java @@ -39,6 +39,7 @@ import com.mercedesbenz.sechub.pds.usecase.UseCaseAdminFetchesJobErrorStream; import com.mercedesbenz.sechub.pds.usecase.UseCaseAdminFetchesJobMetaData; import com.mercedesbenz.sechub.pds.usecase.UseCaseAdminFetchesJobOutputStream; +import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemExecutesJob; import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemHandlesJobCancelRequests; import com.mercedesbenz.sechub.pds.util.PDSResilientRetryExecutor; import com.mercedesbenz.sechub.pds.util.PDSResilientRetryExecutor.ExceptionThrower; @@ -91,6 +92,7 @@ public void throwException(String message, Exception cause) throws IllegalStateE } @Override + @UseCaseSystemExecutesJob(@PDSStep(number = 3, name = "PDS execution call", description = "Central point of PDS job execution.")) public PDSExecutionResult call() throws Exception { LOG.info("Prepare execution of PDS job: {}", pdsJobUUID); PDSExecutionResult result = new PDSExecutionResult(); diff --git a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionEnvironmentService.java b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionEnvironmentService.java index 27fdcaa7eb..3896d68bcc 100644 --- a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionEnvironmentService.java +++ b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionEnvironmentService.java @@ -105,6 +105,10 @@ private void addPdsJobRelatedVariables(UUID pdsJobUUID, Map map) map.put(PDS_JOB_SOURCECODE_ZIP_FILE, locationData.getSourceCodeZipFileLocation()); map.put(PDS_JOB_BINARIES_TAR_FILE, locationData.getBinariesTarFileLocation()); + String extractedAssetsLocation = locationData.getExtractedAssetsLocation(); + + map.put(PDS_JOB_EXTRACTED_ASSETS_FOLDER, extractedAssetsLocation); + String extractedSourcesLocation = locationData.getExtractedSourcesLocation(); map.put(PDS_JOB_SOURCECODE_UNZIPPED_FOLDER, extractedSourcesLocation); diff --git a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionService.java b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionService.java index 71131c1981..28f57eb91e 100644 --- a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionService.java +++ b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/execution/PDSExecutionService.java @@ -37,6 +37,7 @@ import com.mercedesbenz.sechub.pds.job.PDSWorkspaceService; import com.mercedesbenz.sechub.pds.usecase.PDSStep; import com.mercedesbenz.sechub.pds.usecase.UseCaseAdminFetchesMonitoringStatus; +import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemExecutesJob; import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemHandlesJobCancelRequests; import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemSigTermHandling; @@ -205,6 +206,7 @@ public boolean isQueueFull() { } @Async + @UseCaseSystemExecutesJob(@PDSStep(number = 2, name = "Add Job to queue", description = "PDS job is added to queue")) public void addToExecutionQueueAsynchron(UUID jobUUID) { Future former = null; synchronized (jobsInQueue) { diff --git a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceService.java b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceService.java index 329354db30..6fc14bbdfd 100644 --- a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceService.java +++ b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceService.java @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT package com.mercedesbenz.sechub.pds.job; -import static com.mercedesbenz.sechub.commons.core.CommonConstants.FILENAME_BINARIES_TAR; -import static com.mercedesbenz.sechub.commons.core.CommonConstants.FILENAME_SOURCECODE_ZIP; +import static com.mercedesbenz.sechub.commons.core.CommonConstants.*; +import static java.util.Objects.*; import java.io.File; import java.io.FileFilter; @@ -15,6 +15,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; import java.time.Duration; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -34,10 +35,14 @@ import com.mercedesbenz.sechub.commons.archive.ArchiveSupport.ArchiveType; import com.mercedesbenz.sechub.commons.archive.FileSize; import com.mercedesbenz.sechub.commons.archive.SecHubFileStructureDataProvider; +import com.mercedesbenz.sechub.commons.core.security.CheckSumSupport; import com.mercedesbenz.sechub.commons.model.JSONConverter; import com.mercedesbenz.sechub.commons.model.JSONConverterException; import com.mercedesbenz.sechub.commons.model.ScanType; import com.mercedesbenz.sechub.commons.model.SecHubConfigurationModel; +import com.mercedesbenz.sechub.commons.model.template.TemplateType; +import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData; +import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData.PDSAssetData; import com.mercedesbenz.sechub.commons.pds.execution.ExecutionEventData; import com.mercedesbenz.sechub.commons.pds.execution.ExecutionEventDetailIdentifier; import com.mercedesbenz.sechub.commons.pds.execution.ExecutionEventType; @@ -47,18 +52,24 @@ import com.mercedesbenz.sechub.pds.config.PDSServerConfigurationService; import com.mercedesbenz.sechub.pds.storage.PDSMultiStorageService; import com.mercedesbenz.sechub.pds.storage.PDSStorageInfoCollector; +import com.mercedesbenz.sechub.pds.usecase.PDSStep; +import com.mercedesbenz.sechub.pds.usecase.UseCaseSystemExecutesJob; import com.mercedesbenz.sechub.pds.util.PDSArchiveSupportProvider; import com.mercedesbenz.sechub.pds.util.PDSResilientRetryExecutor; import com.mercedesbenz.sechub.pds.util.PDSResilientRetryExecutor.ExceptionThrower; +import com.mercedesbenz.sechub.storage.core.AssetStorage; import com.mercedesbenz.sechub.storage.core.JobStorage; +import com.mercedesbenz.sechub.storage.core.Storage; @Service public class PDSWorkspaceService { public static final String UPLOAD = "upload"; + public static final String ASSETS = "assets"; public static final String EXTRACTED = "extracted"; public static final String EXTRACTED_SOURCES = EXTRACTED + "/sources"; public static final String EXTRACTED_BINARIES = EXTRACTED + "/binaries"; + public static final String EXTRACTED_ASSETS = EXTRACTED + "/assets"; public static final String OUTPUT = "output"; public static final String MESSAGES = "messages"; @@ -120,6 +131,9 @@ public class PDSWorkspaceService { @Autowired PDSWorkspacePreparationResultCalculator preparationResultCalculator; + @Autowired + CheckSumSupport checksumSupport; + private static final ArchiveFilter TAR_FILE_FILTER = new TarFileFilter(); private static final ArchiveFilter ZIP_FILE_FILTER = new SourcecodeZipFileFilter(); @@ -151,6 +165,7 @@ public void throwException(String message, Exception cause) throws IOException { * @return {@link PDSWorkspacePreparationResult}, never null * @throws IOException */ + @UseCaseSystemExecutesJob(@PDSStep(number = 4, name = "PDS workspace preparation", description = "PDS job workspace is prepared here: Directories are created, files are downloaded and extracted etc.")) public PDSWorkspacePreparationResult prepare(UUID pdsJobUUID, PDSJobConfiguration config, String metaData) throws IOException { PDSJobConfigurationSupport configurationSupport = new PDSJobConfigurationSupport(config); @@ -162,6 +177,8 @@ public PDSWorkspacePreparationResult prepare(UUID pdsJobUUID, PDSJobConfiguratio writeMetaData(pdsJobUUID, metaData); + importAndExtractFilesFromAssetStorage(pdsJobUUID, config, configurationSupport, preparationContext); + importWantedFilesFromJobStorage(pdsJobUUID, config, configurationSupport, preparationContext); extractZipFileUploadsWhenConfigured(pdsJobUUID, config, preparationContext); @@ -188,7 +205,7 @@ private void importWantedFilesFromJobStorage(UUID pdsJobUUID, PDSJobConfiguratio PDSResilientRetryExecutor resilientStorageReadExecutor = createResilientReadExecutor(preparationContext); File jobFolder = getUploadFolder(pdsJobUUID); - JobStorage storage = fetchStorage(pdsJobUUID, config); + JobStorage storage = fetchJobStorage(pdsJobUUID, config); Set names = resilientStorageReadExecutor.execute(() -> storage.listNames(), "List storage names for job: " + pdsJobUUID.toString()); @@ -209,6 +226,79 @@ private void importWantedFilesFromJobStorage(UUID pdsJobUUID, PDSJobConfiguratio } finally { storage.close(); } + + } + + private void importAndExtractFilesFromAssetStorage(UUID pdsJobUUID, PDSJobConfiguration config, PDSJobConfigurationSupport configurationSupport, + PDSWorkspacePreparationContext preparationContext) throws IOException { + + List templateMetaDataList = configurationSupport.getTemplateMetaData(); + if (templateMetaDataList.isEmpty()) { + LOG.debug("No template meta data available - will skipp asset data import"); + return; + } + + PDSResilientRetryExecutor resilientStorageReadExecutor = createResilientReadExecutor(preparationContext); + + for (PDSTemplateMetaData metaData : templateMetaDataList) { + TemplateType templateType = requireNonNull(metaData.getTemplateType(), "Template type may not be null"); + PDSAssetData assetData = requireNonNull(metaData.getAssetData(), "Asset data may not be null"); + String fileName = requireNonBlank(assetData.getFileName(), "Filename must be defined"); + String assetId = requireNonBlank(assetData.getAssetId(), "Asset id must be defined"); + String checksumFromSecHub = requireNonBlank(assetData.getChecksum(), "Checksum must be defined."); + + File assetDownloadFile = getAssetFileFromUpload(pdsJobUUID, assetId, fileName); + String assetFilePath = assetId + "/" + fileName; + + resilientStorageReadExecutor.execute(() -> { + + try (AssetStorage storage = storageService.createAssetStorage(assetId)) { + InputStream storageStream = storage.fetch(fileName); + readAndCopyInputStreamToFileSystem(pdsJobUUID, fileName, assetDownloadFile, storageStream); + + String checksumAfterDownloadFromStorage = checksumSupport.createSha256Checksum(assetDownloadFile.toPath()); + + if (!checksumAfterDownloadFromStorage.equals(checksumFromSecHub)) { + throw new IOException("Checksum not as expected for asset file:" + assetFilePath + "\nSecHub checksum:" + checksumFromSecHub + + ", Download checksum:" + checksumAfterDownloadFromStorage); + } + } + }, "Read and copy asset file: " + assetFilePath + " for job: " + pdsJobUUID); + + File extractionFolder = getAssetExtractionFolder(pdsJobUUID, templateType); + if (fileName.toLowerCase().endsWith(".zip")) { + LOG.info("Start extraction of asset file '{}'", fileName); + /* extract ZIP file */ + ArchiveExtractionConstraints archiveExtractionConstraints = createExtractionContraints(); + archiveSupportProvider.getArchiveSupport().extractFileAsIsToFolder(ArchiveType.ZIP, assetDownloadFile, extractionFolder, + archiveExtractionConstraints); + } else { + /* + * This case will only happen in PDS solution development: + * + * At development time, it can happen that there is a need to directly upload + * and change asset files in storage to make things easier/faster turn around + * times. + * + * But SecHub will always use "$pdsProductIdentifier.zip" as the asset file name + * for the product (if there exists one in storage) and send this inside + * parameters! + */ + LOG.warn( + "Asset file name '{}' does not end with '.zip' - will just copy the file to extraction folder. This may ONLY happen at PDS solution development time, but not in production by SecHub!", + fileName); + FileUtils.copyFile(assetDownloadFile, new File(extractionFolder, fileName)); + } + + } + + } + + private static final String requireNonBlank(String target, String message) { + if (requireNonNull(target, message).isBlank()) { + throw new IllegalStateException(message); + } + return target; } private PDSResilientRetryExecutor createResilientReadExecutor(PDSWorkspacePreparationContext preparationContext) { @@ -220,30 +310,34 @@ private PDSResilientRetryExecutor createResilientReadExecutor(PDSWo return resilientExecutor; } - private void readAndCopyStorageToFileSystem(UUID jobUUID, File jobFolder, JobStorage storage, String name) throws IOException { + private void readAndCopyStorageToFileSystem(UUID jobUUID, File jobFolder, Storage storage, String fileName) throws IOException { - File uploadFile = new File(jobFolder, name); + File uploadFile = new File(jobFolder, fileName); - try (InputStream fetchedInputStream = storage.fetch(name)) { + try (InputStream fetchedInputStream = storage.fetch(fileName)) { - try { + readAndCopyInputStreamToFileSystem(jobUUID, fileName, uploadFile, fetchedInputStream); + } - FileUtils.copyInputStreamToFile(fetchedInputStream, uploadFile); + } - LOG.debug("Imported '{}' for job {} from storage to {}", name, jobUUID, uploadFile.getAbsolutePath()); + private void readAndCopyInputStreamToFileSystem(UUID jobUUID, String fileName, File uploadFile, InputStream fetchedInputStream) throws IOException { + try { - } catch (IOException e) { + FileUtils.copyInputStreamToFile(fetchedInputStream, uploadFile); - LOG.error("Was not able to copy stream of uploaded file: {} for job {}, reason: ", name, jobUUID, e.getMessage()); + LOG.debug("Imported '{}' for job {} from storage to {}", fileName, jobUUID, uploadFile.getAbsolutePath()); - if (uploadFile.exists()) { - boolean deleteSuccessful = uploadFile.delete(); - LOG.info("Uploaded file existed. Deleted successfully: {}", deleteSuccessful); - } - throw e; + } catch (IOException e) { + + LOG.error("Was not able to copy stream of uploaded file: {} for job {}, reason: ", fileName, jobUUID, e.getMessage()); + + if (uploadFile.exists()) { + boolean deleteSuccessful = uploadFile.delete(); + LOG.info("Uploaded file existed. Deleted successfully: {}", deleteSuccessful); } + throw e; } - } void extractZipFileUploadsWhenConfigured(UUID jobUUID, PDSJobConfiguration config, PDSWorkspacePreparationContext preparationContext) throws IOException { @@ -288,7 +382,7 @@ private boolean isWantedStorageContent(String name, PDSJobConfigurationSupport c return false; } - private JobStorage fetchStorage(UUID pdsJobUUID, PDSJobConfiguration config) { + private JobStorage fetchJobStorage(UUID pdsJobUUID, PDSJobConfiguration config) { UUID pdsOrSecHubJobUUID; String storagePath; @@ -325,6 +419,41 @@ public File getUploadFolder(UUID pdsJobUUID) { return file; } + /** + * Resolves assets folder - if not existing it will be created + * + * @param pdsJobUUID + * @return assets folder + */ + private Path getAssetUploadFolder(UUID pdsJobUUID) { + File file = new File(getUploadFolder(pdsJobUUID), ASSETS); + file.mkdirs(); + return file.toPath(); + } + + /** + * Resolves asset extraction folder - if not existing it will be created + * + * @param pdsJobUUID + * @param templateType + * @return asset extraction folder + */ + public File getAssetExtractionFolder(UUID pdsJobUUID, TemplateType templateType) { + Path parent = getUploadFolder(pdsJobUUID).toPath(); + Path assetsExtractedPath = parent.resolve(EXTRACTED_ASSETS); + Path templateTypeExtractedPath = assetsExtractedPath.resolve(templateType.getId()); + File file = templateTypeExtractedPath.toFile(); + file.mkdirs(); + return file; + } + + private File getAssetFileFromUpload(UUID pdsJobUUID, String assetId, String fileName) { + Path parent = getAssetUploadFolder(pdsJobUUID); + Path assetIdDownloadPath = parent.resolve(assetId); + assetIdDownloadPath.toFile().mkdirs(); + return assetIdDownloadPath.resolve(fileName).toFile(); + } + private Path getWorkspaceFolderPath(UUID jobUUID) { File workspaceFolder = getWorkspaceFolder(jobUUID); Path workspaceFolderPath = workspaceFolder.toPath(); @@ -418,8 +547,7 @@ private boolean extractArchives(UUID pdsJobUUID, boolean deleteOriginFiles, SecH } ArchiveSupport archiveSupport = archiveSupportProvider.getArchiveSupport(); - ArchiveExtractionConstraints archiveExtractionConstraints = new ArchiveExtractionConstraints(archiveExtractionMaxFileSizeUncompressed, - archiveExtractionMaxEntries, archiveExtractionMaxDirectoryDepth, archiveExtractionTimeout); + ArchiveExtractionConstraints archiveExtractionConstraints = createExtractionContraints(); for (File archiveFile : archiveFiles) { try (FileInputStream archiveFileInputStream = new FileInputStream(archiveFile)) { @@ -443,6 +571,12 @@ private boolean extractArchives(UUID pdsJobUUID, boolean deleteOriginFiles, SecH return true; } + private ArchiveExtractionConstraints createExtractionContraints() { + ArchiveExtractionConstraints archiveExtractionConstraints = new ArchiveExtractionConstraints(archiveExtractionMaxFileSizeUncompressed, + archiveExtractionMaxEntries, archiveExtractionMaxDirectoryDepth, archiveExtractionTimeout); + return archiveExtractionConstraints; + } + public void cleanup(UUID jobUUID, PDSJobConfiguration config) throws IOException { FileUtils.deleteDirectory(getWorkspaceFolder(jobUUID)); LOG.info("Removed workspace folder for job {}", jobUUID); @@ -453,7 +587,7 @@ public void cleanup(UUID jobUUID, PDSJobConfiguration config) throws IOException LOG.info("Removed NOT storage for PDS job {} because sechub storage and will be handled by sechub job {}", jobUUID, config.getSechubJobUUID()); } else { - JobStorage storage = fetchStorage(jobUUID, config); + JobStorage storage = fetchJobStorage(jobUUID, config); storage.deleteAll(); LOG.info("Removed storage for job {}", jobUUID); storage.close(); @@ -549,6 +683,7 @@ public WorkspaceLocationData createLocationData(UUID jobUUID) { locationData.extractedSourcesLocation = createExtractedSourcesLocation(workspaceFolderPath).toString(); locationData.extractedBinariesLocation = createExtractedBinariesLocation(workspaceFolderPath).toString(); + locationData.extractedAssetsLocation = createExtractedAssetsLocation(workspaceFolderPath).toString(); locationData.sourceCodeZipFileLocation = createSourceCodeZipFileLocation(workspaceFolderPath).toString(); locationData.binariesTarFileLocation = createBinariesTarFileLocation(workspaceFolderPath).toString(); @@ -679,6 +814,10 @@ private Path createExtractedSourcesLocation(Path workspaceFolderPath) { return createWorkspacePathAndEnsureParentDirectories(workspaceFolderPath, UPLOAD + "/" + EXTRACTED_SOURCES); } + private Path createExtractedAssetsLocation(Path workspaceFolderPath) { + return createWorkspacePathAndEnsureParentDirectories(workspaceFolderPath, UPLOAD + "/" + EXTRACTED_ASSETS); + } + private Path createWorkspacePathAndEnsureDirectory(Path workspaceLocation, String subPath) { Path path = createWorkspacePathAndEnsureParentDirectories(workspaceLocation, subPath); if (!Files.exists(path)) { diff --git a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/WorkspaceLocationData.java b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/WorkspaceLocationData.java index 6ee704ba75..36a386f261 100644 --- a/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/WorkspaceLocationData.java +++ b/sechub-pds/src/main/java/com/mercedesbenz/sechub/pds/job/WorkspaceLocationData.java @@ -11,6 +11,7 @@ public class WorkspaceLocationData { String userMessagesLocation; String metaDataFileLocation; String eventsLocation; + String extractedAssetsLocation; public String getWorkspaceLocation() { return workspaceLocation; @@ -47,4 +48,8 @@ public String getMetaDataFileLocation() { public String getEventsLocation() { return eventsLocation; } + + public String getExtractedAssetsLocation() { + return extractedAssetsLocation; + } } diff --git a/sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceServiceTest.java b/sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceServiceTest.java index d7791fd554..220567d65f 100644 --- a/sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceServiceTest.java +++ b/sechub-pds/src/test/java/com/mercedesbenz/sechub/pds/job/PDSWorkspaceServiceTest.java @@ -3,27 +3,37 @@ import static com.mercedesbenz.sechub.test.TestConstants.*; import static java.io.File.*; -import static org.junit.jupiter.api.Assertions.*; +import static org.assertj.core.api.Assertions.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; +import java.util.List; import java.util.UUID; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import com.mercedesbenz.sechub.commons.core.security.CheckSumSupport; +import com.mercedesbenz.sechub.commons.model.JSONConverter; +import com.mercedesbenz.sechub.commons.model.template.TemplateType; import com.mercedesbenz.sechub.commons.pds.PDSDefaultParameterKeyConstants; +import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData; +import com.mercedesbenz.sechub.commons.pds.data.PDSTemplateMetaData.PDSAssetData; import com.mercedesbenz.sechub.pds.commons.core.config.PDSProductSetup; import com.mercedesbenz.sechub.pds.config.PDSServerConfigurationService; import com.mercedesbenz.sechub.pds.execution.PDSExecutionParameterEntry; import com.mercedesbenz.sechub.pds.storage.PDSMultiStorageService; import com.mercedesbenz.sechub.pds.storage.PDSStorageInfoCollector; +import com.mercedesbenz.sechub.storage.core.AssetStorage; import com.mercedesbenz.sechub.storage.core.JobStorage; import com.mercedesbenz.sechub.test.TestFileReader; import com.mercedesbenz.sechub.test.TestUtil; @@ -41,6 +51,7 @@ class PDSWorkspaceServiceTest { private PDSWorkspacePreparationResultCalculator preparationResultCalculator; private PDSWorkspacePreparationContext preparationContext; private UUID jobUUID; + private CheckSumSupport checksumSupport; @BeforeAll static void beforeAll() throws IOException { @@ -51,14 +62,15 @@ static void beforeAll() throws IOException { void beforeEach() { jobUUID = UUID.randomUUID(); - storageService = mock(PDSMultiStorageService.class); - storage = mock(JobStorage.class); - storageInfoCollector = mock(PDSStorageInfoCollector.class); - preparationContextFactory = mock(PDSWorkspacePreparationContextFactory.class); - serverConfigService = mock(PDSServerConfigurationService.class); - preparationResultCalculator = mock(PDSWorkspacePreparationResultCalculator.class); + storageService = mock(); + storage = mock(); + storageInfoCollector = mock(); + preparationContextFactory = mock(); + serverConfigService = mock(); + preparationResultCalculator = mock(); + checksumSupport = mock(); - preparationContext = mock(PDSWorkspacePreparationContext.class); + preparationContext = mock(); when(preparationContextFactory.createPreparationContext(any())).thenReturn(preparationContext); PDSProductSetup setup = new PDSProductSetup(); @@ -73,6 +85,7 @@ void beforeEach() { serviceToTest.preparationContextFactory = preparationContextFactory; serviceToTest.serverConfigService = serverConfigService; serviceToTest.preparationResultCalculator = preparationResultCalculator; + serviceToTest.checksumSupport = checksumSupport; config = new PDSJobConfiguration(); @@ -145,7 +158,7 @@ void prepare_returns_result_from_calculator() throws Exception { PDSWorkspacePreparationResult result = serviceToTest.prepare(jobUUID, config, null); /* test */ - assertSame(expected, result); + assertThat(expected).isSameAs(result); } @Test @@ -156,7 +169,7 @@ void when_job_has_no_metadata_no_metadata_file_is_created_in_workspace() throws /* test */ File metaDataFile = serviceToTest.getMetaDataFile(jobUUID); - assertFalse(metaDataFile.exists()); + assertThat(metaDataFile.exists()).isFalse(); } @@ -168,8 +181,8 @@ void when_job_has_metadata_a_metadata_file_is_created_in_workspace_containing_co /* test */ File metaDataFile = serviceToTest.getMetaDataFile(jobUUID); - assertTrue(metaDataFile.exists()); - assertEquals("this is my metadata", TestFileReader.readTextFromFile(metaDataFile)); + assertThat(metaDataFile.exists()).isTrue(); + assertThat(TestFileReader.readTextFromFile(metaDataFile)).isEqualTo("this is my metadata"); } @Test @@ -182,14 +195,15 @@ void createLocationData_contains_expected_pathes_when_using_temp_directory_as_up String expectedWorspaceLocation = workspaceRootFolderPath + separatorChar + jobUUID; /* @formatter:off */ - assertEquals(expectedWorspaceLocation,result.getWorkspaceLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"output"+separatorChar+"result.txt",result.getResultFileLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"output"+separatorChar+"messages",result.getUserMessagesLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"metadata.txt",result.getMetaDataFileLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"upload"+separatorChar+SOURCECODE_ZIP,result.getSourceCodeZipFileLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"upload"+separatorChar+"extracted"+separatorChar+"sources",result.getExtractedSourcesLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"upload"+separatorChar+"extracted"+separatorChar+"binaries",result.getExtractedBinariesLocation()); - assertEquals(expectedWorspaceLocation+separatorChar+"events",result.getEventsLocation()); + assertThat(result.getWorkspaceLocation()).isEqualTo(expectedWorspaceLocation); + assertThat(result.getResultFileLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"output"+separatorChar+"result.txt"); + + assertThat(result.getUserMessagesLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"output"+separatorChar+"messages"); + assertThat(result.getMetaDataFileLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"metadata.txt"); + assertThat(result.getSourceCodeZipFileLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"upload"+separatorChar+SOURCECODE_ZIP); + assertThat(result.getExtractedSourcesLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"upload"+separatorChar+"extracted"+separatorChar+"sources"); + assertThat(result.getExtractedBinariesLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"upload"+separatorChar+"extracted"+separatorChar+"binaries"); + assertThat(result.getEventsLocation()).isEqualTo(expectedWorspaceLocation+separatorChar+"events"); /* @formatter:on */ } @@ -209,6 +223,69 @@ void when_configuration_tells_to_use_sechubstorage_sechub_storage_path_and_sechu verify(storageService).createJobStorageForPath("xyz/abc/project1", config.getSechubJobUUID()); } + @Test + void prepare_downloads_asset_and_stores_file_locally_when_parameter_contains_pds_template_metadata_no_checksum_failure() throws Exception { + /* prepare */ + PDSTemplateMetaData metaData = new PDSTemplateMetaData(); + metaData.setTemplateId("template1"); + metaData.setTemplateType(TemplateType.WEBSCAN_LOGIN); + PDSAssetData assetData = new PDSAssetData(); + assetData.setAssetId("asset1"); + assetData.setChecksum("checksum1"); + assetData.setFileName("file1.txt"); + metaData.setAssetData(assetData); + + String json = JSONConverter.get().toJSON(metaData, false); + config.getParameters().add(createEntry(PDSDefaultParameterKeyConstants.PARAM_KEY_PDS_CONFIG_TEMPLATE_META_DATA_LIST, json)); + + AssetStorage assetStorage = mock(); + when(storageService.createAssetStorage("asset1")).thenReturn(assetStorage); + when(assetStorage.fetch("file1.txt")).thenReturn(new ByteArrayInputStream("testdata".getBytes())); + when(checksumSupport.createSha256Checksum(any(Path.class))).thenReturn("checksum1"); + + /* execute */ + serviceToTest.prepare(jobUUID, config, null); + + /* test */ + ArgumentCaptor pathCaptor = ArgumentCaptor.captor(); + verify(storageService).createAssetStorage("asset1"); + verify(checksumSupport).createSha256Checksum(pathCaptor.capture()); + Path path = pathCaptor.getValue(); + + assertThat(path.getFileName().toString()).isEqualTo("file1.txt"); + + // check file is created + assertThat(Files.exists(path)).isTrue(); + List lines = Files.readAllLines(path); + assertThat(lines).contains("testdata").hasSize(1); + } + + @Test + void prepare_downloads_asset_and_stores_file_locally_when_parameter_contains_pds_template_metadata_checksum_failure() throws Exception { + /* prepare */ + PDSTemplateMetaData metaData = new PDSTemplateMetaData(); + metaData.setTemplateId("template1"); + metaData.setTemplateType(TemplateType.WEBSCAN_LOGIN); + PDSAssetData assetData = new PDSAssetData(); + assetData.setAssetId("asset1"); + assetData.setChecksum("checksum1"); + assetData.setFileName("file1.txt"); + metaData.setAssetData(assetData); + + String json = JSONConverter.get().toJSON(metaData, false); + config.getParameters().add(createEntry(PDSDefaultParameterKeyConstants.PARAM_KEY_PDS_CONFIG_TEMPLATE_META_DATA_LIST, json)); + + AssetStorage assetStorage = mock(); + when(storageService.createAssetStorage("asset1")).thenReturn(assetStorage); + when(assetStorage.fetch("file1.txt")).thenReturn(new ByteArrayInputStream("testdata".getBytes())); + when(checksumSupport.createSha256Checksum(any(Path.class))).thenReturn("checksum-other-means-failure"); + + /* execute + test */ + assertThatThrownBy(() -> serviceToTest.prepare(jobUUID, config, null)).cause().isInstanceOf(IOException.class) + .hasMessageStartingWith("Checksum not as expected"); + + } + private PDSExecutionParameterEntry createEntry(String key, String value) { PDSExecutionParameterEntry entry = new PDSExecutionParameterEntry(); entry.setKey(key); diff --git a/sechub-scan-product-pds/src/main/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataService.java b/sechub-scan-product-pds/src/main/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataService.java index d2c38946f1..b0acdd59e3 100644 --- a/sechub-scan-product-pds/src/main/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataService.java +++ b/sechub-scan-product-pds/src/main/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataService.java @@ -58,8 +58,8 @@ public void ensureTemplateAssetFilesAreAvailableInStorage(List result } /* found */ PDSAssetData assetData = new PDSAssetData(); - assetData.setAsset(assetId); - assetData.setFile(fileData.getFileName()); + assetData.setAssetId(assetId); + assetData.setFileName(fileData.getFileName()); assetData.setChecksum(fileData.getChecksum()); PDSTemplateMetaData metaData = new PDSTemplateMetaData(); - metaData.setTemplate(definition.getId()); - metaData.setType(definition.getType()); + metaData.setTemplateId(definition.getId()); + metaData.setTemplateType(definition.getType()); metaData.setAssetData(assetData); result.add(metaData); diff --git a/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSExecutorConfigSupportTest.java b/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSExecutorConfigSupportTest.java index 99aa876939..987d59c304 100644 --- a/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSExecutorConfigSupportTest.java +++ b/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSExecutorConfigSupportTest.java @@ -469,9 +469,9 @@ private List createTemplateMetaDataServiceExampleResult() { List templateMetaDataServiceResult = new ArrayList<>(); PDSTemplateMetaData pdsTemplateMetaData1 = new PDSTemplateMetaData(); PDSAssetData assetData1 = new PDSAssetData(); - assetData1.setAsset("asset-id-1"); + assetData1.setAssetId("asset-id-1"); assetData1.setChecksum("checksum1"); - assetData1.setFile("file1"); + assetData1.setFileName("file1"); pdsTemplateMetaData1.setAssetData(assetData1); templateMetaDataServiceResult.add(pdsTemplateMetaData1); return templateMetaDataServiceResult; diff --git a/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataServiceTest.java b/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataServiceTest.java index cc7c62ab91..1d5d9ad1d1 100644 --- a/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataServiceTest.java +++ b/sechub-scan-product-pds/src/test/java/com/mercedesbenz/sechub/domain/scan/product/pds/PDSTemplateMetaDataServiceTest.java @@ -89,13 +89,13 @@ void createTemplateMetaData_returns_pds_template_meta_data_when_asset_is_availab /* test */ PDSTemplateMetaData expectedTemplateMetaData1 = new PDSTemplateMetaData(); - expectedTemplateMetaData1.setTemplate(exampleTemplateId1); - expectedTemplateMetaData1.setType(exampleTemplateType); + expectedTemplateMetaData1.setTemplateId(exampleTemplateId1); + expectedTemplateMetaData1.setTemplateType(exampleTemplateType); PDSAssetData assetData1 = new PDSAssetData(); - assetData1.setAsset(exampleAssetId1); + assetData1.setAssetId(exampleAssetId1); assetData1.setChecksum(exampleChecksum1); - assetData1.setFile("test_product_id.zip"); + assetData1.setFileName("test_product_id.zip"); expectedTemplateMetaData1.setAssetData(assetData1); @@ -206,15 +206,15 @@ void ensureTemplateAssetFilesAreAvailableInStorage_calls_asste_service_for_each_ /* prepare */ PDSAssetData assetData1 = new PDSAssetData(); - assetData1.setAsset("asset1"); - assetData1.setFile("file1.txt"); + assetData1.setAssetId("asset1"); + assetData1.setFileName("file1.txt"); PDSTemplateMetaData templateMetaData1 = new PDSTemplateMetaData(); templateMetaData1.setAssetData(assetData1); PDSAssetData assetData2 = new PDSAssetData(); - assetData2.setAsset("asset2"); - assetData2.setFile("file2.txt"); + assetData2.setAssetId("asset2"); + assetData2.setFileName("file2.txt"); PDSTemplateMetaData templateMetaData2 = new PDSTemplateMetaData(); templateMetaData2.setAssetData(assetData2);