From 680a008e8864299b7589fe2392f8e9408936343f Mon Sep 17 00:00:00 2001 From: Andreas Scheibal Date: Mon, 6 May 2024 16:04:31 +0200 Subject: [PATCH 1/3] feature: git upload --- pom.xml | 5 ++ .../config/KdsConfigProperties.java | 10 +++- .../service/did/DidTrustListService.java | 58 +++++++++++++++++++ src/main/resources/application.yml | 4 ++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 37e5123..a9f068f 100644 --- a/pom.xml +++ b/pom.xml @@ -226,6 +226,11 @@ + + org.eclipse.jgit + org.eclipse.jgit + 6.9.0.202403050737-r + diff --git a/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java b/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java index e09b3e3..9ac4d49 100644 --- a/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java +++ b/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java @@ -114,10 +114,11 @@ public static class DidConfig { private Map virtualCountries = new HashMap<>(); private LocalFileConfig localFile = new LocalFileConfig(); + private GitConfig git = new GitConfig(); private DgcGatewayConnectorConfigProperties.KeyStoreWithAlias localKeyStore = new DgcGatewayConnectorConfigProperties.KeyStoreWithAlias(); - + @Getter @Setter public static class LocalFileConfig { @@ -125,6 +126,13 @@ public static class LocalFileConfig { private String directory; } + @Getter + @Setter + public static class GitConfig { + private String path; + private String pat; + private String url; + } } } diff --git a/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java b/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java index 78efe69..9251815 100644 --- a/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java +++ b/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java @@ -28,17 +28,23 @@ import foundation.identity.jsonld.JsonLDObject; import info.weboftrust.ldsignatures.jsonld.LDSecurityKeywords; import info.weboftrust.ldsignatures.signer.JsonWebSignature2020LdSigner; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.PublicKey; import java.security.cert.CertificateEncodingException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.security.interfaces.ECPublicKey; import java.security.interfaces.RSAPublicKey; +import java.time.Instant; import java.util.ArrayList; import java.util.Base64; import java.util.Date; @@ -53,6 +59,9 @@ import lombok.extern.slf4j.Slf4j; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; import org.bouncycastle.cert.X509CertificateHolder; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -102,6 +111,22 @@ public class DidTrustListService { public void job() { String trustList; + + deleteDirectoryAndContents(configProperties.getDid().getGit().getPath()); + try { + Git.cloneRepository() + .setURI(configProperties.getDid().getGit().getUrl()) + .setDirectory(new File(configProperties.getDid().getGit().getPath())) + .setCredentialsProvider( + new UsernamePasswordCredentialsProvider( + "anonymous", configProperties.getDid().getGit().getPat())) + .call(); + } catch (Exception e) { + log.error("Failed to clone repository {}: {}", + configProperties.getDid().getGit().getUrl(), e.getMessage()); + } + + try { trustList = generateTrustList(null); } catch (Exception e) { @@ -139,6 +164,18 @@ public void job() { } log.info("Finished DID Export Process"); + + try { + Git git = Git.open(new File(configProperties.getDid().getGit().getPath())); + git.add().addFilepattern(".").call(); + git.commit().setMessage("Added DID files on " + Instant.now()).call(); + git.push().setCredentialsProvider(new UsernamePasswordCredentialsProvider( + "anonymous", configProperties.getDid().getGit().getPat())).call(); + git.close(); + } catch (GitAPIException | IOException e) { + log.error("Error during Git commit & push: {}",e.getMessage()); + } + } private String getCountryAsLowerCaseAlpha3(String country) { @@ -300,4 +337,25 @@ private Optional searchCsca(X509Certificate dsc, String country .equals(dsc.getIssuerX500Principal())) .findFirst(); } + + private void deleteDirectoryAndContents(String directoryPath) { + Path dir = Paths.get(directoryPath); + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path path : stream) { + if (Files.isDirectory(path)) { + deleteDirectoryAndContents(path.toString()); + } else { + Files.delete(path); + } + } + } catch (IOException e) { + log.error("Error deleting file {} {}",dir,e.getMessage()); + } + try { + Files.delete(dir); + } catch (IOException e) { + log.error("Error deleting directory {} {}",dir,e.getMessage()); + } + } + } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7da19cb..6b5278a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -96,6 +96,10 @@ dgc: localFile: directory: ./did file-name: did.json + git: + path: + url: + pat: didSigningProvider: dummy ld-proof-verification-method: did:web:dummy.net ld-proof-nonce: n0nc3 From 81d79b8d6b387e6ec50ff21082014dfaa298b4de Mon Sep 17 00:00:00 2001 From: Andreas Scheibal Date: Tue, 7 May 2024 01:49:03 +0200 Subject: [PATCH 2/3] fix(tests): rework --- .../config/KdsConfigProperties.java | 3 +- .../service/did/DidTrustListService.java | 58 +------- .../service/did/DummyGitUploader.java | 48 +++++++ .../service/did/GitProvider.java | 27 ++++ .../service/did/GitUploader.java | 136 ++++++++++++++++++ src/main/resources/application.yml | 9 +- src/test/resources/application.yml | 5 + 7 files changed, 226 insertions(+), 60 deletions(-) create mode 100644 src/main/java/tng/trustnetwork/keydistribution/service/did/DummyGitUploader.java create mode 100644 src/main/java/tng/trustnetwork/keydistribution/service/did/GitProvider.java create mode 100644 src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java diff --git a/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java b/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java index 9ac4d49..6d4b939 100644 --- a/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java +++ b/src/main/java/tng/trustnetwork/keydistribution/config/KdsConfigProperties.java @@ -129,7 +129,8 @@ public static class LocalFileConfig { @Getter @Setter public static class GitConfig { - private String path; + private String prefix; + private String workdir; private String pat; private String url; } diff --git a/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java b/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java index 9251815..d803f4b 100644 --- a/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java +++ b/src/main/java/tng/trustnetwork/keydistribution/service/did/DidTrustListService.java @@ -28,23 +28,17 @@ import foundation.identity.jsonld.JsonLDObject; import info.weboftrust.ldsignatures.jsonld.LDSecurityKeywords; import info.weboftrust.ldsignatures.signer.JsonWebSignature2020LdSigner; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.security.PublicKey; import java.security.cert.CertificateEncodingException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.security.interfaces.ECPublicKey; import java.security.interfaces.RSAPublicKey; -import java.time.Instant; import java.util.ArrayList; import java.util.Base64; import java.util.Date; @@ -59,9 +53,6 @@ import lombok.extern.slf4j.Slf4j; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; import org.bouncycastle.cert.X509CertificateHolder; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -102,6 +93,8 @@ public class DidTrustListService { private final CertificateUtils certificateUtils; private final TrustedIssuerService trustedIssuerService; + + private final GitProvider gitProvider; /** * Create and upload DID Document holding Uploaded DSC and Trusted Issuer. @@ -112,21 +105,6 @@ public void job() { String trustList; - deleteDirectoryAndContents(configProperties.getDid().getGit().getPath()); - try { - Git.cloneRepository() - .setURI(configProperties.getDid().getGit().getUrl()) - .setDirectory(new File(configProperties.getDid().getGit().getPath())) - .setCredentialsProvider( - new UsernamePasswordCredentialsProvider( - "anonymous", configProperties.getDid().getGit().getPat())) - .call(); - } catch (Exception e) { - log.error("Failed to clone repository {}: {}", - configProperties.getDid().getGit().getUrl(), e.getMessage()); - } - - try { trustList = generateTrustList(null); } catch (Exception e) { @@ -165,17 +143,7 @@ public void job() { log.info("Finished DID Export Process"); - try { - Git git = Git.open(new File(configProperties.getDid().getGit().getPath())); - git.add().addFilepattern(".").call(); - git.commit().setMessage("Added DID files on " + Instant.now()).call(); - git.push().setCredentialsProvider(new UsernamePasswordCredentialsProvider( - "anonymous", configProperties.getDid().getGit().getPat())).call(); - git.close(); - } catch (GitAPIException | IOException e) { - log.error("Error during Git commit & push: {}",e.getMessage()); - } - + gitProvider.upload(configProperties.getDid().getLocalFile().getDirectory()); } private String getCountryAsLowerCaseAlpha3(String country) { @@ -338,24 +306,4 @@ private Optional searchCsca(X509Certificate dsc, String country .findFirst(); } - private void deleteDirectoryAndContents(String directoryPath) { - Path dir = Paths.get(directoryPath); - try (DirectoryStream stream = Files.newDirectoryStream(dir)) { - for (Path path : stream) { - if (Files.isDirectory(path)) { - deleteDirectoryAndContents(path.toString()); - } else { - Files.delete(path); - } - } - } catch (IOException e) { - log.error("Error deleting file {} {}",dir,e.getMessage()); - } - try { - Files.delete(dir); - } catch (IOException e) { - log.error("Error deleting directory {} {}",dir,e.getMessage()); - } - } - } diff --git a/src/main/java/tng/trustnetwork/keydistribution/service/did/DummyGitUploader.java b/src/main/java/tng/trustnetwork/keydistribution/service/did/DummyGitUploader.java new file mode 100644 index 0000000..bb53be3 --- /dev/null +++ b/src/main/java/tng/trustnetwork/keydistribution/service/did/DummyGitUploader.java @@ -0,0 +1,48 @@ +/*- + * ---license-start + * WorldHealthOrganization / tng-key-distribution + * --- + * Copyright (C) 2021 - 2024 T-Systems International GmbH and all other contributors + * --- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---license-end + */ + +package tng.trustnetwork.keydistribution.service.did; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import tng.trustnetwork.keydistribution.config.KdsConfigProperties; + +@ConditionalOnProperty(name = "dgc.did.didUploadProvider", havingValue = "dummy") +@Service +@Slf4j +@RequiredArgsConstructor +public class DummyGitUploader implements GitProvider { + + private final KdsConfigProperties configProperties; + + /** + * upload dummy method used for unit tests. + * @param sourcePath will only be used for log output + */ + + public void upload(String sourcePath) { + + log.info("Uploaded from {} to {}", sourcePath, configProperties.getDid().getGit().getWorkdir()); + + } + +} diff --git a/src/main/java/tng/trustnetwork/keydistribution/service/did/GitProvider.java b/src/main/java/tng/trustnetwork/keydistribution/service/did/GitProvider.java new file mode 100644 index 0000000..a91fd91 --- /dev/null +++ b/src/main/java/tng/trustnetwork/keydistribution/service/did/GitProvider.java @@ -0,0 +1,27 @@ +/*- + * ---license-start + * WorldHealthOrganization / tng-key-distribution + * --- + * Copyright (C) 2021 - 2024 T-Systems International GmbH and all other contributors + * --- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---license-end + */ + +package tng.trustnetwork.keydistribution.service.did; + +public interface GitProvider { + + void upload(String sourcePath); + +} diff --git a/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java b/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java new file mode 100644 index 0000000..5db521e --- /dev/null +++ b/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java @@ -0,0 +1,136 @@ +/*- + * ---license-start + * WorldHealthOrganization / tng-key-distribution + * --- + * Copyright (C) 2021 - 2024 T-Systems International GmbH and all other contributors + * --- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---license-end + */ + +package tng.trustnetwork.keydistribution.service.did; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import tng.trustnetwork.keydistribution.config.KdsConfigProperties; + +@ConditionalOnProperty(name = "dgc.did.didUploadProvider", havingValue = "local-file") +@Service +@Slf4j +@RequiredArgsConstructor +public class GitUploader implements GitProvider { + + private final KdsConfigProperties configProperties; + + /** + * upload method clones a git repositor, copies the contents of sourcePath to the cloned git repository + * and commits and pushes the contents, replacing the previous contents of the repository. + * @param sourcePath sourcePath from where the files are copied for upload + */ + + public void upload(String sourcePath) { + + Path sourceDirectory = Paths.get(sourcePath); + Path targetDirectory = Paths.get(configProperties.getDid().getGit().getWorkdir() + + File.separator + + configProperties.getDid().getGit().getPrefix()); + + deleteDirectoryAndContents(configProperties.getDid().getGit().getWorkdir()); + + try { + Git.cloneRepository() + .setURI(configProperties.getDid().getGit().getUrl()) + .setDirectory(new File(configProperties.getDid().getGit().getWorkdir())) + .setCredentialsProvider( + new UsernamePasswordCredentialsProvider( + "anonymous", configProperties.getDid().getGit().getPat())) + .call(); + } catch (Exception e) { + log.error("Failed to clone repository {}: {}", + configProperties.getDid().getGit().getUrl(), e.getMessage()); + } + + try { + Files.walkFileTree(sourceDirectory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path targetFile = targetDirectory.resolve(sourceDirectory.relativize(file)); + Files.createDirectories(targetFile.getParent()); + Files.copy(file, targetFile, StandardCopyOption.REPLACE_EXISTING); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path targetDir = targetDirectory.resolve(sourceDirectory.relativize(dir)); + Files.createDirectories(targetDir); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + log.error("Failed to copy files from {} to {}: {}", sourcePath, targetDirectory, e.getMessage()); + } + + try { + Git git = Git.open(new File(configProperties.getDid().getGit().getWorkdir())); + git.add().addFilepattern(".").call(); + git.commit().setMessage("Added DID files on " + Instant.now()).call(); + git.push().setCredentialsProvider(new UsernamePasswordCredentialsProvider( + "anonymous", configProperties.getDid().getGit().getPat())).call(); + git.close(); + } catch (GitAPIException | IOException e) { + log.error("Error during Git commit & push: {}",e.getMessage()); + } + } + + private void deleteDirectoryAndContents(String directoryPath) { + Path dir = Paths.get(directoryPath); + if (dir.toFile().exists()) { + + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path path : stream) { + if (Files.isDirectory(path)) { + deleteDirectoryAndContents(path.toString()); + } else { + Files.delete(path); + } + } + } catch (IOException e) { + log.error("Error deleting file {}",e.getMessage()); + } + try { + Files.delete(dir); + } catch (IOException e) { + log.error("Error deleting root directory {}",e.getMessage()); + } + } else { + log.info("Directory {} does not exist, skippig deletion", dir); + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6b5278a..02fed8c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -94,12 +94,13 @@ dgc: enableDidGeneration: true didUploadProvider: local-file localFile: - directory: ./did + directory: file-name: did.json git: - path: - url: - pat: + workdir: + prefix: + url: + pat: didSigningProvider: dummy ld-proof-verification-method: did:web:dummy.net ld-proof-nonce: n0nc3 diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index 159c29b..7fa7d4e 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -43,6 +43,11 @@ dgc: enableDidGeneration: true didUploadProvider: dummy didSigningProvider: dummy + git: + workdir: "" + prefix: "" + url: "" + pat: "" ld-proof-verification-method: did:web:dummy.net ld-proof-nonce: n0nc3 ld-proof-domain: d0m4in From 8409f861a4fc413c6c1b2e8cf8144640343e3576 Mon Sep 17 00:00:00 2001 From: Andreas Scheibal Date: Fri, 10 May 2024 12:22:20 +0200 Subject: [PATCH 3/3] feat: log info on success --- README.md | 16 +++++++++------- .../keydistribution/service/did/GitUploader.java | 2 ++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 320ca5a..fb5f5a6 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -

+

TNG Key Distribution Service

-

+

-

+
-

+

About • Development • Documentation • @@ -131,7 +131,9 @@ docker-compose up --build After all containers have started, you will be able to reach the service on your [local machine](http://localhost:8080/api/docs) under port 8080. -## Documentation +## Cloud deployment + +## Documentation [OpenAPI Spec](https://worldhealthorganization.github.io/tng-key-distribution/) @@ -147,11 +149,11 @@ The following channels are available for discussions, feedback, and support requ | **Issues** | | | **Other requests** | | -## How to contribute +## How to contribute Contribution and feedback is encouraged and always welcome. For more information about how to contribute, the project structure, as well as additional contribution information, see our [Contribution Guidelines](./CONTRIBUTING.md). By participating in this project, you agree to abide by its [Code of Conduct](./CODE_OF_CONDUCT.md) at all times. -## Contributors +## Contributors Our commitment to open source means that we are enabling -in fact encouraging- all interested parties to contribute and become part of its developer community. diff --git a/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java b/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java index 5db521e..a9080f8 100644 --- a/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java +++ b/src/main/java/tng/trustnetwork/keydistribution/service/did/GitUploader.java @@ -104,6 +104,8 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th git.push().setCredentialsProvider(new UsernamePasswordCredentialsProvider( "anonymous", configProperties.getDid().getGit().getPat())).call(); git.close(); + log.info("Successfully uploaded DID files to Git repository {}", + configProperties.getDid().getGit().getUrl()); } catch (GitAPIException | IOException e) { log.error("Error during Git commit & push: {}",e.getMessage()); }