From 22cef8e62fcbef1f21f155524a71f809476bace9 Mon Sep 17 00:00:00 2001 From: Michael Moen Allport Date: Thu, 31 Aug 2023 15:38:07 +0200 Subject: [PATCH 1/5] Initial setup for version timestamp --- conf/application-local.yml | 1 + doc/requests/examples-sid.http | 2 +- pom.xml | 2 +- .../java/no/ssb/dlp/pseudo/service/sid/SidMapper.java | 9 ++++++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/conf/application-local.yml b/conf/application-local.yml index 6005be5..fdcde4f 100644 --- a/conf/application-local.yml +++ b/conf/application-local.yml @@ -102,4 +102,5 @@ app-roles: - kons-skaar@ssb.no - kons-lunde@ssb.no - mmw@ssb.no + - mic@ssb.no diff --git a/doc/requests/examples-sid.http b/doc/requests/examples-sid.http index da28913..3c31193 100644 --- a/doc/requests/examples-sid.http +++ b/doc/requests/examples-sid.http @@ -43,7 +43,7 @@ Content-Type: application/json { "name": "fnr", "pattern": "**/fnr", - "func": "map-sid(keyId=papis-key-1)" + "func": "map-sid(keyId=papis-key-1,versionTimestamp=10925)" } ] } diff --git a/pom.xml b/pom.xml index 704a144..c5f3ab9 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ no.ssb.dapla.dlp.pseudo dapla-dlp-pseudo-core - 1.2.3 + 1.2.4-SNAPSHOT no.ssb.avro.convert.core diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java index bb8216e..572cd8a 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java @@ -32,6 +32,7 @@ public class SidMapper implements Mapper { private final SidService sidService; private static final int DEFAULT_PARTITION_SIZE = 50000; private final int partitionSize; + private Map config; public SidMapper() { sidService = Application.getContext().getBean(SidService.class); @@ -59,7 +60,8 @@ public Object map(Object data) { for (List bulkFnr: Lists.partition(List.copyOf(fnrs), partitionSize)) { log.info("Execute SID-mapping bulk request"); final ObservableSubscriber> subscriber = ObservableSubscriber.subscribe( - sidService.lookupFnr(bulkFnr, Optional.ofNullable(null))); + sidService.lookupFnr(bulkFnr, Optional.ofNullable( + String.valueOf(this.config.getOrDefault("versionTimestamp", null))))); for (String f: bulkFnr) { bulkRequest.put(f, subscriber); } @@ -85,6 +87,11 @@ public Object map(Object data) { } } + @Override + public void setConfig(Map config) { + this.config = config; + } + @Override public Object restore(Object mapped) { return mapped; From aa245ff8b384080ab85a88fbeae6020213f26ee8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skaar=2C=20Bj=C3=B8rn-Andre?= Date: Thu, 31 Aug 2023 20:31:36 +0200 Subject: [PATCH 2/5] Initialize empty config map --- src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java index 572cd8a..b9a71cf 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java @@ -13,6 +13,7 @@ import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,7 +33,7 @@ public class SidMapper implements Mapper { private final SidService sidService; private static final int DEFAULT_PARTITION_SIZE = 50000; private final int partitionSize; - private Map config; + private Map config = Collections.emptyMap(); public SidMapper() { sidService = Application.getContext().getBean(SidService.class); From f851e4f6714eb949792b307515cec280daf12018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skaar=2C=20Bj=C3=B8rn-Andre?= Date: Mon, 4 Sep 2023 13:42:29 +0200 Subject: [PATCH 3/5] Use SID service to validate versionTimestamp --- doc/requests/examples-sid.http | 2 +- .../service/sid/ExternalSidService.java | 4 ++ .../pseudo/service/sid/LocalSidService.java | 4 ++ .../ssb/dlp/pseudo/service/sid/SidClient.java | 7 +++- .../ssb/dlp/pseudo/service/sid/SidMapper.java | 8 ++++ .../dlp/pseudo/service/sid/SidService.java | 2 + .../dlp/pseudo/service/sid/VersionInfo.java | 14 +++++++ .../dlp/pseudo/service/sid/SidMapperTest.java | 39 +++++++++++++++++++ 8 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java diff --git a/doc/requests/examples-sid.http b/doc/requests/examples-sid.http index 3c31193..29c3b82 100644 --- a/doc/requests/examples-sid.http +++ b/doc/requests/examples-sid.http @@ -43,7 +43,7 @@ Content-Type: application/json { "name": "fnr", "pattern": "**/fnr", - "func": "map-sid(keyId=papis-key-1,versionTimestamp=10925)" + "func": "map-sid(keyId=papis-key-1,versionTimestamp=2023_04_25-12_35_40_649511)" } ] } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java index f6dfc64..975df7a 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java @@ -38,4 +38,8 @@ public Publisher> lookupFnr(List fnrList, Optional< ); } + @Override + public Publisher getSnapshots() { + return sidClient.snapshots(); + } } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java index 509a7d6..36ae0eb 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java @@ -52,6 +52,10 @@ public Publisher> lookupFnr(List fnrList, Optional< ); } + @Override + public Publisher getSnapshots() { + return Publishers.just(VersionInfo.builder().build()); + } public static class NoSidMappingFoundException extends RuntimeException { public NoSidMappingFoundException(String message) { diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java index 054eeb9..0d5cd40 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java @@ -1,14 +1,13 @@ package no.ssb.dlp.pseudo.service.sid; import io.micronaut.http.annotation.Body; +import io.micronaut.http.annotation.Get; import io.micronaut.http.annotation.Post; import io.micronaut.http.client.annotation.Client; import io.micronaut.scheduling.TaskExecutors; import io.micronaut.scheduling.annotation.ExecuteOn; import org.reactivestreams.Publisher; -import java.util.Map; - @Client(id="sid-service") @IdTokenFilterMatcher() public interface SidClient { @@ -21,4 +20,8 @@ public interface SidClient { @ExecuteOn(TaskExecutors.IO) Publisher lookup(@Body MultiSidRequest multiSidRequest); + @Get( "/sid/snapshots") + @ExecuteOn(TaskExecutors.IO) + Publisher snapshots(); + } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java index b9a71cf..95e8996 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java @@ -90,6 +90,14 @@ public Object map(Object data) { @Override public void setConfig(Map config) { + if (config.containsKey("versionTimestamp")) { + VersionInfo snapshots = ObservableSubscriber.subscribe(this.sidService.getSnapshots()).awaitResult() + .orElseThrow(() -> new RuntimeException("SID service did not respond")); + if (!snapshots.getItems().contains(config.get("versionTimestamp").toString())) { + throw new RuntimeException(String.format("Invalid version timestamp. Valid versions are: %s", + String.join(", ", snapshots.getItems()))); + } + } this.config = config; } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java index c933e2c..93c986f 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java @@ -11,4 +11,6 @@ public interface SidService { Publisher lookupSnr(String fnr, Optional snapshot); Publisher> lookupFnr(List fnrList, Optional snapshot); + + Publisher getSnapshots(); } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java new file mode 100644 index 0000000..e9cf9fe --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java @@ -0,0 +1,14 @@ +package no.ssb.dlp.pseudo.service.sid; + +import lombok.Builder; +import lombok.Data; +import lombok.extern.jackson.Jacksonized; + +import java.util.List; + +@Data +@Builder +@Jacksonized +public class VersionInfo { + private final List items; +} diff --git a/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java b/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java index adfb006..315cf11 100644 --- a/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java @@ -7,10 +7,13 @@ import no.ssb.dapla.dlp.pseudo.func.map.Mapper; import no.ssb.dlp.pseudo.service.Application; import org.apache.groovy.util.Maps; +import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import javax.inject.Inject; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.ServiceLoader; @@ -45,6 +48,42 @@ public void testInvokeMapperFunc() { } } + @Test + public void testMapValidVersion() { + when(sidService.lookupFnr(anyList(), any(Optional.class))).thenReturn(Publishers.just( + Maps.of("11854898347", new SidInfo.SidInfoBuilder().snr("0001ha3").build())) + ); + when(sidService.getSnapshots()).thenReturn(Publishers.just( + VersionInfo.builder().items(List.of("12345")).build() + )); + try (var application = mockStatic(Application.class)) { + application.when(Application::getContext).thenReturn(context); + Mapper mapper = ServiceLoader.load(Mapper.class).findFirst().orElseThrow(() -> + new RuntimeException("SidMapper class not found")); + mapper.setConfig(Map.of("versionTimestamp", "12345")); + mapper.init("11854898347"); + Object mappedSid = mapper.map("11854898347"); + verify(sidService, times(1)).getSnapshots(); + verify(sidService, times(1)).lookupFnr(anyList(), any(Optional.class)); + Assertions.assertEquals("0001ha3", mappedSid); + } + } + @Test + public void testMapInvValidVersion() { + when(sidService.getSnapshots()).thenReturn(Publishers.just( + VersionInfo.builder().items(List.of("12345")).build() + )); + try (var application = mockStatic(Application.class)) { + application.when(Application::getContext).thenReturn(context); + Mapper mapper = ServiceLoader.load(Mapper.class).findFirst().orElseThrow(() -> + new RuntimeException("SidMapper class not found")); + Exception exception = Assert.assertThrows(RuntimeException.class, () -> { + mapper.setConfig(Map.of("versionTimestamp", "1234567")); + }); + Assertions.assertEquals("Invalid version timestamp. Valid versions are: 12345", exception.getMessage()); + } + } + @MockBean(SidService.class) SidService sidService() { return mock(SidService.class); From 5a96f3e9784aba6409a95621b5f984f5abf7b78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skaar=2C=20Bj=C3=B8rn-Andre?= Date: Mon, 4 Sep 2023 14:46:19 +0200 Subject: [PATCH 4/5] Avoid hard-coding the versionTimestamp config key --- src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java index 95e8996..768235b 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java @@ -7,6 +7,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.exceptions.HttpClientResponseException; import lombok.extern.slf4j.Slf4j; +import no.ssb.dapla.dlp.pseudo.func.map.MapFuncConfig; import no.ssb.dapla.dlp.pseudo.func.map.Mapper; import no.ssb.dlp.pseudo.service.Application; import org.reactivestreams.Publisher; @@ -62,7 +63,7 @@ public Object map(Object data) { log.info("Execute SID-mapping bulk request"); final ObservableSubscriber> subscriber = ObservableSubscriber.subscribe( sidService.lookupFnr(bulkFnr, Optional.ofNullable( - String.valueOf(this.config.getOrDefault("versionTimestamp", null))))); + String.valueOf(this.config.getOrDefault(MapFuncConfig.Param.VERSION_TIMESTAMP, null))))); for (String f: bulkFnr) { bulkRequest.put(f, subscriber); } @@ -90,10 +91,10 @@ public Object map(Object data) { @Override public void setConfig(Map config) { - if (config.containsKey("versionTimestamp")) { + if (config.containsKey(MapFuncConfig.Param.VERSION_TIMESTAMP)) { VersionInfo snapshots = ObservableSubscriber.subscribe(this.sidService.getSnapshots()).awaitResult() .orElseThrow(() -> new RuntimeException("SID service did not respond")); - if (!snapshots.getItems().contains(config.get("versionTimestamp").toString())) { + if (!snapshots.getItems().contains(config.get(MapFuncConfig.Param.VERSION_TIMESTAMP).toString())) { throw new RuntimeException(String.format("Invalid version timestamp. Valid versions are: %s", String.join(", ", snapshots.getItems()))); } From 4a083314640ce39ed6e90644604310c7500aecf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skaar=2C=20Bj=C3=B8rn-Andre?= Date: Mon, 4 Sep 2023 15:01:28 +0200 Subject: [PATCH 5/5] Use the dapla-dlp-pseudo-core release version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c5f3ab9..da50f50 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ no.ssb.dapla.dlp.pseudo dapla-dlp-pseudo-core - 1.2.4-SNAPSHOT + 1.2.4 no.ssb.avro.convert.core