From 4dd985666178dafb0bb5a5ecd74b2559296d2c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn-Andre=20Skaar?= <31540110+bjornandre@users.noreply.github.com> Date: Tue, 5 Sep 2023 09:20:08 +0200 Subject: [PATCH] Add support for supplying a version timestamp to the 'map-sid' function (#55) * Initial setup for version timestamp * Initialize empty config map * Use SID service to validate versionTimestamp * Use the dapla-dlp-pseudo-core release version --------- Co-authored-by: Michael Moen Allport --- conf/application-local.yml | 1 + doc/requests/examples-sid.http | 2 +- pom.xml | 2 +- .../service/sid/ExternalSidService.java | 4 ++ .../pseudo/service/sid/LocalSidService.java | 4 ++ .../ssb/dlp/pseudo/service/sid/SidClient.java | 7 +++- .../ssb/dlp/pseudo/service/sid/SidMapper.java | 19 ++++++++- .../dlp/pseudo/service/sid/SidService.java | 2 + .../dlp/pseudo/service/sid/VersionInfo.java | 14 +++++++ .../dlp/pseudo/service/sid/SidMapperTest.java | 39 +++++++++++++++++++ 10 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java diff --git a/conf/application-local.yml b/conf/application-local.yml index 6005be5..fdcde4f 100644 --- a/conf/application-local.yml +++ b/conf/application-local.yml @@ -102,4 +102,5 @@ app-roles: - kons-skaar@ssb.no - kons-lunde@ssb.no - mmw@ssb.no + - mic@ssb.no diff --git a/doc/requests/examples-sid.http b/doc/requests/examples-sid.http index da28913..29c3b82 100644 --- a/doc/requests/examples-sid.http +++ b/doc/requests/examples-sid.http @@ -43,7 +43,7 @@ Content-Type: application/json { "name": "fnr", "pattern": "**/fnr", - "func": "map-sid(keyId=papis-key-1)" + "func": "map-sid(keyId=papis-key-1,versionTimestamp=2023_04_25-12_35_40_649511)" } ] } diff --git a/pom.xml b/pom.xml index 704a144..da50f50 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ no.ssb.dapla.dlp.pseudo dapla-dlp-pseudo-core - 1.2.3 + 1.2.4 no.ssb.avro.convert.core diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java index f6dfc64..975df7a 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/ExternalSidService.java @@ -38,4 +38,8 @@ public Publisher> lookupFnr(List fnrList, Optional< ); } + @Override + public Publisher getSnapshots() { + return sidClient.snapshots(); + } } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java index 509a7d6..36ae0eb 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/LocalSidService.java @@ -52,6 +52,10 @@ public Publisher> lookupFnr(List fnrList, Optional< ); } + @Override + public Publisher getSnapshots() { + return Publishers.just(VersionInfo.builder().build()); + } public static class NoSidMappingFoundException extends RuntimeException { public NoSidMappingFoundException(String message) { diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java index 054eeb9..0d5cd40 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidClient.java @@ -1,14 +1,13 @@ package no.ssb.dlp.pseudo.service.sid; import io.micronaut.http.annotation.Body; +import io.micronaut.http.annotation.Get; import io.micronaut.http.annotation.Post; import io.micronaut.http.client.annotation.Client; import io.micronaut.scheduling.TaskExecutors; import io.micronaut.scheduling.annotation.ExecuteOn; import org.reactivestreams.Publisher; -import java.util.Map; - @Client(id="sid-service") @IdTokenFilterMatcher() public interface SidClient { @@ -21,4 +20,8 @@ public interface SidClient { @ExecuteOn(TaskExecutors.IO) Publisher lookup(@Body MultiSidRequest multiSidRequest); + @Get( "/sid/snapshots") + @ExecuteOn(TaskExecutors.IO) + Publisher snapshots(); + } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java index bb8216e..768235b 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidMapper.java @@ -7,12 +7,14 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.exceptions.HttpClientResponseException; import lombok.extern.slf4j.Slf4j; +import no.ssb.dapla.dlp.pseudo.func.map.MapFuncConfig; import no.ssb.dapla.dlp.pseudo.func.map.Mapper; import no.ssb.dlp.pseudo.service.Application; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,6 +34,7 @@ public class SidMapper implements Mapper { private final SidService sidService; private static final int DEFAULT_PARTITION_SIZE = 50000; private final int partitionSize; + private Map config = Collections.emptyMap(); public SidMapper() { sidService = Application.getContext().getBean(SidService.class); @@ -59,7 +62,8 @@ public Object map(Object data) { for (List bulkFnr: Lists.partition(List.copyOf(fnrs), partitionSize)) { log.info("Execute SID-mapping bulk request"); final ObservableSubscriber> subscriber = ObservableSubscriber.subscribe( - sidService.lookupFnr(bulkFnr, Optional.ofNullable(null))); + sidService.lookupFnr(bulkFnr, Optional.ofNullable( + String.valueOf(this.config.getOrDefault(MapFuncConfig.Param.VERSION_TIMESTAMP, null))))); for (String f: bulkFnr) { bulkRequest.put(f, subscriber); } @@ -85,6 +89,19 @@ public Object map(Object data) { } } + @Override + public void setConfig(Map config) { + if (config.containsKey(MapFuncConfig.Param.VERSION_TIMESTAMP)) { + VersionInfo snapshots = ObservableSubscriber.subscribe(this.sidService.getSnapshots()).awaitResult() + .orElseThrow(() -> new RuntimeException("SID service did not respond")); + if (!snapshots.getItems().contains(config.get(MapFuncConfig.Param.VERSION_TIMESTAMP).toString())) { + throw new RuntimeException(String.format("Invalid version timestamp. Valid versions are: %s", + String.join(", ", snapshots.getItems()))); + } + } + this.config = config; + } + @Override public Object restore(Object mapped) { return mapped; diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java index c933e2c..93c986f 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/SidService.java @@ -11,4 +11,6 @@ public interface SidService { Publisher lookupSnr(String fnr, Optional snapshot); Publisher> lookupFnr(List fnrList, Optional snapshot); + + Publisher getSnapshots(); } diff --git a/src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java b/src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java new file mode 100644 index 0000000..e9cf9fe --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/service/sid/VersionInfo.java @@ -0,0 +1,14 @@ +package no.ssb.dlp.pseudo.service.sid; + +import lombok.Builder; +import lombok.Data; +import lombok.extern.jackson.Jacksonized; + +import java.util.List; + +@Data +@Builder +@Jacksonized +public class VersionInfo { + private final List items; +} diff --git a/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java b/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java index adfb006..315cf11 100644 --- a/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/service/sid/SidMapperTest.java @@ -7,10 +7,13 @@ import no.ssb.dapla.dlp.pseudo.func.map.Mapper; import no.ssb.dlp.pseudo.service.Application; import org.apache.groovy.util.Maps; +import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import javax.inject.Inject; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.ServiceLoader; @@ -45,6 +48,42 @@ public void testInvokeMapperFunc() { } } + @Test + public void testMapValidVersion() { + when(sidService.lookupFnr(anyList(), any(Optional.class))).thenReturn(Publishers.just( + Maps.of("11854898347", new SidInfo.SidInfoBuilder().snr("0001ha3").build())) + ); + when(sidService.getSnapshots()).thenReturn(Publishers.just( + VersionInfo.builder().items(List.of("12345")).build() + )); + try (var application = mockStatic(Application.class)) { + application.when(Application::getContext).thenReturn(context); + Mapper mapper = ServiceLoader.load(Mapper.class).findFirst().orElseThrow(() -> + new RuntimeException("SidMapper class not found")); + mapper.setConfig(Map.of("versionTimestamp", "12345")); + mapper.init("11854898347"); + Object mappedSid = mapper.map("11854898347"); + verify(sidService, times(1)).getSnapshots(); + verify(sidService, times(1)).lookupFnr(anyList(), any(Optional.class)); + Assertions.assertEquals("0001ha3", mappedSid); + } + } + @Test + public void testMapInvValidVersion() { + when(sidService.getSnapshots()).thenReturn(Publishers.just( + VersionInfo.builder().items(List.of("12345")).build() + )); + try (var application = mockStatic(Application.class)) { + application.when(Application::getContext).thenReturn(context); + Mapper mapper = ServiceLoader.load(Mapper.class).findFirst().orElseThrow(() -> + new RuntimeException("SidMapper class not found")); + Exception exception = Assert.assertThrows(RuntimeException.class, () -> { + mapper.setConfig(Map.of("versionTimestamp", "1234567")); + }); + Assertions.assertEquals("Invalid version timestamp. Valid versions are: 12345", exception.getMessage()); + } + } + @MockBean(SidService.class) SidService sidService() { return mock(SidService.class);