From 5a3a5aab8bcc88ef2d6f26119404436af3fef1ab Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Sun, 12 Mar 2023 11:50:44 +0100 Subject: [PATCH 1/7] Adapt to tink-fpe --- pom.xml | 7 +- .../pseudo/core/FieldPseudoInterceptor.java | 2 + .../pseudo/core/PseudoFuncConfigFactory.java | 238 ------------------ .../pseudo/core/csv/CsvStreamProcessor.java | 4 - .../pseudo/core/field/FieldPseudonymizer.java | 8 +- .../pseudo/core/file/PseudoFileSource.java | 1 - .../core/func/PseudoFuncConfigFactory.java | 124 +++++++++ .../core/func/PseudoFuncConfigPreset.java | 141 +++++++++++ .../core/func/PseudoFuncDeclaration.java | 70 ++++++ .../dlp/pseudo/core/func/PseudoFuncParam.java | 39 +++ .../core/{ => func}/PseudoFuncRule.java | 2 +- .../core/{ => func}/PseudoFuncRuleMatch.java | 2 +- .../pseudo/core/{ => func}/PseudoFuncs.java | 84 +++++-- .../PseudoFuncRuleTypeConverter.java | 2 +- .../no/ssb/dlp/pseudo/core/util/TinkUtil.java | 4 +- .../core/file/PseudoFileSourceTest.java | 14 +- .../dlp/pseudo/core/func/DaeadFuncTest.java | 63 +++++ .../dlp/pseudo/core/func/Ff31FuncTest.java | 104 ++++++++ .../pseudo/core/func/LegacyFpeFuncTest.java | 64 +++++ .../core/func/PseudoFuncConfigPresetTest.java | 106 ++++++++ .../core/func/PseudoFuncDeclarationTest.java | 62 +++++ .../dlp/pseudo/core/func/RedactFuncTest.java | 50 ++++ .../json/JsonRecordMapSerializerTest.java | 2 - .../dlp/pseudo/core/util/PathJoinerTest.java | 6 +- 24 files changed, 919 insertions(+), 280 deletions(-) delete mode 100644 src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncConfigFactory.java create mode 100644 src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java create mode 100644 src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPreset.java create mode 100644 src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java create mode 100644 src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncParam.java rename src/main/java/no/ssb/dlp/pseudo/core/{ => func}/PseudoFuncRule.java (94%) rename src/main/java/no/ssb/dlp/pseudo/core/{ => func}/PseudoFuncRuleMatch.java (83%) rename src/main/java/no/ssb/dlp/pseudo/core/{ => func}/PseudoFuncs.java (53%) create mode 100644 src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java create mode 100644 src/test/java/no/ssb/dlp/pseudo/core/func/Ff31FuncTest.java create mode 100644 src/test/java/no/ssb/dlp/pseudo/core/func/LegacyFpeFuncTest.java create mode 100644 src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPresetTest.java create mode 100644 src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclarationTest.java create mode 100644 src/test/java/no/ssb/dlp/pseudo/core/func/RedactFuncTest.java diff --git a/pom.xml b/pom.xml index 5925967..ebf321c 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ 1.0.0 - 1.1.0 + 1.1.1-SNAPSHOT 5.0.0 31.1-jre 1.5.1 @@ -167,6 +167,11 @@ junit-jupiter-engine test + + org.junit.jupiter + junit-jupiter-params + test + io.micronaut.test micronaut-test-junit5 diff --git a/src/main/java/no/ssb/dlp/pseudo/core/FieldPseudoInterceptor.java b/src/main/java/no/ssb/dlp/pseudo/core/FieldPseudoInterceptor.java index 35ba714..7e415bf 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/FieldPseudoInterceptor.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/FieldPseudoInterceptor.java @@ -5,6 +5,8 @@ import no.ssb.dapla.dlp.pseudo.func.PseudoFuncInput; import no.ssb.dapla.parquet.FieldInterceptor; import no.ssb.dlp.pseudo.core.field.FieldDescriptor; +import no.ssb.dlp.pseudo.core.func.PseudoFuncRuleMatch; +import no.ssb.dlp.pseudo.core.func.PseudoFuncs; import java.util.*; diff --git a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncConfigFactory.java b/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncConfigFactory.java deleted file mode 100644 index 7d5622c..0000000 --- a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncConfigFactory.java +++ /dev/null @@ -1,238 +0,0 @@ -package no.ssb.dlp.pseudo.core; - -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import lombok.extern.slf4j.Slf4j; -import no.ssb.dapla.dlp.pseudo.func.PseudoFuncConfig; -import no.ssb.dapla.dlp.pseudo.func.daead.DaeadFunc; -import no.ssb.dapla.dlp.pseudo.func.daead.DaeadFuncConfig; -import no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets; -import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFunc; -import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFuncConfig; -import no.ssb.dapla.dlp.pseudo.func.map.MapFunc; -import no.ssb.dapla.dlp.pseudo.func.map.MapFuncConfig; -import no.ssb.dapla.dlp.pseudo.func.redact.RedactFunc; -import no.ssb.dapla.dlp.pseudo.func.redact.RedactFuncConfig; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets.alphabetNameOf; -import static no.ssb.dapla.dlp.pseudo.func.text.CharacterGroup.*; - -// TODO: Move or remove presets - -@Slf4j -class PseudoFuncConfigFactory { - - private static final Map PSEUDO_CONFIG_PRESETS_MAP = new HashMap<>(); - - static { - PSEUDO_CONFIG_PRESETS_MAP.putAll(Maps.uniqueIndex(List.of( - fpePseudoFuncConfigPreset("fpe-text", alphabetNameOf(ALPHANUMERIC, WHITESPACE, SYMBOLS)), - fpePseudoFuncConfigPreset("fpe-text_no", alphabetNameOf(ALPHANUMERIC_NO, WHITESPACE, SYMBOLS)), - fpePseudoFuncConfigPreset("fpe-fnr", alphabetNameOf(DIGITS)), - mappingPseudoFuncConfigPreset("map-sid"), - redactPseudoFuncConfigPreset("redact"), - redactRegexPseudoFuncConfigPreset("redact-regex"), - tinkDaeadPseudoFuncConfigPreset("tink-daead") - - ), PseudoFuncConfigPreset::getFuncName)); - } - - private PseudoFuncConfigFactory() {} - - private static PseudoFuncConfigPreset mappingPseudoFuncConfigPreset(String funcName) { - if (!funcName.startsWith("map-")) { - throw new IllegalArgumentException("Mapping functions must be prefixed with 'map-'"); - } - - return new PseudoFuncConfigPreset(funcName, - Map.of(PseudoFuncConfig.Param.FUNC_IMPL, MapFunc.class.getName()), - List.of(MapFuncConfig.Param.CONTEXT) - ); - } - - private static PseudoFuncConfigPreset redactPseudoFuncConfigPreset(String funcName) { - return new PseudoFuncConfigPreset(funcName, - Map.of(PseudoFuncConfig.Param.FUNC_IMPL, RedactFunc.class.getName()), - List.of(RedactFuncConfig.Param.PLACEHOLDER) - ); - } - - private static PseudoFuncConfigPreset redactRegexPseudoFuncConfigPreset(String funcName) { - return new PseudoFuncConfigPreset(funcName, - Map.of(PseudoFuncConfig.Param.FUNC_IMPL, RedactFunc.class.getName()), - List.of(RedactFuncConfig.Param.PLACEHOLDER, RedactFuncConfig.Param.REGEX) - ); - } - - private static PseudoFuncConfigPreset tinkDaeadPseudoFuncConfigPreset(String funcName) { - if (!funcName.startsWith("tink-")) { - throw new IllegalArgumentException("Tink functions must be prefixed with 'tink-'"); - } - - return new PseudoFuncConfigPreset(funcName, - Map.of(PseudoFuncConfig.Param.FUNC_IMPL, DaeadFunc.class.getName()), - List.of(DaeadFuncConfig.Param.DEK_ID) - ); - } - - private static PseudoFuncConfigPreset fpePseudoFuncConfigPreset(String funcName, String alphabet) { - if (!funcName.startsWith("fpe-")) { - throw new IllegalArgumentException("FPE functions must be prefixed with 'fpe-'"); - } - - return new PseudoFuncConfigPreset(funcName, - Map.of(PseudoFuncConfig.Param.FUNC_IMPL, FpeFunc.class.getName(), FpeFuncConfig.Param.ALPHABET, alphabet), - List.of(FpeFuncConfig.Param.KEY_ID)); - } - - static PseudoFuncConfigPreset getConfigPreset(FuncDeclaration funcDecl) { - String funcName = funcDecl.getFuncName(); - PseudoFuncConfigPreset preset = PSEUDO_CONFIG_PRESETS_MAP.get(funcDecl.getFuncName()); - if (preset != null) { - return preset; - } - - /* - If no preset was defined AND this is an FPE function, then create the preset dynamically - The alphabet to be used will be deduced from the function name (+ separated string with references to - any already defined CharacterGroup, see no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets) - */ - if (funcName.startsWith("fpe-")) { - String alphabetName = funcDecl.getFuncName().substring(4); - log.info("Add dynamic FPE function preset '{}' with alphabet '{}'. Allowed characters: {}", - funcName, alphabetName, new String(Alphabets.fromAlphabetName(alphabetName).availableCharacters())); - preset = fpePseudoFuncConfigPreset(funcName, alphabetName); - PSEUDO_CONFIG_PRESETS_MAP.put(funcName, preset); - return preset; - } - - throw new InvalidPseudoFuncDeclarationException(funcName, "Check spelling. Only FPE functions can be created dynamically"); - } - - public static PseudoFuncConfig get(String funcDeclarationString) { - FuncDeclaration funcDecl = FuncDeclaration.fromString(funcDeclarationString); - return getConfigPreset(funcDecl).toPseudoFuncConfig(funcDecl); - } - - /** - * A PseudoFuncConfig preset that holds a set of common parameters + defines a set of parameters that must be - * supplied externally in order to create a valid PseudoFuncConfig. - */ - static class PseudoFuncConfigPreset { - private final String funcName; - - private final Map defaultParams; - - private final List userDefinedParams; - - public PseudoFuncConfigPreset(String funcName, Map defaultParams, List additionalParams) { - this.funcName = funcName; - this.defaultParams = (defaultParams != null) ? defaultParams : Collections.emptyMap(); - this.userDefinedParams = (additionalParams != null) ? additionalParams : Collections.emptyList(); - } - - public String getFuncName() { - return funcName; - } - - /** - * Params common for all functions - */ - public Map getDefaultParams() { - return defaultParams; - } - - /** - * Names of params that must be supplied externally. Note that ordering is important. - */ - public List getUserDefinedParams() { - return userDefinedParams; - } - - /** - * Construct a PseudoFuncConfig using the supplied user defined arguments - * - * @throws PseudoFuncConfigException if args does not satisfy expected user defined parameters - */ - public PseudoFuncConfig toPseudoFuncConfig(FuncDeclaration funcDecl) { - List args = funcDecl.getArgs(); - if (args == null) { - args = Collections.emptyList(); - } - if (args.size() != userDefinedParams.size()) { - throw new PseudoFuncConfigException("Error creating PseudoFuncConfig for '" + funcName + "'. Expected arguments for " + userDefinedParams + ", but got: " + args); - } - ImmutableMap.Builder params = ImmutableMap.builder(); - params.put(PseudoFuncConfig.Param.FUNC_DECL, funcDecl.toString()); - params.putAll(defaultParams); - for (int i = 0; i < userDefinedParams.size(); i++) { - params.put(userDefinedParams.get(i), args.get(i)); - } - - return new PseudoFuncConfig(params.build()); - } - } - - static class FuncDeclaration { - private static final Splitter COMMA_SPLITTER = Splitter.on(",").trimResults(); - - private final String funcName; - private final List args; - - private FuncDeclaration(String funcName, List args) { - this.funcName = funcName; - this.args = args; - } - - public static FuncDeclaration fromString(String s) { - s = s.replaceAll("[()]", " ").trim(); - if (s.indexOf(' ') == -1) { - return new FuncDeclaration(s, Collections.emptyList()); - } else { - String funcName = s.substring(0, s.indexOf(' ')); - String argsString = s.substring(funcName.length(), s.length()); - List args = COMMA_SPLITTER.splitToList(argsString); - return new FuncDeclaration(funcName, args); - } - } - - @Override - public String toString() { - return funcName + "(" + String.join(",", args) + ")"; - } - - public String getFuncName() { - return funcName; - } - - public List getArgs() { - return args; - } - } - - static class InvalidPseudoFuncDeclarationException extends PseudoException { - public InvalidPseudoFuncDeclarationException(String funcName, String message) { - super("Invalid pseudo function declaration '" + funcName + "': " + message); - } - } - - static class PseudoFuncNotDefinedException extends PseudoException { - public PseudoFuncNotDefinedException(String funcName) { - super("No pseudo func '" + funcName + "' has been defined. Check spelling or supply a function definition to PseudoFuncConfigFactory"); - } - } - - static class PseudoFuncConfigException extends PseudoException { - public PseudoFuncConfigException(String message) { - super(message); - } - } - - -} diff --git a/src/main/java/no/ssb/dlp/pseudo/core/csv/CsvStreamProcessor.java b/src/main/java/no/ssb/dlp/pseudo/core/csv/CsvStreamProcessor.java index 616d101..6ce6c05 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/csv/CsvStreamProcessor.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/csv/CsvStreamProcessor.java @@ -8,12 +8,8 @@ import lombok.RequiredArgsConstructor; import lombok.Value; import lombok.extern.slf4j.Slf4j; -import no.ssb.dlp.pseudo.core.PseudoOperation; import no.ssb.dlp.pseudo.core.StreamProcessor; -import no.ssb.dlp.pseudo.core.StreamPseudonymizer; -import no.ssb.dlp.pseudo.core.json.JsonStreamProcessor; import no.ssb.dlp.pseudo.core.map.RecordMapProcessor; -import no.ssb.dlp.pseudo.core.map.RecordMapPseudonymizer; import no.ssb.dlp.pseudo.core.map.RecordMapSerializer; import java.io.IOException; diff --git a/src/main/java/no/ssb/dlp/pseudo/core/field/FieldPseudonymizer.java b/src/main/java/no/ssb/dlp/pseudo/core/field/FieldPseudonymizer.java index a60e308..612bf64 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/field/FieldPseudonymizer.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/field/FieldPseudonymizer.java @@ -2,7 +2,13 @@ import no.ssb.dapla.dlp.pseudo.func.PseudoFuncInput; import no.ssb.dapla.dlp.pseudo.func.PseudoFuncOutput; -import no.ssb.dlp.pseudo.core.*; +import no.ssb.dlp.pseudo.core.PseudoException; +import no.ssb.dlp.pseudo.core.PseudoKeyset; +import no.ssb.dlp.pseudo.core.PseudoOperation; +import no.ssb.dlp.pseudo.core.PseudoSecret; +import no.ssb.dlp.pseudo.core.func.PseudoFuncRule; +import no.ssb.dlp.pseudo.core.func.PseudoFuncRuleMatch; +import no.ssb.dlp.pseudo.core.func.PseudoFuncs; import java.util.Collection; import java.util.Objects; diff --git a/src/main/java/no/ssb/dlp/pseudo/core/file/PseudoFileSource.java b/src/main/java/no/ssb/dlp/pseudo/core/file/PseudoFileSource.java index 1a92eed..78e3235 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/file/PseudoFileSource.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/file/PseudoFileSource.java @@ -20,7 +20,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; // TODO: Rename mediaType -> sourceContentType and providedMediaType -> receivedFileContentType diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java new file mode 100644 index 0000000..d4308d4 --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java @@ -0,0 +1,124 @@ +package no.ssb.dlp.pseudo.core.func; + +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import no.ssb.crypto.tink.fpe.UnknownCharacterStrategy; +import no.ssb.dapla.dlp.pseudo.func.PseudoFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets; +import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFunc; +import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.map.MapFunc; +import no.ssb.dapla.dlp.pseudo.func.map.MapFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.redact.RedactFunc; +import no.ssb.dapla.dlp.pseudo.func.redact.RedactFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.tink.daead.TinkDaeadFunc; +import no.ssb.dapla.dlp.pseudo.func.tink.daead.TinkDaeadFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.tink.fpe.TinkFpeFunc; +import no.ssb.dapla.dlp.pseudo.func.tink.fpe.TinkFpeFuncConfig; +import no.ssb.dlp.pseudo.core.PseudoException; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets.alphabetNameOf; +import static no.ssb.dapla.dlp.pseudo.func.text.CharacterGroup.*; + +@Slf4j +class PseudoFuncConfigFactory { + + private static final Map PSEUDO_CONFIG_PRESETS_MAP = new HashMap<>(); + + static { + PSEUDO_CONFIG_PRESETS_MAP.putAll(Maps.uniqueIndex(List.of( + tinkDaeadPseudoFuncConfigPreset("daead"), + tinkFpePseudoFuncConfigPreset("ff31"), + sidMappingPseudoFuncConfigPreset("map-sid"), + redactPseudoFuncConfigPreset("redact"), + fpePseudoFuncConfigPreset("fpe-text", alphabetNameOf(ALPHANUMERIC, WHITESPACE, SYMBOLS)), + fpePseudoFuncConfigPreset("fpe-text_no", alphabetNameOf(ALPHANUMERIC_NO, WHITESPACE, SYMBOLS)), + fpePseudoFuncConfigPreset("fpe-fnr", alphabetNameOf(DIGITS)) + + ), PseudoFuncConfigPreset::getFuncName)); + } + + private PseudoFuncConfigFactory() {} + + private static PseudoFuncConfigPreset sidMappingPseudoFuncConfigPreset(String funcName) { + return PseudoFuncConfigPreset.builder(funcName, MapFunc.class) + .staticParam(MapFuncConfig.Param.CONTEXT, "sid") + .requiredParam(String.class, TinkFpeFuncConfig.Param.KEY_ID) + .build(); + } + + private static PseudoFuncConfigPreset redactPseudoFuncConfigPreset(String funcName) { + return PseudoFuncConfigPreset.builder(funcName, RedactFunc.class) + .optionalParam(String.class, RedactFuncConfig.Param.PLACEHOLDER, "*") + .optionalParam(String.class, RedactFuncConfig.Param.REGEX) + .build(); + } + + private static PseudoFuncConfigPreset tinkDaeadPseudoFuncConfigPreset(String funcName) { + return PseudoFuncConfigPreset.builder(funcName, TinkDaeadFunc.class) + .requiredParam(String.class, TinkDaeadFuncConfig.Param.KEY_ID) + .build(); + } + + private static PseudoFuncConfigPreset tinkFpePseudoFuncConfigPreset(String funcName) { + return PseudoFuncConfigPreset.builder(funcName, TinkFpeFunc.class) + .requiredParam(String.class, TinkFpeFuncConfig.Param.KEY_ID) + .optionalParam(UnknownCharacterStrategy.class, TinkFpeFuncConfig.Param.UNKNOWN_CHARACTER_STRATEGY, UnknownCharacterStrategy.FAIL) + .optionalParam(String.class, TinkFpeFuncConfig.Param.TWEAK) + .optionalParam(Character.class, TinkFpeFuncConfig.Param.REDACT_CHAR) + .build(); + } + + private static PseudoFuncConfigPreset fpePseudoFuncConfigPreset(String funcName, String alphabet) { + if (!funcName.startsWith("fpe-")) { + throw new IllegalArgumentException("FPE functions must be prefixed with 'fpe-'"); + } + + return PseudoFuncConfigPreset.builder(funcName, FpeFunc.class) + .staticParam(FpeFuncConfig.Param.ALPHABET, alphabet) + .requiredParam(String.class, FpeFuncConfig.Param.KEY_ID) + .optionalParam(Boolean.class, FpeFuncConfig.Param.REPLACE_ILLEGAL_CHARS, true) + .optionalParam(String.class, FpeFuncConfig.Param.REPLACE_ILLEGAL_CHARS_WITH) + .build(); + } + + static PseudoFuncConfigPreset getConfigPreset(PseudoFuncDeclaration funcDecl) { + String funcName = funcDecl.getFuncName(); + PseudoFuncConfigPreset preset = PSEUDO_CONFIG_PRESETS_MAP.get(funcDecl.getFuncName()); + if (preset != null) { + return preset; + } + + /* + If no preset was defined AND this is an FPE function, then create the preset dynamically + The alphabet to be used will be deduced from the function name (+ separated string with references to + any already defined CharacterGroup, see no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets) + */ + if (funcName.startsWith("fpe-")) { + String alphabetName = funcDecl.getFuncName().substring(4); + log.info("Add dynamic FPE function preset '{}' with alphabet '{}'. Allowed characters: {}", + funcName, alphabetName, new String(Alphabets.fromAlphabetName(alphabetName).availableCharacters())); + preset = fpePseudoFuncConfigPreset(funcName, alphabetName); + PSEUDO_CONFIG_PRESETS_MAP.put(funcName, preset); + return preset; + } + + throw new InvalidPseudoFuncDeclarationException(funcName, "Check spelling. Only FPE functions can be created dynamically"); + } + + public static PseudoFuncConfig get(String funcDeclarationString) { + PseudoFuncDeclaration funcDecl = PseudoFuncDeclaration.fromString(funcDeclarationString); + return getConfigPreset(funcDecl).toPseudoFuncConfig(funcDecl); + } + + static class InvalidPseudoFuncDeclarationException extends PseudoException { + public InvalidPseudoFuncDeclarationException(String funcName, String message) { + super("Invalid pseudo function declaration '" + funcName + "': " + message); + } + } + +} diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPreset.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPreset.java new file mode 100644 index 0000000..d1e57f4 --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPreset.java @@ -0,0 +1,141 @@ +package no.ssb.dlp.pseudo.core.func; + +import com.google.common.collect.ImmutableMap; +import no.ssb.dapla.dlp.pseudo.func.PseudoFunc; +import no.ssb.dapla.dlp.pseudo.func.PseudoFuncConfig; +import no.ssb.dlp.pseudo.core.PseudoException; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * PseudoFuncConfigPreset holds a set of common parameters + defines a set of parameters that can be + * supplied externally in order to create a valid PseudoFuncConfig. + */ +public class PseudoFuncConfigPreset { + + private final String funcName; + + private final Map staticParams; + + private final Set dynamicParams; + + private PseudoFuncConfigPreset(String funcName, Map staticParams, Set dynamicParams) { + this.funcName = funcName; + this.staticParams = (staticParams != null) ? staticParams : Collections.emptyMap(); + this.dynamicParams = (dynamicParams != null) ? dynamicParams : new HashSet<>(); + } + + public String getFuncName() { + return funcName; + } + + /** + * Params common for all functions + */ + public Map getStaticParams() { + return staticParams; + } + + /** + * Names of params that must be supplied externally. + */ + public Set getDynamicParams() { + return dynamicParams; + } + + /** + * Construct a PseudoFuncConfig using the supplied user defined arguments + */ + public PseudoFuncConfig toPseudoFuncConfig(PseudoFuncDeclaration funcDecl) { + Map args = Optional.ofNullable(funcDecl.getArgs()).orElse(Map.of()); + + ImmutableMap.Builder params = ImmutableMap.builder(); + params.put(PseudoFuncConfig.Param.FUNC_DECL, funcDecl.toString()); + params.putAll(staticParams); + + for (PseudoFuncParam param : dynamicParams) { + if (param.isRequired() && !args.containsKey(param.getParamName())) { + throw new MissingPseudoFuncParamException("Missing dynamic required pseudo func param '" + param.getParamName() + "' in function declaration " + funcDecl.toString()); + } + else { + String strValue = args.get(param.getParamName()); + if (strValue == null) { + if (param.getDefaultValue() != null) { + params.put(param.getParamName(), param.getDefaultValue()); + } + } + else { + Object value = param.parseValue(strValue); + if (value != null) { + params.put(param.getParamName(), param.parseValue(strValue)); + } + } + } + } + + Map paramsMap = params.build(); + Set undefined = args.keySet().stream() + .filter(s -> ! paramsMap.keySet().contains(s)) + .collect(Collectors.toSet()); + + if (! undefined.isEmpty()) { + throw new UndefinedPseudoFuncParamException("Encountered param(s) not defined in the PseudoFuncConfigPreset: " + undefined); + } + + return new PseudoFuncConfig(params.build()); + } + + public static Builder builder(String funcName, Class impl) { + return new Builder(funcName, impl); + } + + public static class Builder { + private final String funcName; + + private final Map staticParams = new HashMap<>(); + + private final Set dynamicParams = new HashSet<>(); + + public Builder(String funcName, Class impl) { + this.funcName = funcName; + staticParams.put(PseudoFuncConfig.Param.FUNC_IMPL, impl.getName()); + } + + public Builder staticParam(String name, Object value) { + staticParams.put(name, value); + return this; + } + + public Builder requiredParam(Class type, String name) { + dynamicParams.add(PseudoFuncParam.required(type, name)); + return this; + } + + public Builder optionalParam(Class type, String name) { + dynamicParams.add(PseudoFuncParam.optional(type, name)); + return this; + } + + public Builder optionalParam(Class type, String name, T defaultValue) { + dynamicParams.add(PseudoFuncParam.optional(type, name, defaultValue)); + return this; + } + + public PseudoFuncConfigPreset build() { + return new PseudoFuncConfigPreset(funcName, staticParams, dynamicParams); + } + } + + static class MissingPseudoFuncParamException extends PseudoException { + public MissingPseudoFuncParamException(String message) { + super(message); + } + } + + static class UndefinedPseudoFuncParamException extends PseudoException { + public UndefinedPseudoFuncParamException(String message) { + super(message); + } + } +} diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java new file mode 100644 index 0000000..bc44e0d --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java @@ -0,0 +1,70 @@ +package no.ssb.dlp.pseudo.core.func; + +import no.ssb.dlp.pseudo.core.PseudoException; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +// TODO: Support values wrapped by quotes (for values that need to use special characters like commas or parentheses) + +public class PseudoFuncDeclaration { + private final String funcName; + private final Map args; + + private PseudoFuncDeclaration(String funcName, Map args) { + this.funcName = funcName; + this.args = args; + } + + public static PseudoFuncDeclaration fromString(String s) { + s = s.replaceAll("[()]", " ").trim(); + if (s.indexOf(' ') == -1) { + return new PseudoFuncDeclaration(s, Map.of()); + } else { + String funcName = s.substring(0, s.indexOf(' ')); + String argsString = s.substring(funcName.length(), s.length()); + + Map args = Arrays.stream(argsString.split(",")) + .map(kv -> { + String[] items = kv.split("=", 2); + if (items.length != 2) { + throw new InvalidPseudoFuncParam("Pseudo func param should be on the format 'key=value', but was " + kv); + } + return items; + }) + .collect(Collectors.toMap( + kv -> kv[0].trim(), + kv -> kv[1].trim(), + (a, b) -> b, LinkedHashMap::new)); + + return new PseudoFuncDeclaration(funcName, args); + } + } + + @Override + public String toString() { + return funcName + "(" + args.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining(", ")) + ")"; + } + + public String getFuncName() { + return funcName; + } + + public Map getArgs() { + return args; + } + + static class InvalidPseudoFuncParam extends PseudoException { + public InvalidPseudoFuncParam(String message) { + super(message); + } + } + +} + diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncParam.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncParam.java new file mode 100644 index 0000000..870e287 --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncParam.java @@ -0,0 +1,39 @@ +package no.ssb.dlp.pseudo.core.func; + + +import lombok.NonNull; +import lombok.Value; +import no.ssb.dapla.dlp.pseudo.func.util.FromString; + +@Value +public class PseudoFuncParam { + + @NonNull + private final String paramName; + @NonNull + private final Class type; + private final boolean required; + + private final T defaultValue; + + public boolean isOptional() { + return ! required; + } + + public T parseValue(String stringValue) { + return FromString.convert(stringValue, type); + } + + public static PseudoFuncParam required(Class type, String name) { + return new PseudoFuncParam(name, type, true, null); + } + + public static PseudoFuncParam optional(Class type, String name, T defaultValue) { + return new PseudoFuncParam(name, type, false, defaultValue); + } + + public static PseudoFuncParam optional(Class type, String name) { + return new PseudoFuncParam(name, type, false, null); + } + +} diff --git a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncRule.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncRule.java similarity index 94% rename from src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncRule.java rename to src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncRule.java index 8b38ad6..bc7f0f2 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncRule.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncRule.java @@ -1,4 +1,4 @@ -package no.ssb.dlp.pseudo.core; +package no.ssb.dlp.pseudo.core.func; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncRuleMatch.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncRuleMatch.java similarity index 83% rename from src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncRuleMatch.java rename to src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncRuleMatch.java index 53a8d88..810731c 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncRuleMatch.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncRuleMatch.java @@ -1,4 +1,4 @@ -package no.ssb.dlp.pseudo.core; +package no.ssb.dlp.pseudo.core.func; import lombok.Value; import no.ssb.dapla.dlp.pseudo.func.PseudoFunc; diff --git a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncs.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java similarity index 53% rename from src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncs.java rename to src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java index baf2b03..19ce410 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/PseudoFuncs.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java @@ -1,16 +1,22 @@ -package no.ssb.dlp.pseudo.core; +package no.ssb.dlp.pseudo.core.func; import com.google.crypto.tink.DeterministicAead; import com.google.crypto.tink.JsonKeysetReader; import com.google.crypto.tink.KeysetHandle; import com.google.crypto.tink.KmsClients; +import no.ssb.crypto.tink.fpe.Fpe; import no.ssb.dapla.dlp.pseudo.func.PseudoFunc; import no.ssb.dapla.dlp.pseudo.func.PseudoFuncConfig; import no.ssb.dapla.dlp.pseudo.func.PseudoFuncFactory; -import no.ssb.dapla.dlp.pseudo.func.daead.DaeadFunc; -import no.ssb.dapla.dlp.pseudo.func.daead.DaeadFuncConfig; import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFunc; import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.tink.daead.TinkDaeadFunc; +import no.ssb.dapla.dlp.pseudo.func.tink.daead.TinkDaeadFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.tink.fpe.TinkFpeFunc; +import no.ssb.dapla.dlp.pseudo.func.tink.fpe.TinkFpeFuncConfig; +import no.ssb.dlp.pseudo.core.PseudoException; +import no.ssb.dlp.pseudo.core.PseudoKeyset; +import no.ssb.dlp.pseudo.core.PseudoSecret; import no.ssb.dlp.pseudo.core.exception.NoSuchPseudoKeyException; import no.ssb.dlp.pseudo.core.field.FieldDescriptor; import no.ssb.dlp.pseudo.core.tink.model.EncryptedKeysetWrapper; @@ -45,23 +51,29 @@ static Map initPseudoFuncConfigs(Collection keysetMap, Collection pseudoSecrets) { + String dekId = funcConfig.getRequired(TinkFpeFuncConfig.Param.KEY_ID, String.class); + + // TODO: Support creating new key material instead of failing if none found? + PseudoKeyset keyset = + // Use keyset from provided map + Optional.ofNullable(keysetMap.get(dekId)) + + // Or search for keyset among secrets + .or(() -> { + // Find secret matching key id + PseudoSecret secret = pseudoSecrets.stream().filter(s -> s.getId().endsWith(dekId)).findFirst().orElse(null); + if (secret == null) { + return Optional.empty(); + } + EncryptedKeysetWrapper keysetWrapper = Json.toObject(EncryptedKeysetWrapper.class, new String(secret.getContent())); + return Optional.of(keysetWrapper); + }) + .orElseThrow(() -> new NoSuchPseudoKeyException("No keyset with ID=" + dekId)); + + try { + String keyUri = keyset.getKekUri().toString(); + KeysetHandle keysetHandle = KeysetHandle.read( + JsonKeysetReader.withString(keyset.toJson()), + KmsClients.get(keyUri).getAead(keyUri) + ); + + Fpe fpe = keysetHandle.getPrimitive(Fpe.class); + funcConfig.add(TinkFpeFuncConfig.Param.FPE, fpe); } catch (Exception e) { - throw new RuntimeException("Error populating DaeadFuncConfig", e); + throw new PseudoFuncConfigException("Error populating Tink FpeFuncConfig", e); } } + /** * @return The set of pseudo rules, in the defined order */ @@ -114,4 +162,10 @@ public Optional findPseudoFunc(FieldDescriptor field) { .findFirst(); } + static class PseudoFuncConfigException extends PseudoException { + public PseudoFuncConfigException(String message, Exception e) { + super(message, e); + } + } + } diff --git a/src/main/java/no/ssb/dlp/pseudo/core/typeconverter/PseudoFuncRuleTypeConverter.java b/src/main/java/no/ssb/dlp/pseudo/core/typeconverter/PseudoFuncRuleTypeConverter.java index 52f7dbf..6163817 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/typeconverter/PseudoFuncRuleTypeConverter.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/typeconverter/PseudoFuncRuleTypeConverter.java @@ -3,7 +3,7 @@ import io.micronaut.core.convert.ConversionContext; import io.micronaut.core.convert.ConversionService; import io.micronaut.core.convert.TypeConverter; -import no.ssb.dlp.pseudo.core.PseudoFuncRule; +import no.ssb.dlp.pseudo.core.func.PseudoFuncRule; import java.util.Map; import java.util.Optional; diff --git a/src/main/java/no/ssb/dlp/pseudo/core/util/TinkUtil.java b/src/main/java/no/ssb/dlp/pseudo/core/util/TinkUtil.java index bee4a99..a07846e 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/util/TinkUtil.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/util/TinkUtil.java @@ -9,9 +9,9 @@ @UtilityClass public class TinkUtil { - public static String newWrappedKeyJson(String kekUri) { + public static String newWrappedKeyJson(String kekUri, String keyTemplateName) { try { - KeysetHandle keysetHandle = KeysetHandle.generateNew(KeyTemplates.get("AES256_SIV")); + KeysetHandle keysetHandle = KeysetHandle.generateNew(KeyTemplates.get(keyTemplateName)); return toWrappedKeyJson(keysetHandle, kekUri); } catch (Exception e) { diff --git a/src/test/java/no/ssb/dlp/pseudo/core/file/PseudoFileSourceTest.java b/src/test/java/no/ssb/dlp/pseudo/core/file/PseudoFileSourceTest.java index 04d16a8..e0e589d 100644 --- a/src/test/java/no/ssb/dlp/pseudo/core/file/PseudoFileSourceTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/core/file/PseudoFileSourceTest.java @@ -1,8 +1,6 @@ package no.ssb.dlp.pseudo.core.file; import io.micronaut.http.MediaType; -import no.ssb.dlp.pseudo.core.file.PseudoFileSource; -import no.ssb.dlp.pseudo.core.file.MoreMediaTypes; import org.junit.jupiter.api.Test; import java.io.File; @@ -14,26 +12,26 @@ class PseudoFileSourceTest { @Test - public void zipFileWithMultipleEntries_newPseudoFileSource_shouldReturnCombinedInputStream() throws IOException { + void zipFileWithMultipleEntries_newPseudoFileSource_shouldReturnCombinedInputStream() throws IOException { File file = readFileFromClasspathAndCreateLocalCopy("data/multiple-json-files.zip"); PseudoFileSource pfs = new PseudoFileSource(file); assertThat(pfs.getMediaType()).isEqualTo(MediaType.APPLICATION_JSON_TYPE); assertThat(pfs.getProvidedMediaType()).isEqualTo(MoreMediaTypes.APPLICATION_ZIP_TYPE); - assertThat(pfs.getFiles().size()).isEqualTo(10); + assertThat(pfs.getFiles()).hasSize(10); pfs.cleanup(); } @Test - public void zipFile_pseudoFileSourceCleanup_shouldRemoveFiles() throws IOException { + void zipFile_pseudoFileSourceCleanup_shouldRemoveFiles() throws IOException { File file = readFileFromClasspathAndCreateLocalCopy("data/multiple-json-files.zip"); PseudoFileSource pfs = new PseudoFileSource(file); - assertThat(pfs.getFiles().size()).isEqualTo(10); + assertThat(pfs.getFiles()).hasSize(10); for (File f : pfs.getFiles()) { - assertThat(f.exists()).isTrue(); + assertThat(f).exists(); } pfs.cleanup(); for (File f : pfs.getFiles()) { - assertThat(f.exists()).isFalse(); + assertThat(f).doesNotExist(); } } diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java new file mode 100644 index 0000000..0cb3f5c --- /dev/null +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java @@ -0,0 +1,63 @@ +package no.ssb.dlp.pseudo.core.func; + +import com.google.common.collect.ImmutableList; +import com.google.crypto.tink.CleartextKeysetHandle; +import com.google.crypto.tink.DeterministicAead; +import com.google.crypto.tink.JsonKeysetReader; +import com.google.crypto.tink.KeysetHandle; +import com.google.crypto.tink.daead.DeterministicAeadConfig; +import no.ssb.dapla.dlp.pseudo.func.*; +import no.ssb.dapla.dlp.pseudo.func.tink.daead.TinkDaeadFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.tink.fpe.TinkFpeFuncConfig; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DaeadFuncTest { + + @BeforeAll + static void init() throws Exception { + DeterministicAeadConfig.register(); + } + + private final static String KEYSET_JSON_AES256_SIV = "{\"primaryKeyId\":9876543210,\"key\":[{\"keyData\":{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.AesSivKey\",\"value\":\"EkCIjYUrKTTMAxEZST8xoyBXrfSLtTt+XmfBcE/PQxhr1Ob+YdD84bSMPQDaTGMqD241C4J7oQ+w3RFXaC8vKzbI\",\"keyMaterialType\":\"SYMMETRIC\"},\"status\":\"ENABLED\",\"keyId\":9876543210,\"outputPrefixType\":\"TINK\"}]}\n"; + + private final static Map KEYSETS = Map.of( + "9876543210", KEYSET_JSON_AES256_SIV + ); + + private DeterministicAead daeadPrimitive(String keyId) throws Exception { + if (! KEYSETS.containsKey(keyId)) { + throw new RuntimeException("Unknown keyId: " + keyId); + } + + KeysetHandle keysetHandle = CleartextKeysetHandle.read(JsonKeysetReader.withString(KEYSETS.get(keyId))); + return keysetHandle.getPrimitive(DeterministicAead.class); + } + + private PseudoFunc f(String funcDecl) throws Exception { + PseudoFuncConfig config = PseudoFuncConfigFactory.get(funcDecl); + String keyId = config.getRequired(TinkFpeFuncConfig.Param.KEY_ID, String.class); + config.add(TinkDaeadFuncConfig.Param.DAEAD, daeadPrimitive(keyId)); + return PseudoFuncFactory.create(config); + } + + private void transformAndRestore(Object originalVal, Object expectedVal, PseudoFunc func) { + Iterable expectedElements = (expectedVal instanceof Iterable) ? (Iterable) expectedVal : ImmutableList.of(expectedVal); + Iterable originalElements = (originalVal instanceof Iterable) ? (Iterable) originalVal : ImmutableList.of(originalVal); + PseudoFuncOutput pseudonymized = func.apply(PseudoFuncInput.of(originalVal)); + assertThat(pseudonymized.getValues()).containsExactlyElementsOf(expectedElements); + PseudoFuncOutput depseudonymized = func.restore(PseudoFuncInput.of(pseudonymized.getValues())); + assertThat(depseudonymized.getValues()).containsExactlyElementsOf(originalElements); + } + + @Test + void givenText_daead_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "daead(keyId=9876543210)"; + transformAndRestore("Something", "AUywFurOtYjeblGN+jZeR4w6alLoxuSsaigbZ+am", f(funcDeclStr)); + } + +} diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/Ff31FuncTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/Ff31FuncTest.java new file mode 100644 index 0000000..3a4c5ab --- /dev/null +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/Ff31FuncTest.java @@ -0,0 +1,104 @@ +package no.ssb.dlp.pseudo.core.func; + +import com.google.common.collect.ImmutableList; +import com.google.crypto.tink.CleartextKeysetHandle; +import com.google.crypto.tink.JsonKeysetReader; +import com.google.crypto.tink.KeysetHandle; +import no.ssb.crypto.tink.fpe.Fpe; +import no.ssb.crypto.tink.fpe.FpeConfig; +import no.ssb.crypto.tink.fpe.IncompatiblePlaintextException; +import no.ssb.dapla.dlp.pseudo.func.*; +import no.ssb.dapla.dlp.pseudo.func.tink.fpe.TinkFpeFuncConfig; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class Ff31FuncTest { + + @BeforeAll + static void init() throws Exception { + FpeConfig.register(); + } + + private final static String KEYSET_JSON_FF31_256_ALPHANUMERIC = "{\"primaryKeyId\":1234567890,\"key\":[{\"keyData\":{\"typeUrl\":\"type.googleapis.com/ssb.crypto.tink.FpeFfxKey\",\"value\":\"EiBoBeUFkoew7YJObcgcz1uOmzdhJFkPP7driAxAuS0UiRpCEAIaPkFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5\",\"keyMaterialType\":\"SYMMETRIC\"},\"status\":\"ENABLED\",\"keyId\":1234567890,\"outputPrefixType\":\"RAW\"}]}"; + + private final static Map KEYSETS = Map.of( + "1234567890", KEYSET_JSON_FF31_256_ALPHANUMERIC + ); + + private Fpe fpePrimitive(String keyId) throws Exception { + if (! KEYSETS.containsKey(keyId)) { + throw new RuntimeException("Unknown keyId: " + keyId); + } + + KeysetHandle keysetHandle = CleartextKeysetHandle.read(JsonKeysetReader.withString(KEYSETS.get(keyId))); + return keysetHandle.getPrimitive(Fpe.class); + } + + private PseudoFunc f(String funcDecl) throws Exception { + PseudoFuncConfig config = PseudoFuncConfigFactory.get(funcDecl); + String keyId = config.getRequired(TinkFpeFuncConfig.Param.KEY_ID, String.class); + config.add(TinkFpeFuncConfig.Param.FPE, fpePrimitive(keyId)); + return PseudoFuncFactory.create(config); + } + + private void transformAndRestore(Object originalVal, Object expectedVal, PseudoFunc func) { + Iterable expectedElements = (expectedVal instanceof Iterable) ? (Iterable) expectedVal : ImmutableList.of(expectedVal); + Iterable originalElements = (originalVal instanceof Iterable) ? (Iterable) originalVal : ImmutableList.of(originalVal); + PseudoFuncOutput pseudonymized = func.apply(PseudoFuncInput.of(originalVal)); + assertThat(pseudonymized.getValues()).containsExactlyElementsOf(expectedElements); + PseudoFuncOutput depseudonymized = func.restore(PseudoFuncInput.of(pseudonymized.getValues())); + assertThat(depseudonymized.getValues()).containsExactlyElementsOf(originalElements); + } + + @Test + void givenText_ff31_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "ff31(keyId=1234567890)"; + transformAndRestore("Something", "oADYiZKI3", f(funcDeclStr)); + } + @Test + void givenText_ff31Fail_shouldFailForNonSupportedCharacters() throws Exception { + String funcDeclStr = "ff31(keyId=1234567890)"; // defaults to "strategy=FAIL" + assertThatThrownBy(() -> { + f(funcDeclStr).apply(PseudoFuncInput.of("Ken sent me...")); + }) + .isInstanceOf(IncompatiblePlaintextException.class) + .hasMessageContaining("Plaintext can only contain characters from the alphabet"); + } + + @Test + void givenText_ff31Skip_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "ff31(keyId=1234567890, strategy=SKiP)"; + transformAndRestore("Ken sent me...", "fCR kd95 VR...", f(funcDeclStr)); + } + + @Test + void givenText_ff31Delete_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "ff31(keyId=1234567890, strategy=delete)"; + PseudoFunc func = f(funcDeclStr); + + PseudoFuncOutput pseudonymized = func.apply(PseudoFuncInput.of("Ken sent me...")); + assertThat(pseudonymized.getFirstValue()).isEqualTo("fCRkd95VR"); + + PseudoFuncOutput depseudonymized = func.restore(PseudoFuncInput.of("fCRkd95VR")); + assertThat(depseudonymized.getFirstValue()).isEqualTo("Kensentme"); + } + + @Test + void givenText_ff31Redact_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "ff31(keyId=1234567890, strategy=redact, redactChar=Z)"; + PseudoFunc func = f(funcDeclStr); + + PseudoFuncOutput pseudonymized = func.apply(PseudoFuncInput.of("Ken sent me...")); + assertThat(pseudonymized.getFirstValue()).isEqualTo("KGoDjzQOx4MasT"); + + PseudoFuncOutput depseudonymized = func.restore(PseudoFuncInput.of("KGoDjzQOx4MasT")); + assertThat(depseudonymized.getFirstValue()).isEqualTo("KenZsentZmeZZZ"); + } + + +} diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/LegacyFpeFuncTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/LegacyFpeFuncTest.java new file mode 100644 index 0000000..5035ba3 --- /dev/null +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/LegacyFpeFuncTest.java @@ -0,0 +1,64 @@ +package no.ssb.dlp.pseudo.core.func; + +import com.google.common.collect.ImmutableList; +import no.ssb.dapla.dlp.pseudo.func.*; +import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFunc; +import no.ssb.dapla.dlp.pseudo.func.fpe.FpeFuncConfig; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class LegacyFpeFuncTest { + + private final static Map KEYSETS = Map.of( + "secret1", "C5sn7B4YtwcilAwuVx6NuAsMWLusOSA/ldia40ZugDI=" + ); + + private PseudoFunc f(String funcDecl) throws Exception { + PseudoFuncConfig config = PseudoFuncConfigFactory.get(funcDecl); + String keyId = config.getRequired(FpeFuncConfig.Param.KEY_ID, String.class); + config.add(FpeFuncConfig.Param.KEY_DATA, KEYSETS.get(keyId)); + return PseudoFuncFactory.create(config); + } + + private void transformAndRestore(Object originalVal, Object expectedVal, PseudoFunc func) { + Iterable expectedElements = (expectedVal instanceof Iterable) ? (Iterable) expectedVal : ImmutableList.of(expectedVal); + Iterable originalElements = (originalVal instanceof Iterable) ? (Iterable) originalVal : ImmutableList.of(originalVal); + PseudoFuncOutput pseudonymized = func.apply(PseudoFuncInput.of(originalVal)); + assertThat(pseudonymized.getValues()).containsExactlyElementsOf(expectedElements); + PseudoFuncOutput depseudonymized = func.restore(PseudoFuncInput.of(pseudonymized.getValues())); + assertThat(depseudonymized.getValues()).containsExactlyElementsOf(originalElements); + } + + @Test + void givenText_fpeAnychar_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "fpe-anychar(keyId=secret1)"; + transformAndRestore("Something", "-Æ'GÕT@«L", f(funcDeclStr)); + } + + @Test + void givenText_fpeCustomAlphabet_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "fpe-abcdefghij(keyId=secret1)"; + transformAndRestore("abcdef", "djcjbf", f(funcDeclStr)); + } + + @Test + void givenNonAlphabetText_fpeCustomAlphabet_shouldFail() throws Exception { + String funcDeclStr = "fpe-abcdefghij(keyId=secret1, replaceIllegalChars=false, replaceIllegalCharsWith=X)"; + assertThatThrownBy(() -> { + f(funcDeclStr).apply(PseudoFuncInput.of("abcHELLO")); + }) + .isInstanceOf(FpeFunc.FpePseudoFuncException.class) + .hasMessageContaining("FPE pseudo apply error"); + } + + @Test + void givenDigits_fpeDigits_shouldEncryptAndDecrypt() throws Exception { + String funcDeclStr = "fpe-digits(keyId=secret1)"; + transformAndRestore("1234567890", "7830880047", f(funcDeclStr)); + } + +} diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPresetTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPresetTest.java new file mode 100644 index 0000000..d6c1a5e --- /dev/null +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigPresetTest.java @@ -0,0 +1,106 @@ +package no.ssb.dlp.pseudo.core.func; + +import no.ssb.dapla.dlp.pseudo.func.AbstractPseudoFunc; +import no.ssb.dapla.dlp.pseudo.func.PseudoFuncConfig; +import no.ssb.dapla.dlp.pseudo.func.PseudoFuncInput; +import no.ssb.dapla.dlp.pseudo.func.PseudoFuncOutput; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PseudoFuncConfigPresetTest { + + private enum SomeEnum { + DEFAULT_VAL, VAL1, VAL2; + } + + private static class DummyPseudoFunc extends AbstractPseudoFunc { + public DummyPseudoFunc(PseudoFuncConfig genericConfig) { + super(genericConfig.getFuncDecl()); + } + + @Override + public PseudoFuncOutput apply(PseudoFuncInput input) { + return new PseudoFuncOutput(); + } + + @Override + public PseudoFuncOutput restore(PseudoFuncInput input) { + return new PseudoFuncOutput(); + } + } + + PseudoFuncConfigPreset preset() { + return PseudoFuncConfigPreset.builder("foo", DummyPseudoFunc.class) + .staticParam("staticStr", "yo") + .staticParam("staticLong", 321L) + .staticParam("staticBoolean", Boolean.TRUE) + .requiredParam(String.class, "strParam") + .optionalParam(Integer.class, "optionalIntParam") + .optionalParam(SomeEnum.class, "optionalWithDefaultEnumParam", SomeEnum.DEFAULT_VAL) + .build(); + } + + @Test + void toPseudoFuncConfigFromPseudoFuncDeclaration_withAllOptionalParamsSet() { + String funcDeclString = "foo(strParam=hey ho, optionalWithDefaultEnumParam=val1, optionalIntParam=123)"; + PseudoFuncDeclaration funcDecl = PseudoFuncDeclaration.fromString(funcDeclString); + PseudoFuncConfig config = preset().toPseudoFuncConfig(funcDecl); + Map map = config.asMap(); + + assertThat(map).hasSize(8); + assertThat(map.get("decl")).isEqualTo(funcDecl.toString()); + assertThat(map.get("impl")).isEqualTo(DummyPseudoFunc.class.getName()); + assertThat(map.get("staticStr")).isEqualTo("yo"); + assertThat(map.get("staticLong")).isEqualTo(321L); + assertThat(map.get("staticBoolean")).isEqualTo(true); + assertThat(map.get("strParam")).isEqualTo("hey ho"); + assertThat(map.get("optionalIntParam")).isEqualTo(123); + assertThat(map.get("optionalWithDefaultEnumParam")).isEqualTo(SomeEnum.VAL1); + } + + @Test + void toPseudoFuncConfigFromPseudoFuncDeclaration_withNoOptionalParamsSet() { + String funcDeclString = "foo(strParam=hey ho)"; + PseudoFuncDeclaration funcDecl = PseudoFuncDeclaration.fromString(funcDeclString); + PseudoFuncConfig config = preset().toPseudoFuncConfig(funcDecl); + Map map = config.asMap(); + + assertThat(map).hasSize(7); + assertThat(map.get("decl")).isEqualTo(funcDecl.toString()); + assertThat(map.get("impl")).isEqualTo(DummyPseudoFunc.class.getName()); + assertThat(map.get("staticStr")).isEqualTo("yo"); + assertThat(map.get("staticLong")).isEqualTo(321L); + assertThat(map.get("staticBoolean")).isEqualTo(true); + assertThat(map.get("strParam")).isEqualTo("hey ho"); + assertThat(map.get("optionalWithDefaultEnumParam")).isEqualTo(SomeEnum.DEFAULT_VAL); + } + + @Test + void toPseudoFuncConfigFromPseudoFuncDeclaration_withoutRequiredParamsSet_shouldThrowException() { + String funcDeclString = "foo()"; + PseudoFuncDeclaration funcDecl = PseudoFuncDeclaration.fromString(funcDeclString); + + assertThatThrownBy(() -> { + preset().toPseudoFuncConfig(funcDecl); + }) + .isInstanceOf(PseudoFuncConfigPreset.MissingPseudoFuncParamException.class) + .hasMessageContaining("Missing dynamic required pseudo func param 'strParam' in function declaration foo()"); + } + + @Test + void toPseudoFuncConfigFromPseudoFuncDeclaration_withUndefinedParams_shouldThrowException() { + String funcDeclString = "foo(strParam=hey ho, undefinedParam=blah)"; + PseudoFuncDeclaration funcDecl = PseudoFuncDeclaration.fromString(funcDeclString); + + assertThatThrownBy(() -> { + preset().toPseudoFuncConfig(funcDecl); + }) + .isInstanceOf(PseudoFuncConfigPreset.UndefinedPseudoFuncParamException.class) + .hasMessageContaining("Encountered param(s) not defined in the PseudoFuncConfigPreset: [undefinedParam]"); + } + +} \ No newline at end of file diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclarationTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclarationTest.java new file mode 100644 index 0000000..58be380 --- /dev/null +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclarationTest.java @@ -0,0 +1,62 @@ +package no.ssb.dlp.pseudo.core.func; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PseudoFuncDeclarationTest { + + @Test + void parseFuncDeclStringWithArgs() { + String funcDecl = "ffx31(key=123, strategy=skip)"; + PseudoFuncDeclaration decl = PseudoFuncDeclaration.fromString(funcDecl); + assertThat(decl.getFuncName()).isEqualTo("ffx31"); + assertThat(decl.getArgs().size()).isEqualTo(2); + assertThat(decl.getArgs().get("key")).isEqualTo("123"); + assertThat(decl.getArgs().get("strategy")).isEqualTo("skip"); + } + + @Test + void parseFuncDeclStringWithArgsThatContainSpaces() { + String funcDecl = "foo(an arg=1 2=3, yet another arg=hi ho)"; + PseudoFuncDeclaration decl = PseudoFuncDeclaration.fromString(funcDecl); + assertThat(decl.getFuncName()).isEqualTo("foo"); + assertThat(decl.getArgs().size()).isEqualTo(2); + assertThat(decl.getArgs().get("an arg")).isEqualTo("1 2=3"); + assertThat(decl.getArgs().get("yet another arg")).isEqualTo("hi ho"); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "\n", "()", "( )"}) + void parseEmptyFuncDeclString() { + PseudoFuncDeclaration decl = PseudoFuncDeclaration.fromString(""); + assertThat(decl.getFuncName()).isEqualTo(""); + } + + @ParameterizedTest + @ValueSource(strings = {"ffx31()", "ffx31", "ffx31( )"}) + void parseFuncDeclStringWithoutArgs(String funcDecl) { + PseudoFuncDeclaration decl = PseudoFuncDeclaration.fromString(funcDecl); + assertThat(decl.getFuncName()).isEqualTo("ffx31"); + assertThat(decl.getArgs()).isNotNull(); + assertThat(decl.getArgs().size()).isEqualTo(0); + } + + @ParameterizedTest + @CsvSource(delimiter = ';', value = { + "foo;foo()", + "foo();foo()", + "foo(key=123);foo(key=123)", + "foo( key = 123 );foo(key=123)", + "foo( some thing = my value );foo(some thing=my value)", + "foo(bar = baz=123);foo(bar=baz=123)", + }) + void toStringOfparsedFuncDecl(String funcDecl, String expectedToString) { + PseudoFuncDeclaration decl = PseudoFuncDeclaration.fromString(funcDecl); + assertThat(decl.toString()).isEqualTo(expectedToString); + } + +} diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/RedactFuncTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/RedactFuncTest.java new file mode 100644 index 0000000..d7e6a7d --- /dev/null +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/RedactFuncTest.java @@ -0,0 +1,50 @@ +package no.ssb.dlp.pseudo.core.func; + +import no.ssb.dapla.dlp.pseudo.func.*; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RedactFuncTest { + + private PseudoFunc f(String funcDecl) { + PseudoFuncConfig config = PseudoFuncConfigFactory.get(funcDecl); + return PseudoFuncFactory.create(config); + } + + static void assertEqual(PseudoFuncOutput out, Object expected) { + assertThat(out.getFirstValue()).isEqualTo(expected); + } + + @Test + void givenText_redactWithDefaults_shouldReplaceWithPlaceholder() { + PseudoFuncOutput out = f("redact()").apply(PseudoFuncInput.of("Something")); + assertEqual(out, "*"); + } + + @Test + void givenText_redactWithCustomPlaceholder_shouldReplaceWithPlaceholder() { + PseudoFuncOutput out = f("redact(placeholder=###)").apply(PseudoFuncInput.of("Something")); + assertEqual(out, "###"); + } + + @Test + void givenEmpty_redact_shouldReturnEmpty() { + PseudoFuncOutput out = f("redact()").apply(PseudoFuncInput.of("")); + assertEqual(out, ""); + } + + // TODO: Support regexes with comma, like these: redact(regex='^.{0,4})' + @Test + void givenText_redactWithRegex_shouldReplaceTextPartially() { + PseudoFuncOutput out = f("redact(regex=^Some").apply(PseudoFuncInput.of("Something")); + assertEqual(out, "*thing"); + } + + @Test + void givenText_redactRestore_shouldEchoInput() { + PseudoFuncOutput out = f("redact()").restore(PseudoFuncInput.of("Something")); + assertEqual(out, "Something"); + } + +} diff --git a/src/test/java/no/ssb/dlp/pseudo/core/json/JsonRecordMapSerializerTest.java b/src/test/java/no/ssb/dlp/pseudo/core/json/JsonRecordMapSerializerTest.java index 1cdeee8..d5aa59a 100644 --- a/src/test/java/no/ssb/dlp/pseudo/core/json/JsonRecordMapSerializerTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/core/json/JsonRecordMapSerializerTest.java @@ -9,8 +9,6 @@ import java.util.Map; -import static org.assertj.core.api.Assertions.assertThat; - class JsonRecordMapSerializerTest { @Test diff --git a/src/test/java/no/ssb/dlp/pseudo/core/util/PathJoinerTest.java b/src/test/java/no/ssb/dlp/pseudo/core/util/PathJoinerTest.java index 5a71c3e..3424964 100644 --- a/src/test/java/no/ssb/dlp/pseudo/core/util/PathJoinerTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/core/util/PathJoinerTest.java @@ -2,11 +2,7 @@ import org.junit.jupiter.api.Test; -import static no.ssb.dlp.pseudo.core.util.PathJoiner.joinAndKeepLeadingAndTrailingSlash; -import static no.ssb.dlp.pseudo.core.util.PathJoiner.joinAndKeepLeadingSlash; -import static no.ssb.dlp.pseudo.core.util.PathJoiner.joinAndKeepTrailingSlash; -import static no.ssb.dlp.pseudo.core.util.PathJoiner.joinWithoutLeadingOrTrailingSlash; - +import static no.ssb.dlp.pseudo.core.util.PathJoiner.*; import static org.assertj.core.api.Assertions.assertThat; class PathJoinerTest { From d5cdd29bd4c427a295bef5ce74ddbdfdfd4a9094 Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Mon, 13 Mar 2023 14:16:38 +0100 Subject: [PATCH 2/7] Update deps and tests --- pom.xml | 18 +++++++++--------- .../dlp/pseudo/core/func/DaeadFuncTest.java | 8 ++++---- .../no/ssb/dlp/pseudo/core/util/ZipsTest.java | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pom.xml b/pom.xml index ebf321c..12244f7 100644 --- a/pom.xml +++ b/pom.xml @@ -13,31 +13,31 @@ artifactregistry://europe-north1-maven.pkg.dev/artifact-registry-14da - 1.0.0 - 1.1.1-SNAPSHOT - 5.0.0 + 1.1.0 + 1.2.0 + 5.1.2 31.1-jre 1.5.1 1.4.5 - 1.18.22 + 1.18.26 3.8.0 - 0.2.1.1 + 0.2.2 1.24.1 1.7.0 2.9.1 - 2.11.2 + 2.11.5 2.1.4 - 9.3 - 3.1.2 + 10.8.1 + 3.2.1 3.10.0 3.0.0-M5 3.3.0 3.2.0 3.0.0-M5 3.2.1 - 0.8.7 + 0.8.8 3.9.1.2184 diff --git a/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java b/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java index 0cb3f5c..890dc8b 100644 --- a/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/core/func/DaeadFuncTest.java @@ -23,10 +23,10 @@ static void init() throws Exception { DeterministicAeadConfig.register(); } - private final static String KEYSET_JSON_AES256_SIV = "{\"primaryKeyId\":9876543210,\"key\":[{\"keyData\":{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.AesSivKey\",\"value\":\"EkCIjYUrKTTMAxEZST8xoyBXrfSLtTt+XmfBcE/PQxhr1Ob+YdD84bSMPQDaTGMqD241C4J7oQ+w3RFXaC8vKzbI\",\"keyMaterialType\":\"SYMMETRIC\"},\"status\":\"ENABLED\",\"keyId\":9876543210,\"outputPrefixType\":\"TINK\"}]}\n"; + private final static String KEYSET_JSON_AES256_SIV = "{\"primaryKeyId\":1284924461,\"key\":[{\"keyData\":{\"typeUrl\":\"type.googleapis.com/google.crypto.tink.AesSivKey\",\"value\":\"EkCIjYUrKTTMAxEZST8xoyBXrfSLtTt+XmfBcE/PQxhr1Ob+YdD84bSMPQDaTGMqD241C4J7oQ+w3RFXaC8vKzbI\",\"keyMaterialType\":\"SYMMETRIC\"},\"status\":\"ENABLED\",\"keyId\":1284924461,\"outputPrefixType\":\"TINK\"}]}"; private final static Map KEYSETS = Map.of( - "9876543210", KEYSET_JSON_AES256_SIV + "1284924461", KEYSET_JSON_AES256_SIV ); private DeterministicAead daeadPrimitive(String keyId) throws Exception { @@ -56,8 +56,8 @@ private void transformAndRestore(Object originalVal, Object expectedVal, PseudoF @Test void givenText_daead_shouldEncryptAndDecrypt() throws Exception { - String funcDeclStr = "daead(keyId=9876543210)"; - transformAndRestore("Something", "AUywFurOtYjeblGN+jZeR4w6alLoxuSsaigbZ+am", f(funcDeclStr)); + String funcDeclStr = "daead(keyId=1284924461)"; + transformAndRestore("Something", "AUyWZC3OtYjeblGN+jZeR4w6alLoxuSsaigbZ+am", f(funcDeclStr)); } } diff --git a/src/test/java/no/ssb/dlp/pseudo/core/util/ZipsTest.java b/src/test/java/no/ssb/dlp/pseudo/core/util/ZipsTest.java index 27e9271..2ebefb8 100644 --- a/src/test/java/no/ssb/dlp/pseudo/core/util/ZipsTest.java +++ b/src/test/java/no/ssb/dlp/pseudo/core/util/ZipsTest.java @@ -136,6 +136,7 @@ public void flowable_zipFlowable_shouldCreateMultipartZippedAndEncryptedFile() t @Test public void flowable_zipFlowable_shouldCreateZippedFlowable() throws Exception { + AtomicInteger calls = new AtomicInteger(); Flowable f = Flowable.range(100, 10).map(Object::toString) .doOnCancel(() -> calls.incrementAndGet()) @@ -145,7 +146,6 @@ public void flowable_zipFlowable_shouldCreateZippedFlowable() throws Exception { final Path pathToZip = Files.createTempDirectory("test").resolve(UUID.randomUUID() + ".zip"); pathToZip.toFile().createNewFile(); - assertInvalidZip(pathToZip); flowableZip.subscribe( (byte[] bytes) -> { From d3bacc742711b3305eee7b3c8c970a64bfd0f0bf Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Mon, 13 Mar 2023 14:32:46 +0100 Subject: [PATCH 3/7] Update tika version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 12244f7..72d8be6 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 1.18.26 3.8.0 0.2.2 - 1.24.1 + 1.28.5 1.7.0 2.9.1 2.11.5 From 2a859f72aacc147a3c7aeb52996d0021004a72ac Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Mon, 13 Mar 2023 21:15:25 +0100 Subject: [PATCH 4/7] Update deps --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 72d8be6..9dca07d 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 no.ssb.dapla.dlp.pseudo dapla-dlp-pseudo-core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 17 @@ -14,16 +14,16 @@ 1.1.0 - 1.2.0 + 1.2.1 5.1.2 31.1-jre 1.5.1 1.4.5 1.18.26 - 3.8.0 + 3.8.7 0.2.2 1.28.5 - 1.7.0 + 1.8.0 2.9.1 2.11.5 From 282527397ecdedd5c961290e05384f280b11ca74 Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Mon, 13 Mar 2023 21:16:55 +0100 Subject: [PATCH 5/7] Add constants for some pseudo func names --- .../core/func/PseudoFuncConfigFactory.java | 23 ++++++++++--------- .../dlp/pseudo/core/func/PseudoFuncNames.java | 16 +++++++++++++ 2 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncNames.java diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java index d4308d4..7d3447a 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncConfigFactory.java @@ -23,6 +23,7 @@ import static no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets.alphabetNameOf; import static no.ssb.dapla.dlp.pseudo.func.text.CharacterGroup.*; +import static no.ssb.dlp.pseudo.core.func.PseudoFuncNames.*; @Slf4j class PseudoFuncConfigFactory { @@ -31,13 +32,13 @@ class PseudoFuncConfigFactory { static { PSEUDO_CONFIG_PRESETS_MAP.putAll(Maps.uniqueIndex(List.of( - tinkDaeadPseudoFuncConfigPreset("daead"), - tinkFpePseudoFuncConfigPreset("ff31"), - sidMappingPseudoFuncConfigPreset("map-sid"), - redactPseudoFuncConfigPreset("redact"), - fpePseudoFuncConfigPreset("fpe-text", alphabetNameOf(ALPHANUMERIC, WHITESPACE, SYMBOLS)), - fpePseudoFuncConfigPreset("fpe-text_no", alphabetNameOf(ALPHANUMERIC_NO, WHITESPACE, SYMBOLS)), - fpePseudoFuncConfigPreset("fpe-fnr", alphabetNameOf(DIGITS)) + tinkDaeadPseudoFuncConfigPreset(DAEAD), + tinkFpePseudoFuncConfigPreset(FF31), + sidMappingPseudoFuncConfigPreset(MAP_SID), + redactPseudoFuncConfigPreset(REDACT), + fpePseudoFuncConfigPreset(FPE + "-text", alphabetNameOf(ALPHANUMERIC, WHITESPACE, SYMBOLS)), + fpePseudoFuncConfigPreset(FPE + "-text_no", alphabetNameOf(ALPHANUMERIC_NO, WHITESPACE, SYMBOLS)), + fpePseudoFuncConfigPreset(FPE + "-fnr", alphabetNameOf(DIGITS)) ), PseudoFuncConfigPreset::getFuncName)); } @@ -74,8 +75,8 @@ private static PseudoFuncConfigPreset tinkFpePseudoFuncConfigPreset(String funcN } private static PseudoFuncConfigPreset fpePseudoFuncConfigPreset(String funcName, String alphabet) { - if (!funcName.startsWith("fpe-")) { - throw new IllegalArgumentException("FPE functions must be prefixed with 'fpe-'"); + if (!funcName.startsWith(FPE)) { + throw new IllegalArgumentException("Legacy FPE functions must be prefixed with '" + FPE + "'"); } return PseudoFuncConfigPreset.builder(funcName, FpeFunc.class) @@ -98,8 +99,8 @@ static PseudoFuncConfigPreset getConfigPreset(PseudoFuncDeclaration funcDecl) { The alphabet to be used will be deduced from the function name (+ separated string with references to any already defined CharacterGroup, see no.ssb.dapla.dlp.pseudo.func.fpe.Alphabets) */ - if (funcName.startsWith("fpe-")) { - String alphabetName = funcDecl.getFuncName().substring(4); + if (funcName.startsWith(FPE + "-")) { + String alphabetName = funcDecl.getFuncName().substring((FPE + "-").length()); log.info("Add dynamic FPE function preset '{}' with alphabet '{}'. Allowed characters: {}", funcName, alphabetName, new String(Alphabets.fromAlphabetName(alphabetName).availableCharacters())); preset = fpePseudoFuncConfigPreset(funcName, alphabetName); diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncNames.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncNames.java new file mode 100644 index 0000000..180b1db --- /dev/null +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncNames.java @@ -0,0 +1,16 @@ +package no.ssb.dlp.pseudo.core.func; + +/** + * Pseudo function name constants + */ +public final class PseudoFuncNames { + private PseudoFuncNames() { + } + + public static final String FF31 = "ff31"; + public static final String DAEAD = "daead"; + public static final String MAP_SID = "map-sid"; + public static final String REDACT = "redact"; + public static final String FPE = "fpe"; + +} From c31fade0aa7010ac597e617c220e4b1288e6bbba Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Mon, 13 Mar 2023 21:20:13 +0100 Subject: [PATCH 6/7] Rename accessor mehthods for convenience properties This simplifies usage of EncryptedKeysetWrapper in REST Controllers. We don't want the convenience accessor methods to be included controller payload. --- src/main/java/no/ssb/dlp/pseudo/core/PseudoKeyset.java | 4 ++-- src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java | 2 +- .../dlp/pseudo/core/tink/model/EncryptedKeysetWrapper.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/no/ssb/dlp/pseudo/core/PseudoKeyset.java b/src/main/java/no/ssb/dlp/pseudo/core/PseudoKeyset.java index a6af83e..fc623dc 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/PseudoKeyset.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/PseudoKeyset.java @@ -4,8 +4,8 @@ import java.util.Set; public interface PseudoKeyset { - String getPrimaryKeyId(); - Set getKeyIds(); + String primaryKeyId(); + Set keyIds(); URI getKekUri(); diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java index 19ce410..0cbcc29 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncs.java @@ -43,7 +43,7 @@ static Map initPseudoFuncConfigs(Collection pseudoKeysetMap = pseudoKeysets.stream().collect( - Collectors.toMap(PseudoKeyset::getPrimaryKeyId, Function.identity())); + Collectors.toMap(PseudoKeyset::primaryKeyId, Function.identity())); return pseudoRules.stream().collect(Collectors.toMap( Function.identity(), diff --git a/src/main/java/no/ssb/dlp/pseudo/core/tink/model/EncryptedKeysetWrapper.java b/src/main/java/no/ssb/dlp/pseudo/core/tink/model/EncryptedKeysetWrapper.java index 2631fef..730262b 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/tink/model/EncryptedKeysetWrapper.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/tink/model/EncryptedKeysetWrapper.java @@ -16,12 +16,12 @@ public class EncryptedKeysetWrapper implements PseudoKeyset { private KeysetInfo keysetInfo; @Override - public String getPrimaryKeyId() { + public String primaryKeyId() { return keysetInfo.getPrimaryKeyId().toString(); } @Override - public Set getKeyIds() { + public Set keyIds() { return keysetInfo.getKeyInfo().stream() .map(keyInfo -> keyInfo.getKeyId().toString()) .collect(Collectors.toSet()); From 3b34f606954aab80320cf2c13df36f44f33a7f6e Mon Sep 17 00:00:00 2001 From: Kenneth Leine Schulstad Date: Mon, 13 Mar 2023 21:21:20 +0100 Subject: [PATCH 7/7] Allow invocation of PseudoFuncDeclaration constructor Currently needed in the Pseudo Service --- .../java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java index bc44e0d..ad025c3 100644 --- a/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java +++ b/src/main/java/no/ssb/dlp/pseudo/core/func/PseudoFuncDeclaration.java @@ -15,7 +15,7 @@ public class PseudoFuncDeclaration { private final String funcName; private final Map args; - private PseudoFuncDeclaration(String funcName, Map args) { + public PseudoFuncDeclaration(String funcName, Map args) { this.funcName = funcName; this.args = args; }