Skip to content

Commit

Permalink
MAINT: add json property descriptions for kv configs (opensearch-proj…
Browse files Browse the repository at this point in the history 
…ect#4747)

add json property descriptions for kv configs

Signed-off-by: Katherine Shen <[email protected]>
Signed-off-by: Krishna Kondaka <[email protected]>
  • Loading branch information
@shenkw1
shenkw1 authored and Krishna Kondaka committed Jul 23, 2024
1 parent 3826a82 commit 197574a
Showing 1 changed file with 79 additions and 1 deletion.
80 changes: 79 additions & 1 deletion ...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.dataprepper.plugins.processor.keyvalue;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.AssertTrue;
Expand Down Expand Up @@ -35,87 +36,163 @@ public class KeyValueProcessorConfig {
static final boolean DEFAULT_RECURSIVE = false;

@NotEmpty
@JsonPropertyDescription("The message field to be parsed. Optional. Default value is `message`.")
private String source = DEFAULT_SOURCE;

@JsonPropertyDescription("The destination field for the parsed source. The parsed source overwrites the " +
"preexisting data for that key. Optional. If `destination` is set to `null`, the parsed fields will be " +
"written to the root of the event. Default value is `parsed_message`.")
private String destination = DEFAULT_DESTINATION;

@JsonProperty("field_delimiter_regex")
@JsonPropertyDescription("A regular expression specifying the delimiter that separates key-value pairs. " +
"Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " +
"Cannot be defined at the same time as `field_split_characters`. Optional. " +
"If this option is not defined, `field_split_characters` is used.")
private String fieldDelimiterRegex;

@JsonProperty("field_split_characters")
@JsonPropertyDescription("A string of characters specifying the delimiter that separates key-value pairs. " +
"Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " +
"Cannot be defined at the same time as `field_delimiter_regex`. Optional. Default value is `&`.")
private String fieldSplitCharacters = DEFAULT_FIELD_SPLIT_CHARACTERS;

@JsonProperty("include_keys")
@JsonPropertyDescription("An array specifying the keys that should be added for parsing. " +
"By default, all keys will be added.")
@NotNull
private List<String> includeKeys = DEFAULT_INCLUDE_KEYS;

@JsonProperty("exclude_keys")
@JsonPropertyDescription("An array specifying the parsed keys that should not be added to the event. " +
"By default, no keys will be excluded.")
@NotNull
private List<String> excludeKeys = DEFAULT_EXCLUDE_KEYS;

@JsonProperty("default_values")
@JsonPropertyDescription("A map specifying the default keys and their values that should be added " +
"to the event in case these keys do not exist in the source field being parsed. " +
"If the default key already exists in the message, the value is not changed. " +
"The `include_keys` filter will be applied to the message before `default_values`.")
@NotNull
private Map<String, Object> defaultValues = DEFAULT_DEFAULT_VALUES;

@JsonProperty("key_value_delimiter_regex")
@JsonPropertyDescription("A regular expression specifying the delimiter that separates the key and value " +
"within a key-value pair. Special regular expression characters such as `[` and `]` must be escaped with " +
"`\\\\`. This option cannot be defined at the same time as `value_split_characters`. Optional. " +
"If this option is not defined, `value_split_characters` is used.")
private String keyValueDelimiterRegex;

@JsonProperty("value_split_characters")
@JsonPropertyDescription("A string of characters specifying the delimiter that separates the key and value within " +
"a key-value pair. Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " +
"Cannot be defined at the same time as `key_value_delimiter_regex`. Optional. Default value is `=`.")
private String valueSplitCharacters = DEFAULT_VALUE_SPLIT_CHARACTERS;

@JsonProperty("non_match_value")
@JsonPropertyDescription("When a key-value pair cannot be successfully split, the key-value pair is " +
"placed in the `key` field, and the specified value is placed in the `value` field. " +
"Optional. Default value is `null`.")
private Object nonMatchValue = DEFAULT_NON_MATCH_VALUE;

@JsonPropertyDescription("A prefix to append before all keys. Optional. Default value is an empty string.")
@NotNull
private String prefix = DEFAULT_PREFIX;

@JsonProperty("delete_key_regex")
@JsonPropertyDescription("A regular expression specifying the characters to delete from the key. " +
"Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. Cannot be an " +
"empty string. Optional. No default value.")
@NotNull
private String deleteKeyRegex = DEFAULT_DELETE_KEY_REGEX;

@JsonProperty("delete_value_regex")
@JsonPropertyDescription("A regular expression specifying the characters to delete from the value. " +
"Special regular expression characters such as `[` and `]` must be escaped with `\\\\`. " +
"Cannot be an empty string. Optional. No default value.")
@NotNull
private String deleteValueRegex = DEFAULT_DELETE_VALUE_REGEX;

@JsonProperty("transform_key")
@JsonPropertyDescription("When to lowercase, uppercase, or capitalize keys.")
@NotNull
private String transformKey = DEFAULT_TRANSFORM_KEY;

@JsonProperty("whitespace")
@JsonPropertyDescription("Specifies whether to be lenient or strict with the acceptance of " +
"unnecessary white space surrounding the configured value-split sequence. Default is `lenient`.")
@NotNull
private String whitespace = DEFAULT_WHITESPACE;

@JsonProperty("skip_duplicate_values")
@JsonPropertyDescription("A Boolean option for removing duplicate key-value pairs. When set to `true`, " +
"only one unique key-value pair will be preserved. Default is `false`.")
@NotNull
private boolean skipDuplicateValues = DEFAULT_SKIP_DUPLICATE_VALUES;

@JsonProperty("remove_brackets")
@JsonPropertyDescription("Specifies whether to treat square brackets, angle brackets, and parentheses " +
"as value “wrappers” that should be removed from the value. Default is `false`.")
@NotNull
private boolean removeBrackets = DEFAULT_REMOVE_BRACKETS;

@JsonProperty("value_grouping")
@JsonPropertyDescription("Specifies whether to group values using predefined value grouping delimiters: " +
"`{...}`, `[...]`, `<...>`, `(...)`, `\"...\"`, `'...'`, `http://... (space)`, and `https:// (space)`. " +
"If this flag is enabled, then the content between the delimiters is considered to be one entity and " +
"is not parsed for key-value pairs. Default is `false`. If `value_grouping` is `true`, then " +
"`{\"key1=[a=b,c=d]&key2=value2\"}` parses to `{\"key1\": \"[a=b,c=d]\", \"key2\": \"value2\"}`.")
private boolean valueGrouping = DEFAULT_VALUE_GROUPING;

@JsonProperty("recursive")
@JsonPropertyDescription("Specifies whether to recursively obtain additional key-value pairs from values. " +
"The extra key-value pairs will be stored as sub-keys of the root key. Default is `false`. " +
"The levels of recursive parsing must be defined by different brackets for each level: " +
"`[]`, `()`, and `<>`, in this order. Any other configurations specified will only be applied " +
"to the outmost keys.\n" +
"When `recursive` is `true`:\n" +
"`remove_brackets` cannot also be `true`;\n" +
"`skip_duplicate_values` will always be `true`;\n" +
"`whitespace` will always be `\"strict\"`.")
@NotNull
private boolean recursive = DEFAULT_RECURSIVE;

@JsonProperty("tags_on_failure")
@JsonPropertyDescription("When a `kv` operation causes a runtime exception within the processor, " +
"the operation is safely stopped without crashing the processor, and the event is tagged " +
"with the provided tags.")
private List<String> tagsOnFailure;

@JsonProperty("overwrite_if_destination_exists")
@JsonPropertyDescription("Specifies whether to overwrite existing fields if there are key conflicts " +
"when writing parsed fields to the event. Default is `true`.")
private boolean overwriteIfDestinationExists = true;

@JsonProperty("drop_keys_with_no_value")
@JsonPropertyDescription("Specifies whether keys should be dropped if they have a null value. Default is `false`. " +
"If `drop_keys_with_no_value` is set to `true`, " +
"then `{\"key1=value1&key2\"}` parses to `{\"key1\": \"value1\"}`.")
private boolean dropKeysWithNoValue = false;

@JsonProperty("key_value_when")
@JsonPropertyDescription("Allows you to specify a [conditional expression](https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/), " +
"such as `/some-key == \"test\"`, that will be evaluated to determine whether " +
"the processor should be applied to the event.")
private String keyValueWhen;

@JsonProperty("strict_grouping")
@JsonPropertyDescription("When enabled, groups with unmatched end characters yield errors. " +
"The event is ignored after the errors are logged. " +
"Specifies whether strict grouping should be enabled when the `value_grouping` " +
"or `string_literal_character` options are used. Default is `false`.")
private boolean strictGrouping = false;

@JsonProperty("string_literal_character")
@JsonPropertyDescription("When this option is used, any text contained within the specified quotation " +
"mark character will be ignored and excluded from key-value parsing. " +
"Can be set to either a single quotation mark (`'`) or a double quotation mark (`\"`). " +
"Default is `null`.")
@Size(min = 0, max = 1, message = "string_literal_character may only have character")
private String stringLiteralCharacter = null;

Expand All @@ -124,7 +201,8 @@ boolean isValidValueGroupingAndFieldDelimiterRegex() {
return (!valueGrouping || fieldDelimiterRegex == null);
}

@AssertTrue(message = "Invalid Configuration. String literal character config is valid only when value_grouping is enabled, and only double quote (\") and single quote are (') are valid string literal characters.")
@AssertTrue(message = "Invalid Configuration. String literal character config is valid only when value_grouping is enabled, " +
"and only double quote (\") and single quote are (') are valid string literal characters.")
boolean isValidStringLiteralConfig() {
if (stringLiteralCharacter == null)
return true;
Expand Down

0 comments on commit 197574a

Please sign in to comment.