Skip to content

Commit

Permalink
287 issue support for plaintext without a schema (#293)
Browse files Browse the repository at this point in the history
* sending value string to kafka considering just one json str

* added test and deleted JSON format validation

* fixed codacy error

* fixed to use generator for plainText

* fixed codacy errors

* fixed codacy error in ValueSimpleConfigElementTest

* reduced cyclomatic complexity in configureValueGenerator method

* changed parameter name in getLoadGenerator method

* #287 Fix some style concepts

* #287 Fix some style concepts

* #287 Fix some style concepts

Co-authored-by: Mike Ortiz <[email protected]>
Co-authored-by: Diana Gómez <[email protected]>
Co-authored-by: Jose Enrique García Maciñeiras <[email protected]>
  • Loading branch information
4 people authored Sep 13, 2022
1 parent dcfbbed commit 6f289ec
Show file tree
Hide file tree
Showing 54 changed files with 2,076 additions and 2,277 deletions.
10 changes: 3 additions & 7 deletions checkstyle.xml
Original file line number Diff line number Diff line change
Expand Up @@ -274,8 +274,9 @@
<module name="AvoidStaticImport"/>
<module name="IllegalImport"/>
<module name="CustomImportOrder">
<property name="customImportOrderRules"
value="STATIC###STANDARD_JAVA_PACKAGE###THIRD_PARTY_PACKAGE"/>
<property name="customImportOrderRules" value="STANDARD_JAVA_PACKAGE###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE"/>
<property name="specialImportsRegExp" value="^javax\."/>
<property name="standardPackageRegExp" value="^java\."/>
<property name="separateLineBetweenGroups" value="true"/>
<property name="sortImportsInGroupAlphabetically" value="true"/>
</module>
Expand Down Expand Up @@ -361,11 +362,6 @@
<module name="AbstractClassName"/>
<module name="StaticVariableName"/>
<module name="LocalFinalVariableName"/>
<module name="ImportOrder">
<property name="groups" value="java,javax"/>
<property name="separated" value="true"/>
<property name="separatedStaticGroups" value="true"/>
</module>
</module>
<module name="BeforeExecutionExclusionFileFilter">
<property name="fileNamePattern" value="module\-info\.java$"/>
Expand Down
10 changes: 10 additions & 0 deletions config/checkstyle/OSS_checkstyle_suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,15 @@

<suppressions>
<suppress checks="VisibilityModifier" files="[/\\].*model.*\.java"/>
<suppress checks="VisibilityModifier" files="[/\\].*EnrichedRecord\.java"/>
<suppress checks="DesignForExtension" files="[/\\].*model.*\.java"/>
<suppress checks="AbstractClassName" files="[/\\].*Field\.java"/>
<suppress checks="ClassDataAbstractionCoupling" files="[/\\].*Sampler\.java"/>
<suppress checks="ClassDataAbstractionCoupling" files="[/\\].*PropertyEditor\.java"/>
<suppress checks="ClassDataAbstractionCoupling" files="[/\\].*ProcessorTest\.java"/>
<suppress checks="ClassDataAbstractionCoupling" files="[/\\].*Standalone\.java"/>
<suppress checks="ExecutableStatementCount" files="[/\\].*Standalone\.java"/>
<suppress checks="AnonInnerLength" files="[/\\].*AutoCompletion\.java"/>
<suppress checks="Indentation" files="[/\\].*ExtractorTest\.java"/>
<suppress checks="CyclomaticComplexity" files="[/\\].*\.java"/>
</suppressions>
3 changes: 2 additions & 1 deletion pom-maven-central.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

<artifactId>kloadgen</artifactId>

<version>5.0.1</version>
<version>5.1.0</version>

<name>KLoadGen</name>
<description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data based on Data specification</description>
<url>https://corunet.github.io/kloadgen/</url>
Expand Down
3 changes: 2 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

<artifactId>kloadgen</artifactId>

<version>5.0.1</version>
<version>5.1.0</version>

<name>KLoadGen</name>
<description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data base on Data specification</description>
<url>https://corunet.github.io/kloadgen/</url>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@

public class ValueDeserializedConfigElementBeanInfo extends BeanInfoSupport {

private static final String VALUE_SCHEMA_PROPERTIES = "valueSchemaProperties";

private static final String VALUE_SCHEMA_TYPE = "valueSchemaType";

private static final String VALUE_DESERIALIZER_PROPERTY = "valueDeSerializerConfiguration";
Expand All @@ -33,7 +31,7 @@ public ValueDeserializedConfigElementBeanInfo() {

super(ValueDeserializedConfigElement.class);

createPropertyGroup("value_deserialized_load_generator", new String[]{VALUE_NAME_STRATEGY, VALUE_DESERIALIZER_PROPERTY, VALUE_SCHEMA_PROPERTIES, VALUE_SCHEMA_TYPE});
createPropertyGroup("value_deserialized_load_generator", new String[]{VALUE_NAME_STRATEGY, VALUE_DESERIALIZER_PROPERTY, VALUE_SCHEMA_TYPE});

final PropertyDescriptor nameStrategyPropertyProps = property(VALUE_NAME_STRATEGY);
nameStrategyPropertyProps.setPropertyEditorClass(NameStrategyPropertyEditor.class);
Expand All @@ -50,7 +48,7 @@ public ValueDeserializedConfigElementBeanInfo() {
final PropertyDescriptor schemaType = property(VALUE_SCHEMA_TYPE);
schemaType.setPropertyEditorClass(SchemaTypePropertyEditor.class);
schemaType.setValue(NOT_UNDEFINED, Boolean.TRUE);
schemaType.setValue(DEFAULT, "<avro subject>");
schemaType.setValue(DEFAULT, "");
schemaType.setValue(NOT_EXPRESSION, Boolean.FALSE);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* * License, v. 2.0. If a copy of the MPL was not distributed with this
* * file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/

package net.coru.kloadgen.config.valuesimple;

import java.util.Objects;

import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.coru.kloadgen.util.PropsKeysHelper;
import org.apache.jmeter.config.ConfigTestElement;
import org.apache.jmeter.engine.event.LoopIterationEvent;
import org.apache.jmeter.engine.event.LoopIterationListener;
import org.apache.jmeter.testbeans.TestBean;
import org.apache.jmeter.threads.JMeterContextService;
import org.apache.jmeter.threads.JMeterVariables;

@Slf4j
@AllArgsConstructor
@NoArgsConstructor
public final class ValueSimpleConfigElement extends ConfigTestElement implements TestBean, LoopIterationListener {

private String valueSchemaProperties;

private String schema = "NoSchema";

private String valueSubjectName = "";

private String valueSerializerConfiguration;

@Override
public void iterationStart(final LoopIterationEvent loopIterationEvent) {

final JMeterVariables variables = JMeterContextService.getContext().getVariables();
variables.putObject(PropsKeysHelper.VALUE_SUBJECT_NAME, valueSubjectName);
variables.putObject(PropsKeysHelper.VALUE_SCHEMA_TYPE, schema);
variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, getValueSchemaProperties());
variables.putObject(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY, getValueSerializerConfiguration());
variables.putObject(PropsKeysHelper.SIMPLE_VALUED_MESSAGE_KEY, Boolean.TRUE);
}

public String getValueSchemaProperties() {
return getPropertyAsString("valueSchemaProperties");
}

public String getValueSerializerConfiguration() {
return getPropertyAsString("valueSerializerConfiguration");
}

public void setValueSchemaProperties(final String value) {
setProperty("valueSchemaProperties", value);
this.valueSchemaProperties = value;
}

public void setValueSerializerConfiguration(final String valueSerializerConfiguration) {
setProperty("valueSerializerConfiguration", valueSerializerConfiguration);
this.valueSerializerConfiguration = valueSerializerConfiguration;
}

@Override
public boolean equals(final Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
if (!super.equals(o)) {
return false;
}
final ValueSimpleConfigElement that = (ValueSimpleConfigElement) o;
return Objects.equals(getValueSchemaProperties(), that.getValueSchemaProperties())
&& Objects.equals(getValueSerializerConfiguration(), that.getValueSerializerConfiguration());
}

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), getValueSchemaProperties(), getValueSerializerConfiguration());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* * License, v. 2.0. If a copy of the MPL was not distributed with this
* * file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/

package net.coru.kloadgen.config.valuesimple;

import java.beans.PropertyDescriptor;

import net.coru.kloadgen.property.editor.PlainValueSerializerPropertyEditor;
import org.apache.jmeter.testbeans.BeanInfoSupport;

public class ValueSimpleConfigElementBeanInfo extends BeanInfoSupport {

private static final String VALUE_SCHEMA_PROPERTIES = "valueSchemaProperties";

private static final String VALUE_SERIALIZER_PROPERTY = "valueSerializerConfiguration";

public ValueSimpleConfigElementBeanInfo() {

super(ValueSimpleConfigElement.class);

createPropertyGroup("value_simple_configuration", new String[]{VALUE_SCHEMA_PROPERTIES, VALUE_SERIALIZER_PROPERTY});

final PropertyDescriptor keyValueProp = property(VALUE_SCHEMA_PROPERTIES);
keyValueProp.setValue(DEFAULT, "");
keyValueProp.setValue(NOT_UNDEFINED, Boolean.TRUE);
keyValueProp.setValue(NOT_EXPRESSION, Boolean.FALSE);

final PropertyDescriptor schemaType = property(VALUE_SERIALIZER_PROPERTY);
schemaType.setPropertyEditorClass(PlainValueSerializerPropertyEditor.class);
schemaType.setValue(NOT_UNDEFINED, Boolean.TRUE);
schemaType.setValue(DEFAULT, "org.apache.kafka.common.serialization.StringSerializer");
schemaType.setValue(NOT_EXPRESSION, Boolean.FALSE);

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -54,21 +54,22 @@ private List<FieldValueMapping> extractInternalFields(final ObjectField field, f
}

private Transformer<FieldValueMapping, FieldValueMapping> fixName(final String fieldName, final String splitter) {
final String[] fieldNameClean = new String[1];
return fieldValue -> {
if (fieldName.endsWith("[][]") || fieldName.endsWith("[:][]")) {
fieldNameClean[0] = fieldName.substring(0, fieldName.length() - 2);
} else if (fieldName.endsWith("[][:]") || fieldName.endsWith("[:][:]")) {
fieldNameClean[0] = fieldName.substring(0, fieldName.length() - 3);
} else {
fieldNameClean[0] = fieldName;
}

fieldValue.setFieldName(fieldNameClean[0] + splitter + fieldValue.getFieldName());
fieldValue.setFieldName(extractFieldName(fieldName) + splitter + fieldValue.getFieldName());
return fieldValue;
};
}

private static String extractFieldName(final String fieldName) {
String fieldNameClean = fieldName;
if (fieldName.endsWith("[][]") || fieldName.endsWith("[:][]")) {
fieldNameClean = fieldName.substring(0, fieldName.length() - 2);
} else if (fieldName.endsWith("[][:]") || fieldName.endsWith("[:][:]")) {
fieldNameClean = fieldName.substring(0, fieldName.length() - 3);
}
return fieldNameClean;
}

private List<FieldValueMapping> processField(final Field innerField, final Boolean isRootElement, final Boolean isAncestorRequired) {
final List<FieldValueMapping> completeFieldList = new ArrayList<>();
if (innerField instanceof ObjectField) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;

public class ProtoBufExtractor {

Expand Down Expand Up @@ -61,9 +60,9 @@ public final void processField(final TypeElement field, final List<FieldValueMap
if (nestedTypes.containsKey(subfield.getType())) {
extractNestedTypes(completeFieldList, nestedTypes, subfield, isArray, imports, !isOptional, isAncestorRequired);
} else if (nestedTypes.containsKey(dotType)) {
extractDotTypesWhenIsInNestedType(completeFieldList, nestedTypes, subfield, isArray, dotType, imports, !isOptional, isAncestorRequired);
completeFieldList.addAll(extractDotTypesWhenIsInNestedType(nestedTypes, subfield, isArray, dotType, imports, !isOptional, isAncestorRequired));
} else {
extractDotTypeWhenNotNestedType(completeFieldList, subfield, isArray, dotType, !isOptional, isAncestorRequired);
completeFieldList.add(extractDotTypeWhenNotNestedType(subfield, isArray, dotType, !isOptional, isAncestorRequired));
}
}
}
Expand Down Expand Up @@ -105,15 +104,16 @@ private void extractOneOfs(
}
}

private void extractDotTypeWhenNotNestedType(
final List<FieldValueMapping> completeFieldList, final FieldElement subfield, final boolean isArray, final String dotType, final boolean isRequired,
private FieldValueMapping extractDotTypeWhenNotNestedType(final FieldElement subfield, final boolean isArray, final String dotType, final boolean isRequired,
final boolean isAncestorRequired) {
final FieldValueMapping completeFieldList;
if (isArray) {
completeFieldList.add(
FieldValueMapping.builder().fieldName(subfield.getName() + "[]").fieldType(dotType + ARRAY_POSTFIX).required(isRequired).isAncestorRequired(isAncestorRequired).build());
completeFieldList =
FieldValueMapping.builder().fieldName(subfield.getName() + "[]").fieldType(dotType + ARRAY_POSTFIX).required(isRequired).isAncestorRequired(isAncestorRequired).build();
} else {
completeFieldList.add(FieldValueMapping.builder().fieldName(subfield.getName()).fieldType(dotType).required(isRequired).isAncestorRequired(isAncestorRequired).build());
completeFieldList = FieldValueMapping.builder().fieldName(subfield.getName()).fieldType(dotType).required(isRequired).isAncestorRequired(isAncestorRequired).build();
}
return completeFieldList;
}

private void extractMapType(
Expand Down Expand Up @@ -155,21 +155,23 @@ private void extractPrimitiveTypes(
}
}

private void extractDotTypesWhenIsInNestedType(
final List<FieldValueMapping> completeFieldList, final HashMap<String, TypeElement> nestedTypes, final FieldElement subfield, final boolean isArray, final String dotType,
private List<FieldValueMapping> extractDotTypesWhenIsInNestedType(
final HashMap<String, TypeElement> nestedTypes, final FieldElement subfield, final boolean isArray, final String dotType,
final List<String> imports, final boolean isRequired, final boolean isAncestorRequired) {
final List<FieldValueMapping> finalFieldValueMapping = new ArrayList<>();
final List<FieldValueMapping> fieldValueMappingList = processFieldList(nestedTypes.get(dotType), imports);
for (FieldValueMapping fieldValueMapping : fieldValueMappingList) {
if (isArray) {
completeFieldList.add(
finalFieldValueMapping.add(
FieldValueMapping.builder().fieldName(buildFieldName(subfield.getName(), fieldValueMapping.getFieldName(), "[].")).fieldType(fieldValueMapping.getFieldType())
.valueLength(0).fieldValueList(getValueList(fieldValueMapping)).required(isRequired).isAncestorRequired(isAncestorRequired).build());
} else {
completeFieldList.add(
finalFieldValueMapping.add(
FieldValueMapping.builder().fieldName(buildFieldName(subfield.getName(), fieldValueMapping.getFieldName(), ".")).fieldType(fieldValueMapping.getFieldType())
.valueLength(0).fieldValueList(getValueList(fieldValueMapping)).required(isRequired).isAncestorRequired(isAncestorRequired).build());
}
}
return finalFieldValueMapping;
}

private void extractNestedTypes(
Expand Down Expand Up @@ -242,13 +244,6 @@ private String extractEnums(final EnumElement field) {
return String.join(",", enumConstantList);
}

@NotNull
private String getFieldValueMappingPrepared(final FieldValueMapping fieldValueMapping) {
final String[] splitText = fieldValueMapping.getFieldName().split("\\.");
final List<String> preparedField = Arrays.asList(Arrays.copyOfRange(splitText, 1, splitText.length));
return String.join(".", preparedField);
}

private String checkDotType(final String subfieldType, final List<String> imports) {
String dotType = subfieldType;
if (subfieldType.startsWith(".") || subfieldType.contains(".")) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* * License, v. 2.0. If a copy of the MPL was not distributed with this
* * file, You can obtain one at https://mozilla.org/MPL/2.0/.
*/

package net.coru.kloadgen.loadgen.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import lombok.extern.slf4j.Slf4j;
import net.coru.kloadgen.loadgen.BaseLoadGenerator;
import net.coru.kloadgen.model.FieldValueMapping;
import net.coru.kloadgen.serializer.EnrichedRecord;

@Slf4j
@SuppressWarnings("checkstyle:AbbreviationAsWordInName")
public final class PlainTextLoadGenerator implements SRLoadGenerator, BaseLoadGenerator {

private List<FieldValueMapping> fieldExprMappings = new ArrayList<>();

@Override
public void setUpGenerator(final Map<String, String> originals, final String avroSchemaName, final List<FieldValueMapping> fieldExprMappings) {
this.fieldExprMappings = fieldExprMappings;
}

@Override
public void setUpGenerator(final String schema, final List<FieldValueMapping> fieldExprMappings) {
this.fieldExprMappings = fieldExprMappings;
}

public EnrichedRecord nextMessage() {
final FieldValueMapping fieldValueMapping = fieldExprMappings.get(0);
return EnrichedRecord.builder().genericRecord(fieldValueMapping.getFieldName()).build();
}

}
Loading

0 comments on commit 6f289ec

Please sign in to comment.