Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] specifying missing_field_value value and using it instead of empty_string #53108

Merged
merged 2 commits into from
Mar 5, 2020
Merged

[ML] specifying missing_field_value value and using it instead of empty_string #53108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d37f83f
[ML] specifying nullvalue value and using it instead of empty_string
benwtrent Mar 3, 2020
74e8b1a
fixing tests
benwtrent Mar 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .../src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public class DataFrameDataExtractor {
private static final Logger LOGGER = LogManager.getLogger(DataFrameDataExtractor.class);
private static final TimeValue SCROLL_TIMEOUT = new TimeValue(30, TimeUnit.MINUTES);

private static final String EMPTY_STRING = "";
public static final String NULL_VALUE = "\0";

private final Client client;
private final DataFrameDataExtractorContext context;
Expand Down Expand Up @@ -189,7 +189,7 @@ private Row createRow(SearchHit hit) {
} else {
if (values.length == 0 && context.includeRowsWithMissingValues) {
// if values is empty then it means it's a missing value
extractedValues[i] = EMPTY_STRING;
extractedValues[i] = NULL_VALUE;
} else {
// we are here if we have a missing value but the analysis does not support those
// or the value type is not supported (e.g. arrays, etc.)
Expand Down
5 changes: 2 additions & 3 deletions ...icsearch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ml.dataframe.process.customprocessing;

import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;

import java.util.List;
import java.util.Random;
Expand All @@ -18,8 +19,6 @@
*/
class DatasetSplittingCustomProcessor implements CustomProcessor {

private static final String EMPTY = "";

private final int dependentVariableIndex;
private final double trainingPercent;
private final Random random;
Expand Down Expand Up @@ -47,7 +46,7 @@ public void process(String[] row) {
// Let's make sure we have at least one training row
isFirstRow = false;
} else if (isRandomlyExcludedFromTraining()) {
row[dependentVariableIndex] = EMPTY;
row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE;
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion ...test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,8 @@ public void testMissingValues_GivenShouldInclude() throws IOException {
assertThat(rows.get().size(), equalTo(3));

assertThat(rows.get().get(0).getValues(), equalTo(new String[] {"11", "21"}));
assertThat(rows.get().get(1).getValues(), equalTo(new String[] {"", "22"}));
assertThat(rows.get().get(1).getValues()[0], equalTo(DataFrameDataExtractor.NULL_VALUE));
assertThat(rows.get().get(1).getValues()[1], equalTo("22"));
assertThat(rows.get().get(2).getValues(), equalTo(new String[] {"13", "23"}));

assertThat(rows.get().get(0).shouldSkip(), is(false));
Expand Down
3 changes: 2 additions & 1 deletion ...rch/xpack/ml/dataframe/process/customprocessing/DatasetSplittingCustomProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ml.dataframe.process.customprocessing;

import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor;
import org.junit.Before;

import java.util.ArrayList;
Expand Down Expand Up @@ -98,7 +99,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs
assertThat(processedRow[fieldIndex], equalTo(row[fieldIndex]));
}
}
if (processedRow[dependentVariableIndex].length() > 0) {
if (DataFrameDataExtractor.NULL_VALUE.equals(processedRow[dependentVariableIndex]) == false) {
assertThat(processedRow[dependentVariableIndex], equalTo(row[dependentVariableIndex]));
trainingRows++;
}
Expand Down