Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removes two unused AnalysisConfig options #35645

Merged
merged 7 commits into from
Nov 19, 2018
Merged

Removes two unused AnalysisConfig options #35645

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2c1cf75
ML: Removing result_finalization_window && overlapping_buckets
benwtrent Nov 15, 2018
3332d1d
Merge branch 'master' into feature/ml-remove-result_finalization_window
benwtrent Nov 15, 2018
0ce48c5
Merge branch 'master' into feature/ml-remove-result_finalization_window
benwtrent Nov 16, 2018
8d34dd9
Reverting bad method deletions
benwtrent Nov 16, 2018
1d18a51
Setting to current before backport to try and get a green build
benwtrent Nov 16, 2018
a6dfde3
fixing testBuildAutodetectCommand test
benwtrent Nov 16, 2018
5d79e9e
disabling bwc tests for backport
benwtrent Nov 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 3 additions & 52 deletions .../rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/AnalysisConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ public class AnalysisConfig implements ToXContentObject {
public static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
public static final ParseField DETECTORS = new ParseField("detectors");
public static final ParseField INFLUENCERS = new ParseField("influencers");
public static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
public static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
public static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");

@SuppressWarnings("unchecked")
Expand All @@ -84,8 +82,6 @@ public class AnalysisConfig implements ToXContentObject {
builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS);
PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);
}

Expand All @@ -100,14 +96,11 @@ public class AnalysisConfig implements ToXContentObject {
private final String summaryCountFieldName;
private final List<Detector> detectors;
private final List<String> influencers;
private final Boolean overlappingBuckets;
private final Long resultFinalizationWindow;
private final Boolean multivariateByFields;

private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
Boolean multivariateByFields) {
List<Detector> detectors, List<String> influencers, Boolean multivariateByFields) {
this.detectors = Collections.unmodifiableList(detectors);
this.bucketSpan = bucketSpan;
this.latency = latency;
Expand All @@ -116,8 +109,6 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis
this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
this.summaryCountFieldName = summaryCountFieldName;
this.influencers = Collections.unmodifiableList(influencers);
this.overlappingBuckets = overlappingBuckets;
this.resultFinalizationWindow = resultFinalizationWindow;
this.multivariateByFields = multivariateByFields;
}

Expand All @@ -130,14 +121,6 @@ public TimeValue getBucketSpan() {
return bucketSpan;
}

public String getCategorizationFieldName() {
benwtrent marked this conversation as resolved.
Show resolved Hide resolved
return categorizationFieldName;
}

public List<String> getCategorizationFilters() {
benwtrent marked this conversation as resolved.
Show resolved Hide resolved
return categorizationFilters;
}

public CategorizationAnalyzerConfig getCategorizationAnalyzerConfig() {
return categorizationAnalyzerConfig;
}
Expand Down Expand Up @@ -177,14 +160,6 @@ public List<String> getInfluencers() {
return influencers;
}

public Boolean getOverlappingBuckets() {
return overlappingBuckets;
}

public Long getResultFinalizationWindow() {
return resultFinalizationWindow;
}

public Boolean getMultivariateByFields() {
return multivariateByFields;
}
Expand Down Expand Up @@ -255,12 +230,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();
builder.field(INFLUENCERS.getPreferredName(), influencers);
if (overlappingBuckets != null) {
builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
}
if (resultFinalizationWindow != null) {
builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
}
if (multivariateByFields != null) {
builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
}
Expand All @@ -287,17 +256,14 @@ public boolean equals(Object object) {
Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
Objects.equals(detectors, that.detectors) &&
Objects.equals(influencers, that.influencers) &&
Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
Objects.equals(multivariateByFields, that.multivariateByFields);
}

@Override
public int hashCode() {
return Objects.hash(
bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
multivariateByFields);
summaryCountFieldName, detectors, influencers, multivariateByFields);
}

public static Builder builder(List<Detector> detectors) {
Expand All @@ -314,8 +280,6 @@ public static class Builder {
private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
private String summaryCountFieldName;
private List<String> influencers = new ArrayList<>();
private Boolean overlappingBuckets;
private Long resultFinalizationWindow;
private Boolean multivariateByFields;

public Builder(List<Detector> detectors) {
Expand All @@ -332,8 +296,6 @@ public Builder(AnalysisConfig analysisConfig) {
this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
this.influencers = new ArrayList<>(analysisConfig.influencers);
this.overlappingBuckets = analysisConfig.overlappingBuckets;
this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
this.multivariateByFields = analysisConfig.multivariateByFields;
}

Expand Down Expand Up @@ -391,16 +353,6 @@ public Builder setInfluencers(List<String> influencers) {
return this;
}

public Builder setOverlappingBuckets(Boolean overlappingBuckets) {
this.overlappingBuckets = overlappingBuckets;
return this;
}

public Builder setResultFinalizationWindow(Long resultFinalizationWindow) {
this.resultFinalizationWindow = resultFinalizationWindow;
return this;
}

public Builder setMultivariateByFields(Boolean multivariateByFields) {
this.multivariateByFields = multivariateByFields;
return this;
Expand All @@ -409,8 +361,7 @@ public Builder setMultivariateByFields(Boolean multivariateByFields) {
public AnalysisConfig build() {

return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
resultFinalizationWindow, multivariateByFields);
latency, summaryCountFieldName, detectors, influencers, multivariateByFields);
}
}
}
6 changes: 0 additions & 6 deletions ...-high-level/src/test/java/org/elasticsearch/client/ml/job/config/AnalysisConfigTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,6 @@ public static AnalysisConfig.Builder createRandomized() {
if (randomBoolean()) {
builder.setMultivariateByFields(randomBoolean());
}
if (randomBoolean()) {
builder.setOverlappingBuckets(randomBoolean());
}
if (randomBoolean()) {
builder.setResultFinalizationWindow(randomNonNegativeLong());
}

builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false)));
return builder;
Expand Down
97 changes: 17 additions & 80 deletions .../plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,11 @@ public class AnalysisConfig implements ToXContentObject, Writeable {
private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name");
private static final ParseField DETECTORS = new ParseField("detectors");
private static final ParseField INFLUENCERS = new ParseField("influencers");
private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets");
private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window");
private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields");

public static final String ML_CATEGORY_FIELD = "mlcategory";
public static final Set<String> AUTO_CREATED_FIELDS = new HashSet<>(Collections.singletonList(ML_CATEGORY_FIELD));

public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L;

// These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> LENIENT_PARSER = createParser(true);
public static final ConstructingObjectParser<AnalysisConfig.Builder, Void> STRICT_PARSER = createParser(false);
Expand All @@ -94,8 +90,6 @@ private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createPars
builder.setLatency(TimeValue.parseTimeValue(val, LATENCY.getPreferredName())), LATENCY);
parser.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME);
parser.declareStringArray(Builder::setInfluencers, INFLUENCERS);
parser.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS);
parser.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW);
parser.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS);

return parser;
Expand All @@ -112,14 +106,11 @@ private static ConstructingObjectParser<AnalysisConfig.Builder, Void> createPars
private final String summaryCountFieldName;
private final List<Detector> detectors;
private final List<String> influencers;
private final Boolean overlappingBuckets;
private final Long resultFinalizationWindow;
private final Boolean multivariateByFields;

private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, List<String> categorizationFilters,
CategorizationAnalyzerConfig categorizationAnalyzerConfig, TimeValue latency, String summaryCountFieldName,
List<Detector> detectors, List<String> influencers, Boolean overlappingBuckets, Long resultFinalizationWindow,
Boolean multivariateByFields) {
List<Detector> detectors, List<String> influencers, Boolean multivariateByFields) {
this.detectors = detectors;
this.bucketSpan = bucketSpan;
this.latency = latency;
Expand All @@ -128,8 +119,6 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis
this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters);
this.summaryCountFieldName = summaryCountFieldName;
this.influencers = Collections.unmodifiableList(influencers);
this.overlappingBuckets = overlappingBuckets;
this.resultFinalizationWindow = resultFinalizationWindow;
this.multivariateByFields = multivariateByFields;
}

Expand All @@ -146,8 +135,13 @@ public AnalysisConfig(StreamInput in) throws IOException {
summaryCountFieldName = in.readOptionalString();
detectors = Collections.unmodifiableList(in.readList(Detector::new));
influencers = Collections.unmodifiableList(in.readList(StreamInput::readString));
overlappingBuckets = in.readOptionalBoolean();
resultFinalizationWindow = in.readOptionalLong();

// BWC for result_finalization_window and overlapping_buckets
// TODO Remove in 7.0.0
if (in.getVersion().before(Version.V_6_6_0)) {
benwtrent marked this conversation as resolved.
Show resolved Hide resolved
in.readOptionalBoolean();
in.readOptionalLong();
}
multivariateByFields = in.readOptionalBoolean();

// BWC for removed multiple_bucket_spans
Expand Down Expand Up @@ -185,8 +179,13 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(summaryCountFieldName);
out.writeList(detectors);
out.writeStringList(influencers);
out.writeOptionalBoolean(overlappingBuckets);
out.writeOptionalLong(resultFinalizationWindow);

// BWC for result_finalization_window and overlapping_buckets
// TODO Remove in 7.0.0
if (out.getVersion().before(Version.V_6_6_0)) {
out.writeOptionalBoolean(null);
out.writeOptionalLong(null);
}
out.writeOptionalBoolean(multivariateByFields);

// BWC for removed multiple_bucket_spans
Expand Down Expand Up @@ -291,14 +290,6 @@ public Set<String> extractReferencedFilters() {
.flatMap(Set::stream).collect(Collectors.toSet());
}

public Boolean getOverlappingBuckets() {
return overlappingBuckets;
}

public Long getResultFinalizationWindow() {
return resultFinalizationWindow;
}

public Boolean getMultivariateByFields() {
return multivariateByFields;
}
Expand Down Expand Up @@ -394,12 +385,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();
builder.field(INFLUENCERS.getPreferredName(), influencers);
if (overlappingBuckets != null) {
builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets);
}
if (resultFinalizationWindow != null) {
builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);
}
if (multivariateByFields != null) {
builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields);
}
Expand All @@ -420,18 +405,14 @@ public boolean equals(Object o) {
Objects.equals(summaryCountFieldName, that.summaryCountFieldName) &&
Objects.equals(detectors, that.detectors) &&
Objects.equals(influencers, that.influencers) &&
Objects.equals(overlappingBuckets, that.overlappingBuckets) &&
Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) &&
Objects.equals(multivariateByFields, that.multivariateByFields);
}

@Override
public int hashCode() {
return Objects.hash(
bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig, latency,
summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow,
multivariateByFields
);
summaryCountFieldName, detectors, influencers, multivariateByFields);
}

public static class Builder {
Expand All @@ -446,8 +427,6 @@ public static class Builder {
private CategorizationAnalyzerConfig categorizationAnalyzerConfig;
private String summaryCountFieldName;
private List<String> influencers = new ArrayList<>();
private Boolean overlappingBuckets;
private Long resultFinalizationWindow;
private Boolean multivariateByFields;

public Builder(List<Detector> detectors) {
Expand All @@ -464,8 +443,6 @@ public Builder(AnalysisConfig analysisConfig) {
this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig;
this.summaryCountFieldName = analysisConfig.summaryCountFieldName;
this.influencers = new ArrayList<>(analysisConfig.influencers);
this.overlappingBuckets = analysisConfig.overlappingBuckets;
this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow;
this.multivariateByFields = analysisConfig.multivariateByFields;
}

Expand Down Expand Up @@ -517,14 +494,6 @@ public void setInfluencers(List<String> influencers) {
this.influencers = ExceptionsHelper.requireNonNull(influencers, INFLUENCERS.getPreferredName());
}

public void setOverlappingBuckets(Boolean overlappingBuckets) {
this.overlappingBuckets = overlappingBuckets;
}

public void setResultFinalizationWindow(Long resultFinalizationWindow) {
this.resultFinalizationWindow = resultFinalizationWindow;
}

public void setMultivariateByFields(Boolean multivariateByFields) {
this.multivariateByFields = multivariateByFields;
}
Expand All @@ -536,7 +505,6 @@ public void setMultivariateByFields(Boolean multivariateByFields) {
* <li>Check that if non-null Latency is &lt;= MAX_LATENCY</li>
* <li>Check there is at least one detector configured</li>
* <li>Check all the detectors are configured correctly</li>
* <li>Check that OVERLAPPING_BUCKETS is set appropriately</li>
* <li>Check that MULTIPLE_BUCKETSPANS are set appropriately</li>
* <li>If Per Partition normalization is configured at least one detector
* must have a partition field and no influences can be used</li>
Expand All @@ -555,17 +523,13 @@ public AnalysisConfig build() {
verifyMlCategoryIsUsedWhenCategorizationFieldNameIsSet();
verifyCategorizationAnalyzer();
verifyCategorizationFilters();
checkFieldIsNotNegativeIfSpecified(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow);

verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet();

overlappingBuckets = verifyOverlappingBucketsConfig(overlappingBuckets, detectors);

verifyNoInconsistentNestedFieldNames();

return new AnalysisConfig(bucketSpan, categorizationFieldName, categorizationFilters, categorizationAnalyzerConfig,
latency, summaryCountFieldName, detectors, influencers, overlappingBuckets,
resultFinalizationWindow, multivariateByFields);
latency, summaryCountFieldName, detectors, influencers, multivariateByFields);
}

private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() {
Expand All @@ -576,13 +540,6 @@ private void verifyNoMetricFunctionsWhenSummaryCountFieldNameIsSet() {
}
}

private static void checkFieldIsNotNegativeIfSpecified(String fieldName, Long value) {
if (value != null && value < 0) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, fieldName, 0, value);
throw ExceptionsHelper.badRequestException(msg);
}
}

private void verifyDetectorAreDefined() {
if (detectors == null || detectors.isEmpty()) {
throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS));
Expand Down Expand Up @@ -697,25 +654,5 @@ private static boolean isValidRegex(String exp) {
return false;
}
}

private static Boolean verifyOverlappingBucketsConfig(Boolean overlappingBuckets, List<Detector> detectors) {
// If any detector function is rare/freq_rare, mustn't use overlapping buckets
boolean mustNotUse = false;

List<DetectorFunction> illegalFunctions = new ArrayList<>();
for (Detector d : detectors) {
if (Detector.NO_OVERLAPPING_BUCKETS_FUNCTIONS.contains(d.getFunction())) {
illegalFunctions.add(d.getFunction());
mustNotUse = true;
}
}

if (Boolean.TRUE.equals(overlappingBuckets) && mustNotUse) {
throw ExceptionsHelper.badRequestException(
Messages.getMessage(Messages.JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION, illegalFunctions.toString()));
}

return overlappingBuckets;
}
}
}
Loading