diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index ab908037727a..8b25f4fbbbe2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -281,7 +281,6 @@ public static String extractJobIdFromDocumentId(String docId) { return jobId.equals(docId) ? null : jobId; } - /** * Return the Job Id. * diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java index 72fbcca98ee9..9b906f113e26 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java @@ -15,20 +15,13 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; -import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; -import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ScheduledEventToRuleWriter; import org.elasticsearch.xpack.ml.process.NativeController; import org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils; import org.elasticsearch.xpack.ml.process.ProcessPipes; -import org.elasticsearch.xpack.ml.job.process.autodetect.writer.AnalysisLimitsWriter; -import org.elasticsearch.xpack.ml.job.process.autodetect.writer.FieldConfigWriter; -import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ModelPlotConfigWriter; import java.io.BufferedWriter; import java.io.IOException; @@ -41,12 +34,9 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.Random; import java.util.Set; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils.addIfNotNull; - /** * The autodetect process builder. */ @@ -61,28 +51,15 @@ public class AutodetectBuilder { /* * Arguments used by both autodetect and normalize */ - public static final String BUCKET_SPAN_ARG = "--bucketspan="; public static final String DELETE_STATE_FILES_ARG = "--deleteStateFiles"; public static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput"; public static final String MODEL_CONFIG_ARG = "--modelconfig="; public static final String QUANTILES_STATE_PATH_ARG = "--quantilesState="; - private static final String CONF_EXTENSION = ".conf"; private static final String JSON_EXTENSION = ".json"; - static final String JOB_ID_ARG = "--jobid="; private static final String CONFIG_ARG = "--config="; private static final String EVENTS_CONFIG_ARG = "--eventsconfig="; private static final String FILTERS_CONFIG_ARG = "--filtersconfig="; - private static final String LIMIT_CONFIG_ARG = "--limitconfig="; - private static final String MODEL_PLOT_CONFIG_ARG = "--modelplotconfig="; - private static final String FIELD_CONFIG_ARG = "--fieldconfig="; - static final String LATENCY_ARG = "--latency="; - static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields"; - static final String PERSIST_INTERVAL_ARG = "--persistInterval="; - static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval="; - static final String SUMMARY_COUNT_FIELD_ARG = "--summarycountfield="; - static final String TIME_FIELD_ARG = "--timefield="; - static final String STOP_CATEGORIZATION_ON_WARN_ARG = "--stopCategorizationOnWarnStatus"; /** * Name of the config setting containing the path to the logs directory @@ -96,21 +73,6 @@ public class AutodetectBuilder { public static final Setting MAX_ANOMALY_RECORDS_SETTING_DYNAMIC = Setting.intSetting("xpack.ml.max_anomaly_records", DEFAULT_MAX_NUM_RECORDS, Setting.Property.NodeScope, Setting.Property.Dynamic); - private static final int SECONDS_IN_HOUR = 3600; - - /** - * Roughly how often should the C++ process persist state? A staggering - * factor that varies by job is added to this. - */ - private static final long DEFAULT_BASE_PERSIST_INTERVAL = 10800; // 3 hours - - /** - * Roughly how often should the C++ process output quantiles when no - * anomalies are being detected? A staggering factor that varies by job is - * added to this. - */ - static final int BASE_MAX_QUANTILE_INTERVAL = 21600; // 6 hours - /** * Persisted quantiles are written to disk so they can be read by * the autodetect program. All quantiles files have this extension. @@ -178,19 +140,8 @@ public void build() throws IOException, InterruptedException { buildFiltersConfig(command); buildScheduledEventsConfig(command); - - // While it may appear that the JSON formatted job config file contains data that - // is duplicated in the existing limits, modelPlot and field config files, over time - // the C++ backend will retrieve all its required configuration data from the new - // JSON config file and the old-style configuration files will be removed. buildJobConfig(command); - // Per the comment above, these three lines will eventually be removed once migration - // to the new JSON formatted configuration file has been completed. - buildLimits(command); - buildModelPlotConfig(command); - buildFieldConfig(command); - buildQuantiles(command); processPipes.addArgs(command); @@ -204,43 +155,12 @@ List buildAutodetectCommand() { List command = new ArrayList<>(); command.add(AUTODETECT_PATH); - command.add(JOB_ID_ARG + job.getId()); - - AnalysisConfig analysisConfig = job.getAnalysisConfig(); - if (analysisConfig != null) { - addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command); - addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command); - addIfNotNull(analysisConfig.getSummaryCountFieldName(), SUMMARY_COUNT_FIELD_ARG, command); - if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) { - command.add(MULTIVARIATE_BY_FIELDS_ARG); - } - if (Boolean.TRUE.equals(analysisConfig.getPerPartitionCategorizationConfig().isStopOnWarn())) { - command.add(STOP_CATEGORIZATION_ON_WARN_ARG); - } - } - // Input is always length encoded command.add(LENGTH_ENCODED_INPUT_ARG); // Limit the number of output records command.add(maxAnomalyRecordsArg(settings)); - // always set the time field - String timeFieldArg = TIME_FIELD_ARG + getTimeFieldOrDefault(job); - command.add(timeFieldArg); - - int intervalStagger = calculateStaggeringInterval(job.getId()); - logger.debug("[{}] Periodic operations staggered by {} seconds", job.getId(), intervalStagger); - - // Persist model state every few hours even if the job isn't closed - long persistInterval = (job.getBackgroundPersistInterval() == null) ? - (DEFAULT_BASE_PERSIST_INTERVAL + intervalStagger) : - job.getBackgroundPersistInterval().getSeconds(); - command.add(PERSIST_INTERVAL_ARG + persistInterval); - - int maxQuantileInterval = BASE_MAX_QUANTILE_INTERVAL + intervalStagger; - command.add(MAX_QUANTILE_INTERVAL_ARG + maxQuantileInterval); - if (ProcessBuilderUtils.modelConfigFilePresent(env)) { String modelConfigFile = XPackPlugin.resolveConfigFile(env, ProcessBuilderUtils.ML_MODEL_CONF).toString(); command.add(MODEL_CONFIG_ARG + modelConfigFile); @@ -253,67 +173,6 @@ static String maxAnomalyRecordsArg(Settings settings) { return "--maxAnomalyRecords=" + MAX_ANOMALY_RECORDS_SETTING_DYNAMIC.get(settings); } - private static String getTimeFieldOrDefault(Job job) { - DataDescription dataDescription = job.getDataDescription(); - boolean useDefault = dataDescription == null - || Strings.isNullOrEmpty(dataDescription.getTimeField()); - return useDefault ? DataDescription.DEFAULT_TIME_FIELD : dataDescription.getTimeField(); - } - - /** - * This random time of up to 1 hour is added to intervals at which we - * tell the C++ process to perform periodic operations. This means that - * when there are many jobs there is a certain amount of staggering of - * their periodic operations. A given job will always be given the same - * staggering interval (for a given JVM implementation). - * - * @param jobId The ID of the job to calculate the staggering interval for - * @return The staggering interval - */ - static int calculateStaggeringInterval(String jobId) { - Random rng = new Random(jobId.hashCode()); - return rng.nextInt(SECONDS_IN_HOUR); - } - - private void buildLimits(List command) throws IOException { - if (job.getAnalysisLimits() != null) { - Path limitConfigFile = Files.createTempFile(env.tmpFile(), "limitconfig", CONF_EXTENSION); - filesToDelete.add(limitConfigFile); - writeLimits(job.getAnalysisLimits(), limitConfigFile); - String limits = LIMIT_CONFIG_ARG + limitConfigFile.toString(); - command.add(limits); - } - } - - /** - * Write the Ml autodetect model options to emptyConfFile. - */ - private static void writeLimits(AnalysisLimits options, Path emptyConfFile) throws IOException { - - try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(emptyConfFile), StandardCharsets.UTF_8)) { - new AnalysisLimitsWriter(options, osw).write(); - } - } - - private void buildModelPlotConfig(List command) throws IOException { - if (job.getModelPlotConfig() != null) { - Path modelPlotConfigFile = Files.createTempFile(env.tmpFile(), "modelplotconfig", CONF_EXTENSION); - filesToDelete.add(modelPlotConfigFile); - writeModelPlotConfig(job.getModelPlotConfig(), modelPlotConfigFile); - String modelPlotConfig = MODEL_PLOT_CONFIG_ARG + modelPlotConfigFile.toString(); - command.add(modelPlotConfig); - } - } - - private static void writeModelPlotConfig(ModelPlotConfig config, Path emptyConfFile) - throws IOException { - try (OutputStreamWriter osw = new OutputStreamWriter( - Files.newOutputStream(emptyConfFile), - StandardCharsets.UTF_8)) { - new ModelPlotConfigWriter(config, osw).write(); - } - } - private void buildQuantiles(List command) throws IOException { if (quantiles != null && !quantiles.getQuantileState().isEmpty()) { logger.info("Restoring quantiles for job '" + job.getId() + "'"); @@ -344,22 +203,6 @@ public static Path writeNormalizerInitState(String jobId, String state, Environm return stateFile; } - private void buildFieldConfig(List command) throws IOException { - if (job.getAnalysisConfig() != null) { - // write to a temporary field config file - Path fieldConfigFile = Files.createTempFile(env.tmpFile(), "fieldconfig", CONF_EXTENSION); - filesToDelete.add(fieldConfigFile); - try (OutputStreamWriter osw = new OutputStreamWriter( - Files.newOutputStream(fieldConfigFile), - StandardCharsets.UTF_8)) { - new FieldConfigWriter(job.getAnalysisConfig(), referencedFilters, scheduledEvents, osw, logger).write(); - } - - String fieldConfig = FIELD_CONFIG_ARG + fieldConfigFile.toString(); - command.add(fieldConfig); - } - } - private void buildScheduledEventsConfig(List command) throws IOException { if (scheduledEvents.isEmpty()) { return; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriter.java deleted file mode 100644 index e5efc5b772cb..000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriter.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.process.autodetect.writer; - -import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.Objects; - -import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.EQUALS; -import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.NEW_LINE; - -public class AnalysisLimitsWriter { - /* - * The configuration fields used in limits.conf - */ - private static final String MEMORY_STANZA_STR = "[memory]"; - private static final String RESULTS_STANZA_STR = "[results]"; - private static final String MODEL_MEMORY_LIMIT_CONFIG_STR = "modelmemorylimit"; - private static final String MAX_EXAMPLES_LIMIT_CONFIG_STR = "maxexamples"; - - private final AnalysisLimits limits; - private final OutputStreamWriter writer; - - public AnalysisLimitsWriter(AnalysisLimits limits, OutputStreamWriter writer) { - this.limits = Objects.requireNonNull(limits); - this.writer = Objects.requireNonNull(writer); - } - - public void write() throws IOException { - StringBuilder contents = new StringBuilder(MEMORY_STANZA_STR).append(NEW_LINE); - if (limits.getModelMemoryLimit() != null) { - contents.append(MODEL_MEMORY_LIMIT_CONFIG_STR + EQUALS).append(limits.getModelMemoryLimit()).append(NEW_LINE); - } - - contents.append(RESULTS_STANZA_STR).append(NEW_LINE); - if (limits.getCategorizationExamplesLimit() != null) { - contents.append(MAX_EXAMPLES_LIMIT_CONFIG_STR + EQUALS).append(limits.getCategorizationExamplesLimit()).append(NEW_LINE); - } - - writer.write(contents.toString()); - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java deleted file mode 100644 index ea8b3585f234..000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.process.autodetect.writer; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.DefaultDetectorDescription; -import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; -import org.elasticsearch.xpack.core.ml.job.config.Detector; -import org.elasticsearch.xpack.core.ml.job.config.MlFilter; -import org.elasticsearch.xpack.core.ml.utils.MlStrings; -import org.elasticsearch.xpack.ml.MachineLearning; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Set; - -import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.EQUALS; - -public class FieldConfigWriter { - private static final String DETECTOR_PREFIX = "detector."; - private static final String DETECTOR_CLAUSE_SUFFIX = ".clause"; - private static final String DETECTOR_RULES_SUFFIX = ".rules"; - private static final String INFLUENCER_PREFIX = "influencer."; - private static final String CATEGORIZATION_FIELD_OPTION = " categorizationfield="; - private static final String CATEGORIZATION_FILTER_PREFIX = "categorizationfilter."; - private static final String PER_PARTITION_CATEGORIZATION_OPTION = " perpartitioncategorization="; - - // Note: for the Engine API summarycountfield is currently passed as a - // command line option to autodetect rather than in the field config file - - private static final char NEW_LINE = '\n'; - - private final AnalysisConfig config; - private final Set filters; - private final List scheduledEvents; - private final OutputStreamWriter writer; - private final Logger logger; - - public FieldConfigWriter(AnalysisConfig config, Set filters, List scheduledEvents, - OutputStreamWriter writer, Logger logger) { - this.config = Objects.requireNonNull(config); - this.filters = Objects.requireNonNull(filters); - this.scheduledEvents = Objects.requireNonNull(scheduledEvents); - this.writer = Objects.requireNonNull(writer); - this.logger = Objects.requireNonNull(logger); - } - - /** - * Write the Ml autodetect field options to the outputIndex stream. - */ - public void write() throws IOException { - StringBuilder contents = new StringBuilder(); - - // Filters have to be written before the detectors - writeFilters(contents); - writeDetectors(contents); - writeScheduledEvents(contents); - writeCategorizationFilters(contents); - - // As values are written as entire settings rather than part of a - // clause no quoting is needed - writeAsEnumeratedSettings(INFLUENCER_PREFIX, config.getInfluencers(), contents, false); - - logger.debug("FieldConfig:\n" + contents.toString()); - writer.write(contents.toString()); - } - - @SuppressWarnings("unused") // CATEGORIZATION_TOKENIZATION_IN_JAVA is used for performance testing - private void writeCategorizationFilters(StringBuilder contents) { - if (MachineLearning.CATEGORIZATION_TOKENIZATION_IN_JAVA == false) { - writeAsEnumeratedSettings(CATEGORIZATION_FILTER_PREFIX, config.getCategorizationFilters(), - contents, true); - } - } - - private void writeDetectors(StringBuilder contents) throws IOException { - int counter = 0; - for (Detector detector : config.getDetectors()) { - int detectorId = counter++; - writeDetectorClause(detectorId, detector, contents); - writeDetectorRules(detectorId, detector, contents); - } - } - - private void writeDetectorClause(int detectorId, Detector detector, StringBuilder contents) { - contents.append(DETECTOR_PREFIX).append(detectorId).append(DETECTOR_CLAUSE_SUFFIX).append(EQUALS); - - DefaultDetectorDescription.appendOn(detector, contents); - - if (Strings.isNullOrEmpty(config.getCategorizationFieldName()) == false) { - contents.append(CATEGORIZATION_FIELD_OPTION).append(quoteField(config.getCategorizationFieldName())); - if (Strings.isNullOrEmpty(detector.getPartitionFieldName()) == false && - config.getPerPartitionCategorizationConfig().isEnabled()) { - contents.append(PER_PARTITION_CATEGORIZATION_OPTION).append("true"); - } - } - - contents.append(NEW_LINE); - } - - private void writeDetectorRules(int detectorId, Detector detector, StringBuilder contents) throws IOException { - - List rules = new ArrayList<>(); - if (detector.getRules() != null) { - rules.addAll(detector.getRules()); - } - - if (rules.isEmpty()) { - return; - } - - contents.append(DETECTOR_PREFIX).append(detectorId).append(DETECTOR_RULES_SUFFIX).append(EQUALS); - writeDetectionRulesJson(rules, contents); - contents.append(NEW_LINE); - } - - private void writeDetectionRulesJson(List rules, StringBuilder contents) throws IOException { - contents.append('['); - boolean first = true; - for (DetectionRule rule : rules) { - if (first) { - first = false; - } else { - contents.append(','); - } - try (XContentBuilder contentBuilder = XContentFactory.jsonBuilder()) { - contents.append(Strings.toString(rule.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS))); - } - } - contents.append(']'); - } - - private void writeFilters(StringBuilder buffer) throws IOException { - new MlFilterWriter(filters, buffer).write(); - } - - private void writeScheduledEvents(StringBuilder buffer) throws IOException { - if (scheduledEvents.isEmpty() == false) { - new ScheduledEventsWriter(scheduledEvents, config.getBucketSpan(), buffer).write(); - } - } - - private static void writeAsEnumeratedSettings(String settingName, List values, StringBuilder buffer, boolean quote) { - if (values == null) { - return; - } - - int counter = 0; - for (String value : values) { - buffer.append(settingName).append(counter++).append(EQUALS) - .append(quote ? quoteField(value) : value).append(NEW_LINE); - } - } - - private static String quoteField(String field) { - return MlStrings.doubleQuoteIfNotAlphaNumeric(field); - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java index 37ffd92a3d72..98abb3278b19 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilder.java @@ -24,6 +24,7 @@ public class NormalizerBuilder { */ public static final String NORMALIZE = "normalize"; static final String NORMALIZE_PATH = "./" + NORMALIZE; + public static final String BUCKET_SPAN_ARG = "--bucketspan="; private final Environment env; private final String jobId; @@ -44,7 +45,7 @@ public List build() throws IOException { List command = new ArrayList<>(); command.add(NORMALIZE_PATH); - addIfNotNull(bucketSpan, AutodetectBuilder.BUCKET_SPAN_ARG, command); + addIfNotNull(bucketSpan, BUCKET_SPAN_ARG, command); command.add(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG); if (quantilesState != null) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java index 0c496d36d9e6..a3ed79d9f499 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilderTests.java @@ -29,7 +29,6 @@ import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesPattern; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -88,45 +87,11 @@ public void testBuildAutodetectCommand() { List command = autodetectBuilder(job.build()).buildAutodetectCommand(); assertTrue(command.contains(AutodetectBuilder.AUTODETECT_PATH)); - assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "120")); - assertTrue(command.contains(AutodetectBuilder.LATENCY_ARG + "360")); - assertTrue(command.contains(AutodetectBuilder.SUMMARY_COUNT_FIELD_ARG + "summaryField")); - assertTrue(command.contains(AutodetectBuilder.MULTIVARIATE_BY_FIELDS_ARG)); - assertThat(command.contains(AutodetectBuilder.STOP_CATEGORIZATION_ON_WARN_ARG), is(isPerPartitionCategorization)); assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG)); assertTrue(command.contains(AutodetectBuilder.maxAnomalyRecordsArg(settings))); - assertTrue(command.contains(AutodetectBuilder.TIME_FIELD_ARG + "tf")); - assertTrue(command.contains(AutodetectBuilder.JOB_ID_ARG + "unit-test-job")); - - int expectedPersistInterval = 10800 + AutodetectBuilder.calculateStaggeringInterval(job.getId()); - assertTrue(command.contains(AutodetectBuilder.PERSIST_INTERVAL_ARG + expectedPersistInterval)); - int expectedMaxQuantileInterval = 21600 + AutodetectBuilder.calculateStaggeringInterval(job.getId()); - assertTrue(command.contains(AutodetectBuilder.MAX_QUANTILE_INTERVAL_ARG + expectedMaxQuantileInterval)); - - assertEquals(isPerPartitionCategorization ? 12 : 11, command.size()); - } - - public void testBuildAutodetectCommand_defaultTimeField() { - Job.Builder job = buildJobBuilder("unit-test-job"); - - List command = autodetectBuilder(job.build()).buildAutodetectCommand(); - - assertTrue(command.contains(AutodetectBuilder.TIME_FIELD_ARG + "time")); - } - - public void testBuildAutodetectCommand_givenPersistModelState() { - - Job.Builder job = buildJobBuilder("unit-test-job"); - - int expectedPersistInterval = 10800 + AutodetectBuilder.calculateStaggeringInterval(job.getId()); - - settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); - env = TestEnvironment.newEnvironment(settings); - - List command = autodetectBuilder(job.build()).buildAutodetectCommand(); - assertTrue(command.contains(AutodetectBuilder.PERSIST_INTERVAL_ARG + expectedPersistInterval)); + assertEquals(3, command.size()); } private AutodetectBuilder autodetectBuilder(Job job) { @@ -138,7 +103,7 @@ public void testBuildAutodetect() throws Exception { autodetectBuilder(job.build()).build(); - assertThat(filesToDelete, hasSize(3)); + assertThat(filesToDelete, hasSize(1)); verify(nativeController).startProcess(commandCaptor.capture()); verifyNoMoreInteractions(nativeController); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriterTests.java deleted file mode 100644 index af1c53cb0b06..000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriterTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.process.autodetect.writer; - -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; -import org.junit.After; -import org.junit.Before; -import org.mockito.Mockito; - -import java.io.IOException; -import java.io.OutputStreamWriter; - -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; - -public class AnalysisLimitsWriterTests extends ESTestCase { - private OutputStreamWriter writer; - - @Before - public void setUpMocks() { - writer = Mockito.mock(OutputStreamWriter.class); - } - - @After - public void verifyNoMoreWriterInteractions() { - verifyNoMoreInteractions(writer); - } - - public void testWrite_GivenUnsetValues() throws IOException { - AnalysisLimits limits = new AnalysisLimits(null, null); - AnalysisLimitsWriter analysisLimitsWriter = new AnalysisLimitsWriter(limits, writer); - - analysisLimitsWriter.write(); - - verify(writer).write("[memory]\n[results]\n"); - } - - public void testWrite_GivenModelMemoryLimitWasSet() throws IOException { - AnalysisLimits limits = new AnalysisLimits(10L, null); - AnalysisLimitsWriter analysisLimitsWriter = new AnalysisLimitsWriter(limits, writer); - - analysisLimitsWriter.write(); - - verify(writer).write("[memory]\nmodelmemorylimit = 10\n[results]\n"); - } - - public void testWrite_GivenCategorizationExamplesLimitWasSet() throws IOException { - AnalysisLimits limits = new AnalysisLimits(null, 5L); - AnalysisLimitsWriter analysisLimitsWriter = new AnalysisLimitsWriter(limits, writer); - - analysisLimitsWriter.write(); - - verify(writer).write("[memory]\n[results]\nmaxexamples = 5\n"); - } - - public void testWrite_GivenAllFieldsSet() throws IOException { - AnalysisLimits limits = new AnalysisLimits(1024L, 3L); - AnalysisLimitsWriter analysisLimitsWriter = new AnalysisLimitsWriter(limits, writer); - - analysisLimitsWriter.write(); - - verify(writer).write( - "[memory]\nmodelmemorylimit = 1024\n[results]\nmaxexamples = 3\n"); - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java deleted file mode 100644 index 098295e29c85..000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.process.autodetect.writer; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; -import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; -import org.elasticsearch.xpack.core.ml.job.config.Detector; -import org.elasticsearch.xpack.core.ml.job.config.MlFilter; -import org.elasticsearch.xpack.core.ml.job.config.Operator; -import org.elasticsearch.xpack.core.ml.job.config.PerPartitionCategorizationConfig; -import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.xpack.ml.MachineLearning; -import org.ini4j.Config; -import org.ini4j.Ini; -import org.ini4j.Profile.Section; -import org.junit.Before; -import org.mockito.ArgumentCaptor; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.StringReader; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; - - -public class FieldConfigWriterTests extends ESTestCase { - private AnalysisConfig analysisConfig; - private Set filters; - private List scheduledEvents; - private OutputStreamWriter writer; - - @Before - public void setUpDeps() { - analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder("count", null).build())).build(); - filters = new LinkedHashSet<>(); - scheduledEvents = new ArrayList<>(); - } - - public void testMultipleDetectorsToConfFile() - throws IOException { - List detectors = new ArrayList<>(); - - Detector.Builder d = new Detector.Builder("metric", "Integer_Value"); - d.setByFieldName("ts_hash"); - detectors.add(d.build()); - Detector.Builder d2 = new Detector.Builder("count", null); - d2.setByFieldName("ipaddress"); - detectors.add(d2.build()); - Detector.Builder d3 = new Detector.Builder("max", "Integer_Value"); - d3.setOverFieldName("ts_hash"); - detectors.add(d3.build()); - Detector.Builder d4 = new Detector.Builder("rare", null); - d4.setByFieldName("ipaddress"); - d4.setPartitionFieldName("host"); - detectors.add(d4.build()); - Detector.Builder d5 = new Detector.Builder("rare", null); - d5.setByFieldName("weird field"); - detectors.add(d5.build()); - Detector.Builder d6 = new Detector.Builder("max", "field"); - d6.setOverFieldName("tshash"); - detectors.add(d6.build()); - Detector.Builder d7 = new Detector.Builder("max", "Level 1 (Urgent)"); - d7.setByFieldName("10%"); - d7.setOverFieldName("%10"); - d7.setPartitionFieldName("Percentage (%)"); - detectors.add(d7.build()); - - analysisConfig = new AnalysisConfig.Builder(detectors).build(); - - ByteArrayOutputStream ba = new ByteArrayOutputStream(); - writer = new OutputStreamWriter(ba, StandardCharsets.UTF_8); - - createFieldConfigWriter().write(); - writer.close(); - - // read the ini file - all the settings are in the global section - StringReader reader = new StringReader(ba.toString("UTF-8")); - - Config iniConfig = new Config(); - iniConfig.setLineSeparator(new String(new char[]{WriterConstants.NEW_LINE})); - iniConfig.setGlobalSection(true); - - Ini fieldConfig = new Ini(); - fieldConfig.setConfig(iniConfig); - fieldConfig.load(reader); - - Section section = fieldConfig.get(iniConfig.getGlobalSectionName()); - - assertEquals(detectors.size(), section.size()); - - String value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.0.clause"); - assertEquals("metric(Integer_Value) by ts_hash", value); - value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.1.clause"); - assertEquals("count by ipaddress", value); - value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.2.clause"); - assertEquals("max(Integer_Value) over ts_hash", value); - value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.3.clause"); - assertEquals("rare by ipaddress partitionfield=host", value); - value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.4.clause"); - assertEquals("rare by \"weird field\"", value); - value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.5.clause"); - assertEquals("max(field) over tshash", value); - value = fieldConfig.get(iniConfig.getGlobalSectionName(), "detector.6.clause"); - assertEquals("max(\"Level 1 (Urgent)\") by \"10%\" over \"%10\" partitionfield=\"Percentage (%)\"", value); - // Ini4j meddles with escape characters itself, so the assertion below - // fails even though the raw file is fine. The file is never read by - // Ini4j in the production system. - // Assert.assertEquals("max(\"\\\"quoted\\\" field\") over \"ts\\\\hash\"", value); - } - - public void testWrite_GivenConfigHasCategorizationField() throws IOException { - Detector.Builder d = new Detector.Builder("metric", "Integer_Value"); - d.setByFieldName("mlcategory"); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(d.build())); - builder.setCategorizationFieldName("foo"); - analysisConfig = builder.build(); - writer = mock(OutputStreamWriter.class); - - createFieldConfigWriter().write(); - - verify(writer).write("detector.0.clause = metric(Integer_Value) by mlcategory categorizationfield=foo\n"); - verifyNoMoreInteractions(writer); - } - - public void testWrite_GivenConfigHasPerPartitionCategorization() throws IOException { - Detector.Builder d = new Detector.Builder("metric", "Integer_Value"); - d.setByFieldName("mlcategory"); - d.setPartitionFieldName("event.dataset"); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(d.build())); - builder.setCategorizationFieldName("message"); - builder.setPerPartitionCategorizationConfig(new PerPartitionCategorizationConfig(true, false)); - analysisConfig = builder.build(); - writer = mock(OutputStreamWriter.class); - - createFieldConfigWriter().write(); - - verify(writer).write("detector.0.clause = metric(Integer_Value) by mlcategory partitionfield=\"event.dataset\" " - + "categorizationfield=message perpartitioncategorization=true\n"); - verifyNoMoreInteractions(writer); - } - - public void testWrite_GivenConfigHasInfluencers() throws IOException { - Detector.Builder d = new Detector.Builder("metric", "Integer_Value"); - d.setByFieldName("ts_hash"); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(d.build())); - builder.setInfluencers(Arrays.asList("sun", "moon", "earth")); - analysisConfig = builder.build(); - - writer = mock(OutputStreamWriter.class); - - createFieldConfigWriter().write(); - - verify(writer).write("detector.0.clause = metric(Integer_Value) by ts_hash\n" + - "influencer.0 = sun\n" + - "influencer.1 = moon\n" + - "influencer.2 = earth\n"); - verifyNoMoreInteractions(writer); - } - - public void testWrite_GivenConfigHasCategorizationFieldAndFiltersAndInfluencer() throws IOException { - Detector.Builder d = new Detector.Builder("metric", "Integer_Value"); - d.setByFieldName("mlcategory"); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(d.build())); - builder.setInfluencers(Collections.singletonList("sun")); - builder.setCategorizationFieldName("myCategory"); - builder.setCategorizationFilters(Arrays.asList("foo", " ", "abc,def")); - analysisConfig = builder.build(); - - writer = mock(OutputStreamWriter.class); - - createFieldConfigWriter().write(); - - verify(writer).write( - "detector.0.clause = metric(Integer_Value) by mlcategory categorizationfield=myCategory\n" + - (MachineLearning.CATEGORIZATION_TOKENIZATION_IN_JAVA ? "" : - "categorizationfilter.0 = foo\n" + - "categorizationfilter.1 = \" \"\n" + - "categorizationfilter.2 = \"abc,def\"\n") + - "influencer.0 = sun\n"); - verifyNoMoreInteractions(writer); - } - - public void testWrite_GivenDetectorWithRules() throws IOException { - Detector.Builder detector = new Detector.Builder("mean", "metricValue"); - detector.setByFieldName("metricName"); - detector.setPartitionFieldName("instance"); - RuleCondition ruleCondition = new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.LT, 5); - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).build(); - detector.setRules(Collections.singletonList(rule)); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); - analysisConfig = builder.build(); - - writer = mock(OutputStreamWriter.class); - - createFieldConfigWriter().write(); - - ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); - verify(writer).write(captor.capture()); - String actual = captor.getValue(); - String expectedFirstLine = "detector.0.clause = mean(metricValue) by metricName partitionfield=instance\n"; - assertTrue(actual.startsWith(expectedFirstLine)); - String secondLine = actual.substring(expectedFirstLine.length()); - String expectedSecondLineStart = "detector.0.rules = "; - assertTrue(secondLine.startsWith(expectedSecondLineStart)); - String rulesJson = secondLine.substring(expectedSecondLineStart.length()); - assertEquals("[" + Strings.toString(rule.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + "]\n", rulesJson); - } - - public void testWrite_GivenFilters() throws IOException { - Detector d = new Detector.Builder("count", null).build(); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(d)); - analysisConfig = builder.build(); - - filters.add(MlFilter.builder("filter_1").setItems("a", "b").build()); - filters.add(MlFilter.builder("filter_2").setItems("c", "d").build()); - writer = mock(OutputStreamWriter.class); - - createFieldConfigWriter().write(); - - verify(writer).write("filter.filter_1 = [\"a\",\"b\"]\n" + - "filter.filter_2 = [\"c\",\"d\"]\n" + - "detector.0.clause = count\n"); - verifyNoMoreInteractions(writer); - } - - public void testWrite_GivenScheduledEvents() throws IOException { - Detector d = new Detector.Builder("count", null).build(); - - AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(d)); - analysisConfig = builder.build(); - - scheduledEvents.add(new ScheduledEvent.Builder().description("The Ashes") - .startTime(Instant.ofEpochMilli(1511395200000L)) - .endTime(Instant.ofEpochMilli(1515369600000L)) - .calendarId("calendar_id").build()); - scheduledEvents.add(new ScheduledEvent.Builder().description("elasticon") - .startTime(Instant.ofEpochMilli(1519603200000L)) - .endTime(Instant.ofEpochMilli(1519862400000L)) - .calendarId("calendar_id").build()); - - writer = mock(OutputStreamWriter.class); - createFieldConfigWriter().write(); - - verify(writer).write("detector.0.clause = count\n" + - "scheduledevent.0.description = The Ashes\n" + - "scheduledevent.0.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"],\"conditions\":[{\"applies_to\":\"time\"," + - "\"operator\":\"gte\",\"value\":1.5113952E9},{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5153696E9}]}]\n" + - "scheduledevent.1.description = elasticon\n" + - "scheduledevent.1.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"]," + - "\"conditions\":[{\"applies_to\":\"time\",\"operator\":\"gte\",\"value\":1.5196032E9}," + - "{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5198624E9}]}]\n"); - - verifyNoMoreInteractions(writer); - } - - private FieldConfigWriter createFieldConfigWriter() { - return new FieldConfigWriter(analysisConfig, filters, scheduledEvents, writer, mock(Logger.class)); - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java index 5d9c183c738e..9b04b94261e9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerBuilderTests.java @@ -24,7 +24,7 @@ public void testBuildNormalizerCommand() throws IOException { List command = new NormalizerBuilder(env, jobId, null, 300).build(); assertEquals(3, command.size()); assertTrue(command.contains("./normalize")); - assertTrue(command.contains(AutodetectBuilder.BUCKET_SPAN_ARG + "300")); + assertTrue(command.contains(NormalizerBuilder.BUCKET_SPAN_ARG + "300")); assertTrue(command.contains(AutodetectBuilder.LENGTH_ENCODED_INPUT_ARG)); } }