Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Pass extra JSON config files to autodetect #66736

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ScheduledEventToRuleWriter;
import org.elasticsearch.xpack.ml.process.NativeController;
import org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils;
import org.elasticsearch.xpack.ml.process.ProcessPipes;
Expand All @@ -42,6 +43,7 @@
import java.util.Objects;
import java.util.Random;
import java.util.Set;
import java.util.stream.Collectors;

import static org.elasticsearch.xpack.ml.job.process.ProcessBuilderUtils.addIfNotNull;

Expand Down Expand Up @@ -69,6 +71,8 @@ public class AutodetectBuilder {
private static final String JSON_EXTENSION = ".json";
static final String JOB_ID_ARG = "--jobid=";
private static final String CONFIG_ARG = "--config=";
private static final String EVENTS_CONFIG_ARG = "--eventsconfig=";
private static final String FILTERS_CONFIG_ARG = "--filtersconfig=";
private static final String LIMIT_CONFIG_ARG = "--limitconfig=";
private static final String MODEL_PLOT_CONFIG_ARG = "--modelplotconfig=";
private static final String FIELD_CONFIG_ARG = "--fieldconfig=";
Expand Down Expand Up @@ -178,6 +182,9 @@ public void build() throws IOException, InterruptedException {

List<String> command = buildAutodetectCommand();

buildFiltersConfig(command);
buildScheduledEventsConfig(command);

// While it may appear that the JSON formatted job config file contains data that
// is duplicated in the existing limits, modelPlot and field config files, over time
// the C++ backend will retrieve all its required configuration data from the new
Expand Down Expand Up @@ -365,6 +372,28 @@ private void buildFieldConfig(List<String> command) throws IOException {
}
}

private void buildScheduledEventsConfig(List<String> command) throws IOException {
if (scheduledEvents.isEmpty()) {
return;
}
Path eventsConfigFile = Files.createTempFile(env.tmpFile(), "eventsConfig", JSON_EXTENSION);
filesToDelete.add(eventsConfigFile);

List<ScheduledEventToRuleWriter> scheduledEventToRuleWriters = scheduledEvents.stream()
.map(x -> new ScheduledEventToRuleWriter(x.getDescription(), x.toDetectionRule(job.getAnalysisConfig().getBucketSpan())))
.collect(Collectors.toList());

try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(eventsConfigFile),StandardCharsets.UTF_8);
XContentBuilder jsonBuilder = JsonXContent.contentBuilder()) {
osw.write(Strings.toString(
jsonBuilder.startObject()
.field(ScheduledEvent.RESULTS_FIELD.getPreferredName(), scheduledEventToRuleWriters)
.endObject()));
}

command.add(EVENTS_CONFIG_ARG + eventsConfigFile.toString());
}

private void buildJobConfig(List<String> command) throws IOException {
Path configFile = Files.createTempFile(env.tmpFile(), "config", JSON_EXTENSION);
filesToDelete.add(configFile);
Expand All @@ -377,4 +406,21 @@ private void buildJobConfig(List<String> command) throws IOException {

command.add(CONFIG_ARG + configFile.toString());
}

private void buildFiltersConfig(List<String> command) throws IOException {
if (referencedFilters.isEmpty()) {
return;
}
Path filtersConfigFile = Files.createTempFile(env.tmpFile(), "filtersConfig", JSON_EXTENSION);
filesToDelete.add(filtersConfigFile);

try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(filtersConfigFile),StandardCharsets.UTF_8);
XContentBuilder jsonBuilder = JsonXContent.contentBuilder()) {
osw.write(Strings.toString(
jsonBuilder.startObject()
.field(MlFilter.RESULTS_FIELD.getPreferredName(), referencedFilters)
.endObject()));
}
command.add(FILTERS_CONFIG_ARG + filtersConfigFile.toString());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.job.process.autodetect.writer;

import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;

import java.io.IOException;
import java.util.Objects;

public class ScheduledEventToRuleWriter implements ToXContentObject {

public static final ParseField DESCRIPTION = new ParseField("description");
public static final ParseField RULES = new ParseField("rules");

private final String description;
private final DetectionRule detectionRule;

public ScheduledEventToRuleWriter(String description, DetectionRule detectionRule) {
this.description = Objects.requireNonNull(description);
this.detectionRule = Objects.requireNonNull(detectionRule);
}

public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(DESCRIPTION.getPreferredName(), description);
builder.array(RULES.getPreferredName(), detectionRule);
builder.endObject();
return builder;
}
}