Skip to content

Commit

Permalink
Add DailyModelSnapshotRetentionAfterDays to ml jobs (#4812) (#4832)
Browse files Browse the repository at this point in the history 
Relates: elastic/elasticsearch#56125, #4803

This commit adds the DailyModelSnapshotRetentionAfterDays
to ML jobs. It also updates the XML comment for
ModelSnapshotRetentionDays to align with the new default in
Elasticsearch 7.8.

Co-authored-by: Russ Cam <[email protected]>
  • Loading branch information
@github-actions @russcam
github-actions[bot] and russcam authored Jun 30, 2020
1 parent 9bd92ca commit 1dcb5fc
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 22 deletions.
18 changes: 17 additions & 1 deletion src/Nest/XPack/MachineLearning/Job/Config/Job.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,27 @@ public class Job

/// <summary>
/// The time in days that model snapshots are retained for the job.
/// Older snapshots are deleted. The default value is 1 day.
/// Older snapshots are deleted. The default value is 10 days in Elasticsearch 7.8.0+
/// and 1 day in older versions.
/// </summary>
[DataMember(Name = "model_snapshot_retention_days")]
public long? ModelSnapshotRetentionDays { get; set; }

/// <summary>
/// Specifies a number of days between 0 and the value of <see cref="ModelSnapshotRetentionDays"/>.
/// After this period of time, only the first model snapshot per day is retained for this job.
/// Age is calculated relative to the timestamp of the newest model snapshot. For new jobs, the default
/// value is <c>1</c>, which means that all snapshots are retained for one day. Older snapshots
/// are thinned out such that only one per day is retained. For jobs that were
/// created before this setting was available, the default value matches the
/// <see cref="ModelSnapshotRetentionDays"/> value, which preserves the original behavior
/// and no thinning out of model snapshots occurs.
/// <para />
/// Available in Elasticsearch 7.8.0+
/// </summary>
[DataMember(Name = "daily_model_snapshot_retention_after_days")]
public long? DailyModelSnapshotRetentionAfterDays { get; set; }

/// <summary>
/// Advanced configuration option. The period over which adjustments to the score are applied, as new data
/// is seen. The default value is the longer of 30 days or 100 bucket spans.
Expand Down
6 changes: 6 additions & 0 deletions src/Nest/XPack/MachineLearning/MachineLearningInfo/MachineLearningInfoResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ public class AnomalyDetectors
[DataMember(Name = "model_snapshot_retention_days")]
public int ModelSnapshotRetentionDays { get; internal set; }

/// <summary>
/// Available in Elasticsearch 7.8.0+
/// </summary>
[DataMember(Name = "daily_model_snapshot_retention_after_days")]
public long DailyModelSnapshotRetentionAfterDays { get; internal set; }

[DataMember(Name = "categorization_analyzer")]
public CategorizationAnalyzer CategorizationAnalyzer { get; internal set; }
}
Expand Down
44 changes: 34 additions & 10 deletions src/Nest/XPack/MachineLearning/PutJob/PutJobRequest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,27 @@ public partial interface IPutJobRequest

/// <summary>
/// The time in days that model snapshots are retained for the job.
/// Older snapshots are deleted. The default value is 1 day.
/// Older snapshots are deleted. The default value is 10 days in Elasticsearch 7.8.0+
/// and 1 day in older versions.
/// </summary>
[DataMember(Name ="model_snapshot_retention_days")]
long? ModelSnapshotRetentionDays { get; set; }

/// <summary>
/// Specifies a number of days between 0 and the value of <see cref="ModelSnapshotRetentionDays"/>.
/// After this period of time, only the first model snapshot per day is retained for this job.
/// Age is calculated relative to the timestamp of the newest model snapshot. For new jobs, the default
/// value is <c>1</c>, which means that all snapshots are retained for one day. Older snapshots
/// are thinned out such that only one per day is retained. For jobs that were
/// created before this setting was available, the default value matches the
/// <see cref="ModelSnapshotRetentionDays"/> value, which preserves the original behavior
/// and no thinning out of model snapshots occurs.
/// <para />
/// Available in Elasticsearch 7.8.0+
/// </summary>
[DataMember(Name ="daily_model_snapshot_retention_after_days")]
long? DailyModelSnapshotRetentionAfterDays { get; set; }

/// <summary>
/// The name of the index in which to store the machine learning results.
/// The default value is shared, which corresponds to the index name .ml-anomalies-shared.
Expand Down Expand Up @@ -93,6 +109,9 @@ public partial class PutJobRequest
/// <inheritdoc />
public long? ModelSnapshotRetentionDays { get; set; }

/// <inheritdoc />
public long? DailyModelSnapshotRetentionAfterDays { get; set; }

/// <inheritdoc />
public IndexName ResultsIndexName { get; set; }

Expand All @@ -109,41 +128,46 @@ public partial class PutJobDescriptor<TDocument> where TDocument : class
string IPutJobRequest.Description { get; set; }
IModelPlotConfig IPutJobRequest.ModelPlotConfig { get; set; }
long? IPutJobRequest.ModelSnapshotRetentionDays { get; set; }
long? IPutJobRequest.DailyModelSnapshotRetentionAfterDays { get; set; }
IndexName IPutJobRequest.ResultsIndexName { get; set; }
bool? IPutJobRequest.AllowLazyOpen { get; set; }

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.AnalysisConfig"/>
public PutJobDescriptor<TDocument> AnalysisConfig(Func<AnalysisConfigDescriptor<TDocument>, IAnalysisConfig> selector) =>
Assign(selector, (a, v) => a.AnalysisConfig = v?.Invoke(new AnalysisConfigDescriptor<TDocument>()));

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.AnalysisLimits"/>
public PutJobDescriptor<TDocument> AnalysisLimits(Func<AnalysisLimitsDescriptor, IAnalysisLimits> selector) =>
Assign(selector, (a, v) => a.AnalysisLimits = v?.Invoke(new AnalysisLimitsDescriptor()));

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.DataDescription"/>
public PutJobDescriptor<TDocument> DataDescription(Func<DataDescriptionDescriptor<TDocument>, IDataDescription> selector) =>
Assign(selector.InvokeOrDefault(new DataDescriptionDescriptor<TDocument>()), (a, v) => a.DataDescription = v);

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.Description"/>
public PutJobDescriptor<TDocument> Description(string description) => Assign(description, (a, v) => a.Description = v);

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.ModelPlotConfig"/>
public PutJobDescriptor<TDocument> ModelPlot(Func<ModelPlotConfigDescriptor<TDocument>, IModelPlotConfig> selector) =>
Assign(selector, (a, v) => a.ModelPlotConfig = v?.Invoke(new ModelPlotConfigDescriptor<TDocument>()));

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.ModelSnapshotRetentionDays"/>
public PutJobDescriptor<TDocument> ModelSnapshotRetentionDays(long? modelSnapshotRetentionDays) =>
Assign(modelSnapshotRetentionDays, (a, v) => a.ModelSnapshotRetentionDays = v);

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.DailyModelSnapshotRetentionAfterDays"/>
public PutJobDescriptor<TDocument> DailyModelSnapshotRetentionAfterDays(long? dailyModelSnapshotRetentionAfterDays) =>
Assign(dailyModelSnapshotRetentionAfterDays, (a, v) => a.DailyModelSnapshotRetentionAfterDays = v);

/// <inheritdoc cref="IPutJobRequest.ResultsIndexName"/>
public PutJobDescriptor<TDocument> ResultsIndexName(IndexName indexName) =>
Assign(indexName, (a, v) => a.ResultsIndexName = v);

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.ResultsIndexName"/>
public PutJobDescriptor<TDocument> ResultsIndexName<TIndex>() =>
Assign(typeof(TIndex), (a, v) => a.ResultsIndexName = v);

/// <inheritdoc />
/// <inheritdoc cref="IPutJobRequest.AllowLazyOpen"/>
public PutJobDescriptor<TDocument> AllowLazyOpen(bool? allowLazyOpen = true) =>
Assign(allowLazyOpen, (a, v) => a.AllowLazyOpen = v);
}
Expand Down
45 changes: 34 additions & 11 deletions src/Nest/XPack/MachineLearning/UpdateJob/UpdateJobRequest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,27 @@ public partial interface IUpdateJobRequest

/// <summary>
/// The time in days that model snapshots are retained for the job.
/// Older snapshots are deleted. The default value is 1 day.
/// Older snapshots are deleted. The default value is 10 days in Elasticsearch 7.8.0+
/// and 1 day in older versions.
/// </summary>
[DataMember(Name ="model_snapshot_retention_days")]
long? ModelSnapshotRetentionDays { get; set; }

/// <summary>
/// Specifies a number of days between 0 and the value of <see cref="ModelSnapshotRetentionDays"/>.
/// After this period of time, only the first model snapshot per day is retained for this job.
/// Age is calculated relative to the timestamp of the newest model snapshot. For new jobs, the default
/// value is <c>1</c>, which means that all snapshots are retained for one day. Older snapshots
/// are thinned out such that only one per day is retained. For jobs that were
/// created before this setting was available, the default value matches the
/// <see cref="ModelSnapshotRetentionDays"/> value, which preserves the original behavior
/// and no thinning out of model snapshots occurs.
/// <para />
/// Available in Elasticsearch 7.8.0+
/// </summary>
[DataMember(Name ="daily_model_snapshot_retention_after_days")]
long? DailyModelSnapshotRetentionAfterDays { get; set; }

/// <summary>
/// The period over which adjustments to the score are applied, as new data is seen.
/// </summary>
Expand Down Expand Up @@ -104,6 +120,9 @@ public partial class UpdateJobRequest
/// <inheritdoc />
public long? ModelSnapshotRetentionDays { get; set; }

/// <inheritdoc />
public long? DailyModelSnapshotRetentionAfterDays { get; set; }

/// <inheritdoc />
public long? RenormalizationWindowDays { get; set; }

Expand All @@ -123,43 +142,47 @@ public partial class UpdateJobDescriptor<TDocument> where TDocument : class
string IUpdateJobRequest.Description { get; set; }
IModelPlotConfigEnabled IUpdateJobRequest.ModelPlotConfig { get; set; }
long? IUpdateJobRequest.ModelSnapshotRetentionDays { get; set; }
long? IUpdateJobRequest.DailyModelSnapshotRetentionAfterDays { get; set; }
long? IUpdateJobRequest.RenormalizationWindowDays { get; set; }
long? IUpdateJobRequest.ResultsRetentionDays { get; set; }

bool? IUpdateJobRequest.AllowLazyOpen { get; set; }

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.AnalysisLimits" />
public UpdateJobDescriptor<TDocument> AnalysisLimits(Func<AnalysisMemoryLimitDescriptor, IAnalysisMemoryLimit> selector) =>
Assign(selector, (a, v) => a.AnalysisLimits = v?.Invoke(new AnalysisMemoryLimitDescriptor()));

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.BackgroundPersistInterval" />
public UpdateJobDescriptor<TDocument> BackgroundPersistInterval(Time backgroundPersistInterval) =>
Assign(backgroundPersistInterval, (a, v) => a.BackgroundPersistInterval = v);

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.CustomSettings" />
public UpdateJobDescriptor<TDocument> CustomSettings(Func<FluentDictionary<string, object>, FluentDictionary<string, object>> customSettingsDictionary
) =>
Assign(customSettingsDictionary(new FluentDictionary<string, object>()), (a, v) => a.CustomSettings = v);

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.Description" />
public UpdateJobDescriptor<TDocument> Description(string description) => Assign(description, (a, v) => a.Description = v);

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.ModelPlotConfig" />
public UpdateJobDescriptor<TDocument> ModelPlot(Func<ModelPlotConfigEnabledDescriptor<TDocument>, IModelPlotConfigEnabled> selector) =>
Assign(selector, (a, v) => a.ModelPlotConfig = v?.Invoke(new ModelPlotConfigEnabledDescriptor<TDocument>()));

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.ModelSnapshotRetentionDays" />
public UpdateJobDescriptor<TDocument> ModelSnapshotRetentionDays(long? modelSnapshotRetentionDays) =>
Assign(modelSnapshotRetentionDays, (a, v) => a.ModelSnapshotRetentionDays = v);

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.DailyModelSnapshotRetentionAfterDays" />
public UpdateJobDescriptor<TDocument> DailyModelSnapshotRetentionAfterDays(long? dailyModelSnapshotRetentionAfterDays) =>
Assign(dailyModelSnapshotRetentionAfterDays, (a, v) => a.DailyModelSnapshotRetentionAfterDays = v);

/// <inheritdoc cref="IUpdateJobRequest.RenormalizationWindowDays" />
public UpdateJobDescriptor<TDocument> RenormalizationWindowDays(long? renormalizationWindowDays) =>
Assign(renormalizationWindowDays, (a, v) => a.RenormalizationWindowDays = v);

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.ResultsRetentionDays" />
public UpdateJobDescriptor<TDocument> ResultsRetentionDays(long? resultsRetentionDays) => Assign(resultsRetentionDays, (a, v) => a.ResultsRetentionDays = v);

/// <inheritdoc />
/// <inheritdoc cref="IUpdateJobRequest.AllowLazyOpen" />
public UpdateJobDescriptor<TDocument> AllowLazyOpen(bool? allowLazyOpen = true) =>
Assign(allowLazyOpen, (a, v) => a.AllowLazyOpen = v);
}
Expand Down
3 changes: 3 additions & 0 deletions tests/Tests/XPack/MachineLearning/MachineLearningInfo/MachineLearningInfoApiTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ protected override void ExpectResponse(MachineLearningInfoResponse response)
? 10
: 1);

if (TestClient.Configuration.InRange(">=7.8.0"))
anomalyDetectors.DailyModelSnapshotRetentionAfterDays.Should().Be(1);

response.Defaults.Datafeeds.ScrollSize.Should().Be(1000);

if (Cluster.ClusterConfiguration.Version >= "7.6.0")
Expand Down

0 comments on commit 1dcb5fc

Please sign in to comment.