From d1addb63aa8059ddaf2acb9941ddce77156e1169 Mon Sep 17 00:00:00 2001 From: Christian Groschupp Date: Thu, 22 Feb 2024 11:55:05 +0100 Subject: [PATCH] add datastream support Signed-off-by: Christian Groschupp --- .../api/v1/opensearch_index_types.go | 10 ++++++ .../api/v1/opensearch_indextemplate_types.go | 3 ++ .../api/v1/zz_generated.deepcopy.go | 36 +++++++++++++++++++ ...ch.opster.io_opensearchindextemplates.yaml | 13 +++++++ .../opensearch-gateway/requests/Templates.go | 9 +++++ opensearch-operator/pkg/helpers/translate.go | 14 ++++++++ 6 files changed, 85 insertions(+) diff --git a/opensearch-operator/api/v1/opensearch_index_types.go b/opensearch-operator/api/v1/opensearch_index_types.go index 99f3a93eb..c2f2838da 100644 --- a/opensearch-operator/api/v1/opensearch_index_types.go +++ b/opensearch-operator/api/v1/opensearch_index_types.go @@ -4,6 +4,16 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) +type OpensearchDatastreamTimestampFieldSpec struct { + // Name of the field that are used for the DataStream + Name string `json:"name"` +} + +type OpensearchDatastreamSpec struct { + // TimestampField for dataStream + TimestampField OpensearchDatastreamTimestampFieldSpec `json:"timestamp_field,omitempty"` +} + // Describes the specs of an index type OpensearchIndexSpec struct { // Configuration options for the index diff --git a/opensearch-operator/api/v1/opensearch_indextemplate_types.go b/opensearch-operator/api/v1/opensearch_indextemplate_types.go index 53e1fd9d9..cb3d30ec3 100644 --- a/opensearch-operator/api/v1/opensearch_indextemplate_types.go +++ b/opensearch-operator/api/v1/opensearch_indextemplate_types.go @@ -48,6 +48,9 @@ type OpensearchIndexTemplateSpec struct { // Array of wildcard expressions used to match the names of indices during creation IndexPatterns []string `json:"indexPatterns"` + // The dataStream config that should be applied + DataStream *OpensearchDatastreamSpec `json:"dataStream,omitempty"` + // The template that should be applied Template OpensearchIndexSpec `json:"template,omitempty"` diff --git a/opensearch-operator/api/v1/zz_generated.deepcopy.go b/opensearch-operator/api/v1/zz_generated.deepcopy.go index c4aaa75d3..14abd3410 100644 --- a/opensearch-operator/api/v1/zz_generated.deepcopy.go +++ b/opensearch-operator/api/v1/zz_generated.deepcopy.go @@ -1463,6 +1463,37 @@ func (in *OpensearchComponentTemplateStatus) DeepCopy() *OpensearchComponentTemp return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpensearchDatastreamSpec) DeepCopyInto(out *OpensearchDatastreamSpec) { + *out = *in + out.TimestampField = in.TimestampField +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpensearchDatastreamSpec. +func (in *OpensearchDatastreamSpec) DeepCopy() *OpensearchDatastreamSpec { + if in == nil { + return nil + } + out := new(OpensearchDatastreamSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpensearchDatastreamTimestampFieldSpec) DeepCopyInto(out *OpensearchDatastreamTimestampFieldSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpensearchDatastreamTimestampFieldSpec. +func (in *OpensearchDatastreamTimestampFieldSpec) DeepCopy() *OpensearchDatastreamTimestampFieldSpec { + if in == nil { + return nil + } + out := new(OpensearchDatastreamTimestampFieldSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpensearchISMPolicyStatus) DeepCopyInto(out *OpensearchISMPolicyStatus) { *out = *in @@ -1608,6 +1639,11 @@ func (in *OpensearchIndexTemplateSpec) DeepCopyInto(out *OpensearchIndexTemplate *out = make([]string, len(*in)) copy(*out, *in) } + if in.DataStream != nil { + in, out := &in.DataStream, &out.DataStream + *out = new(OpensearchDatastreamSpec) + **out = **in + } in.Template.DeepCopyInto(&out.Template) if in.ComposedOf != nil { in, out := &in.ComposedOf, &out.ComposedOf diff --git a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml index 4cfc668a9..7c7bc494b 100644 --- a/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml +++ b/opensearch-operator/config/crd/bases/opensearch.opster.io_opensearchindextemplates.yaml @@ -47,6 +47,19 @@ spec: items: type: string type: array + dataStream: + description: The dataStream config that should be applied + properties: + timestamp_field: + description: TimestampField for dataStream + properties: + name: + description: Name of the field that are used for the DataStream + type: string + required: + - name + type: object + type: object indexPatterns: description: Array of wildcard expressions used to match the names of indices during creation diff --git a/opensearch-operator/opensearch-gateway/requests/Templates.go b/opensearch-operator/opensearch-gateway/requests/Templates.go index 9f587addb..327169a36 100644 --- a/opensearch-operator/opensearch-gateway/requests/Templates.go +++ b/opensearch-operator/opensearch-gateway/requests/Templates.go @@ -4,6 +4,7 @@ import apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 type IndexTemplate struct { IndexPatterns []string `json:"index_patterns"` + DataStream *Datastream `json:"data_stream,omitempty"` Template Index `json:"template,omitempty"` ComposedOf []string `json:"composed_of,omitempty"` Priority int `json:"priority,omitempty"` @@ -24,6 +25,14 @@ type Index struct { Aliases map[string]IndexAlias `json:"aliases,omitempty"` } +type DatastreamTimestampFieldSpec struct { + Name string `json:"name"` +} + +type Datastream struct { + TimestampField *DatastreamTimestampFieldSpec `json:"timestamp_field,omitempty"` +} + type IndexAlias struct { Index string `json:"index,omitempty"` Alias string `json:"alias,omitempty"` diff --git a/opensearch-operator/pkg/helpers/translate.go b/opensearch-operator/pkg/helpers/translate.go index b9617b498..f2faad4d1 100644 --- a/opensearch-operator/pkg/helpers/translate.go +++ b/opensearch-operator/pkg/helpers/translate.go @@ -9,6 +9,7 @@ import ( func TranslateIndexTemplateToRequest(spec v1.OpensearchIndexTemplateSpec) requests.IndexTemplate { request := requests.IndexTemplate{ IndexPatterns: spec.IndexPatterns, + DataStream: TranslateDatastreamToRequest(spec.DataStream), Template: TranslateIndexToRequest(spec.Template), Priority: spec.Priority, Version: spec.Version, @@ -37,6 +38,19 @@ func TranslateComponentTemplateToRequest(spec v1.OpensearchComponentTemplateSpec return request } +// TranslateDatastreamToRequest rewrites the CRD format to the gateway format +func TranslateDatastreamToRequest(spec *v1.OpensearchDatastreamSpec) *requests.Datastream { + if spec == nil { + return nil + } + request := requests.Datastream{} + if spec.TimestampField.Name != "" { + request.TimestampField = &requests.DatastreamTimestampFieldSpec{Name: spec.TimestampField.Name} + } + + return &request +} + // TranslateIndexToRequest rewrites the CRD format to the gateway format func TranslateIndexToRequest(spec v1.OpensearchIndexSpec) requests.Index { aliases := make(map[string]requests.IndexAlias)