From 72f602d7b1b3c7e3b81a1b47f8f7215751da7d56 Mon Sep 17 00:00:00 2001 From: lwpk110 Date: Mon, 11 Nov 2024 11:23:36 +0800 Subject: [PATCH] test(vector): optional the elasticsearch sink in vector e2e --- .chainsaw.yaml | 4 +- test/e2e/vector/aggregator-assert.yaml | 13 +-- test/e2e/vector/aggregator.yaml | 92 ++++--------------- test/e2e/vector/chainsaw-test.yaml | 21 +++++ .../e2e/vector/elasticsearch-sink-assert.yaml | 11 +++ test/e2e/vector/elasticsearch-sink.yaml | 59 ++++++++++++ 6 files changed, 113 insertions(+), 87 deletions(-) create mode 100644 test/e2e/vector/elasticsearch-sink-assert.yaml create mode 100644 test/e2e/vector/elasticsearch-sink.yaml diff --git a/.chainsaw.yaml b/.chainsaw.yaml index a775f76..ed37dd4 100644 --- a/.chainsaw.yaml +++ b/.chainsaw.yaml @@ -11,7 +11,7 @@ spec: delete: 120s error: 180s exec: 300s - # skipDelete: false - skipDelete: true + skipDelete: false + # skipDelete: true failFast: true parallel: 1 diff --git a/test/e2e/vector/aggregator-assert.yaml b/test/e2e/vector/aggregator-assert.yaml index 923b3d2..5bb1240 100644 --- a/test/e2e/vector/aggregator-assert.yaml +++ b/test/e2e/vector/aggregator-assert.yaml @@ -1,15 +1,4 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: vector-aggregator ---- -apiVersion: v1 -kind: Service -metadata: - name: vector-aggregator -spec: - type: ClusterIP ---- + apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/test/e2e/vector/aggregator.yaml b/test/e2e/vector/aggregator.yaml index 234d693..2c54863 100644 --- a/test/e2e/vector/aggregator.yaml +++ b/test/e2e/vector/aggregator.yaml @@ -1,64 +1,4 @@ -apiVersion: v1 -kind: Pod -metadata: - name: elasticsearch - labels: - app: elasticsearch -spec: - containers: - - name: es7 - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.7 - resources: - limits: - cpu: 200m - memory: 1Gi - env: - - name: discovery.type - value: "single-node" - - name: bootstrap.memory_lock - value: "true" - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - - name: transport.host - value: "127.0.0.1" - - name: xpack.security.enabled - value: "false" - ports: - - containerPort: 9200 - name: http - - name: kibana7 - image: docker.elastic.co/kibana/kibana:7.17.7 - resources: - limits: - cpu: 200m - memory: 500Mi - env: - - name: ELASTICSEARCH_HOSTS - value: "http://localhost:9200" - - name: i18n.locale - value: "zh-CN" - - name: xpack.security.enabled - value: "false" - ports: - - containerPort: 5601 - name: kibana ---- -## create svc for elasticsearch -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - selector: - app: elasticsearch - type: ClusterIP ---- apiVersion: v1 kind: ConfigMap metadata: @@ -76,20 +16,20 @@ data: - vector encoding: codec: "json" - essink: - type: elasticsearch - inputs: - - vector - api_version: auto - compression: none - doc_type: _doc - endpoints:Copilot features might go through different early access stages, which are typically enabled and configured through settings. + # essink: + # type: elasticsearch + # inputs: + # - vector + # api_version: auto + # compression: none + # doc_type: _doc + # endpoints:Copilot features might go through different early access stages, which are typically enabled and configured through settings. - - http://elasticsearch:9200 - mode: bulk - # query: - # X-Powered-By: Vector + # - http://elasticsearch:9200 + # mode: bulk + # # query: + # # X-Powered-By: Vector sources: vector: address: 0.0.0.0:6000 @@ -159,7 +99,13 @@ spec: - containerPort: 6000 name: vector protocol: TCP - # resources: {} + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi volumeMounts: - mountPath: /vector-data-dir name: data diff --git a/test/e2e/vector/chainsaw-test.yaml b/test/e2e/vector/chainsaw-test.yaml index 680877f..694da28 100644 --- a/test/e2e/vector/chainsaw-test.yaml +++ b/test/e2e/vector/chainsaw-test.yaml @@ -10,6 +10,10 @@ spec: value: test steps: - try: + # - apply: + # file: elasticsearch-sink.yaml + # - assert: + # file: elasticsearch-sink-assert.yaml - apply: file: aggregator.yaml - assert: @@ -32,3 +36,20 @@ spec: cleanup: - sleep: duration: 10s + catch: + - sleep: + duration: 5s + - script: + env: + - name: NAMESPACE + value: ($namespace) + content: | + set -ex + kubectl -n $NAMESPACE get pods + - describe: + apiVersion: v1 + kind: Pod + selector: app.kubernetes.io/instance=test-dolphinscheduler + - podLogs: + selector: app.kubernetes.io/instance=test-dolphinscheduler + tail: -1 diff --git a/test/e2e/vector/elasticsearch-sink-assert.yaml b/test/e2e/vector/elasticsearch-sink-assert.yaml new file mode 100644 index 0000000..bb28fb5 --- /dev/null +++ b/test/e2e/vector/elasticsearch-sink-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator +--- +apiVersion: v1 +kind: Service +metadata: + name: vector-aggregator +spec: + type: ClusterIP diff --git a/test/e2e/vector/elasticsearch-sink.yaml b/test/e2e/vector/elasticsearch-sink.yaml new file mode 100644 index 0000000..d40a36b --- /dev/null +++ b/test/e2e/vector/elasticsearch-sink.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + containers: + - name: es7 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.7 + resources: + limits: + cpu: 200m + memory: 1Gi + env: + - name: discovery.type + value: "single-node" + - name: bootstrap.memory_lock + value: "true" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: transport.host + value: "127.0.0.1" + - name: xpack.security.enabled + value: "false" + ports: + - containerPort: 9200 + name: http + - name: kibana7 + image: docker.elastic.co/kibana/kibana:7.17.7 + resources: + limits: + cpu: 200m + memory: 500Mi + env: + - name: ELASTICSEARCH_HOSTS + value: "http://localhost:9200" + - name: i18n.locale + value: "zh-CN" + - name: xpack.security.enabled + value: "false" + ports: + - containerPort: 5601 + name: kibana +--- +# create svc for elasticsearch +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + selector: + app: elasticsearch + type: ClusterIP