Merge pull request vdaas#4 from vankichi/merge/remote-main-for-flush-api

Merge latest main & Fix proto

.all-contributorsrc

@@ -9,7 +9,7 @@
       "login": "kpango",
       "name": "Yusuke Kato",
       "avatar_url": "https://avatars1.githubusercontent.com/u/9798091?v=4",
-      "profile": "http://kpango.com",
+      "profile": "https://kpango.com",
       "contributions": [
         "code",
         "design",
@@ -111,7 +111,7 @@
       "login": "thedrow",
       "name": "Omer Katz",
       "avatar_url": "https://avatars.githubusercontent.com/u/48936?v=4",
-      "profile": "http://omerkatz.com",
+      "profile": "https://omerkatz.com",
       "contributions": [
         "doc",
         "tutorial"
@@ -152,7 +152,27 @@
       "profile": "https://github.com/takuyaymd",
       "contributions": [
         "bug",
-        "code"
+        "code",
+        "maintenance"
+      ]
+    },
+    {
+      "login": "junsei-ando",
+      "name": "junsei-ando",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1892077?v=4",
+      "profile": "https://github.com/junsei-ando",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "ykadowak",
+      "name": "Yusuke Kadowaki",
+      "avatar_url": "https://avatars.githubusercontent.com/u/60080334?v=4",
+      "profile": "https://github.com/ykadowak",
+      "contributions": [
+        "code",
+        "test"
       ]
     }
   ],

.commit_template

@@ -2,7 +2,7 @@
 # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
+# You may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    https://www.apache.org/licenses/LICENSE-2.0

.deepsource.toml

@@ -2,7 +2,7 @@
 # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
+# You may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    https://www.apache.org/licenses/LICENSE-2.0
@@ -67,7 +67,7 @@ enabled = true
 
 [[analyzers]]
 name = "test-coverage"
-enabled = true
+enabled = false
 
 [[transformers]]
 name = "gofumpt"

.devcontainer/devcontainer.json

@@ -1,24 +1,29 @@
 {
   "name": "Vald",
   "image": "vdaas/vald-dev-container:nightly",
+  // Specify the Dockerfile below directly if you need to make customizations
+  // "dockerFile": "../dockers/dev/Dockerfile",
+  // "context": "..",
   "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
-
-  "settings": {
-    "terminal.integrated.shell.linux": "/bin/bash",
-    "go.useGoProxyToCheckForToolUpdates": false,
-    "go.useLanguageServer": true,
-    "go.autocompleteUnimportedPackages": true,
-    "go.buildOnSave": true,
-    "go.vetOnSave": true,
-    "go.lintOnSave": true,
-    "go.testOnSave": true,
-    "go.gopath": "/go"
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "latest",
+      "enableNonRootDocker": "true",
+      "moby": "true"
+    }
   },
-  "extensions": [
-    "golang.Go",
-    "zxh404.vscode-proto3",
-    "wayou.vscode-todo-highlight",
-    "esbenp.prettier-vscode"
-  ],
-  "postCreateCommand": "go version"
+  "remoteUser": "root",
+  "workspaceFolder": "/go/src/github.com/vdaas/vald",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/go/src/github.com/vdaas/vald,type=bind,consistency=delegated",
+
+  // To access grafana
+  "appPort": "3000:3000",
+
+  // Persist zsh history settings below
+  // overwrite HISTFILE to store history file in the named volume defined below
+  // also setting INC_APPEND_HISTORY to immediately flush command history to the file to keep history when rebuilding the devcontainer
+  "postStartCommand": "echo 'export HISTFILE=/commandhistory/.zsh_history' >> /root/.zshrc && echo 'setopt INC_APPEND_HISTORY' >> /root/.zshrc",
+  // define named volume to store zsh history file
+  "mounts": ["source=zshhistory-named-volume,target=/commandhistory,type=volume"],
+  "postAttachCommand": "mkdir -p /etc/server && ln -s $(pwd)/cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml"
 }

.fossa.yml

@@ -2,7 +2,7 @@
 # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
+# You may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    https://www.apache.org/licenses/LICENSE-2.0

.gitattributes

@@ -2,7 +2,7 @@
 # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
+# You may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    https://www.apache.org/licenses/LICENSE-2.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,7 +22,7 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.20.2
+- Go Version: 1.21.5
 - Docker Version: 20.10.8
-- Kubernetes Version: 1.22.0
-- NGT Version: 2.0.9
+- Kubernetes Version: v1.28.4
+- NGT Version: 2.1.6

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -16,7 +16,7 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.20.2
+- Go Version: 1.21.5
 - Docker Version: 20.10.8
-- Kubernetes Version: 1.22.0
-- NGT Version: 2.0.9
+- Kubernetes Version: v1.28.4
+- NGT Version: 2.1.6

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,10 +16,10 @@
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.20.2
+- Go Version: 1.21.5
 - Docker Version: 20.10.8
-- Kubernetes Version: 1.22.0
-- NGT Version: 2.0.9
+- Kubernetes Version: v1.28.4
+- NGT Version: 2.1.6
 
 ### Checklist:
 

.github/actions/deploy-chaos-mesh/action.yaml

@@ -0,0 +1,63 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Deploy Chaos Mesh"
+description: "A action to deploy Chaos Mesh"
+inputs:
+  chaos_mesh_version:
+    description: "The Chaos Mesh version to use. The default version is `versions/CHAOS_MESH_VERSION`"
+    required: false
+    default: ""
+  helm_extra_options:
+    description: "Extra options that passed to Helm command"
+    required: false
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Fetch Chaos Mesh version
+      id: chaos_mesh_version
+      shell: bash
+      run: |
+        CHAOS_MESH_VERSION=${CHAOS_MESH_VERSION:-`cat versions/CHAOS_MESH_VERSION`}
+
+        echo "version=${CHAOS_MESH_VERSION}" >> $GITHUB_OUTPUT
+      env:
+        CHAOS_MESH_VERSION: ${{ inputs.chaos_mesh_version }}
+
+    - name: Check k3d
+      shell: bash
+      id: check_k3d
+      run: |
+        kubectl cluster-info dump
+
+    - name: Check Helm
+      shell: bash
+      run: |
+        helm version
+
+    - name: Deploy Chaos Mesh
+      shell: bash
+      run: |
+        make kubectl/install
+
+        curl -sSL "https://mirrors.chaos-mesh.org/v${CHAOS_MESH_VERSION}/install.sh" | bash -s -- --k3s
+
+        helm install \
+            ${HELM_EXTRA_OPTIONS} \
+            vald-chaos-test tests/chaos/chart
+      env:
+        HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
+        CHAOS_MESH_VERSION: ${{ steps.chaos_mesh_version.outputs.version }}

.github/actions/detect-docker-image-tags/action.yaml

@@ -2,7 +2,7 @@
 # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
+# You may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    https://www.apache.org/licenses/LICENSE-2.0
@@ -14,26 +14,28 @@
 # limitations under the License.
 #
 name: "Detect Docker image tags"
-description: "Detect Docker image tags"
+description: "A action to detect Docker image tags"
+
 inputs:
   tag_name:
-    description: "tag name to check whether exists or not"
+    description: "Tag name to check whether exists or not"
     required: true
     default: ""
   images:
-    description: "image names"
+    description: "Image names"
     required: false
     default: "vdaas/vald-agent-ngt \
       vdaas/vald-discoverer-k8s \
       vdaas/vald-lb-gateway \
       vdaas/vald-manager-index"
 outputs:
   HELM_EXTRA_OPTIONS:
-    description: "helm extra options that specifies image tags"
+    description: "Helm extra options that specifies image tags"
     value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
   IMAGE_TAGS:
-    description: "specifies image tags"
+    description: "Specifies image tags"
     value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }}
+
 runs:
   using: "composite"
   steps:
@@ -47,6 +49,9 @@ runs:
           ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag"
           ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag"
           ["vdaas/vald-manager-index"]="manager.index.image.tag"
+          ["vdaas/vald-index-creation"]="manager.index.creator.image.tag"
+          ["vdaas/vald-index-save"]="manager.index.saver.image.tag"
+          ["vdaas/vald-helm-operator"]="image.tag"
         )
 
         for image in ${IMAGES}

.github/actions/determine-docker-image-tag/action.yaml

@@ -0,0 +1,57 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Determine Docker image tag"
+description: "A action to determine Docker image tag"
+
+outputs:
+  PRIMARY_TAG:
+    description: "Primary tag"
+    value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Dump Github context
+      shell: bash
+      run: |
+        echo "GITHUB_REF $GITHUB_REF"
+        echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH"
+        echo "GITHUB_EVENT_NAME ${{ github.event_name }}"
+        echo "GITHUB_EVENT_NUMBER  ${{ github.event.number }}"
+
+    - name: Determine tag name
+      shell: bash
+      id: determine_tag_name
+      run: |
+        if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then
+          tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
+          primary_tag="${tag_name}"
+        elif [ "${{ github.event_name }}" = "pull_request" ]; then
+          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+          echo "PR-${pr_num}" > versions/VALD_VERSION
+          primary_tag="pr-${pr_num}"
+        elif [ "${{ github.event_name }}" = "pull_request_target" ]; then
+          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+          echo "PR-${pr_num}" > versions/VALD_VERSION
+          primary_tag="pr-${pr_num}"
+        elif [ "$GITHUB_REF" = "refs/heads/main" ]; then
+          echo "nightly" > versions/VALD_VERSION
+          primary_tag="nightly"
+        else
+          primary_tag="unknown"
+        fi
+        echo "PRIMARY_TAG is determined: ${primary_tag}"
+        echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT