feat(helm): add OpenSearch deployment (#827)

reanahub · Oct 23, 2024 · 21a525a · 21a525a
1 parent 0c0849d
commit 21a525a
Show file tree

Hide file tree

Showing 6 changed files with 335 additions and 0 deletions.
diff --git a/helm/configurations/values-dev.yaml b/helm/configurations/values-dev.yaml
@@ -14,6 +14,8 @@ components:
       image: docker.io/reanahub/reana-workflow-controller
       environment:
         REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES: failed
+        REANA_OPENSEARCH_USE_SSL: false
+        REANA_OPENSEARCH_ENABLED: false
     reana_workflow_engine_cwl:
       image: docker.io/reanahub/reana-workflow-engine-cwl
     reana_workflow_engine_yadage:
@@ -31,3 +33,34 @@ components:
 
 pgbouncer:
   enabled: true
+
+# OpenSearch configuration for dev environment
+opensearch:
+  enabled: false
+  tls:
+    generate: false
+  singleNode: true
+  config:
+    opensearch.yml: |
+      cluster.name: reana-opensearch
+      network.host: 0.0.0.0
+      plugins.security.disabled: true
+  securityConfig:
+    enabled: false
+    internalUsersSecret:
+    rolesSecret:
+    rolesMappingSecret:
+  resources:
+    requests:
+      cpu: "500m"
+      memory: "2Gi"
+  extraEnvs:
+    - name: DISABLE_INSTALL_DEMO_CONFIG
+      value: "false"
+    - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: reana-opensearch-secrets
+          key: OPENSEARCH_INITIAL_ADMIN_PASSWORD
+  secretMounts: []
+  customSecurityConfig:
diff --git a/helm/reana/Chart.yaml b/helm/reana/Chart.yaml
@@ -35,3 +35,7 @@ dependencies:
     condition: traefik.enabled
     tags:
       - ingress
+  - name: opensearch
+    version: 2.22.1
+    repository: https://opensearch-project.github.io/helm-charts/
+    condition: opensearch.enabled
diff --git a/helm/reana/README.md b/helm/reana/README.md
@@ -51,6 +51,14 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `components.reana_workflow_controller.environment.REANA_JOB_HOSTPATH_MOUNTS`   | JSON list of optional hostPath mounts, for all user jobs. Each mount object has a key `name` (name of the mount), `hostPath` (path to the directory to be mounted from the Kubernetes nodes) and `mountPath` (path inside the job containers where the `hostPath` will be mounted)            | None                                            |
 | `components.reana_workflow_controller.environment.REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES` | Keep alive Kubernetes user runtime jobs depending on status (`finished` and/or `failed`). | None |
 | `components.reana_workflow_controller.environment.REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT` | Define max number of unacknowledged deliveries that are permitted on `jobs-status` queue consumer. | 10 |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_ENABLED` | Enable workflow and job log retrieval from OpenSearch.            | false                                           |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_HOST` | OpenSearch host.                                                     | None                                            |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_PORT` | OpenSearch port.                                                     | None                                            |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_URL_PREFIX` | OpenSearch URL prefix.                                         | None                                            |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_USE_SSL` | Use SSL when connecting to OpenSearch instance.                   | true                                            |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_CA_CERTS` | Path to a file with OpenSearch root CA certificates.             | "/code/certs/ca.crt"                            |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_USER` | OpenSearch user name for Basic Authentication.                       | reana                                           |
+| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_PASSWORD` | OpenSearch password for Basic Authentication. Set this value in the Helm command. | ""                             |
 | `components.reana_workflow_engine_cwl.environment`       | [REANA-Workflow-Engine-CWL](https://github.com/reanahub/reana-workflow-engine-cwl) environment variables | `{}`                        |
 | `components.reana_workflow_engine_cwl.image`             | [REANA-Workflow-Engine-CWL image](https://hub.docker.com/r/reanahub/reana-workflow-engine-cwl) to use | `docker.io/reanahub/reana-workflow-engine-cwl:<chart-release-version>` |
 | `components.reana_workflow_engine_serial.environment`    | [REANA-Workflow-Engine-Serial](https://github.com/reanahub/reana-workflow-engine-serial) environment variables | `{}`                  |
@@ -98,6 +106,18 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `reana_hostname`                                         | REANA hostname (e.g. reana.example.org)                                              | None                                            |
 | `namespace_runtime`                                      | Namespace in which the REANA runtime pods (workflow engines, jobs etc...) will run   | `.Release.Namespace`                            |
 | `naming_scheme`                                          | REANA component naming scheme                                                        | None                                            |
+| `opensearch.*`                                           | Pass any value from [OpenSearch Helm chart values](https://github.com/opensearch-project/helm-charts/tree/main/charts/opensearch#configuration) here | - |
+| `opensearch.enabled`                                     | Enable OpenSearch                                                                    | false                                           |
+| `opensearch.tls.generate`                                | Enable the generation of a self-signed TLS certificates for OpenSearch               | true                                            |
+| `opensearch.tls.ca.cn`                                   | OpenSearch root CA certificate common name (CN)                                      | reana.io                                        |
+| `opensearch.tls.ca.ttl`                                  | OpenSearch root CA certificate TTL in days                                           | 365                                             |
+| `opensearch.tls.cert.cn`                                 | OpenSearch node certificate common name (CN)                                         | reana-opensearch-master.default.svc.cluster.local |
+| `opensearch.tls.cert.ttl`                                | OpenSearch node certificate TTL in days                                              | 180                                             |
+| `opensearch.tls.admin.cn`                                | OpenSearch admin certificate common name (CN)                                        | opensearch-admin.reana.io                       |
+| `opensearch.tls.admin.ttl`                               | OpenSearch admin certificate TTL in days                                             | 180                                             |
+| `opensearch.customSecurityConfig.internalUsers`          | Provide YAML users configuration for `internal_users.yaml` file; see [documentation](https://opensearch.org/docs/latest/security/configuration/yaml/#internal_usersyml) | None |
+| `opensearch.customSecurityConfig.roles`                  | Provide YAML roles configuration for `roles.yaml` file; see [documentation](https://opensearch.org/docs/latest/security/configuration/yaml/#rolesyml) | None |
+| `opensearch.customSecurityConfig.rolesMapping`           | Provide YAML roles mapping configuration for `roles_mapping.yaml` file; see [documentation](https://opensearch.org/docs/latest/security/configuration/yaml/#roles_mappingyml) | None |
 | `pgbouncer.enabled`                                      | Instantiate PgBouncer inside the cluster to pool database connections                | false                                           |
 | `pgbouncer.image`                                        | [PgBouncer image](https://hub.docker.com/r/bitnami/pgbouncer/) to use                | `bitnami/pgbouncer:1.23.1`                      |
 | `pgbouncer.pool_mode`                                    | Pool mode to use (session, transaction, statement)                                   | transaction                                     |

diff --git a/helm/reana/templates/reana-workflow-controller.yaml b/helm/reana/templates/reana-workflow-controller.yaml
@@ -1,3 +1,6 @@
+{{- $opensearchEnv := .Values.components.reana_workflow_controller.environment }}
+{{- $opensearchEnabled := $opensearchEnv.REANA_OPENSEARCH_ENABLED }}
+{{- $opensearchTlsEnabled := and $opensearchEnabled $opensearchEnv.REANA_OPENSEARCH_USE_SSL }}
 ---
 apiVersion: v1
 kind: Service
@@ -83,6 +86,11 @@ spec:
             mountPath: {{ $workspace_path._1 }}
           {{- end }}
           {{- end }}
+          {{- if $opensearchTlsEnabled }}
+          - name: {{ include "reana.prefix" . }}-opensearch-tls-secrets
+            mountPath: /code/certs
+            readOnly: true
+          {{- end }}
         envFrom:
           - configMapRef:
               name: {{ include "reana.prefix" . }}-database-config
@@ -216,6 +224,13 @@ spec:
               secretKeyRef:
                 name: {{ include "reana.prefix" . }}-db-secrets
                 key: password
+          {{- if $opensearchEnabled }}
+          - name: REANA_OPENSEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: {{ include "reana.prefix" . }}-opensearch-secrets
+                key: REANA_OPENSEARCH_PASSWORD
+          {{- end }}
       - name: job-status-consumer
         image: {{ .Values.components.reana_workflow_controller.image }}
         imagePullPolicy: {{ .Values.components.reana_workflow_controller.imagePullPolicy }}
@@ -287,6 +302,14 @@ spec:
         hostPath:
           path: {{ .Values.shared_storage.hostpath.root_path }}
         {{- end }}
+      {{- if $opensearchTlsEnabled }}
+      - name: {{ include "reana.prefix" . }}-opensearch-tls-secrets
+        secret:
+          secretName: {{ include "reana.prefix" . }}-opensearch-tls-secrets
+          items:
+            - key: ca.crt
+              path: ca.crt
+      {{- end }}
       {{- if .Values.debug.enabled }}
       - name: reana-code
         hostPath:

diff --git a/helm/reana/templates/secrets.yaml b/helm/reana/templates/secrets.yaml
@@ -76,3 +76,89 @@ data:
   tls.crt: {{ $cert.Cert | b64enc | quote }}
   tls.key: {{ $cert.Key | b64enc | quote }}
 {{- end }}
+---
+{{- if and .Values.opensearch.enabled .Values.opensearch.tls.generate }}
+{{- $prefix := include "reana.prefix" . }}
+{{- $tlsSecretName := printf "%s-%s" $prefix "opensearch-tls-secrets" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $tlsSecretName }}
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/tls
+data:
+  {{- $idx := lookup "v1" "Secret" .Release.Namespace $tlsSecretName -}}
+  {{- if $idx }}
+  tls.crt: {{ index $idx.data "tls.crt" }}
+  tls.key: {{ index $idx.data "tls.key" }}
+  admin.crt: {{ index $idx.data "admin.crt" }}
+  admin.key: {{ index $idx.data "admin.key" }}
+  ca.crt: {{ index $idx.data "ca.crt" }}
+  {{ else }}
+  {{- $ca := genCA .Values.opensearch.tls.ca.cn (.Values.opensearch.tls.ca.ttl | int) }}
+  {{- $cert := genSignedCert .Values.opensearch.tls.cert.cn nil (list .Values.opensearch.tls.cert.cn) (.Values.opensearch.tls.cert.ttl | int) $ca }}
+  {{- $certAdmin := genSignedCert .Values.opensearch.tls.admin.cn nil (list .Values.opensearch.tls.cert.cn) (.Values.opensearch.tls.admin.ttl | int) $ca }}
+  tls.crt: {{ $cert.Cert | b64enc | quote }}
+  tls.key: {{ $cert.Key | b64enc | quote }}
+  admin.crt: {{ $certAdmin.Cert | b64enc | quote }}
+  admin.key: {{ $certAdmin.Key | b64enc | quote }}
+  ca.crt: {{ $ca.Cert | b64enc | quote }}
+  {{- end }}
+{{- end }}
+---
+{{- if and .Values.opensearch.enabled .Values.opensearch.customSecurityConfig }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "reana.prefix" . }}-opensearch-config-secrets
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/opaque
+stringData:
+  {{- if .Values.opensearch.customSecurityConfig.internalUsers }}
+  internal_users.yml: |
+    ---
+    # This is the internal user database
+    # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+    _meta:
+      type: "internalusers"
+      config_version: 2
+    {{ .Values.opensearch.customSecurityConfig.internalUsers | toYaml | nindent 4 }}
+  {{- end }}
+  {{- if .Values.opensearch.customSecurityConfig.rolesMapping }}
+  roles_mapping.yml: |
+    ---
+    # In this file users, backendroles and hosts can be mapped to Security roles.
+    # Permissions for OpenSearch roles are configured in roles.yml
+    _meta:
+      type: "rolesmapping"
+      config_version: 2
+    {{ .Values.opensearch.customSecurityConfig.rolesMapping | toYaml | nindent 4 }}
+  {{- end }}
+  {{- if .Values.opensearch.customSecurityConfig.roles }}
+  roles.yml: |
+    ---
+    _meta:
+      type: "roles"
+      config_version: 2
+    # The security REST API access role is used to assign specific users access to change the security settings through the REST API.
+    security_rest_api_access:
+      reserved: true
+    {{ .Values.opensearch.customSecurityConfig.roles | toYaml | nindent 4 }}
+  {{- end }}
+{{- end }}
+---
+{{- if .Values.opensearch.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "reana.prefix" . }}-opensearch-secrets
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/resource-policy": keep
+type: Opaque
+data:
+  REANA_OPENSEARCH_PASSWORD: {{ .Values.components.reana_workflow_controller.environment.REANA_OPENSEARCH_PASSWORD | default "reana" | b64enc }}
+  {{- if not .Values.opensearch.securityConfig.enabled }}
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ .Values.opensearch.initialAdminPassword | default "reana" | b64enc }} # dev environment only
+  {{- end }}
+{{- end }}