diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 60191587..b29fe98d 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -1391,6 +1391,9 @@ set -x YAML_FOLDER="/kubectl-build-deploy/lagoon/deploymentconfigs-pvcs-cronjobs-backups" mkdir -p $YAML_FOLDER +# handle multiple pvc +. /kubectl-build-deploy/scripts/exec-gather-volumes.sh + for SERVICE_TYPES_ENTRY in "${SERVICE_TYPES[@]}" do IFS=':' read -ra SERVICE_TYPES_ENTRY_SPLIT <<< "$SERVICE_TYPES_ENTRY" diff --git a/legacy/helmcharts/cli-persistent/templates/_helpers.tpl b/legacy/helmcharts/cli-persistent/templates/_helpers.tpl index ddb0f7c5..389a51b2 100644 --- a/legacy/helmcharts/cli-persistent/templates/_helpers.tpl +++ b/legacy/helmcharts/cli-persistent/templates/_helpers.tpl @@ -96,3 +96,7 @@ Generate path for twig storage emptyDir {{- define "cli-persistent.twig-storage.path" -}} {{- printf "%s/php" .Values.persistentStorage.path }} {{- end -}} + +{{- define "custom-pvc.name" -}} +{{- printf "lcv-%s" . | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/legacy/helmcharts/cli-persistent/templates/deployment.yaml b/legacy/helmcharts/cli-persistent/templates/deployment.yaml index 86935666..a42d1047 100644 --- a/legacy/helmcharts/cli-persistent/templates/deployment.yaml +++ b/legacy/helmcharts/cli-persistent/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{- $topValues := .Values }} apiVersion: apps/v1 kind: Deployment metadata: @@ -40,6 +41,19 @@ spec: claimName: {{ .Values.persistentStorage.name }} - name: {{ include "cli-persistent.twig-storage.name" . | quote }} emptyDir: {} + {{- range .Values.customVolumeMounts }} + {{- range $volumeName, $value := . }} + {{- range $i, $servicePathMap := $value }} + {{- range $serviceName, $path := $servicePathMap }} + {{- if eq $serviceName $.Release.Name }} + - name: {{ template "custom-pvc.name" $volumeName }} + persistentVolumeClaim: + claimName: {{ template "custom-pvc.name" $volumeName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if .Values.dynamicSecretVolumes }} {{- toYaml .Values.dynamicSecretVolumes | nindent 8 }} {{- end }} @@ -73,6 +87,18 @@ spec: mountPath: {{ .Values.persistentStorage.path | quote }} - name: {{ include "cli-persistent.twig-storage.name" . | quote }} mountPath: {{ include "cli-persistent.twig-storage.path" . | quote }} + {{- range .Values.customVolumeMounts }} + {{- range $volumeName, $value := . }} + {{- range $i, $servicePathMap := $value }} + {{- range $serviceName, $path := $servicePathMap }} + {{- if eq $serviceName $.Release.Name }} + - name: {{ template "custom-pvc.name" $volumeName }} + mountPath: {{ $path }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if .Values.dynamicSecretMounts }} {{- toYaml .Values.dynamicSecretMounts | nindent 12 }} {{- end }} diff --git a/legacy/helmcharts/cli-persistent/values.yaml b/legacy/helmcharts/cli-persistent/values.yaml index 1df2ddc6..480eefdc 100644 --- a/legacy/helmcharts/cli-persistent/values.yaml +++ b/legacy/helmcharts/cli-persistent/values.yaml @@ -11,7 +11,7 @@ gitSha: "" image: "" persistentStorage: - name: "" + name: "cli" imagePullPolicy: Always diff --git a/legacy/helmcharts/custom-pvc/.helmignore b/legacy/helmcharts/custom-pvc/.helmignore new file mode 100644 index 00000000..fbe01f88 --- /dev/null +++ b/legacy/helmcharts/custom-pvc/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/legacy/helmcharts/custom-pvc/Chart.yaml b/legacy/helmcharts/custom-pvc/Chart.yaml new file mode 100644 index 00000000..65705cb2 --- /dev/null +++ b/legacy/helmcharts/custom-pvc/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v2 +name: custom-pvc +description: A Helm chart for Kubernetes creating lagoon custom-pvc + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 \ No newline at end of file diff --git a/legacy/helmcharts/custom-pvc/templates/_helpers.tpl b/legacy/helmcharts/custom-pvc/templates/_helpers.tpl new file mode 100644 index 00000000..1fe8705f --- /dev/null +++ b/legacy/helmcharts/custom-pvc/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* + +{{/* + Create chart name and version as used by the chart label. + */}} + {{- define "custom-pvc.chart" -}} + {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} + {{- end -}} + + +{{- define "custom-pvc.name" -}} +{{- printf "lcv-%s" .IterationValues | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} + +{{- define "custom-pvc.labels" -}} +helm.sh/chart: {{ template "custom-pvc.chart" .Context }} +{{ include "custom-pvc.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Context.Release.Service }} +{{ include "custom-pvc.lagoonLabels" .Context }} +{{- end -}} + +{{/* +Selector labels +*/}} + +{{- define "custom-pvc.selectorLabels" -}} +app.kubernetes.io/name: {{ include "custom-pvc.name" . }} +app.kubernetes.io/instance: {{ .Context.Release.Name }} +{{- end -}} + +{{/* +Lagoon Labels +*/}} +{{- define "custom-pvc.lagoonLabels" -}} +lagoon.sh/service: {{ .Release.Name }} +lagoon.sh/service-type: {{ .Chart.Name }} +lagoon.sh/project: {{ .Values.project }} +lagoon.sh/environment: {{ .Values.environment }} +lagoon.sh/environmentType: {{ .Values.environmentType }} +lagoon.sh/buildType: {{ .Values.buildType }} +{{- end -}} + + +{{/* +Annotations +*/}} +{{- define "custom-pvc.annotations" -}} +lagoon.sh/version: {{ .Values.lagoonVersion | quote }} +{{- if .Values.branch }} +lagoon.sh/branch: {{ .Values.branch | quote }} +{{- end }} +{{- if .Values.prNumber }} +lagoon.sh/prNumber: {{ .Values.prNumber | quote }} +lagoon.sh/prHeadBranch: {{ .Values.prHeadBranch | quote }} +lagoon.sh/prBaseBranch: {{ .Values.prBaseBranch | quote }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/legacy/helmcharts/custom-pvc/templates/pvcs.yaml b/legacy/helmcharts/custom-pvc/templates/pvcs.yaml new file mode 100644 index 00000000..0cca784f --- /dev/null +++ b/legacy/helmcharts/custom-pvc/templates/pvcs.yaml @@ -0,0 +1,23 @@ +{{ $context := . }} +{{- range .Values.customVolumes }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "custom-pvc.name" (dict "IterationValues" . "Context" $context ) }} + labels: + {{- include "custom-pvc.labels" (dict "IterationValues" . "Context" $context ) | nindent 4 }} +{{/* + annotations: + { { - include "custom-pvc.annotations" $context | nindent 4 } } +*/}} +spec: + accessModes: + - ReadWriteMany + storageClassName: bulk + resources: + requests: + storage: {{ $context.Values.persistentStorage.size | quote }} + +{{- end }} + diff --git a/legacy/helmcharts/custom-pvc/values.yaml b/legacy/helmcharts/custom-pvc/values.yaml new file mode 100644 index 00000000..c8b86a5e --- /dev/null +++ b/legacy/helmcharts/custom-pvc/values.yaml @@ -0,0 +1,11 @@ +# Default values for custom-pvc. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# fastly: +# apiToken: '' +# platformTLSConfiguration: '' + +# this is the default persistentStorage size for PVCs +persistentStorage: + size: 5Gi \ No newline at end of file diff --git a/legacy/helmcharts/nginx-php-persistent/templates/_helpers.tpl b/legacy/helmcharts/nginx-php-persistent/templates/_helpers.tpl index 0d290874..5a1e103c 100644 --- a/legacy/helmcharts/nginx-php-persistent/templates/_helpers.tpl +++ b/legacy/helmcharts/nginx-php-persistent/templates/_helpers.tpl @@ -135,3 +135,8 @@ Generate path for twig storage emptyDir {{- define "nginx-php-persistent.twig-storage.path" -}} {{- printf "%s/php" .Values.persistentStorage.path }} {{- end -}} + + +{{- define "custom-pvc.name" -}} +{{- printf "lcv-%s" . | trunc 63 | trimSuffix "-" -}} +{{- end -}} \ No newline at end of file diff --git a/legacy/helmcharts/nginx-php-persistent/templates/deployment.yaml b/legacy/helmcharts/nginx-php-persistent/templates/deployment.yaml index 579f1158..df4cb967 100644 --- a/legacy/helmcharts/nginx-php-persistent/templates/deployment.yaml +++ b/legacy/helmcharts/nginx-php-persistent/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{- $topValues := .Values }} apiVersion: apps/v1 kind: Deployment metadata: @@ -36,6 +37,19 @@ spec: claimName: {{ include "nginx-php-persistent.persistentStorageName" . }} - name: {{ include "nginx-php-persistent.twig-storage.name" . | quote }} emptyDir: {} + {{- range .Values.customVolumeMounts }} + {{- range $volumeName, $value := . }} + {{- range $i, $servicePathMap := $value }} + {{- range $serviceName, $path := $servicePathMap }} + {{- if eq $serviceName $.Release.Name }} + - name: {{ template "custom-pvc.name" $volumeName }} + persistentVolumeClaim: + claimName: {{ template "custom-pvc.name" $volumeName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if .Values.dynamicSecretVolumes }} {{- toYaml .Values.dynamicSecretVolumes | nindent 8 }} {{- end }} @@ -105,6 +119,18 @@ spec: volumeMounts: - name: {{ include "nginx-php-persistent.persistentStorageName" . }} mountPath: {{ .Values.persistentStorage.path | quote }} + {{- range .Values.customVolumeMounts }} + {{- range $volumeName, $value := . }} + {{- range $i, $servicePathMap := $value }} + {{- range $serviceName, $path := $servicePathMap }} + {{- if eq $serviceName $.Release.Name }} + - name: {{ template "custom-pvc.name" $volumeName }} + mountPath: {{ $path }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if .Values.dynamicSecretMounts }} {{- toYaml .Values.dynamicSecretMounts | nindent 12 }} {{- end }} @@ -142,6 +168,18 @@ spec: mountPath: {{ .Values.persistentStorage.path | quote }} - name: {{ include "nginx-php-persistent.twig-storage.name" . | quote }} mountPath: {{ include "nginx-php-persistent.twig-storage.path" . | quote }} + {{- range .Values.customVolumeMounts }} + {{- range $volumeName, $value := . }} + {{- range $i, $servicePathMap := $value }} + {{- range $serviceName, $path := $servicePathMap }} + {{- if eq $serviceName $.Release.Name }} + - name: {{ template "custom-pvc.name" $volumeName }} + mountPath: {{ $path }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if .Values.dynamicSecretMounts }} {{- toYaml .Values.dynamicSecretMounts | nindent 12 }} {{- end }} diff --git a/legacy/helmcharts/nginx-php-persistent/values.yaml b/legacy/helmcharts/nginx-php-persistent/values.yaml index 6ee09e5b..c7ef9660 100644 --- a/legacy/helmcharts/nginx-php-persistent/values.yaml +++ b/legacy/helmcharts/nginx-php-persistent/values.yaml @@ -81,3 +81,4 @@ cronjobUseSpot: false dynamicSecretMounts: [] dynamicSecretVolumes: [] + diff --git a/legacy/scripts/exec-gather-volumes.sh b/legacy/scripts/exec-gather-volumes.sh new file mode 100755 index 00000000..83493442 --- /dev/null +++ b/legacy/scripts/exec-gather-volumes.sh @@ -0,0 +1,91 @@ +#!/bin/bash + + +if [[ -z "$DOCKER_COMPOSE_YAML" ]]; then + echo "no docker compose file given" + exit +fi + +EXTRA_MOUNT_VALUES_FILE="/${KBD_SERVICE_VALUES_OUTDIR:-kubectl-build-deploy}/extravolumes-values.yaml" +touch ${EXTRA_MOUNT_VALUES_FILE} + +# The prefix below CAN be used, but this should be moved into the helm templates +CUSTOMVOLUME_PREFIX="" # we just use this to distinguish from any other volumes that might be created + +# Parse docker-compose.yml and extract volume names with "lagoon.type: persistent" label +volumes=$(yq e '.volumes | with_entries(select(.value.labels."lagoon.type" == "persistent")) | keys | .[]' "$DOCKER_COMPOSE_YAML") + +# Print the list of volume names +echo "Extra volumes defined:" +echo "$volumes" +echo + +# Create an array to store the volumes that need to be created +volumes_to_create=() +EXTRA_VOLUMES_MOUNT_VALS="\ +customVolumeMounts: +" #this will be output to our values file + +# Iterate over the volumes +for volume in $volumes; do +# echo "Volume: $volume" + EXTRA_VOLUMES_MOUNT_VALS+="\ + - $CUSTOMVOLUME_PREFIX$volume: +" + # Loop through the services and check if they reference the current volume + services=$(yq e '.services | to_entries | .[] | select(.value.labels | has("lagoon.volumes.'$volume'.path")) | .key' "$DOCKER_COMPOSE_YAML") + + # Print the services and their corresponding paths for the current volume + while IFS= read -r service; do + path=$(yq e '.services."'$service'".labels."lagoon.volumes.'$volume'.path"' "$DOCKER_COMPOSE_YAML") +# echo "- Service: $service, Path: $path" + + if [[ "$service" != "" ]]; then + EXTRA_VOLUMES_MOUNT_VALS+="\ + - $service: $path +" + fi + + done <<< "$services" + + # If no services reference the volume, print a message indicating it is not used + if [[ -z "$services" ]]; then + echo "- Not used" + else + # Add the volume to the array of volumes that need to be created + volumes_to_create+=("$volume") + fi + + echo +done + +echo "Volumes to be created:" +echo "${volumes_to_create[@]}" + +EXTRA_VOLUMES_VALUES_YAML="" + + +# Check if volumes_to_create array is not empty before iterating +if [[ ${#volumes_to_create[@]} -gt 0 ]]; then + echo "Volumes to be created:" + EXTRA_VOLUMES_VALUES_YAML+="\ +customVolumes: +" + for volume in "${volumes_to_create[@]}"; do + echo "- $CUSTOMVOLUME_PREFIX$volume" + EXTRA_VOLUMES_VALUES_YAML+="\ + - $volume +" + done +else + echo "No volumes to create." +fi + +echo "$EXTRA_VOLUMES_VALUES_YAML" +echo "$EXTRA_VOLUMES_MOUNT_VALS" + +echo "$EXTRA_VOLUMES_VALUES_YAML" > $EXTRA_MOUNT_VALUES_FILE +echo "$EXTRA_VOLUMES_MOUNT_VALS" >> $EXTRA_MOUNT_VALUES_FILE + + +helm template custom-pvc /kubectl-build-deploy/helmcharts/custom-pvc -f $EXTRA_MOUNT_VALUES_FILE -f /kubectl-build-deploy/values.yaml > $YAML_FOLDER/pvc.yaml \ No newline at end of file diff --git a/legacy/scripts/exec-kubectl-resources-with-images.sh b/legacy/scripts/exec-kubectl-resources-with-images.sh index 669026d5..4e714d6b 100755 --- a/legacy/scripts/exec-kubectl-resources-with-images.sh +++ b/legacy/scripts/exec-kubectl-resources-with-images.sh @@ -16,15 +16,15 @@ if [[ $(helm show values /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} | grep # Add the Image Hash as Parameter of "[SERVICETYPE]_SERVICE_IMAGE" HELM_SET_VALUES+=(--set "images.${line}=${DEPLOYMENT_SERVICETYPE_IMAGE_NAME_HASH}") done < <(cat /kubectl-build-deploy/helmcharts/${SERVICE_TYPE}/values.yaml | shyaml keys images) - helm template ${SERVICE_NAME} /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} -f /kubectl-build-deploy/values.yaml -f /kubectl-build-deploy/${SERVICE_NAME}-values.yaml --set service_name="${SERVICE_NAME}" "${HELM_SET_VALUES[@]}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/${SERVICE_NAME}.yaml + helm template ${SERVICE_NAME} /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} -f /kubectl-build-deploy/values.yaml -f /kubectl-build-deploy/${SERVICE_NAME}-values.yaml -f /kubectl-build-deploy/extravolumes-values.yaml --set service_name="${SERVICE_NAME}" "${HELM_SET_VALUES[@]}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/${SERVICE_NAME}.yaml # check if we need a single image to inject elif [[ $(helm show values /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} | grep image) ]]; then SERVICE_NAME_IMAGE="${MAP_SERVICE_NAME_TO_IMAGENAME[${SERVICE_NAME}]}" SERVICE_NAME_IMAGE_HASH="${IMAGE_HASHES[${SERVICE_NAME_IMAGE}]}" cat /kubectl-build-deploy/${SERVICE_NAME}-values.yaml - helm template ${SERVICE_NAME} /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} -f /kubectl-build-deploy/values.yaml -f /kubectl-build-deploy/${SERVICE_NAME}-values.yaml --set image="${SERVICE_NAME_IMAGE_HASH}" "${HELM_SET_VALUES[@]}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/${SERVICE_NAME}.yaml + helm template ${SERVICE_NAME} /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} -f /kubectl-build-deploy/values.yaml -f /kubectl-build-deploy/${SERVICE_NAME}-values.yaml -f /kubectl-build-deploy/extravolumes-values.yaml --set image="${SERVICE_NAME_IMAGE_HASH}" "${HELM_SET_VALUES[@]}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/${SERVICE_NAME}.yaml elif [ -d /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} ]; then cat /kubectl-build-deploy/${SERVICE_NAME}-values.yaml - helm template ${SERVICE_NAME} /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} -f /kubectl-build-deploy/values.yaml -f /kubectl-build-deploy/${SERVICE_NAME}-values.yaml "${HELM_SET_VALUES[@]}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/${SERVICE_NAME}.yaml + helm template ${SERVICE_NAME} /kubectl-build-deploy/helmcharts/${SERVICE_TYPE} -f /kubectl-build-deploy/values.yaml -f /kubectl-build-deploy/${SERVICE_NAME}-values.yaml -f /kubectl-build-deploy/extravolumes-values.yaml "${HELM_SET_VALUES[@]}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/${SERVICE_NAME}.yaml fi diff --git a/legacy/scripts/kubectl-build-deploy/extravolumes-values.yaml b/legacy/scripts/kubectl-build-deploy/extravolumes-values.yaml new file mode 100644 index 00000000..02a2aebb --- /dev/null +++ b/legacy/scripts/kubectl-build-deploy/extravolumes-values.yaml @@ -0,0 +1,8 @@ + + +customVolumes: + - testmount + +customVolumeMounts: + - testmount: + - cli: /testmount/ \ No newline at end of file