Merge pull request #22 from graphistry/v0.3.5

V0.3.5

.github/workflows/dev-cluster-deployment.yaml

@@ -24,6 +24,12 @@ on:
       grafana:
         description: 'would you like to launch grafana? (true,false)'
         required: true
+      #argo:
+      #  description: 'would you like to install argo? (true,false)'
+      #  required: false
+      #deploy-with-argo:
+      #  description: 'would you like to deploy the cluster with argoCD? (true,false)'
+      #  required: false        
 jobs:
   check-dev-true:
       runs-on: ubuntu-latest
@@ -90,6 +96,16 @@ jobs:
         if: "${{ !contains(github.event.inputs.scale-to-zero , 'true') }}"
         run:  docker-compose -f dev-compose/docker-compose.yml up cluster-setup
 
+# needs to only run once
+#      - name: setup argo
+#        if: "${{ contains(github.event.inputs.argo , 'true') }}"
+#        run:  docker-compose -f dev-compose/docker-compose.yml up deploy-argo
+
+# deploy graphistry-resources
+      - name: deploy graphistry resources
+        if: "${{ !contains(github.event.inputs.scale-to-zero , 'true') }}"
+        run: docker-compose -f dev-compose/docker-compose.yml up graphistry-resources
+
 # deploy to skinny dev cluster
       - name: deploy the dev cluster
         if: "${{ env.CLUSTER_NAME == 'skinny' && !contains(github.event.inputs.scale-to-zero , 'true') }}"

.github/workflows/docs-gen.yaml

@@ -0,0 +1,38 @@
+name: Generate chart Docs
+#description: 
+#  Triggers:
+#     - manual run
+#     - push to PR with label 'ci' and that job successfully running (and pushing to dockerhub)
+#  Results:
+#    - push VERSION as tag to graphistry/graph-app-kit if not already there
+#    - if VERSION tag already in gak repo, instead trigger gak repo dockerhubpublish
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  workflow_run:
+    workflows: [Release Charts]
+    types:
+      - completed
+
+jobs:
+
+  cancel_outstanding:
+    name: Detect and cancel outstanding runs of this workflow
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Cancel Previous Runs
+        if: ${{ (github.event.label.name == 'Release Charts' && !contains(github.event.pull_request.labels.*.name, 'skip-cancel')) }}
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+
+  docs-gen:
+    name: generate docs
+    #description: Public GPU env (VERSION_ENV -> VERSION)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2

.github/workflows/release.yaml

@@ -21,7 +21,12 @@ jobs:
           git config user.name "$GITHUB_ACTOR"
           git config user.email "[email protected]"
 
+## docker compose build
+      - name: get aux charts for release
+        run: bash chart-bundler/bundler.sh
+
       - name: Run chart-releaser
         uses: helm/[email protected]
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          CR_RELEASE_NOTES_FILE: "CHANGELOG.md"

.gitignore

@@ -69,7 +69,7 @@ instance/
 .scrapy
 
 # Sphinx documentation
-docs/_build/
+docs/build/
 
 # PyBuilder
 target/
@@ -130,6 +130,24 @@ dmypy.json
 velero-bootstrap/aws/velero_policy.json
 charts/graphistry-helm/velero_values_primary_cluster.yaml
 cluster-config/cluster-config.yml
-charts/kube-prometheus-stack
+
 charts/eks-dev-values.yaml
-charts/graphistry-jupyter-hub
+
+*.lock
+charts/*/Chart.lock
+
+charts/graphistry-helm/templates/_helpers.tpl
+
+*.tgz
+
+terraform/aws/.terraform
+terraform/aws/terraform.tfstate
+terraform/aws/terraform.tfstate.backup
+terraform/aws/.terraform.tfstate.lock.info
+
+cluster-config/cluster-config.yaml
+
+charts/graphistry-helm/templates/postgres/pg-sts.yaml
+
+
+chart-bundle

.readthedocs.yaml

@@ -0,0 +1,22 @@
+
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+
+# Optionally build your docs in additional formats such as PDF
+formats:
+  - pdf
+  - htmlzip
+  - epub
+
+# Build configuration
+python:
+   install:
+   - requirements: docs/requirements.txt

CHANGELOG.md

@@ -1,4 +1,8 @@
-# Changelog
+
+# graphistry-helm Version Release Notes
+
+## Changelog
+
 
 All notable changes to the graphistry-helm repo are documented in this file. Additional Graphistry components are tracked in the main [Graphistry major release history documentation](https://graphistry.zendesk.com/hc/en-us/articles/360033184174-Enterprise-Release-List-Downloads).
 
@@ -7,13 +11,53 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ## [Development]
 
-## [0.3.4 - 2022-09-10]
+
+## [Version 0.3.5 - 2023-02-13]
 
 ### Breaking 🔥
+*   Removed the nexus migration job and replaced it with a strategy to rollout our deployment exactly 
+    the same way as we do with our docker-compose version.
 
+*   Changed StorageClass name to note be namespaced scoped. 
+    This will allow us to use the same storage class for all namespaces.
 ### Added
 
+*   Added Postgres operator and changed our postgres deployment into an operator controlled PostgresCluster.
+    This allows us to use the Postgres operator to manage our Postgres deployment and also allows us to use the Postgres operator to manage our Postgres backups, as well as giving us the option to run postgres in a Highly Available configuration for redundancy. The Operator also allows us to backup our Postgres database to S3/GCS/Azure storage and other S3 compatible storage. As well as giving us the option to encrypt communication between our database and our services.
+
+*   Added Dask operator to control our Dask cuda Scheduler and Workers. 
+    This will allow us to scale up and down the number of workers as needed. 
+    Temporary workaround for service name issue with the operator, 
+    currently unable to set the service name to `dask-scheduler` in the scheduler.service spec 
+    so we are using a service named `dask-scheduler` instead.
+
+*   Added ArgoCD with a "app of apps" deployment pattern to manage our deployment. This gives us the 
+    the ability to manage our deployment with a gitops pattern.     
+
+*   Created Docs -> [Graphistry Helm Docs](https://readthedocs.org/projects/graphistry-helm/)
+
+*   Added volume names to automatically bind PVC to PV after provisioning upon redeployment
+
+*   Added a volume selector to postgres cluster to bind the PVC to the PV after provisioning upon redeployment
+
+*   Added ability to configure Forge ETL python resource limits and number of workers
+
+*   Added Morpheus and MLFlow plugin charts 
+
+
+
 ### Changed
 
+*   Made network policy optional, with a strict mode and default mode set to false as default (No policy in use by default).
+
+*   Removed the nexus migration job and replaced it with a strategy to rollout our deployment exactly 
+    the same way as we do with our docker-compose version.
+
+*   Reorganized the values.yaml override file to be more readable.
+
+*   Adjusted the PVCs for the charts to be more persistent.
+
 ### Fixed
 
+*   Fixed the pvc retention issue by setting the pvc name in the values.yaml so the PV and PVC automatically bind after provisioning upon redeployment
+

README.md

@@ -88,3 +88,7 @@ and then set the nodeSelector value to the hostname of the selected node along w
      --set domain = <FQDN or node external IP ex: example.com> \
      --set imagePullSecrets=<secrets_name>  (has to go last) 
 > **Note:** different labels can be used for the nodeSelector value, but some labels between the nodes may not be unique.
+
+
+[ReadTheDocs](docs/build/html/index.html)
+

acr-bootstrap/import-image-into-acr-from-dockerhub.sh

@@ -65,7 +65,7 @@ import_if_missing ()
   echo "... Finished handling $IMAGE"
 }
 
-import_nginx_ingress_into_acr(){
+import_into_acr(){
   IMAGE=$1
   TAG=$2
   echo "Importing image if missing: $IMAGE:$TAG from $SOURCE_REGISTRY"
@@ -108,9 +108,9 @@ import_if_missing "redis:6.2.6" "library"
 
 echo "Importing nginx ingress controller images into ACR"
 
-import_nginx_ingress_into_acr "$CONTROLLER_IMAGE" "$CONTROLLER_TAG"
-import_nginx_ingress_into_acr "$PATCH_IMAGE" "$PATCH_TAG"
-import_nginx_ingress_into_acr "$DEFAULTBACKEND_IMAGE" "$DEFAULTBACKEND_TAG"
+import_into_acr "$CONTROLLER_IMAGE" "$CONTROLLER_TAG"
+import_into_acr "$PATCH_IMAGE" "$PATCH_TAG"
+import_into_acr "$DEFAULTBACKEND_IMAGE" "$DEFAULTBACKEND_TAG"
 
 echo "importing k8s wait for container (initcontainer for graphistry) into acr "
 
@@ -121,3 +121,32 @@ echo " importing netshoot container for dns/http optional debugging"
 
 import_if_missing "netshoot:latest" "nicolaka"
 
+
+echo "importing dask operator into acr"
+
+SOURCE_REGISTRY=ghcr.io/dask
+
+import_into_acr "dask-kubernetes-operator" "2022.7.0"
+
+echo "postgres operator into acr"
+
+SOURCE_REGISTRY=registry.developers.crunchydata.com/crunchydata
+
+import_into_acr "crunchy-pgbackrest" "ubi8-2.40-1"
+
+import_into_acr "crunchy-postgres" "ubi8-14.5-1"
+
+import_into_acr "postgres-operator" "ubi8-5.2.0-0"
+
+import_into_acr "postgres-operator-upgrade" "ubi8-5.2.0-0"
+
+#import_into_acr "crunchy-upgrade" "ubi8-5.2.0-0"
+
+#import_into_acr "crunchy-postgres-exporter" "ubi8-5.2.0-0"
+
+#import_into_acr "crunchy-pgbouncer" "ubi8-1.17-1"
+
+#import_into_acr "crunchy-pgadmin4" "ubi8-4.30-4"
+SOURCE_REGISTRY=quay.io/martinhelmich
+
+#import_into_acr "prometheus-nginxlog-exporter" "v1.9.2"

cd/argo-apps/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v2
+name: graphistry-ci
+version: 1.0.0

cd/argo-apps/templates/argo-cd.yaml

@@ -0,0 +1,31 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argo-cd
+  namespace: argo-cd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  labels:
+    name: argo
+spec:
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argo-cd
+  project: default
+  source:
+    path: charts/argo-cd
+    repoURL: https://github.com/graphistry/argo-helm.git
+    targetRevision: HEAD #BRANCH OR HEAD WHEN IN MAIN
+    helm:
+      valueFiles:
+        - ./argo-values.yaml # resolved against source.repoURL and source.path
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+
+##helm template charts/argo-cd/apps/ | kubectl apply -f -
+
+#helm upgrade -i argo ./charts/argo-cd  --namespace argo --create-namespace

cd/argo-apps/templates/graphistry-cd-templates/graphistry-cd.yaml

@@ -0,0 +1,30 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: graphistry-app-cd
+  namespace: argo-cd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  labels:
+    name: graphistry
+spec:
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: graphistry
+  project: default
+  source:
+    path: charts/graphistry-helm
+    repoURL: https://github.com/graphistry/graphistry-helm.git
+    targetRevision: nightly #BRANCH OR HEAD WHEN IN MAIN
+    helm:
+      valueFiles:
+        - ../values-overrides/internal/eks-dev-values.yaml # resolved against source.repoURL and source.path
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+
+
+##helm template cd/argo-apps/ | kubectl apply -f -

cd/argo-apps/templates/graphistry-cd-templates/graphistry-resource-cd.yaml

@@ -0,0 +1,29 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: graphistry-resources-cd
+  namespace: argo-cd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  labels:
+    name: graphistry
+spec:
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: graphistry
+  project: default
+  source:
+    path: charts/graphistry-helm-resources
+    repoURL: https://github.com/graphistry/graphistry-helm.git
+    targetRevision: nightly #BRANCH OR HEAD WHEN IN MAIN
+    helm:
+      valueFiles:
+        - ../values-overrides/internal/eks-dev-values.yaml # resolved against source.repoURL and source.path
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+
+##helm template charts/argo-cd/apps/ | kubectl apply -f -