Merge pull request #342 from bjwswang/dev

feat: able to automatically build and publish lowcode components

Makefile

@@ -263,4 +263,13 @@ catalog-push: ## Push a catalog image.
 # Verify code validity
 .PHONY: verify
 verify:
-	@hack/verify-all.sh
+	@hack/verify-all.sh
+
+.PHONY: deploy-lowcode
+deploy-lowcode: kustomize
+	mkdir -p out && cd pipeline/lowcode && tar -zcf ../../out/template.tgz chart-template
+	kubectl create -nyunti-system --dry-run=client configmap chart-template-cm --from-file=./out/template.tgz -oyaml | kubectl apply -f -
+	$(KUSTOMIZE) build pipeline/lowcode | kubectl apply -f -	
+rerun: 
+	kubectl delete -f pipeline/lowcode/sample/publish.yaml
+	kubectl apply -f pipeline/lowcode/sample/publish.yaml

pipeline/lowcode/README.md

@@ -0,0 +1,165 @@
+# Lowcode pipelines
+
+> NOTE: All lowcode resources managed under namespace `yunti-system` by default
+
+Here we use [tekton](https://tekton.dev/) to build and publish a lowcode component. The overall workflow is:
+
+1. fetch source from minio
+2. build dockerimage for lowcode component and push it to docker registry(dockerhub/private registry)
+3. build helm charts for lowcode component with pre-defined [chart template](./chart-template/) and push this component to a private component repository(chartmuseum)
+
+
+## Dependent components
+
+- Component [Minio](https://github.com/kubebb/components/tree/main/charts/minio) to store lowcode component's source code
+- Component [Tekton](https://github.com/kubebb/components/tree/main/charts/tekton-operator) to run the workflow
+
+## Pipeline
+
+We defined a [pipeline lowcode-publish](./pipeline.yaml) which contains 3 sequencial tasks:
+
+- Task [fetch-source](./task-fetch-source.yaml)
+- Task [build-image](./task-build-image.yaml)
+- Task [build-chart](./task-build-chart.yaml)
+
+
+### Parameters
+
+|  Parameter |   Default |   Description     |
+|---------------|----------------|----------------|
+| SOURCE_MINIO_HOST | my-minio.kubebb-addons.svc.cluster.local | minio host/domain to fetch | 
+| SOURCE_PATH | "" | The path where stores the component lowcode materials |
+| SOURCE_SCHEMA_PATH | "schema.json" | The relative path of schema json file to `bucket/object/` |
+| SOURCE_MINIO_ACCESS_KEY | "" |  |
+| SOURCE_MINIO_SECRET_KEY | "" | |
+| APP_IMAGE | "" |  The component's image name along with tag | 
+| REPOSITORY_URL | http://chartmuseum.kubebb-addons.svc.cluster.local:8080 |  The url for the component repository |
+| REPOSITORY_USER | "" |  The username for repository auth |
+| REPOSITORY_PASSWORD | "" |  The password for the repository auth |
+
+### Chart render
+
+We use this [template](./chart-template/Chart.yaml) to render the chart.Below is the relevant mappings
+|   Key in schema or others  |   Key in Chart  |   Description     |
+|---------------|-----------------------------|----------------|
+| Schema .version           | Chart.yaml version          |  Component portal entry and path |
+| params IMAGE           | values.yaml image          |  Component portal entry and path |
+| Schema .meta.namespace          | Chart.yaml name              |  Component name |
+| Schema .meta.name               | Chart.yaml  annotations.core.kubebb.k8s.com.cn/displayname       |  Component display name |
+| Schema .meta.description        | Chart.yaml  description           |  Component's description    |
+| Schema .meta.git_url              | Chart.yaml  sources             |  Component's git url         |
+| Schema .meta.basename           | templates.portal.yaml .spec.entry & .spec.path           |  Component portal entry and path |
+
+## Sample
+
+Clone this project:
+
+```shell
+git clone https://github.com/kubebb/core.git
+cd core
+```
+
+### Prerequsites
+
+0. deploy kubebb 
+
+Follow [official document](https://kubebb.github.io/website/docs/quick-start/core_quickstart)
+
+1. deploy minio 
+
+```shell
+    kubectl apply -f https://raw.githubusercontent.com/kubebb/components/main/examples/minio/componentplan.yaml
+```
+
+2. deploy chartmuseum 
+
+```shell
+    kubectl apply -f https://raw.githubusercontent.com/kubebb/components/main/examples/chartmuseum/componentplan.yaml
+```
+
+3. deploy tekton-operator with https://github.com/kubebb/components/tree/main/examples/tekton-operator
+
+```shell
+    kubectl apply -f https://raw.githubusercontent.com/kubebb/components/main/examples/tekton-operator/componentplan.yaml
+```
+
+4. deploy the lowcode pipelines
+
+```shell
+make deploy-lowcode
+```
+
+### Build component `hello-world`
+
+#### 1. Upload component's schema to minio
+
+- Bucket: `yunti`
+- Object: `hello-world`
+
+Upload `pipeline/lowcode/sample/schema.json` under `yunti/hello-world`
+
+
+#### 2. Apply resources to pre-publish
+
+1. Update dockerconfig
+
+To push image to image registry,we mount this [`dockerconfig-secret`](./sample/pre-publish.yaml#1) in the kaniko pod. So make sure you have created this secret with the correct auth info.
+
+2. Update Dockerfile
+
+This pipelinerun use `dockerfile` configured in configmap [dockerfile-cm](./sample/pre-publish.yaml#13). Update according to your need.
+
+3. Update pvc 
+
+This pipelinerun requires a PVC `sample-publish-ws-pvc` to store source and artifacts. You need to update it based on your environment.
+
+4. Apply this [pre-publish](./sample/pre-publish.yaml)
+
+```shell
+    kubectl apply -f pipeline/lowcode/sample/pre-publish.yaml
+```
+
+#### 4. Apply resources(pipelinerun) to build and publish this component
+
+```shell
+    kubectl apply -f pipeline/lowcode/sample/publish.yaml
+```
+
+#### 5. Watch pipelinerun
+
+```shell
+    kubectl get pods --watch -nyunti-system
+```
+
+
+#### 6. Get the component 
+
+> Make sure you have add the above chartmuseum repo .In case you didnt, you can add it with this [componentplan](https://github.com/kubebb/components/blob/main/examples/chartmuseum/repository_chartmuseum.yaml)
+
+```shell
+❯ kubectl get components -nkubebb-system -l kubebb.component.repository=chartmuseum
+NAME                          AGE
+chartmuseum.hello-world   4m18s
+```
+
+#### 7. Deploy the above component
+
+```shell
+    kubectl apply -f pipeline/lowcode/sample/install-component.yaml
+```
+
+Check the install status:
+
+```shell
+kubectl get componentplan hello-world -oyaml
+```
+
+When `InstallSuccess`, you can access this component by:
+
+1. do port-forward
+
+```shell
+ kubectl port-forward svc/hello-world 8066:8066
+```
+
+2. access in browser `http://localhost:8066/umi-demo-public`

pipeline/lowcode/chart-template/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v2
+name: TEMPLATE_NAMESPACE
+annotations:
+  core.kubebb.k8s.com.cn/displayname: TEMPLATE_NAME
+description: TEMPLATE_DESCRIPTION
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: TEMPLATE_VERSION
+
+sources:
+  - TEMPLATE_GIT_URL
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application

pipeline/lowcode/chart-template/templates/deployment.yaml

@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: TEMPLATE_NAMESPACE 
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: TEMPLATE_NAMESPACE 
+spec:
+  selector:
+    matchLabels:
+      app: TEMPLATE_NAMESPACE 
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: TEMPLATE_NAMESPACE 
+    spec:
+      containers:
+      - image: {{ .Values.image }}
+        imagePullPolicy: {{ .Values.imagePullPolicy }}
+        name: TEMPLATE_NAMESPACE 
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources: {{ toYaml .Values.resources | nindent 10 }}
+      terminationGracePeriodSeconds: 30

pipeline/lowcode/chart-template/templates/ingress.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.ingress.enable }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    description: TEMPLATE_NAMESPACE-ingress
+    displayName: TEMPLATE_NAMESPACE-ingress
+    httpSend: /
+    ingress-lb: {{ .Values.ingress.ingressClassName }}
+    kubernetes.io/ingress.class: {{ .Values.ingress.ingressClassName }}
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_cache static-cache;
+      proxy_cache_valid 404 10m;
+      proxy_cache_use_stale error timeout updating http_404 http_500 http_502 http_503 http_504;
+      proxy_cache_bypass $http_x_purge;
+      add_header X-Cache-Status $upstream_cache_status; # check X-Cache-Status to see if it's HIT
+      rewrite ^/(TEMPLATE_NAMESPACE-apis)(/|$)(.*)$ /$3 break;
+    nginx.ingress.kubernetes.io/enable-access-log: "false"
+    nginx.ingress.kubernetes.io/enable-rewrite-log: "false"
+    nginx.ingress.kubernetes.io/load-balance: round_robin
+    nginx.ingress.kubernetes.io/proxy-body-size: ""
+    nginx.ingress.kubernetes.io/proxy-buffering: "on"
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "60"
+    nginx.ingress.kubernetes.io/server-alias: ""
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/upstream-vhost: $host
+  labels:
+    ingress-lb: {{ .Values.ingress.ingressClassName }}
+  name: TEMPLATE_NAMESPACE
+  namespace: {{ .Release.Namespace }}
+spec:
+  rules:
+    - host: portal.{{ .Values.ingress.ingressDomain }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: TEMPLATE_NAMESPACE
+                port:
+                  number: 80
+            path: TEMPLATE_BASENAME-public
+            pathType: ImplementationSpecific
+{{- end }}

pipeline/lowcode/chart-template/templates/portal.yaml

@@ -0,0 +1,7 @@
+apiVersion: core.kubebb.k8s.com.cn/v1alpha1
+kind: Portal
+metadata:
+  name: TEMPLATE_NAMESPACE
+spec:
+  entry: /TEMPLATE_NAMESPACE-public/index.html
+  path: /TEMPLATE_NAMESPACE

pipeline/lowcode/chart-template/templates/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: TEMPLATE_NAMESPACE 
+  name: TEMPLATE_NAMESPACE 
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 80
+  selector:
+    app: TEMPLATE_NAMESPACE 
+  sessionAffinity: None
+  type: ClusterIP

pipeline/lowcode/chart-template/values.yaml

@@ -0,0 +1,16 @@
+image: TEMPLATE_IMAGE
+imagePullPolicy: IfNotPresent
+resources:
+  limits:
+    cpu: 5000m
+    memory: 1280Mi
+  requests:
+    cpu: 500m
+    memory: 512Mi
+
+# ingress configurations for component
+ingress:
+  # set enable to `true` to enable ingress
+  enable: false
+  ingressClassName: portal-ingress
+  ingressDomain: 172.18.0.2.nip.io

pipeline/lowcode/kustomization.yaml

@@ -0,0 +1,8 @@
+# Adds namespace to all resources.
+namespace: yunti-system
+
+bases:
+  - ./pipeline.yaml
+  - ./task-fetch-source.yaml
+  - ./task-build-image.yaml
+  - ./task-build-chart.yaml