From c205e75bd255e366165a77b3d06b73b367a9ff6f Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:10:29 +0200 Subject: [PATCH 01/12] Foundation Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-discovery.yml | 48 ++++++- .../stages/helm-chart-verification.yml | 22 +++ .../.helmignore | 21 +++ .../Chart.yaml | 13 ++ .../promitor-agent-resource-discovery/LICENSE | 21 +++ .../README.md | 136 ++++++++++++++++++ .../templates/NOTES.txt | 18 +++ .../templates/_helpers.tpl | 54 +++++++ .../templates/configmap.yaml | 73 ++++++++++ .../templates/deployment.yaml | 79 ++++++++++ .../templates/podsecuritypolicy.yaml | 39 +++++ .../templates/role.yaml | 20 +++ .../templates/rolebinding.yaml | 18 +++ .../templates/secret.yaml | 18 +++ .../templates/service.yaml | 31 ++++ .../templates/serviceaccount.yaml | 14 ++ .../values.yaml | 117 +++++++++++++++ 17 files changed, 741 insertions(+), 1 deletion(-) create mode 100644 build/azure-devops/templates/stages/helm-chart-verification.yml create mode 100644 charts/promitor-agent-resource-discovery/.helmignore create mode 100644 charts/promitor-agent-resource-discovery/Chart.yaml create mode 100644 charts/promitor-agent-resource-discovery/LICENSE create mode 100644 charts/promitor-agent-resource-discovery/README.md create mode 100644 charts/promitor-agent-resource-discovery/templates/NOTES.txt create mode 100644 charts/promitor-agent-resource-discovery/templates/_helpers.tpl create mode 100644 charts/promitor-agent-resource-discovery/templates/configmap.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/deployment.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/podsecuritypolicy.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/role.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/rolebinding.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/secret.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/service.yaml create mode 100644 charts/promitor-agent-resource-discovery/templates/serviceaccount.yaml create mode 100644 charts/promitor-agent-resource-discovery/values.yaml diff --git a/build/azure-devops/agents-ci-discovery.yml b/build/azure-devops/agents-ci-discovery.yml index a7ea1946e..9fd2dad93 100644 --- a/build/azure-devops/agents-ci-discovery.yml +++ b/build/azure-devops/agents-ci-discovery.yml @@ -21,6 +21,10 @@ variables: value: 'promitor-discovery-agent' - name: App.Version value: '0.0.0-$(Image.Tag)' + - name: Helm.Chart.Version + value: '0.0.0-$(App.Version)' + - name: Helm.Chart.CI.Name + value: 'promitor-agent-resource-discovery-ci' stages: - stage: Init displayName: Prepare Build @@ -209,4 +213,46 @@ stages: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - template: templates/docker/push-image.yml parameters: - imageName: '$(Image.TaggedName)' \ No newline at end of file + imageName: '$(Image.TaggedName)' +- template: templates/stages/helm-chart-verification.yml + # parameters: + # name: Full + # testFile: tests/fullSuite.js +# - stage: Helm3 +# displayName: Helm Chart (3.x) +# dependsOn: Docker +# variables: +# Helm.Version: '3.0.0' +# jobs: +# - job: Helm3_x +# displayName: Helm 3.x - Lint, Package & Push Chart +# condition: succeeded() +# pool: +# vmImage: ubuntu-16.04 +# steps: +# - download: current +# artifact: variables +# - template: templates/utils/read-variable-on-linux.yml +# parameters: +# variableName: 'Image.Tag' +# - template: templates/helm/lint-chart.yml +# parameters: +# helmVersion: '$(Helm.Version)' +# chartName: '$(Chart.Scraper.Name)' +# - template: templates/helm/package-preview-chart.yml +# parameters: +# chartName: '$(Helm.Chart.CI.Name)' +# chartVersion: '$(Helm.Chart.Version)' +# appVersion: '$(App.Version)' +# imageName: '$(Image.Name)' +# - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: +# - template: templates/helm/push-chart.yml +# parameters: +# chartName: '$(Helm.Chart.CI.Name)' +# chartVersion: '$(Helm.Chart.Version)' +# - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: +# - task: PublishBuildArtifacts@1 +# displayName: 'Publish Helm Chart' +# inputs: +# PathtoPublish: charts/output/$(Helm.Chart.CI.Name)-$(Helm.Chart.Version).tgz +# ArtifactName: helm diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml new file mode 100644 index 000000000..3f74dbabe --- /dev/null +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -0,0 +1,22 @@ +# parameters: +# name: '' +# testFile: '' + +stages: +- stage: Helm3 + displayName: Helm Chart (3.x) + dependsOn: Docker + variables: + Helm.Version: '3.0.0' + jobs: + - job: Helm3_x + displayName: Helm 3.x - Lint, Package & Push Chart + condition: succeeded() + pool: + vmImage: ubuntu-16.04 + steps: + - download: current + artifact: variables + - template: templates/utils/read-variable-on-linux.yml + parameters: + variableName: 'Image.Tag' \ No newline at end of file diff --git a/charts/promitor-agent-resource-discovery/.helmignore b/charts/promitor-agent-resource-discovery/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/charts/promitor-agent-resource-discovery/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/promitor-agent-resource-discovery/Chart.yaml b/charts/promitor-agent-resource-discovery/Chart.yaml new file mode 100644 index 000000000..f9c2b77cd --- /dev/null +++ b/charts/promitor-agent-resource-discovery/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +version: 0.2.0 +appVersion: 1.0.0 +type: application +name: promitor-agent-resource-discovery +description: Promitor, bringing Azure Monitor metrics where you need them. +home: https://promitor.io +sources: + - https://github.com/tomkerkhove/promitor +maintainers: + - name: Tom Kerkhove + url: https://github.com/tomkerkhove +icon: https://raw.githubusercontent.com/tomkerkhove/promitor/master/docs/media/logos/promitor.png \ No newline at end of file diff --git a/charts/promitor-agent-resource-discovery/LICENSE b/charts/promitor-agent-resource-discovery/LICENSE new file mode 100644 index 000000000..782e6b8ab --- /dev/null +++ b/charts/promitor-agent-resource-discovery/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Tom Kerkhove + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/charts/promitor-agent-resource-discovery/README.md b/charts/promitor-agent-resource-discovery/README.md new file mode 100644 index 000000000..415c5bb2e --- /dev/null +++ b/charts/promitor-agent-resource-discovery/README.md @@ -0,0 +1,136 @@ +# Promitor + +[Promitor](https://promitor.io/) is an Azure Monitor scraper which makes + the metrics available for metric systems such as Atlassian Statuspage, + Prometheus and StatsD. + +## TL;DR + +```console +helm repo add promitor https://promitor.azurecr.io/helm/v1/repo +helm repo update +helm install promitor-agent-resource-discovery promitor/promitor-agent-resource-discovery +``` + +## Introduction + +This chart bootstraps a **Promitor Scraper Agent** deployment on a Kubernetes cluster +using the Helm package manager. It will provide the scraper agent with a Kubernetes +Service so that other Pods can consume it. + +## Prerequisites + +- Kubernetes v1.9 or above +- Azure Subscription +- Azure AD Application to authenticate with ([docs](https://promitor.io/configuration/#authentication-with-azure-monitor)) + +## Installing the Chart + +To install the chart with the release name `promitor-agent-resource-discovery`: + +```console +$ helm install promitor-agent-resource-discovery promitor/promitor-agent-resource-discovery \ + --set azureAuthentication.appId='' \ + --set azureAuthentication.appKey='' \ + --values /path/to/metric-declaration.yaml +``` + +The command deploys Prometheus on the Kubernetes cluster with the specified metrics +declaration, for more information see [our documentation](https://promitor.io/deployment/#using-our-helm-chart). + +## Uninstalling the Chart + +To uninstall/delete the `promitor-agent-resource-discovery` deployment: + +```console +helm uninstall promitor-agent-resource-discovery +``` + +The command removes all the Kubernetes components associated with the chart and +deletes the release. + +## Configuration + +The following table lists the configurable parameters of the Promitor chart and +their default values. + +| Parameter | Description | Default | +|:---------------------------|:-------------------------|:---------------------| +| `image.repository` | Repository which provides the image | `tomkerkhove/promitor-agent-resource-discovery` | +| `image.tag` | Tag of image to use | None, chart app version is used by default | +| `image.pullPolicy` | Policy to pull image | `Always` | +| `azureAuthentication.appId` | Id of the Azure AD entity to authenticate with | | +| `azureAuthentication.appKey` | Secret of the Azure AD entity to authenticate with | | +| `resourceDiscovery.enabled` | Indication whether or not resource discovery is required | `false` | +| `resourceDiscovery.host` | DNS name or IP address of the Promitor Resource Discovery agent | | +| `resourceDiscovery.port` | Port (UDP) address of the Promitor Resource Discovery agent | `80` | +| `metricSinks.atlassianStatuspage.enabled` | Indication whether or not metrics should be emitted to a StatsD server | `false`| +| `metricSinks.atlassianStatuspage.pageId` | Id of Atlassian Statuspage page | | +| `metricSinks.atlassianStatuspage.apiKey` | API key of Atlassian Statuspage page | | +| `metricSinks.atlassianStatuspage.systemMetricMapping` | Mapping of the Atlassian Statuspage system metrics and Promitor metrics. Learn how to configure it [below](#atlassian-statuspage).| None | +| `metricSinks.prometheusScrapingEndpoint.enabled` | Indication whether or not metrics should be exposed as a Prometheus scraping endpoint | `true`| +| `metricSinks.prometheusScrapingEndpoint.baseUriPath` | Path where the scraping endpoint for Prometheus is being exposed | `/metrics` | +| `metricSinks.prometheusScrapingEndpoint.enableMetricTimestamps` | Indication whether or not to include timestamp | `true` | +| `metricSinks.prometheusScrapingEndpoint.metricUnavailableValue` | Value to report in Prometheus when no metric was found whether or not to include timestamp | `NaN` | +| `metricSinks.prometheusScrapingEndpoint.enableServiceDiscovery` | Indication whether or not service discovery with annotations should be enabled ([docs](https://github.com/helm/charts/tree/master/stable/prometheus#scraping-pod-metrics-via-annotations)) | `true` | +| `metricSinks.statsd.enabled` | Indication whether or not metrics should be emitted to a StatsD server | `false`| +| `metricSinks.statsd.host` | DNS name or IP address of StatsD server | | +| `metricSinks.statsd.port` | Port (UDP) address of StatsD server | `8125` | +| `metricSinks.statsd.metricPrefix` | Prefix that will be added to every metric defined in the metric declaration | | +| `telemetry.applicationInsights.enabled` | Indication whether or not to send telemetry to Azure Application Insights | `false` | +| `telemetry.applicationInsights.logLevel` | Minimum level of logging for Azure Application Insights | | +| `telemetry.applicationInsights.key` | Application Insights instrumentation key | | +| `telemetry.containerLogs.enabled` | Indication whether or not to send telemetry to container logs | `true` | +| `telemetry.containerLogs.logLevel` | Minimum level of logging for container logs | | +| `telemetry.defaultLogLevel` | Minimum level of logging for all telemetry sinks, unless specified otherwise | `Error` | +| `azureMetadata.tenantId` | Id of Azure tenant | | +| `azureMetadata.subscriptionId` | Id of Azure subscription | | +| `azureMetadata.resourceGroupName` | Name of resource group | `promitor` | +| `metricDefaults.aggregation.interval` | Default interval which defines over what period measurements of a metric should be aggregated | `00:05:00` | +| `metricDefaults.scraping.schedule` | Cron expression that controls the fequency in which all the configured metrics will be scraped from Azure Monitor | `*/5 * * * *` | +| `metrics` | List of metrics to scrape configured following the [metric declaration docs](https://promitor.io/configuration/metrics/) | | +| `rbac.create` | If true, create & use RBAC resources | `true` | +| `rbac.podSecurityPolicyEnabled` | Create pod security policy resources | `false` | +| `rbac.serviceAccount.create` | Create service account resource | `true` | +| `rbac.serviceAccount.name` | Service account name to use if create is false. If create is true, a name is generated using the fullname template | `default` | +| `rbac.serviceAccount.annotations` | Service account annotations| `{}` | +| `resources` | Pod resource requests & limits | `{}` | +| `secrets.createSecret` | Indication if you want to bring your own secret level of logging | `true` | +| `secrets.appIdSecret` | Name of the secret for Azure AD identity id | `azure-app-id` | +| `secrets.appKeySecret` | Name of the secret for Azure AD identity secret | `azure-app-key` | +| `service.exposeExternally` | Indication whether or not to expose service externally | `false` | +| `service.port` | Port on service for other pods to talk to | `8888` | +| `service.targetPort` | Port on container to serve traffic | `88` | +| `service.labelType` | Label to assign to your service | `infrastructure` | +| `service.selectorType` | Selector type to use for the service | `runtime` | + +Specify each parameter using the `--set key=value[,key=value]` argument to +`helm install`. For example: + +```console +$ helm install promitor-agent-resource-discovery promitor/promitor-agent-resource-discovery \ + --set azureAuthentication.appId='' \ + --set azureAuthentication.appKey='' \ + --set azureMetadata.tenantId='' \ + --set azureMetadata.subscriptionId='' \ + --values C:\Promitor\metric-declaration.yaml +``` + +Alternatively, a YAML file that specifies the values for the above parameters can +be provided while installing the chart. For example, + +```console +helm install promitor-agent-resource-discovery promitor/promitor-agent-resource-discovery -f values.yaml +``` + +### Atlassian Statuspage + +You can easily provide system metric mapping between Atlassian Statuspage system metrics and Promitor metrics. + +```yaml +systemMetricMapping: [] +- id: + promitorMetricName: +``` + +This defines which Promitor metric should be reported as a given Atlassian Statuspage system metrics. diff --git a/charts/promitor-agent-resource-discovery/templates/NOTES.txt b/charts/promitor-agent-resource-discovery/templates/NOTES.txt new file mode 100644 index 000000000..3c039f3d6 --- /dev/null +++ b/charts/promitor-agent-resource-discovery/templates/NOTES.txt @@ -0,0 +1,18 @@ +1. Forward the application port by running these commands: + +export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ include "promitor-agent-resource-discovery.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") +kubectl port-forward --namespace {{ .Release.Namespace }} $POD_NAME 8080:{{ .Values.service.targetPort }} + +2. Check the scraping output at http://0.0.0.0:8080{{ .Values.metricSinks.prometheusScrapingEndpoint.baseUriPath }} + +3. To set up Prometheus in your cluster & pull in metrics from Promitor's scraping output, run: + +cat > promitor-scrape-config.yaml < + # promitorMetricName: + prometheusScrapingEndpoint: + enabled: true + baseUriPath: /metrics + enableMetricTimestamps: true + metricUnavailableValue: NaN + enableServiceDiscovery: true + statsd: + enabled: false + host: "" + port: 8125 + metricPrefix: "" +resourceDiscovery: + enabled: false + host: "" + port: 80 +telemetry: + defaultLogLevel: "Error" + applicationInsights: + enabled: false + key: "" + logLevel: "" + containerLogs: + enabled: true + logLevel: "" + +## Metric Declaration YAML +azureMetadata: + tenantId: "" + subscriptionId: "" + resourceGroupName: promitor +metricDefaults: + aggregation: + interval: 00:05:00 + scraping: + schedule: "*/5 * * * *" +metrics: [] +# Sample metric configuration below +# - name: demo_queue_size +# description: "Amount of active messages of the 'myqueue' queue (determined with ServiceBusQueue provider)" +# resourceType: ServiceBusQueue +# azureMetricConfiguration: +# metricName: ActiveMessages +# aggregation: +# type: Total +# interval: 00:15:00 +# resources: +# - namespace: promitor-messaging +# queueName: orders + +secrets: + # To use your own secret, set createSecret to false and define the name/keys that your secret uses + createSecret: true + secretName: "" + appIdSecret: azure-app-id + appKeySecret: azure-app-key + appInsightsSecret: azure-appinsights-key + atlassianStatuspageApiKey: atlassian-statuspage-apikey + +service: + exposeExternally: False + port: 8888 + targetPort: 88 + labelType: infrastructure + selectorType: runtime + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +## Role-based access control +## https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +rbac: + ## If true, create & use RBAC resources + create: true + + ## If true, create & use Pod Security Policy resources + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicyEnabled: false + + ## Service Account for pods + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + serviceAccount: + ## Specifies whether a service account should be created + create: true + + ## The name of the service account to use if create is false + ## If create is true, a name is generated using the fullname template + name: default + annotations: {} From 7976d358e8635dc0a4512f08429724a1de0a7cbc Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:15:08 +0200 Subject: [PATCH 02/12] Fix path Signed-off-by: Tom Kerkhove --- build/azure-devops/templates/stages/helm-chart-verification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 3f74dbabe..bc3afe98b 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -17,6 +17,6 @@ stages: steps: - download: current artifact: variables - - template: templates/utils/read-variable-on-linux.yml + - template: ./../templates/utils/read-variable-on-linux.yml parameters: variableName: 'Image.Tag' \ No newline at end of file From 2c57684ece76b65476b64fe539cacdb34a90fb24 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:16:33 +0200 Subject: [PATCH 03/12] Fix path Signed-off-by: Tom Kerkhove --- build/azure-devops/templates/stages/helm-chart-verification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index bc3afe98b..30705ba52 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -17,6 +17,6 @@ stages: steps: - download: current artifact: variables - - template: ./../templates/utils/read-variable-on-linux.yml + - template: ./../../templates/utils/read-variable-on-linux.yml parameters: variableName: 'Image.Tag' \ No newline at end of file From 2ebc1e51169270e6fa3a9f09a1f74f245a477558 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:17:03 +0200 Subject: [PATCH 04/12] Fix folder Signed-off-by: Tom Kerkhove --- build/azure-devops/templates/stages/helm-chart-verification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 30705ba52..96a8f06b7 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -17,6 +17,6 @@ stages: steps: - download: current artifact: variables - - template: ./../../templates/utils/read-variable-on-linux.yml + - template: ./../../utils/read-variable-on-linux.yml parameters: variableName: 'Image.Tag' \ No newline at end of file From 8674609018f79d6bedfe919972c3ff8dae5fc86c Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:21:44 +0200 Subject: [PATCH 05/12] Variables! Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-discovery.yml | 46 ++----------------- .../stages/helm-chart-verification.yml | 31 +++++++++++-- 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/build/azure-devops/agents-ci-discovery.yml b/build/azure-devops/agents-ci-discovery.yml index 9fd2dad93..8a337d723 100644 --- a/build/azure-devops/agents-ci-discovery.yml +++ b/build/azure-devops/agents-ci-discovery.yml @@ -215,44 +215,8 @@ stages: parameters: imageName: '$(Image.TaggedName)' - template: templates/stages/helm-chart-verification.yml - # parameters: - # name: Full - # testFile: tests/fullSuite.js -# - stage: Helm3 -# displayName: Helm Chart (3.x) -# dependsOn: Docker -# variables: -# Helm.Version: '3.0.0' -# jobs: -# - job: Helm3_x -# displayName: Helm 3.x - Lint, Package & Push Chart -# condition: succeeded() -# pool: -# vmImage: ubuntu-16.04 -# steps: -# - download: current -# artifact: variables -# - template: templates/utils/read-variable-on-linux.yml -# parameters: -# variableName: 'Image.Tag' -# - template: templates/helm/lint-chart.yml -# parameters: -# helmVersion: '$(Helm.Version)' -# chartName: '$(Chart.Scraper.Name)' -# - template: templates/helm/package-preview-chart.yml -# parameters: -# chartName: '$(Helm.Chart.CI.Name)' -# chartVersion: '$(Helm.Chart.Version)' -# appVersion: '$(App.Version)' -# imageName: '$(Image.Name)' -# - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: -# - template: templates/helm/push-chart.yml -# parameters: -# chartName: '$(Helm.Chart.CI.Name)' -# chartVersion: '$(Helm.Chart.Version)' -# - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: -# - task: PublishBuildArtifacts@1 -# displayName: 'Publish Helm Chart' -# inputs: -# PathtoPublish: charts/output/$(Helm.Chart.CI.Name)-$(Helm.Chart.Version).tgz -# ArtifactName: helm + parameters: + chartName: '$(Helm.Chart.CI.Name)' + chartVersion: '$(Helm.Chart.Version)' + appVersion: '$(App.Version)' + imageName: '$(Image.Name)' \ No newline at end of file diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 96a8f06b7..9ba34b0bf 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -1,6 +1,8 @@ -# parameters: -# name: '' -# testFile: '' +parameters: + chartName: '' + chartVersion: '' + appVersion: '' + imageName: '' stages: - stage: Helm3 @@ -19,4 +21,25 @@ stages: artifact: variables - template: ./../../utils/read-variable-on-linux.yml parameters: - variableName: 'Image.Tag' \ No newline at end of file + variableName: 'Image.Tag' + - template: ./../../helm/lint-chart.yml + parameters: + helmVersion: '$(Helm.Version)' + chartName: '$(Chart.Scraper.Name)' + - template: ./../../helm/package-preview-chart.yml + parameters: + chartName: '${{ parameters.chartName }}' + chartVersion: '${{ parameters.chartVersion }}' + appVersion: '${{ parameters.appVersion }}' + imageName: '${{ parameters.imageName }}' + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: ./../../helm/push-chart.yml + parameters: + chartName: '${{ parameters.chartName }}' + chartVersion: '${{ parameters.chartVersion }}' + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - task: PublishBuildArtifacts@1 + displayName: 'Publish Helm Chart' + inputs: + PathtoPublish: charts/output/${{ parameters.chartName }}-${{ parameters.chartVersion }}.tgz + ArtifactName: helm From 373f71676541cd8b68c7e7b170f491cbd0aa0567 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:23:49 +0200 Subject: [PATCH 06/12] Improve Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-discovery.yml | 4 +++- .../templates/stages/helm-chart-verification.yml | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/build/azure-devops/agents-ci-discovery.yml b/build/azure-devops/agents-ci-discovery.yml index 8a337d723..210ebab9a 100644 --- a/build/azure-devops/agents-ci-discovery.yml +++ b/build/azure-devops/agents-ci-discovery.yml @@ -219,4 +219,6 @@ stages: chartName: '$(Helm.Chart.CI.Name)' chartVersion: '$(Helm.Chart.Version)' appVersion: '$(App.Version)' - imageName: '$(Image.Name)' \ No newline at end of file + imageName: '$(Image.Name)' + # dependsOn: 'Docker' + dependsOn: 'Init' \ No newline at end of file diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 9ba34b0bf..5f9985d7c 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -3,11 +3,12 @@ parameters: chartVersion: '' appVersion: '' imageName: '' + dependsOn: '' stages: - stage: Helm3 displayName: Helm Chart (3.x) - dependsOn: Docker + dependsOn: [${{ parameters.dependsOn }}] variables: Helm.Version: '3.0.0' jobs: @@ -19,13 +20,13 @@ stages: steps: - download: current artifact: variables - - template: ./../../utils/read-variable-on-linux.yml - parameters: - variableName: 'Image.Tag' + # - template: ./../../utils/read-variable-on-linux.yml + # parameters: + # variableName: 'Image.Tag' - template: ./../../helm/lint-chart.yml parameters: helmVersion: '$(Helm.Version)' - chartName: '$(Chart.Scraper.Name)' + chartName: '${{ parameters.chartName }}' - template: ./../../helm/package-preview-chart.yml parameters: chartName: '${{ parameters.chartName }}' From 99e6887a86a3effe2c4b8b2e632398020c8f9b81 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:24:49 +0200 Subject: [PATCH 07/12] Single dependsOn Signed-off-by: Tom Kerkhove --- build/azure-devops/templates/stages/helm-chart-verification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 5f9985d7c..25a2b6c2c 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -8,7 +8,7 @@ parameters: stages: - stage: Helm3 displayName: Helm Chart (3.x) - dependsOn: [${{ parameters.dependsOn }}] + dependsOn: ${{ parameters.dependsOn }} variables: Helm.Version: '3.0.0' jobs: From 80ea8b16fcaaad42e68e214c1b4beb2da48ff8ea Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:29:10 +0200 Subject: [PATCH 08/12] Fix Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-discovery.yml | 2 +- build/azure-devops/templates/helm/lint-chart.yml | 2 +- .../templates/stages/helm-chart-verification.yml | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/build/azure-devops/agents-ci-discovery.yml b/build/azure-devops/agents-ci-discovery.yml index 210ebab9a..e771f47ca 100644 --- a/build/azure-devops/agents-ci-discovery.yml +++ b/build/azure-devops/agents-ci-discovery.yml @@ -24,7 +24,7 @@ variables: - name: Helm.Chart.Version value: '0.0.0-$(App.Version)' - name: Helm.Chart.CI.Name - value: 'promitor-agent-resource-discovery-ci' + value: 'promitor-agent-resource-discovery' stages: - stage: Init displayName: Prepare Build diff --git a/build/azure-devops/templates/helm/lint-chart.yml b/build/azure-devops/templates/helm/lint-chart.yml index 16ac325ef..24a7c6499 100644 --- a/build/azure-devops/templates/helm/lint-chart.yml +++ b/build/azure-devops/templates/helm/lint-chart.yml @@ -7,7 +7,7 @@ parameters: steps: - task: HelmInstaller@1 - displayName: 'Install Helm (${{ parameters.helmVersion }}' + displayName: 'Install Helm (${{ parameters.helmVersion }})' inputs: helmVersionToInstall: '${{ parameters.helmVersion }}' - powershell: | diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 25a2b6c2c..0b3acedc1 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -20,9 +20,10 @@ stages: steps: - download: current artifact: variables - # - template: ./../../utils/read-variable-on-linux.yml - # parameters: - # variableName: 'Image.Tag' + # TODO: Move this to step to pass as "pre-run step" + - template: ./../../utils/read-variable-on-linux.yml + parameters: + variableName: 'Image.Tag' - template: ./../../helm/lint-chart.yml parameters: helmVersion: '$(Helm.Version)' From 8b505da89147d77aec4014917bbfc4ba2a116e1d Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Sun, 27 Sep 2020 10:30:30 +0200 Subject: [PATCH 09/12] Use stage template for Scraper Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-scraper.yml | 47 ++++-------------------- 1 file changed, 8 insertions(+), 39 deletions(-) diff --git a/build/azure-devops/agents-ci-scraper.yml b/build/azure-devops/agents-ci-scraper.yml index 87971d4e5..05464d84b 100644 --- a/build/azure-devops/agents-ci-scraper.yml +++ b/build/azure-devops/agents-ci-scraper.yml @@ -24,7 +24,7 @@ variables: - name: Helm.Chart.Version value: '0.0.0-$(App.Version)' - name: Helm.Chart.CI.Name - value: 'promitor-agent-scraper-ci' + value: 'promitor-agent-scraper' stages: - stage: Init displayName: Prepare Build @@ -286,41 +286,10 @@ stages: parameters: helmVersion: '$(Helm.Version)' chartName: '$(Chart.Scraper.Name)' -- stage: Helm3 - displayName: Helm Chart (3.x) - dependsOn: Docker - variables: - Helm.Version: '3.0.0' - jobs: - - job: Helm3_x - displayName: Helm 3.x - Lint, Package & Push Chart - condition: succeeded() - pool: - vmImage: ubuntu-16.04 - steps: - - download: current - artifact: variables - - template: templates/utils/read-variable-on-linux.yml - parameters: - variableName: 'Image.Tag' - - template: templates/helm/lint-chart.yml - parameters: - helmVersion: '$(Helm.Version)' - chartName: '$(Chart.Scraper.Name)' - - template: templates/helm/package-preview-chart.yml - parameters: - chartName: '$(Helm.Chart.CI.Name)' - chartVersion: '$(Helm.Chart.Version)' - appVersion: '$(App.Version)' - imageName: '$(Image.Name)' - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - template: templates/helm/push-chart.yml - parameters: - chartName: '$(Helm.Chart.CI.Name)' - chartVersion: '$(Helm.Chart.Version)' - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish Helm Chart' - inputs: - PathtoPublish: charts/output/$(Helm.Chart.CI.Name)-$(Helm.Chart.Version).tgz - ArtifactName: helm +- template: templates/stages/helm-chart-verification.yml + parameters: + chartName: '$(Helm.Chart.CI.Name)' + chartVersion: '$(Helm.Chart.Version)' + appVersion: '$(App.Version)' + imageName: '$(Image.Name)' + dependsOn: 'Docker' \ No newline at end of file From 45693a75813851de5f39ba61507651f81208baa1 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Mon, 28 Sep 2020 09:19:23 +0200 Subject: [PATCH 10/12] Spaces Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-discovery.yml | 2 +- build/azure-devops/agents-ci-scraper.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/azure-devops/agents-ci-discovery.yml b/build/azure-devops/agents-ci-discovery.yml index e771f47ca..e13b6f7a8 100644 --- a/build/azure-devops/agents-ci-discovery.yml +++ b/build/azure-devops/agents-ci-discovery.yml @@ -215,7 +215,7 @@ stages: parameters: imageName: '$(Image.TaggedName)' - template: templates/stages/helm-chart-verification.yml - parameters: + parameters: chartName: '$(Helm.Chart.CI.Name)' chartVersion: '$(Helm.Chart.Version)' appVersion: '$(App.Version)' diff --git a/build/azure-devops/agents-ci-scraper.yml b/build/azure-devops/agents-ci-scraper.yml index 05464d84b..7ada8fab5 100644 --- a/build/azure-devops/agents-ci-scraper.yml +++ b/build/azure-devops/agents-ci-scraper.yml @@ -287,7 +287,7 @@ stages: helmVersion: '$(Helm.Version)' chartName: '$(Chart.Scraper.Name)' - template: templates/stages/helm-chart-verification.yml - parameters: + parameters: chartName: '$(Helm.Chart.CI.Name)' chartVersion: '$(Helm.Chart.Version)' appVersion: '$(App.Version)' From bfafa8f9a4366d4f19307f69311b9e569c608eeb Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Mon, 28 Sep 2020 09:23:03 +0200 Subject: [PATCH 11/12] Transform CI Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-scraper.yml | 4 ++-- .../templates/helm/package-preview-chart.yml | 4 +++- .../stages/helm-chart-verification.yml | 1 + build/helm/Transform-Chart.ps1 | 24 ++++++++++--------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/build/azure-devops/agents-ci-scraper.yml b/build/azure-devops/agents-ci-scraper.yml index 7ada8fab5..8dd3b5119 100644 --- a/build/azure-devops/agents-ci-scraper.yml +++ b/build/azure-devops/agents-ci-scraper.yml @@ -23,7 +23,7 @@ variables: value: '0.0.0-$(Image.Tag)' - name: Helm.Chart.Version value: '0.0.0-$(App.Version)' - - name: Helm.Chart.CI.Name + - name: Helm.Chart.Name value: 'promitor-agent-scraper' stages: - stage: Init @@ -288,7 +288,7 @@ stages: chartName: '$(Chart.Scraper.Name)' - template: templates/stages/helm-chart-verification.yml parameters: - chartName: '$(Helm.Chart.CI.Name)' + chartName: '$(Helm.Chart.Name)' chartVersion: '$(Helm.Chart.Version)' appVersion: '$(App.Version)' imageName: '$(Image.Name)' diff --git a/build/azure-devops/templates/helm/package-preview-chart.yml b/build/azure-devops/templates/helm/package-preview-chart.yml index d47b180cb..013651470 100644 --- a/build/azure-devops/templates/helm/package-preview-chart.yml +++ b/build/azure-devops/templates/helm/package-preview-chart.yml @@ -4,6 +4,8 @@ parameters: default: '$(Helm.Version)' - name: chartName type: string +- name: transformedChartName + type: string - name: chartVersion type: string - name: appVersion @@ -21,7 +23,7 @@ steps: inputs: targetType: filePath filePath: ./build/helm/Transform-Chart.ps1 - arguments: '-chartName "${{ parameters.chartName }}" -imageName "${{ parameters.imageName }}" -imageTag "${{ parameters.appVersion }}"' + arguments: '-chartName "${{ parameters.chartName }}" -transformedChartName "${{ parameters.transformedChartName }}" -imageName "${{ parameters.imageName }}" -imageTag "${{ parameters.appVersion }}"' workingDirectory: 'charts' - template: package-official-chart.yml parameters: diff --git a/build/azure-devops/templates/stages/helm-chart-verification.yml b/build/azure-devops/templates/stages/helm-chart-verification.yml index 0b3acedc1..556c244f2 100644 --- a/build/azure-devops/templates/stages/helm-chart-verification.yml +++ b/build/azure-devops/templates/stages/helm-chart-verification.yml @@ -34,6 +34,7 @@ stages: chartVersion: '${{ parameters.chartVersion }}' appVersion: '${{ parameters.appVersion }}' imageName: '${{ parameters.imageName }}' + transformedChartName: '${{ parameters.chartName }}-ci' - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - template: ./../../helm/push-chart.yml parameters: diff --git a/build/helm/Transform-Chart.ps1 b/build/helm/Transform-Chart.ps1 index d74b96119..72cd2ea9e 100644 --- a/build/helm/Transform-Chart.ps1 +++ b/build/helm/Transform-Chart.ps1 @@ -2,34 +2,36 @@ Param( [String] [Parameter(Mandatory = $true)] $chartName = $(throw "Chart name is required"), [String] + [Parameter(Mandatory = $true)] $transformedChartName = $(throw "Chart name of the transformed output is required"), + [String] [Parameter(Mandatory = $true)] $imageName = $(throw "Image name is required"), [String] [Parameter(Mandatory = $true)] $imageTag = $(throw "Image tag is required") ) echo "Copying Chart folder" -cp promitor-agent-scraper/ $chartName/ -r +cp $chartName/ $transformedChartName/ -r echo "Determining image version" $imageVersion = "$imageTag".ToLower() echo "Image version is $imageVersion" -echo "Changing name of chart to $chartName" -((Get-Content -path $chartName/Chart.yaml -Raw) -replace 'promitor-agent-scraper', $chartName) | Set-Content -Path $chartName/Chart.yaml +echo "Changing name of chart to $transformedChartName" +((Get-Content -path $transformedChartName/Chart.yaml -Raw) -replace 'promitor-agent-scraper', $transformedChartName) | Set-Content -Path $transformedChartName/Chart.yaml echo "Changing image tag to $imageVersion" -((Get-Content -path $chartName/values.yaml -Raw) -replace 'latest', $imageVersion) | Set-Content -Path $chartName/values.yaml +((Get-Content -path $transformedChartName/values.yaml -Raw) -replace 'latest', $imageVersion) | Set-Content -Path $transformedChartName/values.yaml echo "Changing repo name to $imageName" -((Get-Content -path $chartName/values.yaml -Raw) -replace 'tomkerkhove/promitor-agent-scraper', $imageName) | Set-Content -Path $chartName/values.yaml +((Get-Content -path $transformedChartName/values.yaml -Raw) -replace 'tomkerkhove/promitor-agent-scraper', $imageName) | Set-Content -Path $transformedChartName/values.yaml -echo "Change name of chart in README to $chartName" -((Get-Content -path $chartName/README.md -Raw) -replace 'promitor-agent-scraper', $chartName) | Set-Content -Path $chartName/README.md +echo "Change name of chart in README to $transformedChartName" +((Get-Content -path $transformedChartName/README.md -Raw) -replace 'promitor-agent-scraper', $transformedChartName) | Set-Content -Path $transformedChartName/README.md echo "Change version of image in README to $imageVersion" -((Get-Content -path $chartName/README.md -Raw) -replace 'latest', $imageVersion) | Set-Content -Path $chartName/README.md +((Get-Content -path $transformedChartName/README.md -Raw) -replace 'latest', $imageVersion) | Set-Content -Path $transformedChartName/README.md echo "Outputting transformed content" -Get-Content -path $chartName/Chart.yaml -Raw -Get-Content -path $chartName/values.yaml -Raw -Get-Content -path $chartName/README.md -Raw \ No newline at end of file +Get-Content -path $transformedChartName/Chart.yaml -Raw +Get-Content -path $transformedChartName/values.yaml -Raw +Get-Content -path $transformedChartName/README.md -Raw \ No newline at end of file From 61570c2819f23b0e83f27aedb634c90a049b29ca Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Mon, 28 Sep 2020 10:05:05 +0200 Subject: [PATCH 12/12] Fix depends on Signed-off-by: Tom Kerkhove --- build/azure-devops/agents-ci-discovery.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/azure-devops/agents-ci-discovery.yml b/build/azure-devops/agents-ci-discovery.yml index e13b6f7a8..084752b40 100644 --- a/build/azure-devops/agents-ci-discovery.yml +++ b/build/azure-devops/agents-ci-discovery.yml @@ -220,5 +220,4 @@ stages: chartVersion: '$(Helm.Chart.Version)' appVersion: '$(App.Version)' imageName: '$(Image.Name)' - # dependsOn: 'Docker' - dependsOn: 'Init' \ No newline at end of file + dependsOn: 'Docker' \ No newline at end of file