diff --git a/.abs/main.yaml b/.abs/main.yaml new file mode 100644 index 00000000..12ff9a18 --- /dev/null +++ b/.abs/main.yaml @@ -0,0 +1,8 @@ +replace-app-version-with-git: true +replace-chart-version-with-git: true +generate-metadata: true +chart-dir: ./helm/app-exporter +destination: ./build + +# CI overwrites this, check .circleci/config.yaml +catalog-base-url: https://giantswarm.github.io/control-plane-catalog/ diff --git a/.circleci/config.yml b/.circleci/config.yml index 062dd623..2c47ae67 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - architect: giantswarm/architect@5.10.1 + architect: giantswarm/architect@5.11.1 workflows: build: @@ -28,6 +28,7 @@ workflows: - master - architect/push-to-app-catalog: context: architect + executor: app-build-suite name: push-to-app-catalog app_catalog: control-plane-catalog app_catalog_test: control-plane-test-catalog diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b19d0fc..cf64f877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add `cluster_id` field to `app_operator_app_info` metrics that contains the value of the `giantswarm.io/cluster` label from the App CR, empty otherwise. + +### Removed + +- Removed PSP support and thus support for pre v1.25 Kubernetes clusters. + - Removed `.global.podSecurityStandards.enforced` Helm value. +- Removed `.project.branch` and `project.commit` Helm values. + ### Changed -- Remove operatorkit dependency. +- Bump `architect-orb` to `v5.11.1`. +- Updated build pipeline to use `app-build-suite`. + - Changed value for `application.giantswarm.io/branch` label to point to `.Chart.AppVersion` instead as ABS does not support mangling the templates anymore. + - Changed value for `application.giantswarm.io/commit` label to point to `.Chart.AppVersion` instead as ABS does not support mangling the templates anymore. +- Defaulted `.image.tag` to be an empty string and default that to `.Chart.AppVersion` in the deployment. +- Removed `operatorkit` dependency. ## [0.20.0] - 2024-04-30 diff --git a/go.mod b/go.mod index b8c93d91..d75ed198 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/prometheus/client_golang v1.20.5 github.com/spf13/viper v1.19.0 + k8s.io/api v0.31.3 k8s.io/apimachinery v0.31.3 k8s.io/client-go v0.31.3 sigs.k8s.io/controller-runtime v0.19.2 @@ -97,7 +98,6 @@ require ( gopkg.in/resty.v1 v1.12.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.3 // indirect k8s.io/apiextensions-apiserver v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect diff --git a/helm/app-exporter/Chart.yaml b/helm/app-exporter/Chart.yaml index 56bf4f4e..90f710ca 100644 --- a/helm/app-exporter/Chart.yaml +++ b/helm/app-exporter/Chart.yaml @@ -1,10 +1,10 @@ apiVersion: "v1" -appVersion: "[[ .AppVersion ]]" +appVersion: "0.20.1-dev" description: "Chart holding app-exporter." home: "https://github.com/giantswarm/app-exporter" icon: https://s.giantswarm.io/app-icons/prometheus/1/light.svg name: "app-exporter" -version: "[[ .Version ]]" +version: "0.20.1-dev" annotations: application.giantswarm.io/team: "honeybadger" config.giantswarm.io/version: 1.x.x diff --git a/helm/app-exporter/templates/_helpers.tpl b/helm/app-exporter/templates/_helpers.tpl index 3aecc8c8..8f66bbcc 100644 --- a/helm/app-exporter/templates/_helpers.tpl +++ b/helm/app-exporter/templates/_helpers.tpl @@ -19,8 +19,8 @@ Common labels {{- define "labels.common" -}} app: {{ include "name" . | quote }} {{ include "labels.selector" . }} -application.giantswarm.io/branch: {{ .Values.project.branch | replace "#" "-" | replace "/" "-" | replace "." "-" | trunc 63 | trimSuffix "-" | quote }} -application.giantswarm.io/commit: {{ .Values.project.commit | quote }} +application.giantswarm.io/branch: {{ .Chart.AppVersion | replace "#" "-" | replace "/" "-" | replace "." "-" | trunc 63 | trimSuffix "-" | quote }} +application.giantswarm.io/commit: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service | quote }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} application.giantswarm.io/team: {{ index .Chart.Annotations "application.giantswarm.io/team" | quote }} @@ -34,3 +34,14 @@ Selector labels app.kubernetes.io/name: {{ include "name" . | quote }} app.kubernetes.io/instance: {{ .Release.Name | quote }} {{- end -}} + +{{/* +Define image tag. +*/}} +{{- define "image.tag" -}} +{{- if .Values.image.tag }} +{{- .Values.image.tag }} +{{- else }} +{{- .Chart.AppVersion }} +{{- end }} +{{- end }} diff --git a/helm/app-exporter/templates/deployment.yaml b/helm/app-exporter/templates/deployment.yaml index 4df93270..18b91b50 100644 --- a/helm/app-exporter/templates/deployment.yaml +++ b/helm/app-exporter/templates/deployment.yaml @@ -44,7 +44,7 @@ spec: {{- end }} containers: - name: {{ include "name" . }} - image: "{{ .Values.registry.domain }}/{{ .Values.image.name }}:{{ .Values.image.tag }}" + image: "{{ .Values.registry.domain }}/{{ .Values.image.name }}:{{ include "image.tag" . }}" args: - daemon - --config.dirs=/var/run/{{ include "name" . }}/configmap/ @@ -52,6 +52,8 @@ spec: volumeMounts: - name: {{ include "name" . }}-configmap mountPath: /var/run/{{ include "name" . }}/configmap/ + ports: + - containerPort: {{ .Values.config.listenPort }} livenessProbe: httpGet: path: /healthz diff --git a/helm/app-exporter/templates/psp.yaml b/helm/app-exporter/templates/psp.yaml deleted file mode 100644 index b773bafb..00000000 --- a/helm/app-exporter/templates/psp.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- if not (((.Values.global).podSecurityStandards).enforced) }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ include "resource.psp.name" . }} - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - labels: - {{- include "labels.common" . | nindent 4 }} -spec: - privileged: false - fsGroup: - rule: MustRunAs - ranges: - - min: 1 - max: 65535 - runAsUser: - rule: MustRunAsNonRoot - runAsGroup: - rule: MustRunAs - ranges: - - min: 1 - max: 65535 - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - 'secret' - - 'configMap' - allowPrivilegeEscalation: false - hostNetwork: false - hostIPC: false - hostPID: false -{{- end }} diff --git a/helm/app-exporter/templates/rbac.yaml b/helm/app-exporter/templates/rbac.yaml index 8258475f..50002c07 100644 --- a/helm/app-exporter/templates/rbac.yaml +++ b/helm/app-exporter/templates/rbac.yaml @@ -40,36 +40,3 @@ roleRef: kind: ClusterRole name: {{ include "resource.default.name" . }} apiGroup: rbac.authorization.k8s.io ---- -{{- if not (((.Values.global).podSecurityStandards).enforced) }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "resource.psp.name" . }} - labels: - {{- include "labels.common" . | nindent 4 }} -rules: - - apiGroups: - - extensions - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - {{ include "resource.psp.name" . }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "resource.psp.name" . }} - labels: - {{- include "labels.common" . | nindent 4 }} -subjects: - - kind: ServiceAccount - name: {{ include "resource.default.name" . }} - namespace: {{ include "resource.default.namespace" . }} -roleRef: - kind: ClusterRole - name: {{ include "resource.psp.name" . }} - apiGroup: rbac.authorization.k8s.io -{{- end }} diff --git a/helm/app-exporter/values.schema.json b/helm/app-exporter/values.schema.json index 5a88c0f2..1095c310 100644 --- a/helm/app-exporter/values.schema.json +++ b/helm/app-exporter/values.schema.json @@ -49,19 +49,6 @@ } } }, - "global": { - "type": "object", - "properties": { - "podSecurityStandards": { - "type": "object", - "properties": { - "enforced": { - "type": "boolean" - } - } - } - } - }, "image": { "type": "object", "properties": { @@ -107,17 +94,6 @@ } } }, - "project": { - "type": "object", - "properties": { - "branch": { - "type": "string" - }, - "commit": { - "type": "string" - } - } - }, "provider": { "type": "object", "properties": { @@ -154,6 +130,9 @@ } } }, + "readOnlyRootFilesystem": { + "type": "boolean" + }, "runAsNonRoot": { "type": "boolean" }, diff --git a/helm/app-exporter/values.yaml b/helm/app-exporter/values.yaml index 9c16f411..fef01c8e 100644 --- a/helm/app-exporter/values.yaml +++ b/helm/app-exporter/values.yaml @@ -8,7 +8,7 @@ deployment: image: name: "giantswarm/app-exporter" - tag: "[[ .Version ]]" + tag: "" registry: domain: gsoci.azurecr.io @@ -23,10 +23,6 @@ pod: group: id: 1000 -project: - branch: "[[ .Branch ]]" - commit: "[[ .SHA ]]" - config: debug: true listenPort: 8000 @@ -54,6 +50,7 @@ podSecurityContext: # Add seccomp profile to container security context securityContext: + readOnlyRootFilesystem: true allowPrivilegeEscalation: false capabilities: drop: @@ -61,7 +58,3 @@ securityContext: runAsNonRoot: true seccompProfile: type: RuntimeDefault - -global: - podSecurityStandards: - enforced: false diff --git a/service/collector/app.go b/service/collector/app.go index d7221dbe..8a6549d6 100644 --- a/service/collector/app.go +++ b/service/collector/app.go @@ -42,6 +42,7 @@ var ( labelUpgradeAvailable, labelVersion, labelVersionMismatch, + labelClusterId, }, nil, ) @@ -166,9 +167,16 @@ func (a *App) collectAppStatus(ctx context.Context, ch chan<- prometheus.Metric) appSpecVersion := key.Version(app) appStatusVersion := expkey.FormatVersion(app.Status.Version) + clusterLabel := key.ClusterLabel(app) + + var clusterId string + { + clusterId = clusterLabel + } + // clusterMissing is true if `giantswarm.io/cluster` label is missing - // on the org-namespaced app. Otherwise it's false. - clusterMissing := key.IsInOrgNamespace(app) && key.ClusterLabel(app) == "" + // on the org-namespaced app. Otherwise, it's false. + clusterMissing := key.IsInOrgNamespace(app) && clusterLabel == "" // For optional apps in public catalogs we check if an upgrade // is available. @@ -198,6 +206,7 @@ func (a *App) collectAppStatus(ctx context.Context, ch chan<- prometheus.Metric) // Getting version from spec, not status since the version in the spec is the desired version. appSpecVersion, strconv.FormatBool(appSpecVersion != appStatusVersion), + clusterId, ) if !key.IsAppCordoned(app) { diff --git a/service/collector/app_test.go b/service/collector/app_test.go index 55570202..95a29614 100644 --- a/service/collector/app_test.go +++ b/service/collector/app_test.go @@ -95,43 +95,53 @@ func Test_collectAppStatus(t *testing.T) { { name: "flawless", apps: []*v1alpha1.App{ - newApp("hello-world-app", "giantswarm", "hello-world", "0.3.0", "", ""), - newApp("example", "customer", "default", "1.0.0", "", ""), + newApp("hello-world-app", "giantswarm", "hello-world", "0.3.0", "", "", map[string]string{}), + newApp("example", "customer", "default", "1.0.0", "", "", map[string]string{}), + newApp("test-app", "default", "test-app", "1.0.0", "", "", map[string]string{ + label.Cluster: "foo", + }), }, catalogs: []*v1alpha1.Catalog{ newCatalog("giantswarm", "default"), newCatalog("customer", "default"), + newCatalog("default", "giantswarm"), }, catalogsEntries: []*v1alpha1.AppCatalogEntry{ newACE("hello-world-app", "giantswarm", "default", "0.3.0", "", "", true), newACE("example", "customer", "default", "1.0.0", "", "", true), + newACE("test-app", "default", "giantswarm", "1.0.0", "", "", true), }, expectedMetrics: "testdata/expected.1", - expectedMetricsCount: 2, + expectedMetricsCount: 3, }, { name: "flawless with v* versions", apps: []*v1alpha1.App{ - newApp("hello-world-app", "giantswarm", "hello-world", "v0.3.0", "", ""), - newApp("example", "customer", "default", "v1.0.0", "", ""), + newApp("hello-world-app", "giantswarm", "hello-world", "v0.3.0", "", "", map[string]string{}), + newApp("example", "customer", "default", "v1.0.0", "", "", map[string]string{}), + newApp("test-app", "default", "test-app", "v1.0.0", "", "", map[string]string{ + label.Cluster: "foo", + }), }, catalogs: []*v1alpha1.Catalog{ newCatalog("giantswarm", "default"), newCatalog("customer", "default"), + newCatalog("default", "giantswarm"), }, catalogsEntries: []*v1alpha1.AppCatalogEntry{ newACE("hello-world-app", "giantswarm", "default", "v0.3.0", "", "", true), newACE("example", "customer", "default", "v1.0.0", "", "", true), + newACE("test-app", "default", "giantswarm", "1.0.0", "", "", true), }, expectedMetrics: "testdata/expected.1", - expectedMetricsCount: 2, + expectedMetricsCount: 3, }, // It is RECOMMENDED to use mixed versions in the next tests { name: "app pending update", apps: []*v1alpha1.App{ - newApp("hello-world-app", "giantswarm", "hello-world", "0.3.0", "", ""), - newApp("example", "customer", "default", "v1.0.0", "v0.9.0", ""), + newApp("hello-world-app", "giantswarm", "hello-world", "0.3.0", "", "", map[string]string{}), + newApp("example", "customer", "default", "v1.0.0", "v0.9.0", "", map[string]string{}), }, catalogs: []*v1alpha1.Catalog{ newCatalog("giantswarm", "default"), @@ -338,8 +348,8 @@ func Test_getTeamMappings(t *testing.T) { { name: "flawless", apps: []v1alpha1.App{ - *newApp("hello-world-app", "giantswarm", "hello-world", "0.2.0", "", ""), - *newApp("example", "customer", "default", "1.0.0", "", ""), + *newApp("hello-world-app", "giantswarm", "hello-world", "0.2.0", "", "", map[string]string{}), + *newApp("example", "customer", "default", "1.0.0", "", "", map[string]string{}), }, catalogs: []*v1alpha1.Catalog{ newCatalog("giantswarm", "default"), @@ -367,8 +377,8 @@ func Test_getTeamMappings(t *testing.T) { { name: "flawless with v* versions", apps: []v1alpha1.App{ - *newApp("hello-world-app", "giantswarm", "hello-world", "0.2.0", "", ""), - *newApp("example", "customer", "default", "v1.0.0", "", ""), + *newApp("hello-world-app", "giantswarm", "hello-world", "0.2.0", "", "", map[string]string{}), + *newApp("example", "customer", "default", "v1.0.0", "", "", map[string]string{}), }, catalogs: []*v1alpha1.Catalog{ newCatalog("giantswarm", "default"), @@ -396,8 +406,8 @@ func Test_getTeamMappings(t *testing.T) { { name: "flawless with team mappings", apps: []v1alpha1.App{ - *newApp("hello-world-app", "giantswarm", "hello-world", "0.3.0", "", ""), - *newApp("example", "customer", "default", "v1.0.0", "", ""), + *newApp("hello-world-app", "giantswarm", "hello-world", "0.3.0", "", "", map[string]string{}), + *newApp("example", "customer", "default", "v1.0.0", "", "", map[string]string{}), }, catalogs: []*v1alpha1.Catalog{ newCatalog("giantswarm", "default"), @@ -532,7 +542,7 @@ func newACE(app, catalog, namespace, version, owners, team string, latest bool) return &ace } -func newApp(name, catalog, namespace, version, statusVersion, statusRelease string) *v1alpha1.App { +func newApp(name, catalog, namespace, version, statusVersion, statusRelease string, labels map[string]string) *v1alpha1.App { if statusVersion == "" { statusVersion = version } @@ -547,7 +557,7 @@ func newApp(name, catalog, namespace, version, statusVersion, statusRelease stri APIVersion: "application.giantswarm.io/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{}, + Labels: labels, Name: name, Namespace: namespace, }, diff --git a/service/collector/collector.go b/service/collector/collector.go index 44122702..b1fa2faa 100644 --- a/service/collector/collector.go +++ b/service/collector/collector.go @@ -20,4 +20,5 @@ const ( labelUpgradeAvailable = "upgrade_available" labelVersion = "version" labelVersionMismatch = "version_mismatch" + labelClusterId = "cluster_id" ) diff --git a/service/collector/testdata/expected.1 b/service/collector/testdata/expected.1 index a9b026a5..488da2bb 100644 --- a/service/collector/testdata/expected.1 +++ b/service/collector/testdata/expected.1 @@ -1,4 +1,5 @@ # HELP app_operator_app_info Managed apps status. # TYPE app_operator_app_info gauge -app_operator_app_info{app="example",app_version="",catalog="customer",cluster_missing="false",deployed_version="1.0.0",latest_version="1.0.0",name="example",namespace="default",status="deployed",team="honeybadger",upgrade_available="false",version="1.0.0",version_mismatch="false"} 1 -app_operator_app_info{app="hello-world-app",app_version="",catalog="giantswarm",cluster_missing="false",deployed_version="0.3.0",latest_version="0.3.0",name="hello-world-app",namespace="hello-world",status="deployed",team="honeybadger",upgrade_available="false",version="0.3.0",version_mismatch="false"} 1 +app_operator_app_info{app="example",app_version="",catalog="customer",cluster_id="",cluster_missing="false",deployed_version="1.0.0",latest_version="1.0.0",name="example",namespace="default",status="deployed",team="honeybadger",upgrade_available="false",version="1.0.0",version_mismatch="false"} 1 +app_operator_app_info{app="hello-world-app",app_version="",catalog="giantswarm",cluster_id="",cluster_missing="false",deployed_version="0.3.0",latest_version="0.3.0",name="hello-world-app",namespace="hello-world",status="deployed",team="honeybadger",upgrade_available="false",version="0.3.0",version_mismatch="false"} 1 +app_operator_app_info{app="test-app",app_version="",catalog="default",cluster_id="foo",cluster_missing="false",deployed_version="1.0.0",latest_version="1.0.0",name="test-app",namespace="test-app",status="deployed",team="honeybadger",upgrade_available="false",version="1.0.0",version_mismatch="false"} 1 diff --git a/service/collector/testdata/expected.2 b/service/collector/testdata/expected.2 index 56bd7a87..4f6eb614 100644 --- a/service/collector/testdata/expected.2 +++ b/service/collector/testdata/expected.2 @@ -1,4 +1,4 @@ # HELP app_operator_app_info Managed apps status. # TYPE app_operator_app_info gauge -app_operator_app_info{app="example",app_version="",catalog="customer",cluster_missing="false",deployed_version="0.9.0",latest_version="1.0.0",name="example",namespace="default",status="deployed",team="honeybadger",upgrade_available="false",version="1.0.0",version_mismatch="true"} 1 -app_operator_app_info{app="hello-world-app",app_version="",catalog="giantswarm",cluster_missing="false",deployed_version="0.3.0",latest_version="0.3.0",name="hello-world-app",namespace="hello-world",status="deployed",team="honeybadger",upgrade_available="false",version="0.3.0",version_mismatch="false"} 1 +app_operator_app_info{app="example",app_version="",catalog="customer",cluster_id="",cluster_missing="false",deployed_version="0.9.0",latest_version="1.0.0",name="example",namespace="default",status="deployed",team="honeybadger",upgrade_available="false",version="1.0.0",version_mismatch="true"} 1 +app_operator_app_info{app="hello-world-app",app_version="",catalog="giantswarm",cluster_id="",cluster_missing="false",deployed_version="0.3.0",latest_version="0.3.0",name="hello-world-app",namespace="hello-world",status="deployed",team="honeybadger",upgrade_available="false",version="0.3.0",version_mismatch="false"} 1 diff --git a/tests/ats/metrics_test.go b/tests/ats/metrics_test.go index ab025d4e..4a2cb728 100644 --- a/tests/ats/metrics_test.go +++ b/tests/ats/metrics_test.go @@ -14,6 +14,9 @@ import ( "testing" "time" + "github.com/giantswarm/k8smetadata/pkg/label" + v1 "k8s.io/api/core/v1" + "github.com/giantswarm/apiextensions-application/api/v1alpha1" "github.com/giantswarm/backoff" "github.com/giantswarm/k8sclient/v8/pkg/k8sclient" @@ -70,6 +73,105 @@ func TestMetrics(t *testing.T) { } } + catalogCR := &v1alpha1.Catalog{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "giantswarm", + Labels: map[string]string{ + label.AppOperatorVersion: "0.0.0", + }, + }, + Spec: v1alpha1.CatalogSpec{ + Description: "default", + Title: "default", + Repositories: []v1alpha1.CatalogSpecRepository{ + { + Type: "helm", + URL: "https://giantswarm.github.io/default-catalog", + }, + }, + Storage: v1alpha1.CatalogSpecStorage{ + Type: "helm", + URL: "https://giantswarm.github.io/default-catalog", + }, + }, + } + err = k8sClients.CtrlClient().Create(ctx, catalogCR) + if err != nil { + t.Fatalf("failed to create default catalog: %#v", err) + } + + { + err := k8sClients.CtrlClient().Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + }, + }) + if err != nil { + t.Fatalf("failed to create test-app namespace: %#v", err) + } + } + + { + testAppUserValues := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app-user-values", + Namespace: "test-app", + }, + Data: map[string]string{ + "values": "namespace: test-app", + }, + } + + _, err := k8sClients.K8sClient().CoreV1().ConfigMaps("test-app").Create(ctx, testAppUserValues, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create test-app-user-valies config map: %#v", err) + } + } + + { + err := k8sClients.CtrlClient().Create(ctx, &v1alpha1.App{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-app", + Namespace: "test-app", + Labels: map[string]string{ + label.AppOperatorVersion: "0.0.0", + label.AppKubernetesName: "test-app", + label.App: "test-app", + label.Cluster: "kind", + "foo": "bar", + }, + }, + Spec: v1alpha1.AppSpec{ + Catalog: "default", + KubeConfig: v1alpha1.AppSpecKubeConfig{ + InCluster: true, + }, + Name: "test-app", + Namespace: "test-app", + Version: "1.0.0", + UserConfig: v1alpha1.AppSpecUserConfig{ + ConfigMap: v1alpha1.AppSpecUserConfigConfigMap{ + Name: "test-app-user-values", + Namespace: "test-app", + }, + }, + }, + }) + if err != nil { + t.Fatalf("failed to create test-app app cr: %#v", err) + } + } + + t.Logf("Waiting for test-app to come up...") + + testAppPodName, err := waitForPod(ctx, k8sClients, "test-app", "test-app") + if err != nil { + t.Fatalf("could not get test-app pod %#v", err) + } + + t.Logf("Waited for test-app to come up: %s", testAppPodName) + var fw *k8sportforward.Forwarder { c := k8sportforward.ForwarderConfig{ @@ -84,33 +186,33 @@ func TestMetrics(t *testing.T) { var podName string { - logger.Debugf(ctx, "waiting for %#q pod", project.Name()) + t.Logf("waiting for %#q pod", project.Name()) - podName, err = waitForPod(ctx, k8sClients) + podName, err = waitForPod(ctx, k8sClients, namespace, project.Name()) if err != nil { t.Fatalf("could not get %#q pod %#v", project.Name(), err) } - logger.Debugf(ctx, "waited for %#q pod", project.Name()) + t.Logf("waited for %#q pod: %s", project.Name(), podName) } var tunnel *k8sportforward.Tunnel { - logger.Debugf(ctx, "creating tunnel to pod %#q on port %d", podName, serverPort) + t.Logf("creating tunnel to pod %#q on port %d", podName, serverPort) tunnel, err = fw.ForwardPort(namespace, podName, serverPort) if err != nil { t.Fatalf("could not create tunnel %v", err) } - logger.Debugf(ctx, "created tunnel to pod %#q on port %d", podName, serverPort) + t.Logf("created tunnel to pod %#q on port %d", podName, serverPort) } var metricsResp *http.Response { metricsURL := fmt.Sprintf("http://%s/metrics", tunnel.LocalAddress()) - logger.Debugf(ctx, "getting metrics from %#q", metricsURL) + t.Logf("getting metrics from %#q", metricsURL) metricsResp, err = waitForServer(metricsURL) if err != nil { @@ -121,10 +223,11 @@ func TestMetrics(t *testing.T) { t.Fatalf("expected http status %#q got %#q", http.StatusOK, metricsResp.StatusCode) } - logger.Debugf(ctx, "got metrics from %#q", metricsURL) + t.Logf("got metrics from %#q", metricsURL) } var app *v1alpha1.App + var testApp *v1alpha1.App { app = &v1alpha1.App{} err = k8sClients.CtrlClient().Get(ctx, types.NamespacedName{Namespace: namespace, Name: project.Name()}, app) @@ -138,21 +241,45 @@ func TestMetrics(t *testing.T) { appVersion = expkey.FormatVersion(app.Status.AppVersion) } - expectedAppMetric := fmt.Sprintf("app_operator_app_info{app=\"%s\",app_version=\"%s\",catalog=\"%s\",cluster_missing=\"%s\",deployed_version=\"%s\",latest_version=\"%s\",name=\"%s\",namespace=\"%s\",status=\"%s\",team=\"noteam\",upgrade_available=\"%s\",version=\"%s\",version_mismatch=\"%s\"} 1", + expectedAppExporterMetric := fmt.Sprintf("app_operator_app_info{app=\"%s\",app_version=\"%s\",catalog=\"%s\",cluster_id=\"%s\",cluster_missing=\"%s\",deployed_version=\"%s\",latest_version=\"%s\",name=\"%s\",namespace=\"%s\",status=\"%s\",team=\"noteam\",upgrade_available=\"%s\",version=\"%s\",version_mismatch=\"%s\"} 1", app.Spec.Name, appVersion, app.Spec.Catalog, + "", "false", expkey.FormatVersion(app.Status.Version), // deployed_version "", // latest_version is empty app.Name, app.Namespace, app.Status.Release.Status, - "false", // upgrade_avaiable is false + "false", // upgrade_available is false expkey.FormatVersion(app.Spec.Version), // version is the desired version strconv.FormatBool(app.Spec.Version != app.Status.Version)) - logger.Debugf(ctx, "checking for expected app metric\n%s", expectedAppMetric) + t.Logf("Expected app-exporter metrics:\n%s", expectedAppExporterMetric) + + testApp = &v1alpha1.App{} + err = k8sClients.CtrlClient().Get(ctx, types.NamespacedName{Namespace: "test-app", Name: "test-app"}, testApp) + if err != nil { + t.Fatalf("expected nil got %#q", err) + } + + expectedTestAppMetric := fmt.Sprintf("app_operator_app_info{app=\"%s\",app_version=\"%s\",catalog=\"%s\",cluster_id=\"%s\",cluster_missing=\"%s\",deployed_version=\"%s\",latest_version=\"%s\",name=\"%s\",namespace=\"%s\",status=\"%s\",team=\"honeybadger\",upgrade_available=\"%s\",version=\"%s\",version_mismatch=\"%s\"} 1", + testApp.Spec.Name, + "2.13.0", + testApp.Spec.Catalog, + "kind", + "false", + expkey.FormatVersion(testApp.Status.Version), // deployed_version + "", // latest_version is empty + testApp.Name, + testApp.Namespace, + testApp.Status.Release.Status, + "false", // upgrade_available is false + expkey.FormatVersion(testApp.Spec.Version), // version is the desired version + strconv.FormatBool(testApp.Spec.Version != testApp.Status.Version)) + + t.Logf("Expected test-app metrics:\n%s", expectedTestAppMetric) respBytes, err := ioutil.ReadAll(metricsResp.Body) if err != nil { @@ -160,32 +287,41 @@ func TestMetrics(t *testing.T) { } metrics := string(respBytes) - if !strings.Contains(metrics, expectedAppMetric) { - t.Fatalf("expected app metric\n\n%s\n\nnot found in response\n\n%s", expectedAppMetric, metrics) + + t.Logf("METRICS RESPONSE START") + t.Logf("%s", metrics) + t.Logf("METROCS RESPONSE END") + + if !strings.Contains(metrics, expectedAppExporterMetric) { + t.Fatalf("expected app (app-exporter) metric\n\n%s\n\nnot found in response\n\n%s", expectedAppExporterMetric, metrics) + } + + if !strings.Contains(metrics, expectedTestAppMetric) { + t.Fatalf("expected app (test-app) metric\n\n%s\n\nnot found in response\n\n%s", expectedTestAppMetric, metrics) } - logger.Debugf(ctx, "found expected app metric") + t.Logf("found expected app metric") expectedAppOperatorMetric := "app_operator_ready_total{namespace=\"giantswarm\",version=\"0.0.0\"} 1" - logger.Debugf(ctx, "checking for expected app-operator metric\n%s", expectedAppOperatorMetric) + t.Logf("checking for expected app-operator metric\n%s", expectedAppOperatorMetric) if !strings.Contains(metrics, expectedAppOperatorMetric) { t.Fatalf("expected app metric\n\n%s\n\nnot found in response\n\n%s", expectedAppOperatorMetric, metrics) } - logger.Debugf(ctx, "found expected app-operator metric") + t.Logf("found expected app-operator metric") } } -func waitForPod(ctx context.Context, k8sClients *k8sclient.Clients) (string, error) { +func waitForPod(ctx context.Context, k8sClients *k8sclient.Clients, namespace string, appLabelValue string) (string, error) { var err error var podName string o := func() error { lo := metav1.ListOptions{ FieldSelector: "status.phase=Running", - LabelSelector: fmt.Sprintf("app=%s", project.Name()), + LabelSelector: fmt.Sprintf("app=%s", appLabelValue), } pods, err := k8sClients.K8sClient().CoreV1().Pods(namespace).List(ctx, lo) if err != nil {