From f81fbf4d2dc86aa739001a215550006cef92a015 Mon Sep 17 00:00:00 2001 From: Elliott Baron Date: Fri, 20 Sep 2024 16:29:34 -0400 Subject: [PATCH] Allow proxy to work without TLS --- api/v1beta2/cryostat_types.go | 2 +- ...yostat-operator.clusterserviceversion.yaml | 2 +- .../operator.cryostat.io_cryostats.yaml | 2 +- .../bases/operator.cryostat.io_cryostats.yaml | 2 +- .../resource_definitions.go | 68 +++--- internal/controllers/configmaps.go | 196 ++++++++++-------- internal/controllers/constants/constants.go | 5 +- internal/controllers/reconciler.go | 2 +- internal/controllers/reconciler_test.go | 26 ++- internal/controllers/services.go | 7 +- internal/test/resources.go | 146 ++++++++++--- 11 files changed, 290 insertions(+), 168 deletions(-) diff --git a/api/v1beta2/cryostat_types.go b/api/v1beta2/cryostat_types.go index 4148c579..36885a1c 100644 --- a/api/v1beta2/cryostat_types.go +++ b/api/v1beta2/cryostat_types.go @@ -314,7 +314,7 @@ type ReportsServiceConfig struct { // traffic from Cryostat agents to the Cryostat application. type AgentServiceConfig struct { // HTTP port number for the Cryostat agent API service. - // Defaults to 8443. + // Defaults to 8282. // +optional HTTPPort *int32 `json:"httpPort,omitempty"` ServiceConfig `json:",inline"` diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index 730c6ead..326f8a19 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -30,7 +30,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:4.0.0-dev - createdAt: "2024-09-20T18:00:33Z" + createdAt: "2024-09-20T20:29:31Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { diff --git a/bundle/manifests/operator.cryostat.io_cryostats.yaml b/bundle/manifests/operator.cryostat.io_cryostats.yaml index 05092905..5eceac38 100644 --- a/bundle/manifests/operator.cryostat.io_cryostats.yaml +++ b/bundle/manifests/operator.cryostat.io_cryostats.yaml @@ -9142,7 +9142,7 @@ spec: httpPort: description: |- HTTP port number for the Cryostat agent API service. - Defaults to 8443. + Defaults to 8282. format: int32 type: integer labels: diff --git a/config/crd/bases/operator.cryostat.io_cryostats.yaml b/config/crd/bases/operator.cryostat.io_cryostats.yaml index a963d40b..109b114f 100644 --- a/config/crd/bases/operator.cryostat.io_cryostats.yaml +++ b/config/crd/bases/operator.cryostat.io_cryostats.yaml @@ -9129,7 +9129,7 @@ spec: httpPort: description: |- HTTP port number for the Cryostat agent API service. - Defaults to 8443. + Defaults to 8282. format: int32 type: integer labels: diff --git a/internal/controllers/common/resource_definitions/resource_definitions.go b/internal/controllers/common/resource_definitions/resource_definitions.go index 672b24cd..a93eddf5 100644 --- a/internal/controllers/common/resource_definitions/resource_definitions.go +++ b/internal/controllers/common/resource_definitions/resource_definitions.go @@ -271,6 +271,7 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima NewStorageContainer(cr, imageTags.StorageImageTag, tls), newDatabaseContainer(cr, imageTags.DatabaseImageTag, tls), *authProxy, + newAgentProxyContainer(cr, imageTags.AgentProxyImageTag, tls), } volumes := newVolumeForCR(cr) @@ -303,8 +304,6 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima } if tls != nil { - containers = append(containers, newAgentProxyContainer(cr, imageTags.AgentProxyImageTag, tls)) - volSources = append(volSources, corev1.VolumeProjection{ // Add Cryostat self-signed CA Secret: &corev1.SecretProjection{ @@ -340,17 +339,6 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima }, }, }, - corev1.Volume{ - Name: "agent-proxy-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cr.Name + "-agent-proxy", - }, - DefaultMode: &readOnlyMode, - }, - }, - }, corev1.Volume{ Name: "keystore", VolumeSource: corev1.VolumeSource{ @@ -378,7 +366,21 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima }, }, } - volumes = append(volumes, certVolume) + + // Add agent proxy config map as a volume + agentProxyVolume := corev1.Volume{ + Name: "agent-proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cr.Name + "-agent-proxy", + }, + DefaultMode: &readOnlyMode, + }, + }, + } + + volumes = append(volumes, certVolume, agentProxyVolume) if !openshift { // if not deploying openshift-oauth-proxy then we must be deploying oauth2_proxy instead @@ -1564,7 +1566,23 @@ func newAgentProxyContainer(cr *model.CryostatInstance, imageTag string, tls *TL } } - healthCheckPort := int32(8081) + // Mount the config map containing the nginx.conf (and DH params if TLS is enabled) + mounts := []corev1.VolumeMount{ + { + Name: "agent-proxy-config", + MountPath: constants.AgentProxyConfigFilePath, + ReadOnly: true, + }, + } + if tls != nil { + // Mount the TLS secret for the agent proxy + mounts = append(mounts, corev1.VolumeMount{ + Name: "agent-proxy-tls-secret", + MountPath: "/var/run/secrets/operator.cryostat.io/" + tls.AgentProxySecret, + ReadOnly: true, + }) + } + return corev1.Container{ Name: cr.Name + "-agent-proxy", Image: imageTag, @@ -1574,37 +1592,27 @@ func newAgentProxyContainer(cr *model.CryostatInstance, imageTag string, tls *TL ContainerPort: constants.AgentProxyContainerPort, }, { - ContainerPort: healthCheckPort, + ContainerPort: constants.AgentProxyHealthPort, }, }, + // Override the command to run nginx pointed at our config file. See: + // https://github.com/sclorg/nginx-container/blob/e7d8db9bc5299a4c4e254f8a82e917c7c136468b/1.24/README.md#direct-usage-with-a-mounted-directory Command: []string{ "nginx", "-c", fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyConfigFileName), "-g", "daemon off;"}, - // Can't use HTTP probe since the port is not exposed over the network LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", - Port: intstr.FromInt32(healthCheckPort), + Port: intstr.FromInt32(constants.AgentProxyHealthPort), Scheme: corev1.URISchemeHTTP, }, }, }, SecurityContext: securityContext, Resources: *newAgentProxyContainerResource(cr), - VolumeMounts: []corev1.VolumeMount{ - { - Name: "agent-proxy-tls-secret", - MountPath: "/var/run/secrets/operator.cryostat.io/" + tls.AgentProxySecret, - ReadOnly: true, - }, - { - Name: "agent-proxy-config", - MountPath: constants.AgentProxyConfigFilePath, - ReadOnly: true, - }, - }, + VolumeMounts: mounts, } } diff --git a/internal/controllers/configmaps.go b/internal/controllers/configmaps.go index 7f3ba423..f7254035 100644 --- a/internal/controllers/configmaps.go +++ b/internal/controllers/configmaps.go @@ -86,86 +86,96 @@ type alphaConfigUpstream struct { } func (r *Reconciler) reconcileOAuth2ProxyConfig(ctx context.Context, cr *model.CryostatInstance, tls *resources.TLSConfig) error { + bindHost := "0.0.0.0" + immutable := true + cfg := &oauth2ProxyAlphaConfig{ + Server: alphaConfigServer{}, + UpstreamConfig: alphaConfigUpstreamConfig{ProxyRawPath: true, Upstreams: []alphaConfigUpstream{ + { + Id: "cryostat", + Path: "/", + Uri: fmt.Sprintf("http://localhost:%d", constants.CryostatHTTPContainerPort), + }, + { + Id: "grafana", + Path: "/grafana/", + Uri: fmt.Sprintf("http://localhost:%d", constants.GrafanaContainerPort), + }, + { + Id: "storage", + Path: "^/storage/(.*)$", + RewriteTarget: "/$1", + Uri: fmt.Sprintf("http://localhost:%d", constants.StoragePort), + PassHostHeader: false, + ProxyWebSockets: false, + }, + }}, + Providers: []alphaConfigProvider{{Id: "dummy", Name: "Unused - Sign In Below", ClientId: "CLIENT_ID", ClientSecret: "CLIENT_SECRET", Provider: "google"}}, + } + + if tls != nil { + cfg.Server.SecureBindAddress = fmt.Sprintf("https://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) + cfg.Server.TLS = proxyTLS{ + Key: tlsSecretSource{ + FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSPrivateKeyKey), + }, + Cert: tlsSecretSource{ + FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSCertKey), + }, + } + } else { + cfg.Server.BindAddress = fmt.Sprintf("http://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) + } + + data := make(map[string]string) + json, err := json.Marshal(cfg) + if err != nil { + return err + } + data[resources.OAuth2ConfigFileName] = string(json) cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: cr.Name + "-oauth2-proxy-cfg", Namespace: cr.InstallNamespace, }, + Immutable: &immutable, + Data: data, } if r.IsOpenShift { return r.deleteConfigMap(ctx, cm) + } else { + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + return nil + }) } - - return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { - bindHost := "0.0.0.0" - immutable := true - cfg := &oauth2ProxyAlphaConfig{ - Server: alphaConfigServer{}, - UpstreamConfig: alphaConfigUpstreamConfig{ProxyRawPath: true, Upstreams: []alphaConfigUpstream{ - { - Id: "cryostat", - Path: "/", - Uri: fmt.Sprintf("http://localhost:%d", constants.CryostatHTTPContainerPort), - }, - { - Id: "grafana", - Path: "/grafana/", - Uri: fmt.Sprintf("http://localhost:%d", constants.GrafanaContainerPort), - }, - { - Id: "storage", - Path: "^/storage/(.*)$", - RewriteTarget: "/$1", - Uri: fmt.Sprintf("http://localhost:%d", constants.StoragePort), - PassHostHeader: false, - ProxyWebSockets: false, - }, - }}, - Providers: []alphaConfigProvider{{Id: "dummy", Name: "Unused - Sign In Below", ClientId: "CLIENT_ID", ClientSecret: "CLIENT_SECRET", Provider: "google"}}, - } - - if tls != nil { - cfg.Server.SecureBindAddress = fmt.Sprintf("https://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) - cfg.Server.TLS = proxyTLS{ - Key: tlsSecretSource{ - FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSPrivateKeyKey), - }, - Cert: tlsSecretSource{ - FromFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.CryostatSecret, corev1.TLSCertKey), - }, - } - } else { - cfg.Server.BindAddress = fmt.Sprintf("http://%s:%d", bindHost, constants.AuthProxyHttpContainerPort) - } - - data := make(map[string]string) - json, err := json.Marshal(cfg) - if err != nil { - return err - } - data[resources.OAuth2ConfigFileName] = string(json) - - cm.Immutable = &immutable - cm.Data = data - return nil - }) } type nginxConfParams struct { - ServerName string - TLSCertFile string - TLSKeyFile string - CACertFile string - DHParamFile string + // Hostname of the server + ServerName string + // Whether TLS is enabled + TLSEnabled bool + // Path to certificate for HTTPS + TLSCertFile string + // Path to private key for HTTPS + TLSKeyFile string + // Path to CA certificate + CACertFile string + // Diffie-Hellman parameters file + DHParamFile string + // Nginx proxy container port ContainerPort int32 - CryostatPort int32 - AllowedPaths []string + // Nginx health container port + HealthPort int32 + // Cryostat HTTP container port + CryostatPort int32 + // Only these path prefixes will be proxied, others will return 404 + AllowedPathPrefixes []string } // Reference: https://ssl-config.mozilla.org -var nginxConfTemplate = template.Must(template.New("").Parse(` -worker_processes auto; +var nginxConfTemplate = template.Must(template.New("").Parse(`worker_processes auto; error_log stderr notice; pid /run/nginx.pid; @@ -192,9 +202,11 @@ http { default_type application/octet-stream; server { + server_name {{ .ServerName }}; + + {{ if .TLSEnabled -}} listen {{ .ContainerPort }} ssl; listen [::]:{{ .ContainerPort }} ssl; - server_name {{ .ServerName }}; ssl_certificate {{ .TLSCertFile }}; ssl_certificate_key {{ .TLSKeyFile }}; @@ -223,7 +235,14 @@ http { ssl_client_certificate {{ .CACertFile }}; ssl_verify_client on; - {{ range .AllowedPaths -}} + {{- else -}} + + listen {{ .ContainerPort }}; + listen [::]:{{ .ContainerPort }}; + + {{- end }} + + {{ range .AllowedPathPrefixes -}} location {{ . }} { proxy_pass http://127.0.0.1:{{ $.CryostatPort }}$request_uri; } @@ -237,7 +256,8 @@ http { # Heatlh Check server { - listen 8081; + listen {{ .HealthPort }}; + listen [::]:{{ .HealthPort }}; location = /healthz { return 200; @@ -249,8 +269,10 @@ http { } }`)) -// From https://ssl-config.mozilla.org/ffdhe2048.txt -const dhParams = `-----BEGIN DH PARAMETERS----- +const ( + dhFileName = "dhparam.pem" + // From https://ssl-config.mozilla.org/ffdhe2048.txt + dhParams = `-----BEGIN DH PARAMETERS----- MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz +8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a 87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 @@ -258,6 +280,7 @@ YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== -----END DH PARAMETERS-----` +) func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.CryostatInstance, tls *resources.TLSConfig) error { cm := &corev1.ConfigMap{ @@ -267,39 +290,44 @@ func (r *Reconciler) reconcileAgentProxyConfig(ctx context.Context, cr *model.Cr }, } - if tls == nil { // TODO make this work without TLS - return r.deleteConfigMap(ctx, cm) - } - + data := map[string]string{} buf := &bytes.Buffer{} params := &nginxConfParams{ ServerName: fmt.Sprintf("%s-agent.%s.svc", cr.Name, cr.InstallNamespace), - TLSCertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey), - TLSKeyFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey), - CACertFile: fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey), - DHParamFile: fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, constants.AgentProxyDHFileName), ContainerPort: constants.AgentProxyContainerPort, + HealthPort: constants.AgentProxyHealthPort, CryostatPort: constants.CryostatHTTPContainerPort, - AllowedPaths: []string{ + AllowedPathPrefixes: []string{ "/api/v2.2/discovery/", "/api/v2.2/credentials/", "/api/beta/recordings/", "/health/", }, } + if tls != nil { + params.TLSEnabled = true + params.TLSCertFile = fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSCertKey) + params.TLSKeyFile = fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, corev1.TLSPrivateKeyKey) + params.CACertFile = fmt.Sprintf("/var/run/secrets/operator.cryostat.io/%s/%s", tls.AgentProxySecret, constants.CAKey) + params.DHParamFile = fmt.Sprintf("%s/%s", constants.AgentProxyConfigFilePath, dhFileName) + + // Add Diffie-Hellman parameters to config map + data[dhFileName] = dhParams + } + + // Create an nginx.conf where: + // 1. If TLS is enabled, requires client certificate authentication against our CA + // 2. Proxies only those API endpoints required by the agent err := nginxConfTemplate.Execute(buf, params) if err != nil { return err } - return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { - immutable := true // TODO breaks update - cm.Immutable = &immutable + // Add generated nginx.conf to config map + data[constants.AgentProxyConfigFileName] = buf.String() - cm.Data = map[string]string{ - constants.AgentProxyConfigFileName: buf.String(), - constants.AgentProxyDHFileName: dhParams, - } + return r.createOrUpdateConfigMap(ctx, cm, cr.Object, func() error { + cm.Data = data return nil }) } diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go index 37c2f723..59b7bd9b 100644 --- a/internal/controllers/constants/constants.go +++ b/internal/controllers/constants/constants.go @@ -27,8 +27,8 @@ const ( ReportsContainerPort int32 = 10000 StoragePort int32 = 8333 DatabasePort int32 = 5432 - AgentProxyContainerPort int32 = 8443 - AgentProxyHealthPort int32 = 8081 + AgentProxyContainerPort int32 = 8282 + AgentProxyHealthPort int32 = 8281 LoopbackAddress string = "127.0.0.1" OperatorNamePrefix string = "cryostat-operator-" OperatorDeploymentName string = "cryostat-operator-controller" @@ -46,7 +46,6 @@ const ( AgentProxyConfigFilePath string = "/etc/nginx-cryostat" AgentProxyConfigFileName string = "nginx.conf" - AgentProxyDHFileName string = "dhparam.pem" // TODO non-exported targetNamespaceCRLabelPrefix = "operator.cryostat.io/" TargetNamespaceCRNameLabel = targetNamespaceCRLabelPrefix + "name" diff --git a/internal/controllers/reconciler.go b/internal/controllers/reconciler.go index 4474d4f6..37c5226f 100644 --- a/internal/controllers/reconciler.go +++ b/internal/controllers/reconciler.go @@ -269,7 +269,7 @@ func (r *Reconciler) reconcileCryostat(ctx context.Context, cr *model.CryostatIn if err != nil { return requeueIfIngressNotReady(reqLogger, err) } - err = r.reconcileAgentService(ctx, cr, tlsConfig, serviceSpecs) + err = r.reconcileAgentService(ctx, cr) if err != nil { return reconcile.Result{}, err } diff --git a/internal/controllers/reconciler_test.go b/internal/controllers/reconciler_test.go index f4dc86d4..612c69f4 100644 --- a/internal/controllers/reconciler_test.go +++ b/internal/controllers/reconciler_test.go @@ -1264,6 +1264,9 @@ func (c *controllerTest) commonTests() { It("should create routes with edge TLS termination", func() { t.expectRoutes() }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("with cert-manager not configured in CR", func() { BeforeEach(func() { @@ -1286,6 +1289,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "AllCertificatesReady") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("with DISABLE_SERVICE_TLS=true", func() { BeforeEach(func() { @@ -1315,6 +1321,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "CertManagerDisabled") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("Disable cert-manager after being enabled", func() { BeforeEach(func() { @@ -1342,6 +1351,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "CertManagerDisabled") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("Enable cert-manager after being disabled", func() { BeforeEach(func() { @@ -1373,6 +1385,9 @@ func (c *controllerTest) commonTests() { t.checkConditionPresent(operatorv1beta2.ConditionTypeTLSSetupComplete, metav1.ConditionTrue, "AllCertificatesReady") }) + It("should create the agent proxy config map", func() { + t.expectAgentProxyConfigMap() + }) }) Context("cert-manager missing", func() { JustBeforeEach(func() { @@ -2734,6 +2749,7 @@ func (t *cryostatTestInput) expectAgentProxyConfigMap() { Expect(err).ToNot(HaveOccurred()) t.checkMetadata(cm, expected) + Expect(cm.Data).To(Equal(expected.Data)) } func (t *cryostatTestInput) expectPVC(expectedPVC *corev1.PersistentVolumeClaim) { @@ -2991,6 +3007,7 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment, Expect(template.Spec.SecurityContext).To(Equal(t.NewPodSecurityContext(cr))) // Check that the networking environment variables are set correctly + Expect(len(template.Spec.Containers)).To(Equal(7)) coreContainer := template.Spec.Containers[0] port := int32(10000) if cr.Spec.ServiceOptions != nil && cr.Spec.ServiceOptions.ReportsConfig != nil && @@ -3045,12 +3062,9 @@ func (t *cryostatTestInput) checkMainPodTemplate(deployment *appsv1.Deployment, authProxyContainer := template.Spec.Containers[5] t.checkAuthProxyContainer(&authProxyContainer, t.NewAuthProxyContainerResource(cr), t.NewAuthProxySecurityContext(cr), cr.Spec.AuthorizationOptions) - // TODO make this work without TLS - if t.TLS { - // Check that Agent Proxy is configured properly - agentProxyContainer := template.Spec.Containers[6] - t.checkAgentProxyContainer(&agentProxyContainer, t.NewAgentProxyContainerResource(cr), t.NewAgentProxySecurityContext(cr)) - } + // Check that Agent Proxy is configured properly + agentProxyContainer := template.Spec.Containers[6] + t.checkAgentProxyContainer(&agentProxyContainer, t.NewAgentProxyContainerResource(cr), t.NewAgentProxySecurityContext(cr)) // Check that the proper Service Account is set Expect(template.Spec.ServiceAccountName).To(Equal(t.Name)) diff --git a/internal/controllers/services.go b/internal/controllers/services.go index 75697c2a..58ec16f5 100644 --- a/internal/controllers/services.go +++ b/internal/controllers/services.go @@ -111,8 +111,7 @@ func (r *Reconciler) reconcileReportsService(ctx context.Context, cr *model.Cryo return nil } -func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.CryostatInstance, - tls *resource_definitions.TLSConfig, specs *resource_definitions.ServiceSpecs) error { +func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.CryostatInstance) error { config := configureAgentService(cr) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -121,10 +120,6 @@ func (r *Reconciler) reconcileAgentService(ctx context.Context, cr *model.Cryost }, } - if tls == nil { - // Delete service if it exists - return r.deleteService(ctx, svc) - } return r.createOrUpdateService(ctx, svc, cr.Object, &config.ServiceConfig, func() error { svc.Spec.Selector = map[string]string{ "app": cr.Name, diff --git a/internal/test/resources.go b/internal/test/resources.go index 03dd9a51..53c2b2f0 100644 --- a/internal/test/resources.go +++ b/internal/test/resources.go @@ -832,8 +832,8 @@ func (r *TestResources) NewAgentProxyService() *corev1.Service { Ports: []corev1.ServicePort{ { Name: "http", - Port: 8443, - TargetPort: intstr.FromInt(8443), + Port: 8282, + TargetPort: intstr.FromInt(8282), }, }, }, @@ -1346,10 +1346,10 @@ func (r *TestResources) NewAuthProxyPorts() []corev1.ContainerPort { func (r *TestResources) NewAgentProxyPorts() []corev1.ContainerPort { return []corev1.ContainerPort{ { - ContainerPort: 8443, + ContainerPort: 8281, }, { - ContainerPort: 8081, + ContainerPort: 8282, }, } } @@ -2138,7 +2138,7 @@ func (r *TestResources) NewAgentProxyLivenessProbe() *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Port: intstr.IntOrString{IntVal: 8081}, + Port: intstr.IntOrString{IntVal: 8281}, Path: "/healthz", Scheme: corev1.URISchemeHTTP, }, @@ -2341,6 +2341,17 @@ func (r *TestResources) newVolumes(certProjections []corev1.VolumeProjection) [] }, }, }, + { + Name: "agent-proxy-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.Name + "-agent-proxy", + }, + DefaultMode: &readOnlymode, + }, + }, + }, } projs := append([]corev1.VolumeProjection{}, certProjections...) if r.TLS { @@ -2393,17 +2404,6 @@ func (r *TestResources) newVolumes(certProjections []corev1.VolumeProjection) [] }, }, }, - corev1.Volume{ - Name: "agent-proxy-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.Name + "-agent-proxy", - }, - DefaultMode: &readOnlymode, - }, - }, - }, ) } @@ -3293,9 +3293,7 @@ func (r *TestResources) NewLockConfigMap() *corev1.ConfigMap { } } -func (r *TestResources) NewAgentProxyConfigMap() *corev1.ConfigMap { - nginxConf := fmt.Sprintf(` -worker_processes auto; +const nginxFormatTLS = `worker_processes auto; error_log stderr notice; pid /run/nginx.pid; @@ -3322,12 +3320,13 @@ http { default_type application/octet-stream; server { - listen 8081 ssl; - listen [::]:8181 ssl; server_name %s-agent.%s.svc; - ssl_certificate /var/run/secrets/operator.cryostat.io/%s-agent/tls.crt; - ssl_certificate_key /var/run/secrets/operator.cryostat.io/%s-agent/tls.key; + listen 8282 ssl; + listen [::]:8282 ssl; + + ssl_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/tls.crt; + ssl_certificate_key /var/run/secrets/operator.cryostat.io/%s-agent-tls/tls.key; ssl_session_timeout 5m; ssl_session_cache shared:SSL:20m; @@ -3347,10 +3346,10 @@ http { ssl_stapling on; ssl_stapling_verify on; - ssl_trusted_certificate /var/run/secrets/operator.cryostat.io/%s-agent/ca.crt; + ssl_trusted_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/ca.crt; # Client certificate authentication - ssl_client_certificate /var/run/secrets/operator.cryostat.io/%s-agent/ca.crt; + ssl_client_certificate /var/run/secrets/operator.cryostat.io/%s-agent-tls/ca.crt; ssl_verify_client on; location /api/v2.2/discovery/ { @@ -3376,7 +3375,8 @@ http { # Heatlh Check server { - listen 8081; + listen 8281; + listen [::]:8281; location = /healthz { return 200; @@ -3386,15 +3386,81 @@ http { return 404; } } -}`, r.Name, r.Namespace, r.Name, r.Name, r.Name, r.Name) +}` - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: r.Name + "-agent-proxy", - Namespace: r.Namespace, - }, - Data: map[string]string{ - "nginx.conf": nginxConf, +const nginxFormatNoTLS = `worker_processes auto; +error_log stderr notice; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /dev/stdout main; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + types_hash_max_size 4096; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + server { + server_name %s-agent.%s.svc; + + listen 8282; + listen [::]:8282; + + location /api/v2.2/discovery/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /api/v2.2/credentials/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /api/beta/recordings/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location /health/ { + proxy_pass http://127.0.0.1:8181$request_uri; + } + + location / { + return 404; + } + } + + # Heatlh Check + server { + listen 8281; + listen [::]:8281; + + location = /healthz { + return 200; + } + + location / { + return 404; + } + } +}` + +func (r *TestResources) NewAgentProxyConfigMap() *corev1.ConfigMap { + var data map[string]string + if r.TLS { + data = map[string]string{ + "nginx.conf": fmt.Sprintf(nginxFormatTLS, r.Name, r.Namespace, r.Name, r.Name, r.Name, r.Name), "dhparam.pem": `-----BEGIN DH PARAMETERS----- MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz +8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a @@ -3403,7 +3469,19 @@ YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== -----END DH PARAMETERS-----`, + } + } else { + data = map[string]string{ + "nginx.conf": fmt.Sprintf(nginxFormatNoTLS, r.Name, r.Namespace), + } + } + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Name + "-agent-proxy", + Namespace: r.Namespace, }, + Data: data, } }