[release-v1.6] Tests for encryption with Kourier local gateway (knati…

…ve#1235)

* Enable internal-tls on ocp-tls (knative#1203)

* Enable internal-tls on OCP 4.8

* Use tls to match JOB name

* Add a target to enable internal-tls in Makefile (knative#1224)

* Add a target to enable internal-tls in Makefile

* Update CI template for internal-tls enabled

* Tests for encryption with Kourier local gateway (knative#13263)

* Generate Secrets

* Commit generated cert-secret.yaml

* httpproxy enables tls client

* httpproxy uses https when CA_CERT specified

* Pass CA_CERT and SERVER_NAME env variables properly to tests

* Avoid using cluster-local certificates for external services

* Enable tls tests for cluster-local Kourier gateway

* Need to create test resources including the test namespace first
before installing Knative so that applying
test/config/tls/cert-secret.yaml succeeds

* TMP: Enable tls in the standard e2e make target - test purposes

* Fix indentation

* Use knative-serving-ingress ns for deploying server-certs

* Deploy certificates at test phase

* Separate test and install of installing certs

* Wait for knative-serving-ingress to exist

* Revert "TMP: Enable tls in the standard e2e make target - test purposes"

This reverts commit 54fabb3.

Co-authored-by: Kenjiro Nakayama <[email protected]>

.github/workflows/kind-e2e.yaml

@@ -312,6 +312,8 @@ jobs:
         echo "SYSTEM_NAMESPACE=$SYSTEM_NAMESPACE" >> $GITHUB_ENV
         echo "GATEWAY_OVERRIDE=$GATEWAY_OVERRIDE" >> $GITHUB_ENV
         echo "GATEWAY_NAMESPACE_OVERRIDE=$GATEWAY_NAMESPACE_OVERRIDE" >> $GITHUB_ENV
+        echo "CA_CERT=$CA_CERT" >> $GITHUB_ENV
+        echo "SERVER_NAME=$SERVER_NAME" >> $GITHUB_ENV
 
     - name: Test ${{ matrix.test-suite }}
       run: |

Makefile

@@ -30,6 +30,10 @@ test-e2e:
 	./openshift/e2e-tests.sh
 .PHONY: test-e2e
 
+test-e2e-tls:
+	ENABLE_INTERNAL_TLS="true" ./openshift/e2e-tests.sh
+.PHONY: test-e2e-tls
+
 test-images:
 	for img in $(TEST_IMAGES); do \
 		KO_DOCKER_REPO=$(DOCKER_REPO_OVERRIDE) ko resolve --tags=$(TEST_IMAGE_TAG) $(KO_FLAGS) -RBf $$img ; \

openshift/ci-operator/generate-ci-config.sh

@@ -4,6 +4,7 @@ branch=${1-'knative-v0.6.0'}
 openshift=${2-'4.3'}
 promotion_disabled=${3-false}
 generate_continuous=${4-false}
+internal_tls_enabled=${5-false}
 
 if [[ "$branch" == "knative-next" ]]; then
   promotion_name="knative-nightly"
@@ -150,7 +151,9 @@ EOF
 
   print_single_test    "e2e-aws-ocp-${openshift//./}"             "make test-e2e"         "" "true" "generic-claim" ""
 
-  if [[ "$generate_continuous" == true ]]; then
+  if [[ "$internal_tls_enabled" == true ]]; then
+    print_single_test "e2e-aws-ocp-${openshift//./}-continuous"  "make test-e2e-tls"          "" "true" "generic-claim" "${cron}"
+  elif [[ "$generate_continuous" == true ]]; then
     print_single_test "e2e-aws-ocp-${openshift//./}-continuous"  "make test-e2e"          "" "true" "generic-claim" "${cron}"
   fi
 

openshift/ci-operator/update-ci.sh

@@ -39,7 +39,7 @@ CONFIG=$CONFIGDIR/openshift-knative-serving-release-$VERSION
 PERIODIC_CONFIG=$PERIODIC_CONFIGDIR/openshift-knative-serving-release-$VERSION-periodics.yaml
 CURDIR=$(dirname $0)
 
-# $1=branch $2=openshift $3=promotion_disabled $4=generate_continuous
+# $1=branch $2=openshift $3=promotion_disabled $4=generate_continuous $5=internal_tls_enabled(optional)
 $CURDIR/generate-ci-config.sh knative-$VERSION 4.6 true false > ${CONFIG}__46.yaml
 $CURDIR/generate-ci-config.sh knative-$VERSION 4.7 true false > ${CONFIG}__47.yaml
 $CURDIR/generate-ci-config.sh knative-$VERSION 4.8 true false > ${CONFIG}__48.yaml

openshift/e2e-common.sh

@@ -229,6 +229,23 @@ spec:
       logging.enable-request-log: "true"
 EOF
 
+  # TODO: Only one cluster enables internal-tls but it should be enabled by default when the feature is stable.
+  if [[ ${ENABLE_INTERNAL_TLS} == "true" ]]; then
+    oc patch knativeserving knative-serving \
+        -n "${SERVING_NAMESPACE}" \
+        --type merge --patch '{"spec": {"config": {"network": {"internal-encryption": "true"}}}}'
+    oc patch knativeserving knative-serving \
+        -n "${SERVING_NAMESPACE}" \
+        --type merge --patch '{"spec": {"config": {"kourier": {"cluster-cert-secret": "server-certs"}}}}'
+    # Deploy certificates for testing TLS with cluster-local gateway
+    timeout 600 '[[ $(oc get ns $SERVING_INGRESS_NAMESPACE -oname | wc -l) == 0 ]]' || return 1
+    yq read --doc 1 ./test/config/tls/cert-secret.yaml | sed "s/knative-serving/${SERVING_INGRESS_NAMESPACE}/" | oc apply -f -
+    echo "Restart activator to mount the certificates"
+    oc delete pod -n ${SERVING_NAMESPACE} -l app=activator
+    oc wait --timeout=60s --for=condition=Available deployment  -n ${SERVING_NAMESPACE} activator
+    echo "internal-encryption is enabled"
+  fi
+
   # Wait for 4 pods to appear first
   timeout 600 '[[ $(oc get pods -n $SERVING_NAMESPACE --no-headers | wc -l) -lt 4 ]]' || return 1
   wait_until_pods_running $SERVING_NAMESPACE || return 1
@@ -269,6 +286,15 @@ function prepare_knative_serving_tests_nightly {
   export GATEWAY_OVERRIDE=kourier
   export GATEWAY_NAMESPACE_OVERRIDE="$SERVING_INGRESS_NAMESPACE"
   export INGRESS_CLASS=kourier.ingress.networking.knative.dev
+
+  if [[ ${ENABLE_INTERNAL_TLS} == "true" ]]; then
+    # Deploy CA cert for testing TLS with cluster-local gateway
+    yq read --doc 0 ./test/config/tls/cert-secret.yaml | oc apply -f -
+    # This needs to match the name of Secret in test/config/tls/cert-secret.yaml
+    export CA_CERT=ca-cert
+    # This needs to match $san from test/config/tls/generate.sh
+    export SERVER_NAME=knative.dev
+  fi
 }
 
 function run_e2e_tests(){

openshift/e2e-tests.sh

@@ -9,6 +9,8 @@ env
 
 failed=0
 
+export ENABLE_INTERNAL_TLS="${ENABLE_INTERNAL_TLS:-false}"
+
 (( !failed )) && install_knative || failed=1
 (( !failed )) && prepare_knative_serving_tests_nightly || failed=2
 (( !failed )) && run_e2e_tests || failed=3

test/config/tls/cert-secret.yaml

@@ -0,0 +1,30 @@
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ca-cert
+  namespace: serving-tests
+data:
+  ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURSVENDQWkyZ0F3SUJBZ0lVQ2pESkw2RStjQ3FVVGdOdXNkY0g2Z2pNRStNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRWFNQmdHQTFVRUNnd1JTMjVoZEdsMlpTQkRiMjF0ZFc1cGRIa3hGREFTQmdOVkJBTU1DMlY0WVcxdwpiR1V1WTI5dE1CNFhEVEl5TURnek1UQTVNVE14TVZvWERUSXpNRGd6TVRBNU1UTXhNVm93TWpFYU1CZ0dBMVVFCkNnd1JTMjVoZEdsMlpTQkRiMjF0ZFc1cGRIa3hGREFTQmdOVkJBTU1DMlY0WVcxd2JHVXVZMjl0TUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBbkdWdVROa3pkcm10dEdqOWRWWjBiYlp2d3UwaQpjd1I5MUZ1U3NyRmVYZFNuV212MWJxTWZ6SGJWVkFoYTRRV2xwWktqc2M4bG9OalV6WEdLZ1o3Ulp0ajA0endKCnVubHJmSm5obTZOL2ZnYVNKem1RaEpjek9ZZ3VLM290dzN6dGxtZ3RpWE56VTBoUDc0bWNBMjlPS0NwbDdTNkkKWmwrV2xNUTdIWldqemY0ZUR3TmRqMDNkWm0zSGVweHVyWUFCMHZmMEk0L25ETHpxL0Jha2drVkpTTmdCWDFBLwo4TzluN0dCeGQ2NEU2WWxmY294UmZybHdNNkg0L2RmeTRMdTZYdlAzU3RvNWhWNWZsb2RzU1lNZ2xJNFE0dDZ0CmdTSUpjSGpyRExXL0RpSUkxQmw0aHpHelFtY0N3ZWFXbzQ4YkFsaFhOK1NFWm5iMDlRcE1oMHZ5N1FJREFRQUIKbzFNd1VUQWRCZ05WSFE0RUZnUVVaTFF2aGZvWldPaUY4ZWRlTVA1UUxOejJsUFl3SHdZRFZSMGpCQmd3Rm9BVQpaTFF2aGZvWldPaUY4ZWRlTVA1UUxOejJsUFl3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCCkFRc0ZBQU9DQVFFQW16dUJHZUxJWGkyOTBhUXh6UU1pSjRhdWtJTUQzNXRFa0IxSUFTT2RiQWVRWEhGb3I3NFMKbENwMSt1Rm5nUjJuY1EydUFjSjM3VUZhVm04d2t2UkN2bXg5bWN4anpwQ1F1bXpsMU1QY3pHdWxybjEreHArVAoxYlk2QUg1RFo0VXkrQ0JvMUpEaE0xUFVzdlJMOTcyQUpPU2tYWEpYajRQcWdSdStBUlU1eDk4K2QvbUFIemhPCkNCdW5iQTZTZEhLYjVMOTdxbCtqSzNFb0I2VERLNmNvNW5LSStHcnBDcTRKcllUZ1R1NG1uTkp0WHJEdWNpWnEKbG5DTEdWTTAvaFJ2NGRGMWZ4eGdLVkRMUmVpaitaM1lGQk1YYlFmb3B0c1o3RC9Tay85UnVVbkgzNUFtQnZRMwpNL21pRERaSTZDTVNtREQ5TCtsNmhMb2N3Zi9rS1BOVkJRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: server-certs
+  namespace: knative-serving
+data:
+  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRGhaYk8rUTJtbVhHb2QKeHZTMDJncW9LVHoxWWRTRVJhQzgwbElxMzB3NHNlblVXRFJva1J5dDBSOGU5R29Qd2hneGxnWGZoM296ODZYZQp4WncrclZJd1RZQVE3UHp0TnFFU04vQXRMQ1dBWHV0OTRTcSt2Z1JyTVY0dGFNUHBBV3VmaDB0WFhxeFR3TTJBCk43dE5yUi9oaXVrdkoxSTl2empDbGVJK2RXeEtWWWpNWFRWMXRwMEpxSThZVlNKOXp2TGxSODhzRCtqSFk0am0KN3JERnlTREVVQVd1Vlp1NVdJdEpqY1N6NnQycXlReVdMcDNWcC9GRU9rbGdQQTRkelg5ZlRETjlaMzdTR2JSaQppaXRNMEYxdTNSYUdJeGFhS0NFYXg3UW1zODJ1YXdwNyt0cWUwQlpkYXlQS01UUUJJM3VhWXVIMjRTZWQ3dkVnCjRaYUI0bnFGQWdNQkFBRUNnZ0VBS0RPRWlKWHJmUkdVbDdVSnBrd1JoSWErYWFIR1RzVkFjdzBzUEp0Uk0vZC8KbGpFWlArRko5VEtNVTNBU2pyYjJxN2x2V0x3SUxzWHhPcmVTTTVla1JoczhrVWhEb0dlUytQWGpMNXRsSU8xTgpJVW1NM3pKekJVOXIxYnVPM2JzMEgrTDRyQitscXRhRGtLL2dCMjJ1ZHdMWXJtRmNDTWxYYlZWZ1lmVjltQkF0CnFRZ0NOdzlRVGhtVEpCY0VwWUtnNVdoem9pdjNUQmdWWXRmQ1I0d0VSOGJFaVA0ckhTQ001TFpqZ1VwQ1ZzVUMKRkxzY2hOVDl1SS9EcDhXN2tGMGJ0TjFIcjNuZWZ4MzN4cERxRmxvSzloeXM0ODhnUit1cXUyZGZMN1NMZ1JPZApCaTVFVytwRWZaQzlZckNlcGI1QnBabEM5NkF4US9kUGhQMER2b3lBYlFLQmdRRDhKazVzVlIvc3k2YjMyWE5jCld6VnpmclZNaFFYa1ZzMDh5UmNTWmpxeWYvTjhqczVRMjR3SzcvLzlhYjVnRWtVcGgxZUVmQU5TaEJEVFQrQlkKK3QzVDZUSHNpMVArdEc1NFJEcFVIU1EvSW53bmoxcSt5Q3hsNjM5VE1OVUU2bFRjWkJMRW9ZUmExcklsdXg2VwpVaHZZdVpKN3E4dk5HZk9yT2Z0MWQyZkQzd0tCZ1FEazF0RUZCbHlUeUtuazZlY2ZvZmdxbnhHd0s3N2pDaUJPCmRpcGhMT1NkVzh0ZWt5d0J6R1ViU3d5YUhYcXB4UEhQblp1eXZ0dWxJak9waW82VGVQMzl6emZFQVZuVEw3b2IKb0t4S3dHbXFBRUpMUkF4NWtZMUsybSt0a1hYYUwzeDZJMCtESGlmSXdoR3hPSk1xd3ZZM2FkbW11bkkzUmdMYwpydTRzTFJsdUd3S0JnQ0VHdzI3ZEYzbGtrMUlUWVZEUGdZakhKK2dGNUdlc0Z1WEhVUVpQN1pCRHdoaW1lOCtMCmNpUmNteU1PSHFsbXV6aGRTZEZJalFiWjFYcFlGQUtUbVVxUVdNR3EzaTJXWklITUxmZW1lWURyZTJlVEYwZTEKNEZyWkphdzMwUzc3b25IYmlibkhqaFozMkkyb25MRUR3REg2M0h6bVc2TlpxdGphbDEwamJxdnhBb0dCQU1ZbQpiSjlaUHRpSXJQUVd4WmJTZlQwS3VCcEFCdTQ1V25nV1FlUWJKMnBLamZLNnBTUjVoQ0w1L1ZPRnF5MU41OFRLCnlJTWlXTGJJd3N0UHV0MWZxeThYTzBaeGxRSVZGYVhPbnVHcmN0Tk5uaG5tTnBjZHZhYlBObHlvMDgrMXhxZEwKNUJHNUR1SDdpYTVYT3JlUVVmcnhvUkdKNkZTTVB2WXdVdlBWcVd0NUFvR0FGSmx2MEFEaTY5eXpVaUU5Y2tZOQplYTJzVEFlZmJSbHNsaVBNdXNuWjdPSE81UzAzOURJZ01xSk9VOHlXVlRYR3FmTFhsZDJoZG9GeDRCSURwY1F3CkJGeHJ3d0s4eGVwY21zd29BNUk5Qlg0WTBMclQzeU5JbFdod0NJaE9RdlZnOXR3ZDdKOGRlY21IbVpFaXVRbTEKVUFpRHo5UHhlL0MwdC9sb2JGWHpDYTg9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURERENDQWZTZ0F3SUJBZ0lVUXFEUTkrWWRPdDUydWFaT0JMYStTNWdGWnJ3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd01qRWFNQmdHQTFVRUNnd1JTMjVoZEdsMlpTQkRiMjF0ZFc1cGRIa3hGREFTQmdOVkJBTU1DMlY0WVcxdwpiR1V1WTI5dE1CNFhEVEl5TURnek1UQTVNVE14TVZvWERUSXpNRGd6TVRBNU1UTXhNVm93TWpFVU1CSUdBMVVFCkF3d0xaWGhoYlhCc1pTNWpiMjB4R2pBWUJnTlZCQW9NRVV0dVlYUnBkbVVnUTI5dGJYVnVhWFI1TUlJQklqQU4KQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBNFdXenZrTnBwbHhxSGNiMHROb0txQ2s4OVdIVQpoRVdndk5KU0t0OU1PTEhwMUZnMGFKRWNyZEVmSHZScUQ4SVlNWllGMzRkNk0vT2wzc1djUHExU01FMkFFT3o4CjdUYWhFamZ3TFN3bGdGN3JmZUVxdnI0RWF6RmVMV2pENlFGcm40ZExWMTZzVThETmdEZTdUYTBmNFlycEx5ZFMKUGI4NHdwWGlQblZzU2xXSXpGMDFkYmFkQ2FpUEdGVWlmYzd5NVVmUExBL294Mk9JNXU2d3hja2d4RkFGcmxXYgp1VmlMU1kzRXMrcmRxc2tNbGk2ZDFhZnhSRHBKWUR3T0hjMS9YMHd6ZldkKzBobTBZb29yVE5CZGJ0MFdoaU1XCm1pZ2hHc2UwSnJQTnJtc0tlL3JhbnRBV1hXc2p5akUwQVNON21tTGg5dUVubmU3eElPR1dnZUo2aFFJREFRQUIKb3hvd0dEQVdCZ05WSFJFRUR6QU5nZ3RyYm1GMGFYWmxMbVJsZGpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQQpHN2xraFI5VC9OMFIxLzc1WGJ5dWdlOWgrYXFHTzdxblAyTHg2aHo3bHdjOFM2VTVNS09EaW9zRXhMOStKYzhSCkorbTJjTmFsR3pOTG1ER3FDbDBJU3g4SGRhc1ZQbW9McFlpamFlRWNXS1JoVzZaTUNyVlorOEpnQWVmcTFSaFQKMnBqM05OTllWZ2NaWVJsV2NydlBnWlFZZ3JYTGhCWCtPKzRQWmNXUlZzem1yYncwWms1aE53ZWNWc1lvbHRLVwpCdEpCNmx5WDBLcDdNcURqUXBnZUR4WnVTbElMaktTK1J6ejU0L3ZxTDhpQTJrN0Z6TWNUS1IydXhTRGtwK2lZCkQrQ2cyenBnb3c2RFJxeEowWmJ3YXZRY3ozWHQrc0FUYjZIcG10WFNyVHk1R2E4L0R2U1ZvSGl2YlNTMkJ5ZloKQ3BBZkhLT0FBUGdHbUFOaGJpQ2VhZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K

test/config/tls/generate.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script generates test/config/tls/cert-secret.yaml.
+
+san="knative.dev"
+
+# Create CA key and cert
+openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -subj '/O=Knative Community/CN=example.com' -keyout rootCAKey.pem -out rootCACert.pem
+
+# Create server key
+openssl req -out tls.csr -newkey rsa:2048 -nodes -keyout tls.key -subj "/CN=example.com/O=Knative Community" -addext "subjectAltName = DNS:$san"
+
+# Create server certs
+openssl x509 -req -extfile <(printf "subjectAltName=DNS:$san") -days 365 -in tls.csr -CA rootCACert.pem -CAkey rootCAKey.pem -CAcreateserial -out tls.crt
+
+CA_CERT=$(cat rootCACert.pem | base64  | tr -d '\n')
+TLS_KEY=$(cat tls.key | base64  | tr -d '\n')
+TLS_CERT=$(cat tls.crt | base64  | tr -d '\n')
+
+cat <<EOF > cert-secret.yaml
+# Copyright 2022 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ca-cert
+  namespace: serving-tests
+data:
+  ca.crt: ${CA_CERT}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: server-certs
+  namespace: knative-serving
+data:
+  tls.key: ${TLS_KEY}
+  tls.crt: ${TLS_CERT}
+EOF
+
+# Clean up
+rm -f rootCACert.pem rootCAKey.pem tls.key tls.crt tls.csr rootCACert.srl

test/e2e-common.sh

@@ -307,6 +307,10 @@ function install() {
     YTT_FILES+=("${REPO_ROOT_DIR}/test/config/resource-quota/resource-quota.yaml")
   fi
 
+  if (( ENABLE_TLS )); then
+    YTT_FILES+=("${REPO_ROOT_DIR}/test/config/tls/cert-secret.yaml")
+  fi
+
   local ytt_result=$(mktemp)
   local ytt_post_install_result=$(mktemp)
   local ytt_flags=""
@@ -360,11 +364,15 @@ function install() {
 
   if (( ENABLE_TLS )); then
     echo "Patch to config-network to enable internal encryption"
-    kubectl patch configmap/config-network \
-      -n ${SYSTEM_NAMESPACE} \
-      --type merge \
-      -p '{"data":{"internal-encryption":"true"}}'
-
+    toggle_feature internal-encryption true config-network
+    if [[ "$INGRESS_CLASS" == "kourier.ingress.networking.knative.dev" ]]; then
+      echo "Point Kourier local gateway to custom server certificates"
+      toggle_feature cluster-cert-secret server-certs config-kourier
+      # This needs to match the name of Secret in test/config/tls/cert-secret.yaml
+      export CA_CERT=ca-cert
+      # This needs to match $san from test/config/tls/generate.sh
+      export SERVER_NAME=knative.dev
+    fi
     echo "Restart activator to mount the certificates"
     kubectl delete pod -n ${SYSTEM_NAMESPACE} -l app=activator
     kubectl wait --timeout=60s --for=condition=Available deployment  -n ${SYSTEM_NAMESPACE} activator

test/e2e/service_to_service_test.go

@@ -24,12 +24,14 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
 	netapi "knative.dev/networking/pkg/apis/networking"
+	ptr "knative.dev/pkg/ptr"
 	pkgTest "knative.dev/pkg/test"
 	ingress "knative.dev/pkg/test/ingress"
 	"knative.dev/pkg/test/logstream"
@@ -45,6 +47,8 @@ const (
 	targetHostEnv      = "TARGET_HOST"
 	gatewayHostEnv     = "GATEWAY_HOST"
 	helloworldResponse = "Hello World! How about some tasty noodles?"
+	caCertDirectory    = "/var/lib/knative/ca"
+	caCertPath         = caCertDirectory + "/ca.crt"
 )
 
 // testCases for table-driven testing.
@@ -109,6 +113,16 @@ func testProxyToHelloworld(t *testing.T, clients *test.Clients, helloworldURL *u
 		})
 	}
 
+	caSecretName := os.Getenv("CA_CERT")
+
+	// External services use different TLS certificates than cluster-local services.
+	// Not passing CA_CERT will make the httpproxy use plain http to connect to the
+	// target service.
+	if caSecretName != "" && !accessibleExternal {
+		envVars = append(envVars, []corev1.EnvVar{{Name: "CA_CERT", Value: caCertPath},
+			{Name: "SERVER_NAME", Value: os.Getenv("SERVER_NAME")}}...)
+	}
+
 	// Set up httpproxy app.
 	t.Log("Creating a Service for the httpproxy test app.")
 	names := test.ResourceNames{
@@ -118,11 +132,24 @@ func testProxyToHelloworld(t *testing.T, clients *test.Clients, helloworldURL *u
 
 	test.EnsureTearDown(t, clients, &names)
 
-	resources, err := v1test.CreateServiceReady(t, clients, &names,
+	serviceOptions := []rtesting.ServiceOption{
 		rtesting.WithEnv(envVars...),
 		rtesting.WithConfigAnnotations(map[string]string{
 			"sidecar.istio.io/inject": strconv.FormatBool(inject),
-		}))
+		}),
+	}
+
+	if caSecretName != "" && !accessibleExternal {
+		serviceOptions = append(serviceOptions, rtesting.WithVolume("ca-certs", caCertDirectory, corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: caSecretName,
+				Optional:   ptr.Bool(false),
+			}}),
+		)
+	}
+
+	resources, err := v1test.CreateServiceReady(t, clients, &names, serviceOptions...)
+
 	if err != nil {
 		t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err)
 	}

test/test_images/httpproxy/httpproxy.go

@@ -17,6 +17,9 @@ limitations under the License.
 package main
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
 	"flag"
 	"log"
 	"os"
@@ -61,12 +64,17 @@ func main() {
 		routedHost = gateway
 	}
 
+	scheme := "http"
+	if caCert := os.Getenv("CA_CERT"); caCert != "" {
+		scheme = "https"
+	}
 	target := &url.URL{
-		Scheme: "http",
+		Scheme: scheme,
 		Host:   routedHost,
 	}
 	log.Print("target is ", target)
 	proxy := httputil.NewSingleHostReverseProxy(target)
+	proxy.Transport = newTLSEnabledTransport()
 	proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) {
 		log.Print("error reverse proxying request: ", err)
 		http.Error(w, err.Error(), http.StatusBadGateway)
@@ -84,3 +92,39 @@ func main() {
 	log.Print("Listening on address: ", address)
 	test.ListenAndServeGracefully(address, handler)
 }
+
+func newTLSEnabledTransport() http.RoundTripper {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	if caCert := os.Getenv("CA_CERT"); caCert != "" {
+		rootCAs, err := createRootCAs(caCert)
+		if err != nil {
+			log.Fatal(err)
+			return transport
+		}
+		transport.TLSClientConfig = &tls.Config{
+			RootCAs: rootCAs,
+			// If SERVER_NAME is not set the empty value will make the
+			// TLS client infer the ServerName from the hostname.
+			ServerName: os.Getenv("SERVER_NAME"),
+		}
+	}
+	return transport
+}
+
+func createRootCAs(caCertFile string) (*x509.CertPool, error) {
+	pemData, err := os.ReadFile(caCertFile)
+	if err != nil {
+		return nil, err
+	}
+	rootCAs, err := x509.SystemCertPool()
+	if rootCAs == nil || err != nil {
+		if err != nil {
+			log.Printf("Failed to load cert poll from system: %v. Will create a new cert pool.", err)
+		}
+		rootCAs = x509.NewCertPool()
+	}
+	if !rootCAs.AppendCertsFromPEM(pemData) {
+		return nil, errors.New("failed to add the certificate to the root CA")
+	}
+	return rootCAs, nil
+}