From 3b8bf2f018434c8b242a2639ab4d7a4f8548ed9e Mon Sep 17 00:00:00 2001 From: Nikhil Palaskar Date: Mon, 3 Jul 2023 10:10:50 -0400 Subject: [PATCH] Add TLS on Keycloak server (#3427) PBENCH-1138 1. Run Keycloak containers with SSL support 2. Make necessary changes in the pbench-server to connect to the Keycloak running on SSL. 3. Note: run-pbench-in-a-can script creates self signed pbench-server certificate and the same certificate is used for Keycloak configuration. --- exec-tests | 2 +- jenkins/run-server-func-tests | 10 +---- lib/pbench/client/__init__.py | 4 -- lib/pbench/client/oidc_admin.py | 2 +- lib/pbench/server/auth/__init__.py | 13 +++++-- lib/pbench/test/unit/server/auth/test_auth.py | 39 ++++++++++++------- server/lib/config/pbench-server-default.cfg | 4 ++ server/pbenchinacan/container-build.sh | 4 ++ server/pbenchinacan/deploy | 1 + server/pbenchinacan/deploy-dependencies | 6 ++- .../etc/pbench-server/pbench-server.cfg | 6 ++- server/pbenchinacan/load_keycloak.sh | 32 ++++++++++----- server/pbenchinacan/run-pbench-in-a-can | 3 +- tox.ini | 1 - 14 files changed, 81 insertions(+), 46 deletions(-) diff --git a/exec-tests b/exec-tests index a99dc828f7..69f04eb936 100755 --- a/exec-tests +++ b/exec-tests @@ -243,7 +243,7 @@ if [[ "${major}" == "all" || "${major}" == "server" ]]; then shift posargs="${@}" # We use SQLALCHEMY_SILENCE_UBER_WARNING here ... (see above). - SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True PBENCH_SERVER=${server_arg} KEEP_DATASETS="${keep_datasets}" pytest --tb=native -v -s -rs --pyargs ${posargs} pbench.test.functional.server + REQUESTS_CA_BUNDLE=${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True PBENCH_SERVER=${server_arg} KEEP_DATASETS="${keep_datasets}" pytest --tb=native -v -s -rs --pyargs ${posargs} pbench.test.functional.server rc=${?} fi fi diff --git a/jenkins/run-server-func-tests b/jenkins/run-server-func-tests index c01ec49894..a52c7ee132 100755 --- a/jenkins/run-server-func-tests +++ b/jenkins/run-server-func-tests @@ -8,15 +8,7 @@ export PB_SERVER_IMAGE_TAG=${PB_SERVER_IMAGE_TAG:-"$(cat jenkins/branch.name)"} export PB_POD_NAME=${PB_POD_NAME:-"pbench-in-a-can_${PB_SERVER_IMAGE_TAG}"} export PB_SERVER_CONTAINER_NAME=${PB_SERVER_CONTAINER_NAME:-"${PB_POD_NAME}-pbenchserver"} -# Note: the value of PB_HOST_IP will be used to generate the TLS certificate -# and so it (not `localhost`) must also be used to access the Pbench Server; -# otherwise, the TLS validation will fail due to a host mismatch. -if [[ -z "${PB_HOST_IP}" ]]; then - host_ip_list=$(hostname -I) - PB_HOST_IP=${host_ip_list%% *} - export PB_HOST_IP -fi -SERVER_URL="https://${PB_HOST_IP}:8443" +SERVER_URL="https://localhost:8443" SERVER_API_ENDPOINTS="${SERVER_URL}/api/v1/endpoints" # Have Curl use the Pbench CA certificate to validate the TLS/SSL connection diff --git a/lib/pbench/client/__init__.py b/lib/pbench/client/__init__.py index c67b418348..1e89dad819 100644 --- a/lib/pbench/client/__init__.py +++ b/lib/pbench/client/__init__.py @@ -1,5 +1,4 @@ from enum import Enum -import os from pathlib import Path from typing import Iterator, Optional from urllib import parse @@ -319,9 +318,6 @@ def connect(self, headers: Optional[dict[str, str]] = None) -> None: url = parse.urljoin(self.url, "api/v1/endpoints") self.session = requests.Session() - # Use the same CA as Curl to do TLS verification; - # if it's not defined then disable TLS verification. - self.session.verify = os.environ.get("CURL_CA_BUNDLE", False) if headers: self.session.headers.update(headers) response = self.session.get(url) diff --git a/lib/pbench/client/oidc_admin.py b/lib/pbench/client/oidc_admin.py index fb47a77b83..055de7ee32 100644 --- a/lib/pbench/client/oidc_admin.py +++ b/lib/pbench/client/oidc_admin.py @@ -12,7 +12,7 @@ class OIDCAdmin(Connection): ADMIN_PASSWORD = os.getenv("OIDC_ADMIN_PASSWORD", "admin") def __init__(self, server_url: str): - super().__init__(server_url, verify=False) + super().__init__(server_url) def get_admin_token(self) -> dict: """pbench-server realm admin user login. diff --git a/lib/pbench/server/auth/__init__.py b/lib/pbench/server/auth/__init__.py index e3bb986ca6..aa05d91b14 100644 --- a/lib/pbench/server/auth/__init__.py +++ b/lib/pbench/server/auth/__init__.py @@ -205,6 +205,11 @@ def wait_for_oidc_server( """ try: oidc_server = server_config.get("openid", "server_url") + oidc_realm = server_config.get("openid", "realm") + # Get a custom cert location to verify Keycloak ssl if its define + # in the config file. Otherwise, we default to using system-wide + # certificates. + cert = server_config.get("openid", "cert_location", fallback=True) except (NoOptionError, NoSectionError) as exc: raise OpenIDClient.NotConfigured() from exc @@ -224,18 +229,20 @@ def wait_for_oidc_server( status_forcelist=tuple(int(x) for x in HTTPStatus if x != 200), ) adapter = HTTPAdapter(max_retries=retry) - session.mount("http://", adapter) session.mount("https://", adapter) # We will also need to retry the connection if the health status is not UP. connected = False for _ in range(5): try: - response = session.get(f"{oidc_server}/health") + response = session.get( + f"{oidc_server}/realms/{oidc_realm}/.well-known/openid-configuration", + verify=cert, + ) response.raise_for_status() except Exception as exc: raise OpenIDClient.ServerConnectionError() from exc - if response.json().get("status") == "UP": + if response.json().get("issuer") == f"{oidc_server}/realms/{oidc_realm}": logger.debug("OIDC server connection verified") connected = True break diff --git a/lib/pbench/test/unit/server/auth/test_auth.py b/lib/pbench/test/unit/server/auth/test_auth.py index 177e7772e6..fb1f0d90d4 100644 --- a/lib/pbench/test/unit/server/auth/test_auth.py +++ b/lib/pbench/test/unit/server/auth/test_auth.py @@ -293,29 +293,36 @@ def test_wait_for_oidc_server_fail(self, make_logger): with pytest.raises(OpenIDClient.NotConfigured): OpenIDClient.wait_for_oidc_server(config, make_logger) # Missing "server_url" - section = {} - config["openid"] = section + config["openid"] = {} with pytest.raises(OpenIDClient.NotConfigured): OpenIDClient.wait_for_oidc_server(config, make_logger) - section = {"server_url": "https://example.com"} - config["openid"] = section + config["openid"] = { + "server_url": "https://example.com", + "realm": "realm", + "cert_location": "/ca.crt", + } + + # Keycloak well-known endpoint without any response + with pytest.raises(OpenIDClient.ServerConnectionError): + OpenIDClient.wait_for_oidc_server(config, make_logger) - # Keycloak health returning response but status is not UP + # Keycloak well-known endpoint returning response without a valid issuer responses.add( responses.GET, - "https://example.com/health", - body='{"status": "DOWN","checks": []}', + "https://example.com/realms/realm/.well-known/openid-configuration", + body="{}", content_type="application/json", ) with pytest.raises(OpenIDClient.ServerConnectionTimedOut): OpenIDClient.wait_for_oidc_server(config, make_logger) - # Keycloak health returning network exception and no content + # Keycloak well-known endpoint returning network exception and no + # content responses.add( responses.GET, - "https://example.com/health", + "https://example.com/realms/realm/.well-known/openid-configuration", body=Exception("some network exception"), ) @@ -327,14 +334,18 @@ def test_wait_for_oidc_server_succ(self, make_logger): """Verify .wait_for_oidc_server() success mode""" config = configparser.ConfigParser() - section = {"server_url": "https://example.com"} - config["openid"] = section - # Keycloak health returning response with status UP + config["openid"] = { + "server_url": "https://example.com", + "realm": "realm", + "cert_location": "/ca.crt", + } + + # Keycloak well-known endpoint returning response with valid issuer responses.add( responses.GET, - "https://example.com/health", - body='{"status": "UP","checks": []}', + "https://example.com/realms/realm/.well-known/openid-configuration", + body='{"issuer": "https://example.com/realms/realm"}', content_type="application/json", ) diff --git a/server/lib/config/pbench-server-default.cfg b/server/lib/config/pbench-server-default.cfg index 418e52815d..d4f231a92a 100644 --- a/server/lib/config/pbench-server-default.cfg +++ b/server/lib/config/pbench-server-default.cfg @@ -100,6 +100,10 @@ realm = pbench-server # Client entity name requesting OIDC to authenticate a user. client = pbench-client +# Cert location for connecting to the OIDC client +# If you want to use a custom CA then its location path should be recorded. +#cert_location = /path/CA + [logging] logger_type = devlog logging_level = INFO diff --git a/server/pbenchinacan/container-build.sh b/server/pbenchinacan/container-build.sh index 2a6aa8d45a..443c3acd49 100755 --- a/server/pbenchinacan/container-build.sh +++ b/server/pbenchinacan/container-build.sh @@ -63,6 +63,10 @@ buildah run $container rm -f /tmp/pbench-server.rpm buildah copy --chown root:root --chmod 0644 $container \ ${PBINC_SERVER}/lib/config/nginx.conf /etc/nginx/nginx.conf +# Add our Pbench Server CA certificate. +buildah copy --chown root:root --chmod 0444 $container \ + ${PBINC_INACAN}/etc/pki/tls/certs/pbench_CA.crt /etc/pki/tls/certs/pbench_CA.crt + # Since we configure Nginx to log via syslog directly, remove Nginx log rotation # configuration as it emits unnecessary "Permission denied" errors. buildah run $container rm /etc/logrotate.d/nginx diff --git a/server/pbenchinacan/deploy b/server/pbenchinacan/deploy index b76f6ee438..334a03a227 100755 --- a/server/pbenchinacan/deploy +++ b/server/pbenchinacan/deploy @@ -44,6 +44,7 @@ PB_DEPLOY_FILES=${PB_DEPLOY_FILES:-${HOME}/Deploy} SRV_PBENCH=${SRV_PBENCH:-/srv/pbench} PB_SSL_CERT_FILE=${PB_SSL_CERT_FILE:-${PB_DEPLOY_FILES}/pbench-server.crt} PB_SSL_KEY_FILE=${PB_SSL_KEY_FILE:-${PB_DEPLOY_FILES}/pbench-server.key} +PB_SSL_CA_FILE=${PB_SSL_CA_FILE:-${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt} # Locations inside the container # diff --git a/server/pbenchinacan/deploy-dependencies b/server/pbenchinacan/deploy-dependencies index b2155f4eed..49c46c1b9d 100755 --- a/server/pbenchinacan/deploy-dependencies +++ b/server/pbenchinacan/deploy-dependencies @@ -91,10 +91,14 @@ podman run \ --name "${PB_POD_NAME}-keycloak" \ --pull ${PB_KEYCLOAK_IMAGE_PULL_POLICY} \ --restart no \ + -v ${PB_DEPLOY_FILES}/pbench-server.crt:/opt/keycloak/conf/keycloak.crt:Z \ + -v ${PB_DEPLOY_FILES}/pbench-server.key:/opt/keycloak/conf/keycloak.key:Z \ -e KEYCLOAK_ADMIN=admin \ -e KEYCLOAK_ADMIN_PASSWORD=admin \ -d \ ${PB_KEYCLOAK_IMAGE} \ - start-dev --health-enabled=true --http-port=8090 + start-dev --health-enabled=true --https-port=8090 \ + --https-certificate-file=/opt/keycloak/conf/keycloak.crt \ + --https-certificate-key-file=/opt/keycloak/conf/keycloak.key server/pbenchinacan/load_keycloak.sh diff --git a/server/pbenchinacan/etc/pbench-server/pbench-server.cfg b/server/pbenchinacan/etc/pbench-server/pbench-server.cfg index 05468ce5e0..5e2c98cfc9 100644 --- a/server/pbenchinacan/etc/pbench-server/pbench-server.cfg +++ b/server/pbenchinacan/etc/pbench-server/pbench-server.cfg @@ -24,7 +24,11 @@ uri = postgresql://pbenchcontainer:pbench@localhost:5432/pbenchcontainer secret-key = "pbench-in-a-can secret shhh" [openid] -server_url = http://localhost:8090 +server_url = https://localhost:8090 + +# Override the default cert value to use for pbenchinacan Keycloak container +# connection. +cert_location = /etc/pki/tls/certs/pbench_CA.crt ########################################################################### # The rest will come from the default config file. diff --git a/server/pbenchinacan/load_keycloak.sh b/server/pbenchinacan/load_keycloak.sh index 748e17ae39..e91f0aef40 100755 --- a/server/pbenchinacan/load_keycloak.sh +++ b/server/pbenchinacan/load_keycloak.sh @@ -17,7 +17,7 @@ # "https://localhost:8443/*" unless specified otherwise by 'KEYCLOAK_REDIRECT_URI' # env variable. -KEYCLOAK_HOST_PORT=${KEYCLOAK_HOST_PORT:-"http://localhost:8090"} +KEYCLOAK_HOST_PORT=${KEYCLOAK_HOST_PORT:-"https://localhost:8090"} KEYCLOAK_REDIRECT_URI=${KEYCLOAK_REDIRECT_URI:-"https://localhost:8443/*"} ADMIN_USERNAME=${ADMIN_USERNAME:-"admin"} ADMIN_PASSWORD=${ADMIN_PASSWORD:-"admin"} @@ -26,13 +26,19 @@ ADMIN_PASSWORD=${ADMIN_PASSWORD:-"admin"} REALM=${KEYCLOAK_REALM:-"pbench-server"} CLIENT=${KEYCLOAK_CLIENT:-"pbench-client"} +TMP_DIR=${TMP_DIR:-${WORKSPACE_TMP:-/var/tmp/pbench}} +PB_DEPLOY_FILES=${PB_DEPLOY_FILES:-${TMP_DIR}/pbench_server_deployment} + +export CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-"${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt"} + end_in_epoch_secs=$(date --date "2 minutes" +%s) # Run the custom configuration ADMIN_TOKEN="" while true; do - ADMIN_TOKEN=$(curl -s -f -X POST "${KEYCLOAK_HOST_PORT}/realms/master/protocol/openid-connect/token" \ + ADMIN_TOKEN=$(curl -s -f -X POST \ + "${KEYCLOAK_HOST_PORT}/realms/master/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "username=${ADMIN_USERNAME}" \ -d "password=${ADMIN_PASSWORD}" \ @@ -53,7 +59,8 @@ echo echo "Keycloak connection successful on : ${KEYCLOAK_HOST_PORT}" echo -status_code=$(curl -f -s -o /dev/null -w "%{http_code}" -X POST "${KEYCLOAK_HOST_PORT}/admin/realms" \ +status_code=$(curl -f -s -o /dev/null -w "%{http_code}" -X POST \ + "${KEYCLOAK_HOST_PORT}/admin/realms" \ -H "Authorization: Bearer ${ADMIN_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"realm": "'${REALM}'", "enabled": true}') @@ -70,7 +77,8 @@ fi # a token from Keycloak using a . # Having in the aud claim of the token is essential for the token # to be validated. -curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/client-scopes" \ +curl -si -f -X POST \ + "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/client-scopes" \ -H "Authorization: Bearer ${ADMIN_TOKEN}" \ -H "Content-Type: application/json" \ -d '{ @@ -99,7 +107,8 @@ curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/client-scopes" }' -CLIENT_CONF=$(curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients" \ +CLIENT_CONF=$(curl -si -f -X POST \ + "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients" \ -H "Authorization: Bearer ${ADMIN_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"clientId": "'${CLIENT}'", @@ -111,7 +120,7 @@ CLIENT_CONF=$(curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/c "attributes": {"post.logout.redirect.uris": "'${KEYCLOAK_REDIRECT_URI}'"}, "redirectUris": ["'${KEYCLOAK_REDIRECT_URI}'"]}') -CLIENT_ID=$(grep -o -e 'http://[^[:space:]]*' <<< ${CLIENT_CONF} | sed -e 's|.*/||') +CLIENT_ID=$(grep -o -e 'https://[^[:space:]]*' <<< ${CLIENT_CONF} | sed -e 's|.*/||') if [[ -z "${CLIENT_ID}" ]]; then echo "${CLIENT} id is empty" exit 1 @@ -119,7 +128,8 @@ else echo "Created ${CLIENT} client" fi -status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients/${CLIENT_ID}/roles" \ +status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ + "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients/${CLIENT_ID}/roles" \ -H "Authorization: Bearer ${ADMIN_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name": "ADMIN"}') @@ -139,12 +149,13 @@ if [[ -z "${ROLE_ID}" ]]; then exit 1 fi -USER=$(curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users" \ +USER=$(curl -si -f -X POST \ + "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users" \ -H "Authorization: Bearer ${ADMIN_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"username": "admin", "enabled": true, "credentials": [{"type": "password", "value": "123", "temporary": false}]}') -USER_ID=$(grep -o -e 'http://[^[:space:]]*' <<< ${USER} | sed -e 's|.*/||') +USER_ID=$(grep -o -e 'https://[^[:space:]]*' <<< ${USER} | sed -e 's|.*/||') if [[ -z "${USER_ID}" ]]; then echo "User id is empty" @@ -153,7 +164,8 @@ else echo "Created an 'admin' user inside ${REALM} realm" fi -status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users/${USER_ID}/role-mappings/clients/${CLIENT_ID}" \ +status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ + "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users/${USER_ID}/role-mappings/clients/${CLIENT_ID}" \ -H "Authorization: Bearer ${ADMIN_TOKEN}" \ -H "Content-Type: application/json" \ -d '[{"id":"'${ROLE_ID}'","name":"ADMIN"}]') diff --git a/server/pbenchinacan/run-pbench-in-a-can b/server/pbenchinacan/run-pbench-in-a-can index 35b62f0ac3..0bdbd4616b 100755 --- a/server/pbenchinacan/run-pbench-in-a-can +++ b/server/pbenchinacan/run-pbench-in-a-can @@ -131,9 +131,10 @@ podman run \ -addext "authorityKeyIdentifier = keyid,issuer" \ -addext "basicConstraints=CA:FALSE" \ -addext "keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment" \ - -addext "subjectAltName = IP.2:${PB_HOST_IP}" \ + -addext "subjectAltName = IP.2:${PB_HOST_IP}, DNS:localhost" \ 2>&1 | sed -E -e '/^[.+*-]*$/ d' +chmod 0640 ${PB_DEPLOY_FILES}/pbench-server.key #+ # Start the services which the Pbench Server depends upon and then start the # Pbench Server itself. diff --git a/tox.ini b/tox.ini index 301ea97cd7..577b7934f5 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,6 @@ passenv = USER WORKSPACE WORKSPACE_TMP - setenv = VIRTUAL_ENV = {envdir} XDG_CACHE_HOME = {envdir}