diff --git a/agent/config/pbench-agent-default.cfg b/agent/config/pbench-agent-default.cfg index aff4ac3c14..ae05f6f609 100644 --- a/agent/config/pbench-agent-default.cfg +++ b/agent/config/pbench-agent-default.cfg @@ -20,7 +20,7 @@ ssh_opts = -o BatchMode=yes -o StrictHostKeyChecking=no # REST API entrypoint api_version = 1 rest_endpoint = api/v%(api_version)s -server_rest_url = http://%(pbench_web_server)s/%(rest_endpoint)s +server_rest_url = https://%(pbench_web_server)s/%(rest_endpoint)s [pbench/tools] light-tool-set = vmstat diff --git a/docs/Server/API/V1/contents.md b/docs/Server/API/V1/contents.md index b8d7794ecd..61c10487d9 100644 --- a/docs/Server/API/V1/contents.md +++ b/docs/Server/API/V1/contents.md @@ -77,12 +77,12 @@ Pbench returns a JSON object with two list fields: { "name": "dir1", "type": "dir", - "uri": "http://hostname/api/v1/datasets//contents/dir1" + "uri": "https://hostname/api/v1/datasets//contents/dir1" }, { "name": "dir2", "type": "dir", - "uri": "http://hostname/api/v1/datasets//contents/dir2" + "uri": "https://hostname/api/v1/datasets//contents/dir2" }, ... ], @@ -93,7 +93,7 @@ Pbench returns a JSON object with two list fields: "size": 24, "mode": "0o644", "type": "reg", - "uri": "http://hostname/api/v1/datasets//inventory/file.txt" + "uri": "https://hostname/api/v1/datasets//inventory/file.txt" }, { "name": "data.lis", @@ -101,7 +101,7 @@ Pbench returns a JSON object with two list fields: "size": 18, "mode": "0o644", "type": "reg", - "uri": "http://hostname/api/v1/datasets//inventory/data.lis" + "uri": "https://hostname/api/v1/datasets//inventory/data.lis" }, ... ] @@ -126,12 +126,12 @@ The `type` codes are: { "name": "reference-result", "type": "sym", - "uri": "http://hostname/api/v1/datasets//contents/linkresult" + "uri": "https://hostname/api/v1/datasets//contents/linkresult" }, { "name": "directory", "type": "dir", - "uri": "http://hostname/api/v1/datasets//contents/directory" + "uri": "https://hostname/api/v1/datasets//contents/directory" } ``` @@ -159,6 +159,6 @@ URI returning the linked file's byte stream. "size": 18, "mode": "0o644", "type": "reg", - "uri": "http://hostname/api/v1/datasets//inventory/" + "uri": "https://hostname/api/v1/datasets//inventory/" } ``` diff --git a/docs/Server/API/V1/endpoints.md b/docs/Server/API/V1/endpoints.md index 103c1dd179..2b96911afd 100644 --- a/docs/Server/API/V1/endpoints.md +++ b/docs/Server/API/V1/endpoints.md @@ -48,7 +48,7 @@ URI template and parameters for the API. ##### `template` The API's URI template pattern, with URI parameters in the form `{}`, as in -`http://host:port/api/v1/datasets/{dataset}/metadata`. +`https://host:port/api/v1/datasets/{dataset}/metadata`. ##### `params` @@ -104,7 +104,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}" }, "datasets_contents": { "params": { @@ -115,11 +115,11 @@ let uri = uriTemplate( "type": "path" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}/contents/{target}" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}/contents/{target}" }, "datasets_daterange": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/datasets/daterange" + "template": "https://10.1.1.1:8443/api/v1/datasets/daterange" }, "datasets_detail": { "params": { @@ -127,7 +127,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}/detail" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}/detail" }, "datasets_inventory": { "params": { @@ -138,11 +138,11 @@ let uri = uriTemplate( "type": "path" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}/inventory/{target}" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}/inventory/{target}" }, "datasets_list": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/datasets" + "template": "https://10.1.1.1:8443/api/v1/datasets" }, "datasets_mappings": { "params": { @@ -150,7 +150,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/mappings/{dataset_view}" + "template": "https://10.1.1.1:8443/api/v1/datasets/mappings/{dataset_view}" }, "datasets_metadata": { "params": { @@ -158,7 +158,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}/metadata" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}/metadata" }, "datasets_namespace": { "params": { @@ -169,11 +169,11 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}/namespace/{dataset_view}" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}/namespace/{dataset_view}" }, "datasets_search": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/datasets/search" + "template": "https://10.1.1.1:8443/api/v1/datasets/search" }, "datasets_values": { "params": { @@ -184,27 +184,27 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/datasets/{dataset}/values/{dataset_view}" + "template": "https://10.1.1.1:8443/api/v1/datasets/{dataset}/values/{dataset_view}" }, "endpoints": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/endpoints" + "template": "https://10.1.1.1:8443/api/v1/endpoints" }, "login": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/login" + "template": "https://10.1.1.1:8443/api/v1/login" }, "logout": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/logout" + "template": "https://10.1.1.1:8443/api/v1/logout" }, "register": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/register" + "template": "https://10.1.1.1:8443/api/v1/register" }, "server_audit": { "params": {}, - "template": "http://10.1.1.1:8080/api/v1/server/audit" + "template": "https://10.1.1.1:8443/api/v1/server/audit" }, "server_settings": { "params": { @@ -212,7 +212,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/server/settings/{key}" + "template": "https://10.1.1.1:8443/api/v1/server/settings/{key}" }, "upload": { "params": { @@ -220,7 +220,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/upload/{filename}" + "template": "https://10.1.1.1:8443/api/v1/upload/{filename}" }, "user": { "params": { @@ -228,7 +228,7 @@ let uri = uriTemplate( "type": "string" } }, - "template": "http://10.1.1.1:8080/api/v1/user/{target_username}" + "template": "https://10.1.1.1:8443/api/v1/user/{target_username}" } } } diff --git a/docs/Server/API/V1/list.md b/docs/Server/API/V1/list.md index 66868118c0..a7687030de 100644 --- a/docs/Server/API/V1/list.md +++ b/docs/Server/API/V1/list.md @@ -264,12 +264,12 @@ display purposes and must not be assumed to be unique or definitive. JSON object in this field. For example, the query -`GET http://host/api/v1/datasets/list?metadata=user.dashboard.favorite&limit=3` +`GET https://host/api/v1/datasets/list?metadata=user.dashboard.favorite&limit=3` might return: ```json { - "next_url": "http://pbench.example.com/api/v1/datasets?limit=3&metadata=user.dashboard.favorite&offset=3", + "next_url": "https://pbench.example.com/api/v1/datasets?limit=3&metadata=user.dashboard.favorite&offset=3", "results": [ { "metadata": { diff --git a/jenkins/run-server-func-tests b/jenkins/run-server-func-tests index dec42ac7b0..c01ec49894 100755 --- a/jenkins/run-server-func-tests +++ b/jenkins/run-server-func-tests @@ -8,9 +8,20 @@ export PB_SERVER_IMAGE_TAG=${PB_SERVER_IMAGE_TAG:-"$(cat jenkins/branch.name)"} export PB_POD_NAME=${PB_POD_NAME:-"pbench-in-a-can_${PB_SERVER_IMAGE_TAG}"} export PB_SERVER_CONTAINER_NAME=${PB_SERVER_CONTAINER_NAME:-"${PB_POD_NAME}-pbenchserver"} -SERVER_URL="http://localhost:8080" +# Note: the value of PB_HOST_IP will be used to generate the TLS certificate +# and so it (not `localhost`) must also be used to access the Pbench Server; +# otherwise, the TLS validation will fail due to a host mismatch. +if [[ -z "${PB_HOST_IP}" ]]; then + host_ip_list=$(hostname -I) + PB_HOST_IP=${host_ip_list%% *} + export PB_HOST_IP +fi +SERVER_URL="https://${PB_HOST_IP}:8443" SERVER_API_ENDPOINTS="${SERVER_URL}/api/v1/endpoints" +# Have Curl use the Pbench CA certificate to validate the TLS/SSL connection +export CURL_CA_BUNDLE="${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt" + cleanup_flag=0 keep_flag=0 exit_status=0 diff --git a/lib/pbench/client/__init__.py b/lib/pbench/client/__init__.py index c7dd0426e7..61686cbbd0 100644 --- a/lib/pbench/client/__init__.py +++ b/lib/pbench/client/__init__.py @@ -1,4 +1,5 @@ from enum import Enum +import os from pathlib import Path from typing import Iterator, Optional from urllib import parse @@ -57,7 +58,7 @@ class API(Enum): class PbenchServerClient: - DEFAULT_SCHEME = "http" + DEFAULT_SCHEME = "https" DEFAULT_PAGE_SIZE = 100 def __init__(self, host: str): @@ -316,6 +317,10 @@ def connect(self, headers: Optional[dict[str, str]] = None) -> None: """ url = parse.urljoin(self.url, "api/v1/endpoints") self.session = requests.Session() + + # Use the same CA as Curl to do TLS verification; + # if it's not defined then disable TLS verification. + self.session.verify = os.environ.get("CURL_CA_BUNDLE", False) if headers: self.session.headers.update(headers) response = self.session.get(url) diff --git a/lib/pbench/server/api/resources/endpoint_configure.py b/lib/pbench/server/api/resources/endpoint_configure.py index 5238cfc7dd..884ca4313c 100644 --- a/lib/pbench/server/api/resources/endpoint_configure.py +++ b/lib/pbench/server/api/resources/endpoint_configure.py @@ -64,7 +64,7 @@ def get(self): API to get or modify metadata is: { - "template": "http://host/api/v1/datasets/{dataset}/metadata", + "template": "https://host/api/v1/datasets/{dataset}/metadata", "params": {"dataset": {"type": "string"}} } diff --git a/lib/pbench/server/auth/__init__.py b/lib/pbench/server/auth/__init__.py index 3dad4f59a1..e3bb986ca6 100644 --- a/lib/pbench/server/auth/__init__.py +++ b/lib/pbench/server/auth/__init__.py @@ -225,6 +225,7 @@ def wait_for_oidc_server( ) adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) + session.mount("https://", adapter) # We will also need to retry the connection if the health status is not UP. connected = False diff --git a/lib/pbench/test/unit/agent/task/test_results_move.py b/lib/pbench/test/unit/agent/task/test_results_move.py index cbe4dfa591..2b0ab4702f 100644 --- a/lib/pbench/test/unit/agent/task/test_results_move.py +++ b/lib/pbench/test/unit/agent/task/test_results_move.py @@ -51,11 +51,11 @@ class TestResultsMove: TOKN_TEXT = "what is a token but 139 characters of gibberish" ACCESS_TEXT = "private" RELAY_TEXT = "http://relay.example.com" - SRVR_TEXT = "http://pbench.test.example.com" + SRVR_TEXT = "https://pbench.test.example.com" META_SWITCH = "--metadata" META_TEXT_FOO = "pbench.sat:FOO" META_TEXT_TEST = "pbench.test:TEST" - URL = "http://pbench.example.com/api/v1" + URL = "https://pbench.example.com/api/v1" ENDPOINT = "/login" @staticmethod @@ -203,7 +203,7 @@ def test_results_move(monkeypatch, caplog, setup): responses.add( responses.PUT, - f"http://pbench.example.com/api/v1/upload/{name}.tar.xz", + f"{TestResultsMove.URL}/upload/{name}.tar.xz", status=200, ) @@ -292,7 +292,7 @@ def test_results_move_server(monkeypatch, caplog, setup): responses.add( responses.PUT, - f"{TestResultsMove.SRVR_TEXT}/api/v1/upload/{script}_{config}_{date}.tar.xz", + f"{TestResultsMove.SRVR_TEXT}/api/v1/upload/{name}.tar.xz", status=200, ) diff --git a/lib/pbench/test/unit/agent/task/test_results_push.py b/lib/pbench/test/unit/agent/task/test_results_push.py index 4ff975782a..051c9d1d95 100644 --- a/lib/pbench/test/unit/agent/task/test_results_push.py +++ b/lib/pbench/test/unit/agent/task/test_results_push.py @@ -25,8 +25,8 @@ class TestResultsPush: RELAY_SWITCH = "--relay" SRVR_SWITCH = "--server" RELAY_TEXT = "http://relay.example.com" - SRVR_TEXT = "http://pbench.test.example.com" - URL = "http://pbench.example.com/api/v1" + SRVR_TEXT = "https://pbench.test.example.com" + URL = "https://pbench.example.com/api/v1" @staticmethod def server_mock( diff --git a/lib/pbench/test/unit/client/conftest.py b/lib/pbench/test/unit/client/conftest.py index a3e87f3d21..9704e533dd 100644 --- a/lib/pbench/test/unit/client/conftest.py +++ b/lib/pbench/test/unit/client/conftest.py @@ -19,7 +19,7 @@ def connect(): "api": {}, "uri": {}, "openid": { - "server": "http://oidc_server", + "server": "https://oidc_server", "realm": "pbench-server", "client": "pbench-client", }, @@ -27,7 +27,7 @@ def connect(): ) responses.add( responses.POST, - "http://oidc_server/realms/master/protocol/openid-connect/token", + "https://oidc_server/realms/master/protocol/openid-connect/token", json={ "access_token": "admin_token", }, diff --git a/lib/pbench/test/unit/client/test_connect.py b/lib/pbench/test/unit/client/test_connect.py index 670e2d3b69..e82a0e3299 100644 --- a/lib/pbench/test/unit/client/test_connect.py +++ b/lib/pbench/test/unit/client/test_connect.py @@ -7,7 +7,7 @@ class TestConnect: def test_construct_host(self): pbench = PbenchServerClient("10.1.100.2") assert pbench.host == "10.1.100.2" - assert pbench.scheme == "http" + assert pbench.scheme == "https" assert pbench.session is None assert pbench.endpoints is None @@ -22,7 +22,7 @@ def test_construct_url(self): def test_connect(self): pbench = PbenchServerClient("10.1.100.2") url = f"{pbench.url}/api/v1/endpoints" - openid_dict = {"server": "http://oidc_server", "client": "pbench_client"} + openid_dict = {"server": "https://oidc_server", "client": "pbench_client"} with responses.RequestsMock() as rsp: rsp.add( diff --git a/lib/pbench/test/unit/server/conftest.py b/lib/pbench/test/unit/server/conftest.py index 38ca75abfd..b448db3ec1 100644 --- a/lib/pbench/test/unit/server/conftest.py +++ b/lib/pbench/test/unit/server/conftest.py @@ -49,7 +49,7 @@ secret-key = my_precious [openid] -server_url = http://openid.example.com +server_url = https://openid.example.com [logging] logger_type = null @@ -189,6 +189,7 @@ def client(monkeypatch, server_config, add_auth_connection_mock): the db_session fixture instead, which adds DB cleanup after the test. """ app = create_app(server_config) + app.config["PREFERRED_URL_SCHEME"] = "https" app_client = app.test_client() app_client.logger = app.logger diff --git a/lib/pbench/test/unit/server/test_datasets_list.py b/lib/pbench/test/unit/server/test_datasets_list.py index 300c3395bd..7ba6fd0346 100644 --- a/lib/pbench/test/unit/server/test_datasets_list.py +++ b/lib/pbench/test/unit/server/test_datasets_list.py @@ -151,7 +151,7 @@ def convert(k: str, v: Any) -> Any: else: query["offset"] = str(next_offset) next_url = ( - f"http://localhost{server_config.rest_uri}/datasets?" + f"https://localhost{server_config.rest_uri}/datasets?" + urlencode_json(query) ) else: diff --git a/server/lib/config/nginx.conf b/server/lib/config/nginx.conf index 148a4d0b87..d23ae63188 100644 --- a/server/lib/config/nginx.conf +++ b/server/lib/config/nginx.conf @@ -70,8 +70,6 @@ http { default_type application/octet-stream; server { - listen 8080; - listen [::]:8080; listen 8443 ssl; listen [::]:8443 ssl; server_name _; diff --git a/server/pbenchinacan/load_keycloak.sh b/server/pbenchinacan/load_keycloak.sh index 35eaa9ed81..748e17ae39 100755 --- a/server/pbenchinacan/load_keycloak.sh +++ b/server/pbenchinacan/load_keycloak.sh @@ -14,11 +14,11 @@ # The script defaults to using master realm username/password as admin/admin # unless specified otherwise by 'ADMIN_USERNAME' and 'ADMIN_PASSWORD' env # variables. The script also defaults the keycloak redirect URI as -# "http://localhost:8080/*" unless specified otherwise by 'KEYCLOAK_REDIRECT_URI' +# "https://localhost:8443/*" unless specified otherwise by 'KEYCLOAK_REDIRECT_URI' # env variable. KEYCLOAK_HOST_PORT=${KEYCLOAK_HOST_PORT:-"http://localhost:8090"} -KEYCLOAK_REDIRECT_URI=${KEYCLOAK_REDIRECT_URI:-"http://localhost:8080/*"} +KEYCLOAK_REDIRECT_URI=${KEYCLOAK_REDIRECT_URI:-"https://localhost:8443/*"} ADMIN_USERNAME=${ADMIN_USERNAME:-"admin"} ADMIN_PASSWORD=${ADMIN_PASSWORD:-"admin"} # These values must match the options "realm" and "client in the diff --git a/server/pbenchinacan/run-pbench-in-a-can b/server/pbenchinacan/run-pbench-in-a-can index b968bbf1d2..35b62f0ac3 100755 --- a/server/pbenchinacan/run-pbench-in-a-can +++ b/server/pbenchinacan/run-pbench-in-a-can @@ -27,9 +27,16 @@ export PB_DASHBOARD_DIR="${PB_DASHBOARD_DIR:-${PWD}/dashboard/build/}" export KEYCLOAK_REALM=${KEYCLOAK_REALM:-"pbench-server"} export KEYCLOAK_CLIENT=${KEYCLOAK_CLIENT:-"pbench-client"} +# Note: the value of PB_HOST_IP will be used to generate the TLS certificate +# and so it (not `localhost`) must also be used to access the Pbench Server; +# otherwise, the TLS validation will fail due to a host mismatch. +if [[ -z "${PB_HOST_IP}" ]]; then + host_ip_list=$(hostname -I) + PB_HOST_IP=${host_ip_list%% *} + export PB_HOST_IP +fi + host_name=${PB_HOST_NAME:-$(hostname --fqdn)} -host_ip_list=${PB_HOST_IP:-$(hostname -I)} -host_ip=${host_ip_list%% *} # Set up TMP_DIR, if it's not already defined, to point to WORKSPACE_TMP, if it # is defined (e.g., by the CI), or to `/var/tmp/pbench` as a fallback. @@ -103,6 +110,12 @@ podman run \ # # We do this in the Pbench Server container so that we get a known version of # openssl (the native one on the Jenkins executors appears to be ancient). +# +# Note that this command, if successful, dumps the status of the factorization +# search to stderr which produces a bunch of junk in the output; thus, we +# redirect stderr to stdout and then pipe it through sed to remove any complete +# lines consisting solely of any combination of periods, plus signs, asterisks, +# and hyphens. podman run \ --rm \ --volume ${PB_DEPLOY_FILES}:/data:Z \ @@ -118,7 +131,8 @@ podman run \ -addext "authorityKeyIdentifier = keyid,issuer" \ -addext "basicConstraints=CA:FALSE" \ -addext "keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment" \ - -addext "subjectAltName = IP.2:${host_ip}" + -addext "subjectAltName = IP.2:${PB_HOST_IP}" \ + 2>&1 | sed -E -e '/^[.+*-]*$/ d' #+ # Start the services which the Pbench Server depends upon and then start the