From 082d9579160dc393117c62c7068bd39996405935 Mon Sep 17 00:00:00 2001 From: Sean O'Neill <78733408+soneillf5@users.noreply.github.com> Date: Fri, 9 Apr 2021 12:37:37 +0100 Subject: [PATCH] Add automated tests for TransportServer status (#1516) --- .github/workflows/edge.yml | 10 ++ .../rejected-invalid.yaml | 15 ++ .../rejected-warning.yaml | 15 ++ .../transport-server-status/standard/dns.yaml | 64 +++++++ .../standard/global-configuration.yaml | 12 ++ .../standard/transport-server.yaml | 14 ++ tests/pytest.ini | 1 + tests/suite/custom_resources_utils.py | 167 ++++++++++++++++++ tests/suite/fixtures.py | 61 +++++++ tests/suite/test_transport_server_status.py | 112 ++++++++++++ 10 files changed, 471 insertions(+) create mode 100644 tests/data/transport-server-status/rejected-invalid.yaml create mode 100644 tests/data/transport-server-status/rejected-warning.yaml create mode 100644 tests/data/transport-server-status/standard/dns.yaml create mode 100644 tests/data/transport-server-status/standard/global-configuration.yaml create mode 100644 tests/data/transport-server-status/standard/transport-server.yaml create mode 100644 tests/suite/test_transport_server_status.py diff --git a/.github/workflows/edge.yml b/.github/workflows/edge.yml index b6c1ee6f60..9889502d42 100644 --- a/.github/workflows/edge.yml +++ b/.github/workflows/edge.yml @@ -243,6 +243,11 @@ jobs: tag: ${{ github.sha }} marker: 'vs' type: oss + - os: ubuntu-20.04 + image: nginx-ingress + tag: ${{ github.sha }} + marker: 'ts' + type: oss - os: ubuntu-20.04 image: nginx-ingress tag: ${{ github.sha }} @@ -263,6 +268,11 @@ jobs: tag: ${{ github.sha }} marker: 'vs' type: plus + - os: ubuntu-20.04 + image: nginx-plus-ingress + tag: ${{ github.sha }} + marker: 'ts' + type: plus - os: ubuntu-20.04 image: nginx-plus-ingress tag: ${{ github.sha }} diff --git a/tests/data/transport-server-status/rejected-invalid.yaml b/tests/data/transport-server-status/rejected-invalid.yaml new file mode 100644 index 0000000000..5f51486d99 --- /dev/null +++ b/tests/data/transport-server-status/rejected-invalid.yaml @@ -0,0 +1,15 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: TransportServer +metadata: + name: transport-server +spec: + listener: + name: dns-tcp + # we specify an invalid protocol to generate an 'Invalid' state + protocol: invalid-protocol + upstreams: + - name: dns-app + service: coredns + port: 5353 + action: + pass: dns-app diff --git a/tests/data/transport-server-status/rejected-warning.yaml b/tests/data/transport-server-status/rejected-warning.yaml new file mode 100644 index 0000000000..b3e3176c47 --- /dev/null +++ b/tests/data/transport-server-status/rejected-warning.yaml @@ -0,0 +1,15 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: TransportServer +metadata: + name: transport-server +spec: + listener: + # we specify a missing listener to generate a 'Warning' state + name: invalid-listener + protocol: TCP + upstreams: + - name: dns-app + service: coredns + port: 5353 + action: + pass: dns-app diff --git a/tests/data/transport-server-status/standard/dns.yaml b/tests/data/transport-server-status/standard/dns.yaml new file mode 100644 index 0000000000..d81e644b86 --- /dev/null +++ b/tests/data/transport-server-status/standard/dns.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns +data: + Corefile: | + .:5353 { + forward . 8.8.8.8:53 + log + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns +spec: + replicas: 2 + selector: + matchLabels: + app: coredns + template: + metadata: + labels: + app: coredns + spec: + containers: + - name: coredns + image: coredns/coredns:1.6.7 + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 5353 + name: dns + protocol: UDP + - containerPort: 5353 + name: dns-tcp + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns +spec: + selector: + app: coredns + ports: + - name: dns + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-status/standard/global-configuration.yaml b/tests/data/transport-server-status/standard/global-configuration.yaml new file mode 100644 index 0000000000..63a5213668 --- /dev/null +++ b/tests/data/transport-server-status/standard/global-configuration.yaml @@ -0,0 +1,12 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: GlobalConfiguration +metadata: + name: nginx-configuration +spec: + listeners: + - name: dns-udp + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-status/standard/transport-server.yaml b/tests/data/transport-server-status/standard/transport-server.yaml new file mode 100644 index 0000000000..277568ed44 --- /dev/null +++ b/tests/data/transport-server-status/standard/transport-server.yaml @@ -0,0 +1,14 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: TransportServer +metadata: + name: transport-server +spec: + listener: + name: dns-tcp + protocol: TCP + upstreams: + - name: dns-app + service: coredns + port: 5353 + action: + pass: dns-app diff --git a/tests/pytest.ini b/tests/pytest.ini index ea33a95e07..61da48387d 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -6,6 +6,7 @@ markers = vsr: mark test as a VirtualServerRoute test policies: mark test as an AccessControl policy test vs: mark test as a VirtualServer test + ts: mark test as a TransportServer test ingresses: mark test as an Ingresses test appprotect: mark test as an AppProtect test rewrite: mark test as an uri rewrite test diff --git a/tests/suite/custom_resources_utils.py b/tests/suite/custom_resources_utils.py index caba0065d3..754f0147eb 100644 --- a/tests/suite/custom_resources_utils.py +++ b/tests/suite/custom_resources_utils.py @@ -88,6 +88,36 @@ def read_custom_resource(custom_objects: CustomObjectsApi, namespace, plural, na raise +def read_custom_resource_v1alpha1(custom_objects: CustomObjectsApi, namespace, plural, name) -> object: + """ + Get CRD information (kubectl describe output) + + :param custom_objects: CustomObjectsApi + :param namespace: The custom resource's namespace + :param plural: the custom resource's plural name + :param name: the custom object's name + :return: object + """ + print(f"Getting info for v1alpha1 crd {name} in namespace {namespace}") + try: + response = custom_objects.get_namespaced_custom_object( + "k8s.nginx.org", "v1alpha1", namespace, plural, name + ) + pprint(response) + return response + + except ApiException: + logging.exception(f"Exception occurred while reading CRD") + raise + + +def read_ts(custom_objects: CustomObjectsApi, namespace, name) -> object: + """ + Read TransportService resource. + """ + return read_custom_resource_v1alpha1(custom_objects, namespace, "transportservers", name) + + def read_ap_crd(custom_objects: CustomObjectsApi, namespace, plural, name) -> object: """ Get AppProtect CRD information (kubectl describe output) @@ -249,6 +279,117 @@ def create_virtual_server_from_yaml( raise +def patch_ts_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace) -> dict: + """ + Create a TransportServer Resource based on yaml file. + + :param custom_objects: CustomObjectsApi + :param yaml_manifest: an absolute path to file + :param namespace: + :return: a dictionary representing the resource + """ + return create_resource_from_yaml(custom_objects, yaml_manifest, namespace, "transportservers") + + +def create_gc_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace) -> dict: + """ + Create a GlobalConfiguration Resource based on yaml file. + + :param custom_objects: CustomObjectsApi + :param yaml_manifest: an absolute path to file + :param namespace: + :return: a dictionary representing the resource + """ + return create_resource_from_yaml(custom_objects, yaml_manifest, namespace, "globalconfigurations") + + +def create_resource_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace, plural) -> dict: + """ + Create a Resource based on yaml file. + + :param custom_objects: CustomObjectsApi + :param yaml_manifest: an absolute path to file + :param namespace: + :param plural: the plural of the resource + :return: a dictionary representing the resource + """ + + with open(yaml_manifest) as f: + body = yaml.safe_load(f) + try: + print("Create a Custom Resource: " + body["kind"]) + group, version = body["apiVersion"].split("/") + custom_objects.create_namespaced_custom_object( + group, version, namespace, plural, body + ) + print(f"Custom resource {body['kind']} created with name '{body['metadata']['name']}'") + return body + except ApiException as ex: + logging.exception( + f"Exception: {ex} occurred while creating {body['kind']}: {body['metadata']['name']}" + ) + raise + + +def delete_ts(custom_objects: CustomObjectsApi, resource, namespace) -> None: + """ + Delete a TransportServer Resource. + + :param custom_objects: CustomObjectsApi + :param namespace: namespace + :param resource: a dictionary representation of the resource yaml + :param namespace: + :return: + """ + return delete_resource(custom_objects, resource, namespace, "transportservers") + + +def delete_gc(custom_objects: CustomObjectsApi, resource, namespace) -> None: + """ + Delete a GlobalConfiguration Resource. + + :param custom_objects: CustomObjectsApi + :param namespace: namespace + :param resource: a dictionary representation of the resource yaml + :param namespace: + :return: + """ + return delete_resource(custom_objects, resource, namespace, "globalconfigurations") + + +def delete_resource(custom_objects: CustomObjectsApi, resource, namespace, plural) -> None: + """ + Delete a Resource. + + :param custom_objects: CustomObjectsApi + :param namespace: namespace + :param resource: a dictionary representation of the resource yaml + :param namespace: + :param plural: the plural of the resource + :return: + """ + + name = resource['metadata']['name'] + kind = resource['kind'] + group, version = resource["apiVersion"].split("/") + + print(f"Delete a: {kind}, name: {name}") + delete_options = client.V1DeleteOptions() + + custom_objects.delete_namespaced_custom_object( + group, version, namespace, plural, name, delete_options + ) + ensure_item_removal( + custom_objects.get_namespaced_custom_object, + group, + version, + namespace, + plural, + name, + ) + print(f"Resource:{kind} was removed with name '{name}'") + + def create_ap_logconf_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace) -> str: """ Create a logconf for AppProtect based on yaml file. @@ -448,6 +589,32 @@ def patch_virtual_server_from_yaml( raise +def patch_ts( + custom_objects: CustomObjectsApi, name, yaml_manifest, namespace +) -> None: + """ + Patch a TransportServer based on yaml manifest + """ + return patch_custom_resource_v1alpha1(custom_objects, name, yaml_manifest, namespace, "transportservers") + + +def patch_custom_resource_v1alpha1(custom_objects: CustomObjectsApi, name, yaml_manifest, namespace, plural) -> None: + """ + Patch a custom resource based on yaml manifest + """ + print(f"Update a Resource: {name}") + with open(yaml_manifest) as f: + dep = yaml.safe_load(f) + + try: + custom_objects.patch_namespaced_custom_object( + "k8s.nginx.org", "v1alpha1", namespace, plural, name, dep + ) + except ApiException: + logging.exception(f"Failed with exception while patching custom resource: {name}") + raise + + def delete_and_create_vs_from_yaml( custom_objects: CustomObjectsApi, name, yaml_manifest, namespace ) -> None: diff --git a/tests/suite/fixtures.py b/tests/suite/fixtures.py index 6b99e3348a..5e878bd5b8 100644 --- a/tests/suite/fixtures.py +++ b/tests/suite/fixtures.py @@ -24,6 +24,10 @@ delete_v_s_route, create_crd_from_yaml, delete_crd, + patch_ts_from_yaml, + create_gc_from_yaml, + delete_ts, + delete_gc, ) from suite.kube_config_utils import ensure_context_in_config, get_current_context_name from suite.resources_utils import ( @@ -57,6 +61,8 @@ create_configmap_from_yaml, create_secret_from_yaml, configure_rbac_with_ap, + create_items_from_yaml, + delete_items_from_yaml, ) from suite.yaml_utils import ( get_first_vs_host_from_yaml, @@ -727,6 +733,61 @@ def fin(): ) +class TransportServerSetup: + """ + Encapsulate Transport Server Example details. + + Attributes: + name (str): + namespace (str): + """ + + def __init__(self, name, namespace): + self.name = name + self.namespace = namespace + + +@pytest.fixture(scope="class") +def transport_server_setup( + request, kube_apis, test_namespace +) -> TransportServerSetup: + """ + Prepare Transport Server Example. + + :param request: internal pytest fixture to parametrize this method + :param kube_apis: client apis + :param test_namespace: + :return: TransportServerSetup + """ + print( + "------------------------- Deploy Transport Server Example -----------------------------------" + ) + + # deploy global config + global_config_file = f"{TEST_DATA}/{request.param['example']}/standard/global-configuration.yaml" + gc_resource = create_gc_from_yaml(kube_apis.custom_objects, global_config_file, "nginx-ingress") + + # deploy dns + dns_file = f"{TEST_DATA}/{request.param['example']}/standard/dns.yaml" + create_items_from_yaml(kube_apis, dns_file, test_namespace) + + # deploy transport server + transport_server_file = f"{TEST_DATA}/{request.param['example']}/standard/transport-server.yaml" + ts_resource = patch_ts_from_yaml(kube_apis.custom_objects, transport_server_file, test_namespace) + + wait_until_all_pods_are_ready(kube_apis.v1, test_namespace) + + def fin(): + print("Clean up TransportServer Example:") + delete_ts(kube_apis.custom_objects, ts_resource, test_namespace) + delete_items_from_yaml(kube_apis, dns_file, test_namespace) + delete_gc(kube_apis.custom_objects, gc_resource, "nginx-ingress") + + request.addfinalizer(fin) + + return TransportServerSetup(ts_resource['metadata']['name'], test_namespace) + + @pytest.fixture(scope="class") def v_s_route_app_setup(request, kube_apis, v_s_route_setup) -> None: """ diff --git a/tests/suite/test_transport_server_status.py b/tests/suite/test_transport_server_status.py new file mode 100644 index 0000000000..936ebe6278 --- /dev/null +++ b/tests/suite/test_transport_server_status.py @@ -0,0 +1,112 @@ +import pytest + +from suite.resources_utils import wait_before_test +from suite.custom_resources_utils import ( + read_ts, + patch_ts, +) +from settings import TEST_DATA + + +@pytest.mark.ts +@pytest.mark.parametrize( + "crd_ingress_controller, transport_server_setup", + [ + ( + { + "type": "complete", + "extra_args": + [ + "-enable-custom-resources", + "-global-configuration=nginx-ingress/nginx-configuration", + "-enable-leader-election=false" + ] + }, + {"example": "transport-server-status", "app_type": "simple"}, + ) + ], + indirect=True, +) +class TestTransportServerStatus: + + def restore_ts(self, kube_apis, transport_server_setup) -> None: + """ + Function to revert a TransportServer resource to a valid state. + """ + patch_src = f"{TEST_DATA}/transport-server-status/standard/transport-server.yaml" + patch_ts( + kube_apis.custom_objects, + transport_server_setup.name, + patch_src, + transport_server_setup.namespace, + ) + + @pytest.mark.smoke + def test_status_valid( + self, kube_apis, crd_ingress_controller, transport_server_setup, + ): + """ + Test TransportServer status with valid fields in yaml. + """ + response = read_ts( + kube_apis.custom_objects, + transport_server_setup.namespace, + transport_server_setup.name, + ) + assert ( + response["status"] + and response["status"]["reason"] == "AddedOrUpdated" + and response["status"]["state"] == "Valid" + ) + + def test_status_warning( + self, kube_apis, crd_ingress_controller, transport_server_setup, + ): + """ + Test TransportServer status with a missing listener. + """ + patch_src = f"{TEST_DATA}/transport-server-status/rejected-warning.yaml" + patch_ts( + kube_apis.custom_objects, + transport_server_setup.name, + patch_src, + transport_server_setup.namespace, + ) + wait_before_test() + response = read_ts( + kube_apis.custom_objects, + transport_server_setup.namespace, + transport_server_setup.name, + ) + self.restore_ts(kube_apis, transport_server_setup) + assert ( + response["status"] + and response["status"]["reason"] == "Rejected" + and response["status"]["state"] == "Warning" + ) + + def test_status_invalid( + self, kube_apis, crd_ingress_controller, transport_server_setup, + ): + """ + Test TransportServer status with an invalid protocol. + """ + patch_src = f"{TEST_DATA}/transport-server-status/rejected-invalid.yaml" + patch_ts( + kube_apis.custom_objects, + transport_server_setup.name, + patch_src, + transport_server_setup.namespace, + ) + wait_before_test() + response = read_ts( + kube_apis.custom_objects, + transport_server_setup.namespace, + transport_server_setup.name, + ) + self.restore_ts(kube_apis, transport_server_setup) + assert ( + response["status"] + and response["status"]["reason"] == "Rejected" + and response["status"]["state"] == "Invalid" + )