diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a9046d023..d00a9fb0e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,7 +64,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.0.5 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.1.0 with: cache: true @@ -96,7 +96,7 @@ jobs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v23.0.5 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v23.1.0 with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} architecture: ${{ matrix.architecture }} @@ -106,5 +106,6 @@ jobs: juju-snap-channel: ${{ matrix.juju.snap_channel }} libjuju-version-constraint: ${{ matrix.juju.libjuju }} _beta_allure_report: ${{ matrix.juju.allure_on_amd64 && matrix.architecture == 'amd64' }} + metallb-addon: true permissions: contents: write # Needed for Allure Report beta diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9e5a394cf..ad6ce0146 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,14 +15,14 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.0.5 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.1.0 release: name: Release charm needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v23.0.5 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v23.1.0 with: channel: 8.0/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 1e0fe95d0..173f04567 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -10,7 +10,7 @@ on: jobs: sync-docs: name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v23.0.5 + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v23.1.0 with: reviewers: a-velasco,izmalk permissions: diff --git a/config.yaml b/config.yaml new file mode 100644 index 000000000..001f56ecd --- /dev/null +++ b/config.yaml @@ -0,0 +1,17 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +options: + expose-external: + description: | + String to determine how to expose the MySQLRouter externally from the Kubernetes cluster. + Possible values: 'false', 'nodeport', 'loadbalancer' + type: string + default: "false" + + loadbalancer-extra-annotations: + description: | + A JSON string representing extra annotations for the Kubernetes service created + and managed by the charm. + type: string + default: "{}" diff --git a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py index 2604c39e6..1e7ff8405 100644 --- a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py +++ b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py @@ -269,7 +269,7 @@ def _remove_stale_otel_sdk_packages(): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 1 +LIBPATCH = 2 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"] @@ -371,10 +371,6 @@ class UntraceableObjectError(TracingError): """Raised when an object you're attempting to instrument cannot be autoinstrumented.""" -class TLSError(TracingError): - """Raised when the tracing endpoint is https but we don't have a cert yet.""" - - def _get_tracing_endpoint( tracing_endpoint_attr: str, charm_instance: object, @@ -484,10 +480,15 @@ def wrap_init(self: CharmBase, framework: Framework, *args, **kwargs): ) if tracing_endpoint.startswith("https://") and not server_cert: - raise TLSError( + logger.error( "Tracing endpoint is https, but no server_cert has been passed." - "Please point @trace_charm to a `server_cert` attr." + "Please point @trace_charm to a `server_cert` attr. " + "This might also mean that the tracing provider is related to a " + "certificates provider, but this application is not (yet). " + "In that case, you might just have to wait a bit for the certificates " + "integration to settle. " ) + return exporter = OTLPSpanExporter( endpoint=tracing_endpoint, diff --git a/lib/charms/tempo_coordinator_k8s/v0/tracing.py b/lib/charms/tempo_coordinator_k8s/v0/tracing.py index 4af379a5d..2035dffd6 100644 --- a/lib/charms/tempo_coordinator_k8s/v0/tracing.py +++ b/lib/charms/tempo_coordinator_k8s/v0/tracing.py @@ -34,7 +34,7 @@ def __init__(self, *args): `TracingEndpointRequirer.request_protocols(*protocol:str, relation:Optional[Relation])` method. Using this method also allows you to use per-relation protocols. -Units of provider charms obtain the tempo endpoint to which they will push their traces by calling +Units of requirer charms obtain the tempo endpoint to which they will push their traces by calling `TracingEndpointRequirer.get_endpoint(protocol: str)`, where `protocol` is, for example: - `otlp_grpc` - `otlp_http` @@ -44,7 +44,10 @@ def __init__(self, *args): If the `protocol` is not in the list of protocols that the charm requested at endpoint set-up time, the library will raise an error. -## Requirer Library Usage +We recommend that you scale up your tracing provider and relate it to an ingress so that your tracing requests +go through the ingress and get load balanced across all units. Otherwise, if the provider's leader goes down, your tracing goes down. + +## Provider Library Usage The `TracingEndpointProvider` object may be used by charms to manage relations with their trace sources. For this purposes a Tempo-like charm needs to do two things @@ -107,7 +110,7 @@ def __init__(self, *args): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 1 +LIBPATCH = 3 PYDEPS = ["pydantic"] @@ -985,11 +988,16 @@ def charm_tracing_config( is_https = endpoint.startswith("https://") if is_https: - if cert_path is None: - raise TracingError("Cannot send traces to an https endpoint without a certificate.") - elif not Path(cert_path).exists(): - # if endpoint is https BUT we don't have a server_cert yet: - # disable charm tracing until we do to prevent tls errors + if cert_path is None or not Path(cert_path).exists(): + # disable charm tracing until we obtain a cert to prevent tls errors + logger.error( + "Tracing endpoint is https, but no server_cert has been passed." + "Please point @trace_charm to a `server_cert` attr. " + "This might also mean that the tracing provider is related to a " + "certificates provider, but this application is not (yet). " + "In that case, you might just have to wait a bit for the certificates " + "integration to settle. " + ) return None, None return endpoint, str(cert_path) else: diff --git a/poetry.lock b/poetry.lock index 9cb44e9c5..09da853b0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "allure-pytest" @@ -1833,7 +1833,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/pyproject.toml b/pyproject.toml index 470982aec..dee7b8b88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ tenacity = "^8.5.0" allure-pytest = "^2.13.5" allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.0.2", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} - [tool.coverage.run] branch = true diff --git a/src/abstract_charm.py b/src/abstract_charm.py index c51494497..3958b7b22 100644 --- a/src/abstract_charm.py +++ b/src/abstract_charm.py @@ -48,6 +48,7 @@ def __init__(self, *args) -> None: self._database_requires = relations.database_requires.RelationEndpoint(self) self._database_provides = relations.database_provides.RelationEndpoint(self) self._cos_relation = relations.cos.COSRelation(self, self._container) + self._ha_cluster = None self.framework.observe(self.on.update_status, self.reconcile) self.framework.observe( self.on[upgrade.PEER_RELATION_ENDPOINT_NAME].relation_changed, self.reconcile @@ -99,23 +100,29 @@ def _logrotate(self) -> logrotate.LogRotate: @property @abc.abstractmethod - def _read_write_endpoint(self) -> str: + def _read_write_endpoints(self) -> str: """MySQL Router read-write endpoint""" @property @abc.abstractmethod - def _read_only_endpoint(self) -> str: + def _read_only_endpoints(self) -> str: """MySQL Router read-only endpoint""" @property @abc.abstractmethod - def _exposed_read_write_endpoint(self) -> str: - """The exposed read-write endpoint""" + def _exposed_read_write_endpoint(self) -> typing.Optional[str]: + """The exposed read-write endpoint. + + Only defined in vm charm. + """ @property @abc.abstractmethod - def _exposed_read_only_endpoint(self) -> str: - """The exposed read-only endpoint""" + def _exposed_read_only_endpoint(self) -> typing.Optional[str]: + """The exposed read-only endpoint. + + Only defined in vm charm. + """ @abc.abstractmethod def is_externally_accessible(self, *, event) -> typing.Optional[bool]: @@ -124,6 +131,14 @@ def is_externally_accessible(self, *, event) -> typing.Optional[bool]: Only defined in vm charm to return True/False. In k8s charm, returns None. """ + @property + @abc.abstractmethod + def _status(self) -> ops.StatusBase: + """Status of the charm. + + Only applies to Kubernetes charm + """ + @property def _tls_certificate_saved(self) -> bool: """Whether a TLS certificate is available to use""" @@ -201,6 +216,8 @@ def _determine_app_status(self, *, event) -> ops.StatusBase: # (Relations should not be modified during upgrade.) return upgrade_status statuses = [] + if self._status: + statuses.append(self._status) for endpoint in (self._database_requires, self._database_provides): if status := endpoint.get_status(event): statuses.append(status) @@ -212,6 +229,9 @@ def _determine_unit_status(self, *, event) -> ops.StatusBase: workload_status = self.get_workload(event=event).status if self._upgrade: statuses.append(self._upgrade.get_unit_juju_status(workload_status=workload_status)) + # only in machine charms + if self._ha_cluster: + statuses.append(self._ha_cluster.get_unit_juju_status()) statuses.append(workload_status) return self._prioritize_statuses(statuses) @@ -232,12 +252,19 @@ def wait_until_mysql_router_ready(self, *, event) -> None: """ @abc.abstractmethod - def _reconcile_node_port(self, *, event) -> None: - """Reconcile node port. + def _reconcile_service(self) -> None: + """Reconcile service. Only applies to Kubernetes charm """ + def _check_service_connectivity(self) -> bool: + """Check if the service is available (connectable with a socket). + + Returns false if Machine charm. Overridden in Kubernetes charm. + """ + return False + @abc.abstractmethod def _reconcile_ports(self, *, event) -> None: """Reconcile exposed ports. @@ -245,6 +272,10 @@ def _reconcile_ports(self, *, event) -> None: Only applies to Machine charm """ + @abc.abstractmethod + def _update_endpoints(self) -> None: + """Update the endpoints in the provider relation if necessary.""" + # ======================= # Handlers # ======================= @@ -311,6 +342,10 @@ def reconcile(self, event=None) -> None: # noqa: C901 f"{self._cos_relation.is_relation_breaking(event)=}" ) + # only in machine charms + if self._ha_cluster: + self._ha_cluster.set_vip(self.config.get("vip")) + try: if self._unit_lifecycle.authorized_leader: if self._database_requires.is_relation_breaking(event): @@ -324,15 +359,17 @@ def reconcile(self, event=None) -> None: # noqa: C901 and isinstance(workload_, workload.AuthenticatedWorkload) and workload_.container_ready ): - self._reconcile_node_port(event=event) + self._reconcile_service() self._database_provides.reconcile_users( event=event, - router_read_write_endpoint=self._read_write_endpoint, - router_read_only_endpoint=self._read_only_endpoint, + router_read_write_endpoints=self._read_write_endpoints, + router_read_only_endpoints=self._read_only_endpoints, exposed_read_write_endpoint=self._exposed_read_write_endpoint, exposed_read_only_endpoint=self._exposed_read_only_endpoint, shell=workload_.shell, ) + self._update_endpoints() + if workload_.container_ready: workload_.reconcile( event=event, diff --git a/src/kubernetes_charm.py b/src/kubernetes_charm.py index 4b5ff26a1..9e502d6ee 100755 --- a/src/kubernetes_charm.py +++ b/src/kubernetes_charm.py @@ -6,6 +6,9 @@ """MySQL Router Kubernetes charm""" +import enum +import functools +import json import logging import socket import typing @@ -25,6 +28,7 @@ import relations.cos import relations.database_provides import relations.database_requires +import relations.secrets import rock import upgrade import workload @@ -34,6 +38,14 @@ logging.getLogger("httpcore").setLevel(logging.WARNING) +class _ServiceType(enum.Enum): + """Supported K8s service types""" + + CLUSTER_IP = "ClusterIP" + NODE_PORT = "NodePort" + LOAD_BALANCER = "LoadBalancer" + + @trace_charm( tracing_endpoint="tracing_endpoint", extra_types=( @@ -51,11 +63,23 @@ class KubernetesRouterCharm(abstract_charm.MySQLRouterCharm): """MySQL Router Kubernetes charm""" + _PEER_RELATION_NAME = "mysql-router-peers" + + _K8S_SERVICE_CONNECTION_TIMEOUT = 3 + _K8S_SERVICE_INITIALIZED_KEY = "k8s-service-initialized" + _K8S_SERVICE_CREATING_KEY = "k8s-service-creating" + def __init__(self, *args) -> None: super().__init__(*args) self._namespace = self.model.name + self.service_name = f"{self.app.name}-service" + self._lightkube_client = lightkube.Client() + + self._peer_data = relations.secrets.RelationSecrets(self, self._PEER_RELATION_NAME) + self.framework.observe(self.on.install, self._on_install) + self.framework.observe(self.on.config_changed, self.reconcile) self.framework.observe( self.on[rock.CONTAINER_NAME].pebble_ready, self._on_workload_container_pebble_ready ) @@ -80,15 +104,198 @@ def _upgrade(self) -> typing.Optional[kubernetes_upgrade.Upgrade]: except upgrade.PeerRelationNotReady: pass + @property + def _status(self) -> ops.StatusBase: + if self.config.get("expose-external", "false") not in [ + "false", + "nodeport", + "loadbalancer", + ]: + return ops.BlockedStatus("Invalid expose-external config value") + if ( + self._peer_data.get_value( + relations.secrets.APP_SCOPE, self._K8S_SERVICE_INITIALIZED_KEY + ) + and not self._check_service_connectivity() + ): + if self._peer_data.get_value( + relations.secrets.APP_SCOPE, self._K8S_SERVICE_CREATING_KEY + ): + return ops.MaintenanceStatus("Waiting for K8s service connectivity") + else: + return ops.BlockedStatus("K8s service not connectible") + def is_externally_accessible(self, *, event) -> typing.Optional[bool]: """No-op since this charm is exposed with node-port""" - def _reconcile_node_port(self, *, event) -> None: - self._patch_service(event) + def _get_service(self) -> typing.Optional[lightkube.resources.core_v1.Service]: + """Get the managed k8s service.""" + try: + service = self._lightkube_client.get( + res=lightkube.resources.core_v1.Service, + name=self.service_name, + namespace=self.model.name, + ) + except lightkube.core.exceptions.ApiError as e: + if e.status.code == 404: + return None + raise + + return service + + @functools.cache + def _get_pod(self, unit_name: str) -> lightkube.resources.core_v1.Pod: + """Get the pod for the provided unit name.""" + return self._lightkube_client.get( + res=lightkube.resources.core_v1.Pod, + name=unit_name.replace("/", "-"), + namespace=self.model.name, + ) + + @functools.cache + def _get_node(self, unit_name: str) -> lightkube.resources.core_v1.Node: + """Return the node for the provided unit name.""" + node_name = self._get_pod(unit_name).spec.nodeName + return self._lightkube_client.get( + res=lightkube.resources.core_v1.Node, + name=node_name, + namespace=self.model.name, + ) + + def _reconcile_service(self) -> None: + expose_external = self.config.get("expose-external", "false") + if expose_external not in ["false", "nodeport", "loadbalancer"]: + logger.warning(f"Invalid config value {expose_external=}") + return + + desired_service_type = { + "false": _ServiceType.CLUSTER_IP, + "nodeport": _ServiceType.NODE_PORT, + "loadbalancer": _ServiceType.LOAD_BALANCER, + }[expose_external] + + service = self._get_service() + service_exists = service is not None + service_type = service_exists and _ServiceType(service.spec.type) + if service_exists and service_type == desired_service_type: + return + + pod0 = self._get_pod(f"{self.app.name}/0") + + annotations = ( + json.loads(self.config.get("loadbalancer-extra-annotations", "{}")) + if desired_service_type == _ServiceType.LOAD_BALANCER + else {} + ) + + desired_service = lightkube.resources.core_v1.Service( + metadata=lightkube.models.meta_v1.ObjectMeta( + name=self.service_name, + namespace=self.model.name, + ownerReferences=pod0.metadata.ownerReferences, # the stateful set + labels={"app.kubernetes.io/name": self.app.name}, + annotations=annotations, + ), + spec=lightkube.models.core_v1.ServiceSpec( + ports=[ + lightkube.models.core_v1.ServicePort( + name="mysql-rw", + port=self._READ_WRITE_PORT, + targetPort=self._READ_WRITE_PORT, + ), + lightkube.models.core_v1.ServicePort( + name="mysql-ro", + port=self._READ_ONLY_PORT, + targetPort=self._READ_ONLY_PORT, + ), + ], + type=desired_service_type.value, + selector={"app.kubernetes.io/name": self.app.name}, + ), + ) + + # Delete and re-create until https://bugs.launchpad.net/juju/+bug/2084711 resolved + if service_exists: + logger.info(f"Issuing delete service {service_type=}") + self._lightkube_client.delete( + res=lightkube.resources.core_v1.Service, + name=self.service_name, + namespace=self.model.name, + ) + logger.info(f"Deleting service {service_type=}") + + try: + for attempt in tenacity.Retrying( + reraise=True, + stop=tenacity.stop_after_delay(10), + wait=tenacity.wait_fixed(1), + ): + with attempt: + assert self._get_service() is not None + except AssertionError: + logger.warning("Deletion of service took longer than expected") + return + else: + logger.debug(f"Deleted service {service_type=}") + + logger.info(f"Creating desired service {desired_service_type=}") + self._lightkube_client.apply(desired_service, field_manager=self.app.name) + + self._peer_data.set_value( + relations.secrets.APP_SCOPE, self._K8S_SERVICE_CREATING_KEY, "true" + ) + self._peer_data.set_value( + relations.secrets.APP_SCOPE, self._K8S_SERVICE_INITIALIZED_KEY, "true" + ) + + logger.info(f"Request to create desired service {desired_service_type=} dispatched") + + def _check_service_connectivity(self) -> bool: + if not self._get_service() or not isinstance( + self.get_workload(event=None), workload.AuthenticatedWorkload + ): + logger.debug("No service or unauthenticated workload") + return False + + for endpoints in ( + self._read_write_endpoints, + self._read_only_endpoints, + ): + if endpoints == "": + logger.debug( + f"Empty endpoints {self._read_write_endpoints=} {self._read_only_endpoints=}" + ) + return False + + for endpoint in endpoints.split(","): + with socket.socket() as s: + s.settimeout(self._K8S_SERVICE_CONNECTION_TIMEOUT) + + host, port = endpoint.split(":") + + try: + socket_connect_code = s.connect_ex((host, int(port))) + except socket.gaierror: + # Sometimes, it may take LB hostname record to propagate + logger.info(f"Unable to resolve {endpoint=}") + return False + + if socket_connect_code != 0: + logger.info(f"Unable to connect to {endpoint=}") + return False + + return True def _reconcile_ports(self, *, event) -> None: """Needed for VM, so no-op""" + def _update_endpoints(self) -> None: + if self._check_service_connectivity(): + self._database_provides.update_endpoints( + router_read_write_endpoints=self._read_write_endpoints, + router_read_only_endpoints=self._read_only_endpoints, + ) + def wait_until_mysql_router_ready(self, *, event=None) -> None: logger.debug("Waiting until MySQL Router is ready") self.unit.status = ops.MaintenanceStatus("MySQL Router starting") @@ -127,97 +334,88 @@ def model_service_domain(self) -> str: @property def _host(self) -> str: """K8s service hostname for MySQL Router""" - # Example: mysql-router-k8s.my-model.svc.cluster.local - return f"{self.app.name}.{self.model_service_domain}" + # Example: mysql-router-k8s-service.my-model.svc.cluster.local + return f"{self.service_name}.{self.model_service_domain}" - @property - def _read_write_endpoint(self) -> str: - return f"{self._host}:{self._READ_WRITE_PORT}" + def _get_node_hosts(self) -> set[str]: + """Return the node ports of nodes where units of this app are scheduled.""" + peer_relation = self.model.get_relation(self._PEER_RELATION_NAME) + if not peer_relation: + return set() + + def _get_node_address(node) -> str: + # OpenStack will return an internal hostname, not externally accessible + # Preference: ExternalIP > InternalIP > Hostname + for typ in ["ExternalIP", "InternalIP", "Hostname"]: + for address in node.status.addresses: + if address.type == typ: + return address.address + + hosts = set() + for unit in peer_relation.units | {self.model.unit}: + node = self._get_node(unit.name) + hosts.add(_get_node_address(node)) + return hosts + + def _get_hosts_ports(self, port_type: str) -> str: # noqa: C901 + """Gets the host and port for the endpoint depending of type of service.""" + if port_type not in ["rw", "ro"]: + raise ValueError("Invalid port type") + + service = self._get_service() + if not service: + return "" + + port = self._READ_WRITE_PORT if port_type == "rw" else self._READ_ONLY_PORT + + service_type = _ServiceType(service.spec.type) + + if service_type == _ServiceType.CLUSTER_IP: + return f"{self._host}:{port}" + elif service_type == _ServiceType.NODE_PORT: + hosts = self._get_node_hosts() + + for p in service.spec.ports: + if p.name == f"mysql-{port_type}": + node_port = p.nodePort + + return ",".join(sorted({f"{host}:{node_port}" for host in hosts})) + elif service_type == _ServiceType.LOAD_BALANCER and service.status.loadBalancer.ingress: + if len(service.status.loadBalancer.ingress) != 0: + ip = service.status.loadBalancer.ingress[0].ip + hostname = service.status.loadBalancer.ingress[0].hostname + + if ip: + return f"{ip}:{port}" + + if hostname: + return f"{hostname}:{port}" + + return "" @property - def _read_only_endpoint(self) -> str: - return f"{self._host}:{self._READ_ONLY_PORT}" + def _read_write_endpoints(self) -> str: + return self._get_hosts_ports("rw") @property - def _exposed_read_write_endpoint(self) -> str: - return f"{self._node_ip}:{self._node_port('rw')}" + def _read_only_endpoints(self) -> str: + return self._get_hosts_ports("ro") @property - def _exposed_read_only_endpoint(self) -> str: - return f"{self._node_ip}:{self._node_port('ro')}" - - def _patch_service(self, event=None) -> None: - """Patch Juju-created k8s service. - - The k8s service will be tied to pod-0 so that the service is auto cleaned by - k8s when the last pod is scaled down. - """ - logger.debug("Patching k8s service") - client = lightkube.Client() - pod0 = client.get( - res=lightkube.resources.core_v1.Pod, - name=self.app.name + "-0", - namespace=self.model.name, - ) - service = lightkube.resources.core_v1.Service( - metadata=lightkube.models.meta_v1.ObjectMeta( - name=self.app.name, - namespace=self.model.name, - ownerReferences=pod0.metadata.ownerReferences, - labels={ - "app.kubernetes.io/name": self.app.name, - }, - ), - spec=lightkube.models.core_v1.ServiceSpec( - ports=[ - lightkube.models.core_v1.ServicePort( - name="mysql-rw", - port=self._READ_WRITE_PORT, - targetPort=self._READ_WRITE_PORT, # Value ignored if NodePort - ), - lightkube.models.core_v1.ServicePort( - name="mysql-ro", - port=self._READ_ONLY_PORT, - targetPort=self._READ_ONLY_PORT, # Value ignored if NodePort - ), - ], - type=( - "NodePort" - if self._database_provides.external_connectivity(event) - else "ClusterIP" - ), - selector={"app.kubernetes.io/name": self.app.name}, - ), - ) - client.patch( - res=lightkube.resources.core_v1.Service, - obj=service, - name=service.metadata.name, - namespace=service.metadata.namespace, - force=True, - field_manager=self.app.name, - ) - logger.debug("Patched k8s service") + def _exposed_read_write_endpoint(self) -> typing.Optional[str]: + """Only applies to VM charm, so no-op.""" + pass @property - def _node_name(self) -> str: - """Return the node name for this unit's pod ip.""" - pod = lightkube.Client().get( - lightkube.resources.core_v1.Pod, - name=self.unit.name.replace("/", "-"), - namespace=self._namespace, - ) - return pod.spec.nodeName + def _exposed_read_only_endpoint(self) -> typing.Optional[str]: + """Only applies to VM charm, so no-op.""" + pass def get_all_k8s_node_hostnames_and_ips( self, ) -> typing.Tuple[typing.List[str], typing.List[str]]: """Return all node hostnames and IPs registered in k8s.""" - node = lightkube.Client().get( - lightkube.resources.core_v1.Node, - name=self._node_name, - namespace=self._namespace, - ) + node = self._get_node(self.unit.name) hostnames = [] ips = [] for a in node.status.addresses: @@ -227,47 +425,6 @@ def get_all_k8s_node_hostnames_and_ips( hostnames.append(a.address) return hostnames, ips - @property - def _node_ip(self) -> typing.Optional[str]: - """Return node IP.""" - node = lightkube.Client().get( - lightkube.resources.core_v1.Node, - name=self._node_name, - namespace=self._namespace, - ) - # [ - # NodeAddress(address='192.168.0.228', type='InternalIP'), - # NodeAddress(address='example.com', type='Hostname') - # ] - # Remember that OpenStack, for example, will return an internal hostname, which is not - # accessible from the outside. Give preference to ExternalIP, then InternalIP first - # Separated, as we want to give preference to ExternalIP, InternalIP and then Hostname - for typ in ["ExternalIP", "InternalIP", "Hostname"]: - for a in node.status.addresses: - if a.type == typ: - return a.address - - def _node_port(self, port_type: str) -> int: - """Return node port.""" - service = lightkube.Client().get( - lightkube.resources.core_v1.Service, self.app.name, namespace=self._namespace - ) - if not service or not service.spec.type == "NodePort": - return -1 - # svc.spec.ports - # [ServicePort(port=3306, appProtocol=None, name=None, nodePort=31438, protocol='TCP', targetPort=3306)] - if port_type == "rw": - port = self._READ_WRITE_PORT - elif port_type == "ro": - port = self._READ_ONLY_PORT - else: - raise ValueError(f"Invalid {port_type=}") - logger.debug(f"Looking for NodePort for {port_type} in {service.spec.ports}") - for svc_port in service.spec.ports: - if svc_port.port == port: - return svc_port.nodePort - raise Exception(f"NodePort not found for {port_type}") - # ======================= # Handlers # ======================= @@ -279,11 +436,6 @@ def _on_install(self, _) -> None: self.unit.open_port("tcp", port) if not self.unit.is_leader(): return - try: - self._patch_service() - except lightkube.ApiError: - logger.exception("Failed to patch k8s service") - raise def _on_workload_container_pebble_ready(self, _) -> None: self.unit.set_workload_version(self.get_workload(event=None).version) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 08cb70484..ef548eb98 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -40,9 +40,14 @@ def __init__(self, *, app_name: str, endpoint_name: str) -> None: class _Relation: """Relation to one application charm""" - def __init__(self, *, relation: ops.Relation) -> None: + def __init__( + self, *, relation: ops.Relation, interface: data_interfaces.DatabaseProvides + ) -> None: self._id = relation.id + # Application charm databag + self._databag = remote_databag.RemoteDatabag(interface=interface, relation=relation) + def __eq__(self, other) -> bool: if not isinstance(other, _Relation): return False @@ -63,19 +68,13 @@ class _RelationThatRequestedUser(_Relation): def __init__( self, *, relation: ops.Relation, interface: data_interfaces.DatabaseProvides, event ) -> None: - super().__init__(relation=relation) + super().__init__(relation=relation, interface=interface) self._interface = interface if event and isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id: raise _RelationBreaking - # Application charm databag - databag = remote_databag.RemoteDatabag(interface=interface, relation=relation) - self._database: str = databag["database"] - # Whether endpoints should be externally accessible - # (e.g. when related to `data-integrator` charm) - # Implements DA073 - Add Expose Flag to the Database Interface - # https://docs.google.com/document/d/1Y7OZWwMdvF8eEMuVKrqEfuFV3JOjpqLHL7_GPqJpRHU - self.external_connectivity = databag.get("external-node-connectivity") == "true" - if databag.get("extra-user-roles"): + + self._database: str = self._databag["database"] + if self._databag.get("extra-user-roles"): raise _UnsupportedExtraUserRole( app_name=relation.app.name, endpoint_name=relation.name ) @@ -85,28 +84,26 @@ def _set_databag( *, username: str, password: str, - router_read_write_endpoint: str, - router_read_only_endpoint: str, + router_read_write_endpoints: str, + router_read_only_endpoints: str, ) -> None: """Share connection information with application charm.""" logger.debug( - f"Setting databag {self._id=} {self._database=}, {username=}, {router_read_write_endpoint=}, {router_read_only_endpoint=}" + f"Setting databag {self._id=} {self._database=}, {username=}, {router_read_write_endpoints=}, {router_read_only_endpoints=}" ) self._interface.set_database(self._id, self._database) self._interface.set_credentials(self._id, username, password) - self._interface.set_endpoints(self._id, router_read_write_endpoint) - self._interface.set_read_only_endpoints(self._id, router_read_only_endpoint) + self._interface.set_endpoints(self._id, router_read_write_endpoints) + self._interface.set_read_only_endpoints(self._id, router_read_only_endpoints) logger.debug( - f"Set databag {self._id=} {self._database=}, {username=}, {router_read_write_endpoint=}, {router_read_only_endpoint=}" + f"Set databag {self._id=} {self._database=}, {username=}, {router_read_write_endpoints=}, {router_read_only_endpoints=}" ) def create_database_and_user( self, *, - router_read_write_endpoint: str, - router_read_only_endpoint: str, - exposed_read_write_endpoint: str, - exposed_read_only_endpoint: str, + router_read_write_endpoints: str, + router_read_only_endpoints: str, shell: mysql_shell.Shell, ) -> None: """Create database & user and update databag.""" @@ -123,19 +120,11 @@ def create_database_and_user( username=username, database=self._database ) - rw_endpoint = ( - exposed_read_write_endpoint - if self.external_connectivity - else router_read_write_endpoint - ) - ro_endpoint = ( - exposed_read_only_endpoint if self.external_connectivity else router_read_only_endpoint - ) self._set_databag( username=username, password=password, - router_read_write_endpoint=rw_endpoint, - router_read_only_endpoint=ro_endpoint, + router_read_write_endpoints=router_read_write_endpoints, + router_read_only_endpoints=router_read_only_endpoints, ) @@ -149,13 +138,29 @@ class _RelationWithSharedUser(_Relation): def __init__( self, *, relation: ops.Relation, interface: data_interfaces.DatabaseProvides ) -> None: - super().__init__(relation=relation) + super().__init__(relation=relation, interface=interface) self._interface = interface self._local_databag = self._interface.fetch_my_relation_data([relation.id])[relation.id] for key in ("database", "username", "password", "endpoints", "read-only-endpoints"): if key not in self._local_databag: raise _UserNotShared + def update_endpoints( + self, + *, + router_read_write_endpoints: str, + router_read_only_endpoints: str, + ) -> None: + """Update the endpoins in the databag.""" + logger.debug( + f"Updating endpoints {self._id} {router_read_write_endpoints=} {router_read_only_endpoints=}" + ) + self._interface.set_endpoints(self._id, router_read_write_endpoints) + self._interface.set_read_only_endpoints(self._id, router_read_only_endpoints) + logger.debug( + f"Updated endpoints {self._id} {router_read_write_endpoints=} {router_read_only_endpoints=}" + ) + def delete_databag(self) -> None: """Remove connection information from databag.""" logger.debug(f"Deleting databag {self._id=}") @@ -182,24 +187,6 @@ def __init__(self, charm_: "abstract_charm.MySQLRouterCharm") -> None: charm_.framework.observe(self._interface.on.database_requested, charm_.reconcile) charm_.framework.observe(charm_.on[self._NAME].relation_broken, charm_.reconcile) - def external_connectivity(self, event) -> bool: - """Whether any of the relations are marked as external.""" - requested_users = [] - for relation in self._interface.relations: - try: - requested_users.append( - _RelationThatRequestedUser( - relation=relation, interface=self._interface, event=event - ) - ) - except ( - _RelationBreaking, - remote_databag.IncompleteDatabag, - _UnsupportedExtraUserRole, - ): - pass - return any(relation.external_connectivity for relation in requested_users) - @property # TODO python3.10 min version: Use `list` instead of `typing.List` def _shared_users(self) -> typing.List[_RelationWithSharedUser]: @@ -213,15 +200,27 @@ def _shared_users(self) -> typing.List[_RelationWithSharedUser]: pass return shared_users + def update_endpoints( + self, + *, + router_read_write_endpoints: str, + router_read_only_endpoints: str, + ) -> None: + """Update endpoints in the databags.""" + for relation in self._shared_users: + relation.update_endpoints( + router_read_write_endpoints=router_read_write_endpoints, + router_read_only_endpoints=router_read_only_endpoints, + ) + def reconcile_users( self, *, event, - router_read_write_endpoint: str, - router_read_only_endpoint: str, - exposed_read_write_endpoint: str, - exposed_read_only_endpoint: str, + router_read_write_endpoints: str, + router_read_only_endpoints: str, shell: mysql_shell.Shell, + **_, ) -> None: """Create requested users and delete inactive users. @@ -230,8 +229,7 @@ def reconcile_users( relation is broken. """ logger.debug( - f"Reconciling users {event=}, {router_read_write_endpoint=}, {router_read_only_endpoint=}, " - f"{exposed_read_write_endpoint=}, {exposed_read_only_endpoint=}" + f"Reconciling users {event=}, {router_read_write_endpoints=}, {router_read_only_endpoints=}" ) requested_users = [] for relation in self._interface.relations: @@ -251,18 +249,15 @@ def reconcile_users( for relation in requested_users: if relation not in self._shared_users: relation.create_database_and_user( - router_read_write_endpoint=router_read_write_endpoint, - router_read_only_endpoint=router_read_only_endpoint, - exposed_read_write_endpoint=exposed_read_write_endpoint, - exposed_read_only_endpoint=exposed_read_only_endpoint, + router_read_write_endpoints=router_read_write_endpoints, + router_read_only_endpoints=router_read_only_endpoints, shell=shell, ) for relation in self._shared_users: if relation not in requested_users: relation.delete_user(shell=shell) logger.debug( - f"Reconciled users {event=}, {router_read_write_endpoint=}, {router_read_only_endpoint=}, " - f"{exposed_read_write_endpoint=}, {exposed_read_only_endpoint=}" + f"Reconciled users {event=}, {router_read_write_endpoints=}, {router_read_only_endpoints=}" ) def delete_all_databags(self) -> None: diff --git a/src/relations/tls.py b/src/relations/tls.py index 8397a5200..b2cae77cd 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -112,6 +112,7 @@ def save_certificate(self, event: tls_certificates.CertificateAvailableEvent) -> def _generate_csr(self, key: bytes) -> bytes: """Generate certificate signing request (CSR).""" + service_name = self._charm.service_name unit_name = self._charm.unit.name.replace("/", "-") extra_hosts, extra_ips = self._charm.get_all_k8s_node_hostnames_and_ips() return tls_certificates.generate_csr( @@ -122,13 +123,22 @@ def _generate_csr(self, key: bytes) -> bytes: organization=self._charm.app.name, sans_dns=[ socket.getfqdn(), + service_name, unit_name, + f"{service_name}.{self._charm.app.name}-endpoints", f"{unit_name}.{self._charm.app.name}-endpoints", + f"{self._charm.app.name}.{self._charm.app.name}-endpoints", + f"{service_name}.{self._charm.app.name}-endpoints.{self._charm.model_service_domain}", f"{unit_name}.{self._charm.app.name}-endpoints.{self._charm.model_service_domain}", + f"{self._charm.app.name}.{self._charm.app.name}-endpoints.{self._charm.model_service_domain}", f"{self._charm.app.name}-endpoints", f"{self._charm.app.name}-endpoints.{self._charm.model_service_domain}", + f"{service_name}.{self._charm.app.name}", f"{unit_name}.{self._charm.app.name}", + f"{self._charm.app.name}.{self._charm.app.name}", + f"{service_name}.{self._charm.app.name}.{self._charm.model_service_domain}", f"{unit_name}.{self._charm.app.name}.{self._charm.model_service_domain}", + f"{self._charm.app.name}.{self._charm.app.name}.{self._charm.model_service_domain}", self._charm.app.name, f"{self._charm.app.name}.{self._charm.model_service_domain}", *extra_hosts, diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index a4b885445..f3b0315db 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -103,6 +103,18 @@ async def get_inserted_data_by_application(unit: Unit) -> Optional[str]: return (await run_action(unit, "get-inserted-data")).get("data") +async def get_credentials(unit: Unit) -> Dict: + """Helper to run an action on data-integrator to get credentials. + + Args: + unit: The data-integrator unit to run action against + + Returns: + A dictionary with the credentials + """ + return await run_action(unit, "get-credentials") + + async def get_unit_address(ops_test: OpsTest, unit_name: str) -> str: """Get unit IP address. @@ -391,8 +403,9 @@ def is_connection_possible(credentials: Dict, **extra_opts) -> bool: with MySQLConnector(config) as cursor: cursor.execute("SELECT 1") return cursor.fetchone()[0] == 1 - except (DatabaseError, InterfaceError, OperationalError, ProgrammingError): + except (DatabaseError, InterfaceError, OperationalError, ProgrammingError) as e: # Errors raised when the connection is not possible + logger.error(e) return False diff --git a/tests/integration/test_expose_external.py b/tests/integration/test_expose_external.py new file mode 100644 index 000000000..812df81a0 --- /dev/null +++ b/tests/integration/test_expose_external.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import asyncio +import logging +import time +from pathlib import Path + +import pytest +import tenacity +import yaml +from pytest_operator.plugin import OpsTest + +from . import architecture, juju_ +from .helpers import ( + APPLICATION_DEFAULT_APP_NAME, + MYSQL_DEFAULT_APP_NAME, + MYSQL_ROUTER_DEFAULT_APP_NAME, + get_credentials, + is_connection_possible, +) + +logger = logging.getLogger(__name__) + +METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) + +MYSQL_APP_NAME = MYSQL_DEFAULT_APP_NAME +MYSQL_ROUTER_APP_NAME = MYSQL_ROUTER_DEFAULT_APP_NAME +APPLICATION_APP_NAME = APPLICATION_DEFAULT_APP_NAME +DATA_INTEGRATOR = "data-integrator" +SLOW_TIMEOUT = 15 * 60 +MODEL_CONFIG = {"logging-config": "=INFO;unit=DEBUG"} +TEST_DATABASE_NAME = "testdatabase" + +TLS_SETUP_SLEEP_TIME = 30 +if juju_.is_3_or_higher: + TLS_APP_NAME = "self-signed-certificates" + if architecture.architecture == "arm64": + TLS_CHANNEL = "latest/edge" + else: + TLS_CHANNEL = "latest/stable" + TLS_CONFIG = {"ca-common-name": "Test CA"} +else: + TLS_APP_NAME = "tls-certificates-operator" + if architecture.architecture == "arm64": + TLS_CHANNEL = "legacy/edge" + else: + TLS_CHANNEL = "legacy/stable" + TLS_CONFIG = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} + + +async def confirm_cluster_ip_endpoints(ops_test: OpsTest) -> None: + """Helper function to test the cluster ip endpoints""" + for attempt in tenacity.Retrying( + reraise=True, + stop=tenacity.stop_after_delay(SLOW_TIMEOUT), + wait=tenacity.wait_fixed(10), + ): + with attempt: + data_integrator_unit = ops_test.model.applications[DATA_INTEGRATOR].units[0] + credentials = await get_credentials(data_integrator_unit) + + assert credentials["mysql"]["database"] == TEST_DATABASE_NAME, "Database is empty" + assert credentials["mysql"]["username"] is not None, "Username is empty" + assert credentials["mysql"]["password"] is not None, "Password is empty" + + endpoint_name = f"mysql-router-k8s-service.{ops_test.model.name}.svc.cluster.local" + assert credentials["mysql"]["endpoints"] == f"{endpoint_name}:6446", "Endpoint is unexpected" + assert ( + credentials["mysql"]["read-only-endpoints"] == f"{endpoint_name}:6447" + ), "Read-only endpoint is unexpected" + + +async def confirm_endpoint_connectivity(ops_test: OpsTest) -> None: + """Helper to confirm endpoint connectivity""" + for attempt in tenacity.Retrying( + reraise=True, + stop=tenacity.stop_after_delay(SLOW_TIMEOUT), + wait=tenacity.wait_fixed(10), + ): + with attempt: + data_integrator_unit = ops_test.model.applications[DATA_INTEGRATOR].units[0] + credentials = await get_credentials(data_integrator_unit) + assert credentials["mysql"]["endpoints"] is not None, "Endpoints missing" + + connection_config = { + "username": credentials["mysql"]["username"], + "password": credentials["mysql"]["password"], + "host": credentials["mysql"]["endpoints"].split(",")[0].split(":")[0], + } + + extra_connection_options = { + "port": credentials["mysql"]["endpoints"].split(":")[1], + "ssl_disabled": False, + } + + assert is_connection_possible( + connection_config, **extra_connection_options + ), "Connection not possible through endpoints" + + +@pytest.mark.group(1) +@pytest.mark.abort_on_fail +async def test_expose_external(ops_test) -> None: + """Test the expose-external config option.""" + logger.info("Building mysql-router-k8s charm") + mysql_router_charm = await ops_test.build_charm(".") + await ops_test.model.set_config(MODEL_CONFIG) + + mysql_router_resources = { + "mysql-router-image": METADATA["resources"]["mysql-router-image"]["upstream-source"] + } + + logger.info("Deploying mysql-k8s, mysql-router-k8s and data-integrator") + await asyncio.gather( + ops_test.model.deploy( + MYSQL_APP_NAME, + channel="8.0/edge", + application_name=MYSQL_APP_NAME, + config={"profile": "testing"}, + base="ubuntu@22.04", + num_units=1, + trust=True, + ), + ops_test.model.deploy( + mysql_router_charm, + application_name=MYSQL_ROUTER_APP_NAME, + base="ubuntu@22.04", + resources=mysql_router_resources, + num_units=1, + trust=True, + ), + ops_test.model.deploy( + DATA_INTEGRATOR, + channel="latest/edge", + application_name=DATA_INTEGRATOR, + base="ubuntu@22.04", + config={"database-name": TEST_DATABASE_NAME}, + num_units=1, + ), + ) + + logger.info("Relating mysql-k8s, mysql-router-k8s and data-integrator") + async with ops_test.fast_forward("60s"): + await ops_test.model.relate( + f"{MYSQL_APP_NAME}:database", f"{MYSQL_ROUTER_APP_NAME}:backend-database" + ) + await ops_test.model.relate( + f"{MYSQL_ROUTER_APP_NAME}:database", f"{DATA_INTEGRATOR}:mysql" + ) + + await ops_test.model.wait_for_idle( + apps=[MYSQL_APP_NAME, MYSQL_ROUTER_APP_NAME, DATA_INTEGRATOR], + status="active", + timeout=SLOW_TIMEOUT, + ) + + logger.info("Testing endpoint when expose-external=false (default)") + await confirm_cluster_ip_endpoints(ops_test) + + logger.info("Testing endpoint when expose-external=nodeport") + mysql_router_application = ops_test.model.applications[MYSQL_ROUTER_APP_NAME] + + await mysql_router_application.set_config({"expose-external": "nodeport"}) + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="active", + timeout=SLOW_TIMEOUT, + ) + + await confirm_endpoint_connectivity(ops_test) + + logger.info("Testing endpoint when expose-external=loadbalancer") + await mysql_router_application.set_config({"expose-external": "loadbalancer"}) + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="active", + timeout=SLOW_TIMEOUT, + ) + + await confirm_endpoint_connectivity(ops_test) + + +@pytest.mark.group(1) +@pytest.mark.abort_on_fail +async def test_expose_external_with_tls(ops_test: OpsTest) -> None: + """Test endpoints when mysql-router-k8s is related to a TLS operator.""" + mysql_router_application = ops_test.model.applications[MYSQL_ROUTER_APP_NAME] + + logger.info("Resetting expose-external=false") + await mysql_router_application.set_config({"expose-external": "false"}) + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="active", + timeout=SLOW_TIMEOUT, + ) + + logger.info("Deploying TLS operator") + await ops_test.model.deploy( + TLS_APP_NAME, + channel=TLS_CHANNEL, + config=TLS_CONFIG, + base="ubuntu@22.04", + ) + async with ops_test.fast_forward("60s"): + await ops_test.model.wait_for_idle( + apps=[TLS_APP_NAME], + status="active", + timeout=SLOW_TIMEOUT, + ) + + logger.info("Relate mysql-router-k8s with TLS operator") + await ops_test.model.relate(MYSQL_ROUTER_APP_NAME, TLS_APP_NAME) + + time.sleep(TLS_SETUP_SLEEP_TIME) + + logger.info("Testing endpoint when expose-external=false(default)") + await confirm_cluster_ip_endpoints(ops_test) + + logger.info("Testing endpoint when expose-external=nodeport") + await mysql_router_application.set_config({"expose-external": "nodeport"}) + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="active", + timeout=SLOW_TIMEOUT, + ) + + await confirm_endpoint_connectivity(ops_test) + + logger.info("Testing endpoint when expose-external=loadbalancer") + await mysql_router_application.set_config({"expose-external": "loadbalancer"}) + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="active", + timeout=SLOW_TIMEOUT, + ) + + await confirm_endpoint_connectivity(ops_test) diff --git a/tests/integration/test_node_port.py b/tests/integration/test_node_port.py deleted file mode 100644 index e51a81fa3..000000000 --- a/tests/integration/test_node_port.py +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -import asyncio -import logging -import subprocess -from pathlib import Path - -import pytest -import yaml -from pytest_operator.plugin import OpsTest - -from . import architecture, markers -from .helpers import ( - APPLICATION_DEFAULT_APP_NAME, - MYSQL_DEFAULT_APP_NAME, - MYSQL_ROUTER_DEFAULT_APP_NAME, - execute_queries_against_unit, - get_inserted_data_by_application, - get_server_config_credentials, - get_tls_ca, - get_unit_address, - is_connection_possible, -) - -logger = logging.getLogger(__name__) - -METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) - -MYSQL_APP_NAME = MYSQL_DEFAULT_APP_NAME -MYSQL_ROUTER_APP_NAME = MYSQL_ROUTER_DEFAULT_APP_NAME -SELF_SIGNED_CERTIFICATE_NAME = "self-signed-certificates" -APPLICATION_APP_NAME = APPLICATION_DEFAULT_APP_NAME -DATA_INTEGRATOR = "data-integrator" -SLOW_TIMEOUT = 15 * 60 -MODEL_CONFIG = {"logging-config": "=INFO;unit=DEBUG"} -if architecture.architecture == "arm64": - tls_channel = "latest/edge" -else: - tls_channel = "latest/stable" -TLS_CONFIG = {"ca-common-name": "Test CA"} - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -@markers.skip_if_lower_than_3_1 -async def test_build_and_deploy(ops_test: OpsTest): - """Test the deployment of the charm.""" - # Build and deploy applications - mysqlrouter_charm = await ops_test.build_charm(".") - await ops_test.model.set_config(MODEL_CONFIG) - - mysqlrouter_resources = { - "mysql-router-image": METADATA["resources"]["mysql-router-image"]["upstream-source"] - } - - logger.info("Deploying mysql, mysqlrouter and application") - await asyncio.gather( - ops_test.model.deploy( - MYSQL_APP_NAME, - channel="8.0/edge", - application_name=MYSQL_APP_NAME, - config={"profile": "testing"}, - base="ubuntu@22.04", - num_units=3, - trust=True, # Necessary after a6f1f01: Fix/endpoints as k8s services (#142) - ), - ops_test.model.deploy( - mysqlrouter_charm, - application_name=MYSQL_ROUTER_APP_NAME, - base="ubuntu@22.04", - resources=mysqlrouter_resources, - num_units=1, - trust=True, - ), - ops_test.model.deploy( - APPLICATION_APP_NAME, - channel="latest/edge", - application_name=APPLICATION_APP_NAME, - base="ubuntu@22.04", - num_units=1, - ), - ops_test.model.deploy( - DATA_INTEGRATOR, - channel="latest/edge", - application_name=DATA_INTEGRATOR, - base="ubuntu@22.04", - config={"database-name": "test"}, - num_units=1, - ), - ops_test.model.deploy( - SELF_SIGNED_CERTIFICATE_NAME, - channel=tls_channel, - application_name=SELF_SIGNED_CERTIFICATE_NAME, - config=TLS_CONFIG, - base="ubuntu@22.04", - num_units=1, - ), - ) - - async with ops_test.fast_forward(): - logger.info("Relating mysql, mysqlrouter and application") - ( - await ops_test.model.relate( - f"{MYSQL_APP_NAME}", f"{SELF_SIGNED_CERTIFICATE_NAME}:certificates" - ), - ) - ( - await ops_test.model.relate( - f"{MYSQL_ROUTER_APP_NAME}", f"{SELF_SIGNED_CERTIFICATE_NAME}:certificates" - ), - ) - - # Relate the database with mysqlrouter - await ops_test.model.relate( - f"{MYSQL_ROUTER_APP_NAME}:backend-database", f"{MYSQL_APP_NAME}:database" - ) - # Relate mysqlrouter with data integrator - await ops_test.model.relate( - f"{DATA_INTEGRATOR}:mysql", f"{MYSQL_ROUTER_APP_NAME}:database" - ) - # Relate mysqlrouter with application next - await ops_test.model.relate( - f"{APPLICATION_APP_NAME}:database", f"{MYSQL_ROUTER_APP_NAME}:database" - ) - - # Now, we should have one - await ops_test.model.wait_for_idle( - apps=[ - MYSQL_APP_NAME, - MYSQL_ROUTER_APP_NAME, - APPLICATION_APP_NAME, - DATA_INTEGRATOR, - SELF_SIGNED_CERTIFICATE_NAME, - ], - status="active", - timeout=SLOW_TIMEOUT, - ) - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -@markers.skip_if_lower_than_3_1 -async def test_tls(ops_test: OpsTest): - """Test the database relation.""" - logger.info("Assert TLS file exists") - assert await get_tls_ca( - ops_test, MYSQL_ROUTER_APP_NAME + "/0" - ), "No CA found after TLS relation" - - # After relating to only encrypted connection should be possible - logger.info("Asserting connections after relation") - unit = ops_test.model.units.get(DATA_INTEGRATOR + "/0") - action = await unit.run_action("get-credentials") - creds = (await asyncio.wait_for(action.wait(), 60)).results["mysql"] - config = { - "username": creds["username"], - "password": creds["password"], - "host": creds["endpoints"].split(":")[0], - } - - extra_opts = { - "ssl_disabled": False, - "port": creds["endpoints"].split(":")[1], - } - assert is_connection_possible( - config, **extra_opts - ), f"Encryption enabled - connection not possible to unit {MYSQL_ROUTER_APP_NAME}/0" - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -@markers.skip_if_lower_than_3_1 -async def test_node_port_and_clusterip_setup(): - """Test the nodeport.""" - for app_name in [DATA_INTEGRATOR, APPLICATION_APP_NAME]: - try: - relation_info = yaml.safe_load( - subprocess.check_output([ - "juju", - "show-unit", - f"{app_name}/0", - ]) - )[f"{app_name}/0"]["relation-info"] - if app_name == DATA_INTEGRATOR: - endpoint = list(filter(lambda x: x["endpoint"] == "mysql", relation_info))[0][ - "application-data" - ]["endpoints"] - assert "svc.cluster.local" not in endpoint - else: - endpoint = list(filter(lambda x: x["endpoint"] == "database", relation_info))[0][ - "application-data" - ]["endpoints"] - assert "svc.cluster.local" in endpoint - except subprocess.CalledProcessError as e: - logger.error(f"Failed to get the unit info for {app_name}: {e.output}") - raise - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -@markers.skip_if_lower_than_3_1 -async def test_data_integrator(ops_test: OpsTest): - """Test the nodeport.""" - application_app = ops_test.model.applications.get(APPLICATION_APP_NAME) - mysql_app = ops_test.model.applications.get(MYSQL_APP_NAME) - - # Ensure that the data inserted by sample application is present in the database - application_unit = application_app.units[0] - inserted_data = await get_inserted_data_by_application(application_unit) - - mysql_unit = mysql_app.units[0] - mysql_unit_address = await get_unit_address(ops_test, mysql_unit.name) - - server_config_credentials = await get_server_config_credentials(mysql_unit) - - select_inserted_data_sql = [ - f"SELECT data FROM continuous_writes_database.random_data WHERE data = '{inserted_data}'", - ] - selected_data = await execute_queries_against_unit( - mysql_unit_address, - server_config_credentials["username"], - server_config_credentials["password"], - select_inserted_data_sql, - ) - - assert len(selected_data) > 0 - assert inserted_data == selected_data[0] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 038067624..7cbc1b191 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -44,7 +44,9 @@ def patch(monkeypatch): @pytest.fixture(autouse=True) def kubernetes_patch(monkeypatch): - monkeypatch.setattr("kubernetes_charm.KubernetesRouterCharm.model_service_domain", "") + monkeypatch.setattr( + "kubernetes_charm.KubernetesRouterCharm.model_service_domain", "my-model.svc.cluster.local" + ) monkeypatch.setattr( "rock.Rock._run_command", lambda *args, **kwargs: "null", # Use "null" for `json.loads()` @@ -56,16 +58,21 @@ def kubernetes_patch(monkeypatch): monkeypatch.setattr("rock._Path.rmtree", lambda *args, **kwargs: None) monkeypatch.setattr("lightkube.Client", lambda *args, **kwargs: None) monkeypatch.setattr( - "kubernetes_charm.KubernetesRouterCharm._patch_service", lambda *args, **kwargs: None + "kubernetes_charm.KubernetesRouterCharm._reconcile_service", lambda *args, **kwargs: None ) - monkeypatch.setattr("kubernetes_charm.KubernetesRouterCharm._node_ip", None) - monkeypatch.setattr("kubernetes_charm.KubernetesRouterCharm._node_name", None) monkeypatch.setattr( - "kubernetes_charm.KubernetesRouterCharm.get_all_k8s_node_hostnames_and_ips", - lambda *args, **kwargs: None, + "kubernetes_charm.KubernetesRouterCharm._get_hosts_ports", + lambda _, port_type: "mysql-router-k8s-service.my-model.svc.cluster.local:6446" + if port_type == "rw" + else "mysql-router-k8s-service.my-model.svc.cluster.local:6447", ) monkeypatch.setattr( - "kubernetes_charm.KubernetesRouterCharm._node_port", lambda *args, **kwargs: None + "kubernetes_charm.KubernetesRouterCharm._check_service_connectivity", + lambda *args, **kwargs: True, + ) + monkeypatch.setattr( + "kubernetes_charm.KubernetesRouterCharm.get_all_k8s_node_hostnames_and_ips", + lambda *args, **kwargs: None, ) monkeypatch.setattr("kubernetes_upgrade._Partition.get", lambda *args, **kwargs: 0) monkeypatch.setattr("kubernetes_upgrade._Partition.set", lambda *args, **kwargs: None) diff --git a/tests/unit/scenario_/database_relations/test_database_relations.py b/tests/unit/scenario_/database_relations/test_database_relations.py index 6614e428e..ad32c8e5d 100644 --- a/tests/unit/scenario_/database_relations/test_database_relations.py +++ b/tests/unit/scenario_/database_relations/test_database_relations.py @@ -19,6 +19,12 @@ def model_service_domain(monkeypatch, request): monkeypatch.setattr( "kubernetes_charm.KubernetesRouterCharm.model_service_domain", request.param ) + monkeypatch.setattr( + "kubernetes_charm.KubernetesRouterCharm._get_hosts_ports", + lambda _, port_type: f"mysql-router-k8s-service.{request.param}:6446" + if port_type == "rw" + else f"mysql-router-k8s-service.{request.param}:6447", + ) return request.param @@ -32,7 +38,11 @@ def output_states(*, relations: list[scenario.Relation]) -> typing.Iterable[scen context = scenario.Context(kubernetes_charm.KubernetesRouterCharm) container = scenario.Container("mysql-router", can_connect=True) input_state = scenario.State( - relations=[*relations, scenario.PeerRelation(endpoint="upgrade-version-a")], + relations=[ + *relations, + scenario.PeerRelation(endpoint="mysql-router-peers"), + scenario.PeerRelation(endpoint="upgrade-version-a"), + ], containers=[container], leader=True, ) @@ -76,8 +86,8 @@ def assert_complete_local_app_databag( ) assert local_app_data == { "database": provides.remote_app_data["database"], - "endpoints": f"mysql-router-k8s.{model_service_domain}:6446", - "read-only-endpoints": f"mysql-router-k8s.{model_service_domain}:6447", + "endpoints": f"mysql-router-k8s-service.{model_service_domain}:6446", + "read-only-endpoints": f"mysql-router-k8s-service.{model_service_domain}:6447", } diff --git a/tests/unit/scenario_/database_relations/test_database_relations_breaking.py b/tests/unit/scenario_/database_relations/test_database_relations_breaking.py index bf0fee96a..1504a99e1 100644 --- a/tests/unit/scenario_/database_relations/test_database_relations_breaking.py +++ b/tests/unit/scenario_/database_relations/test_database_relations_breaking.py @@ -18,7 +18,11 @@ def output_state( context = scenario.Context(kubernetes_charm.KubernetesRouterCharm) container = scenario.Container("mysql-router", can_connect=True) input_state = scenario.State( - relations=[*relations, scenario.PeerRelation(endpoint="upgrade-version-a")], + relations=[ + *relations, + scenario.PeerRelation(endpoint="mysql-router-peers"), + scenario.PeerRelation(endpoint="upgrade-version-a"), + ], containers=[container], secrets=secrets, leader=True, @@ -45,8 +49,8 @@ def test_breaking_requires_and_complete_provides( relation = relation.replace( local_app_data={ "database": "foobar", - "endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6446", - "read-only-endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6447", + "endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6446", + "read-only-endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6447", "secret-user": secret.id, } ) @@ -55,8 +59,8 @@ def test_breaking_requires_and_complete_provides( relation = relation.replace( local_app_data={ "database": "foobar", - "endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6446", - "read-only-endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6447", + "endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6446", + "read-only-endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6447", "username": "foouser", "password": "foobar", } @@ -96,8 +100,8 @@ def test_complete_requires_and_breaking_provides( relation = relation.replace( local_app_data={ "database": "foobar", - "endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6446", - "read-only-endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6447", + "endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6446", + "read-only-endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6447", "secret-user": secret.id, } ) @@ -106,8 +110,8 @@ def test_complete_requires_and_breaking_provides( relation = relation.replace( local_app_data={ "database": "foobar", - "endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6446", - "read-only-endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6447", + "endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6446", + "read-only-endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6447", "username": "foouser", "password": "foobar", } @@ -143,14 +147,14 @@ def test_complete_requires_and_breaking_provides( assert rev_contents == {"username": "foouser", "password": "foobar"} assert local_app_data == { "database": "foobar", - "endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6446", - "read-only-endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6447", + "endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6446", + "read-only-endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6447", } else: assert relation.local_app_data == { "database": "foobar", - "endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6446", - "read-only-endpoints": "mysql-router-k8s.my-model.svc.cluster.local:6447", + "endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6446", + "read-only-endpoints": "mysql-router-k8s-service.my-model.svc.cluster.local:6447", "username": "foouser", "password": "foobar", } diff --git a/tests/unit/scenario_/test_start.py b/tests/unit/scenario_/test_start.py index 32da304f0..afeb3728f 100644 --- a/tests/unit/scenario_/test_start.py +++ b/tests/unit/scenario_/test_start.py @@ -18,7 +18,10 @@ def test_start_sets_status_if_no_relations(leader, can_connect, unit_status): input_state = scenario.State( containers=[scenario.Container("mysql-router", can_connect=can_connect)], leader=leader, - relations=[scenario.PeerRelation(endpoint="upgrade-version-a")], + relations=[ + scenario.PeerRelation(endpoint="mysql-router-peers"), + scenario.PeerRelation(endpoint="upgrade-version-a"), + ], ) output_state = context.run("start", input_state) if leader: