From 2556a563afc87cf57aa7328b9be622be43d71771 Mon Sep 17 00:00:00 2001 From: PietroPasotti Date: Mon, 2 Dec 2024 14:58:21 +0100 Subject: [PATCH] grafana_datasource interface specification (#204) * added basic schema and interface spec * initial draft * base model * flip requirer and provider * flip requirer and provider * newer scenario syntax * app name fix * provider tests * source host field type fix * extra fields optional * json schemas * json schemas * schema fix for interface.yaml * default * stripped branch names --- README_INTERFACE_TESTS.md | 2 +- .../grafana_datasource/v0/provider.json | 133 ++++++++++++++++++ .../grafana_datasource/v0/requirer.json | 52 +++++++ interfaces/grafana_datasource/v0/README.md | 85 +++++++++++ .../grafana_datasource/v0/interface.yaml | 24 ++++ .../v0/interface_tests/test_provider.py | 24 ++++ .../v0/interface_tests/test_requirer.py | 80 +++++++++++ interfaces/grafana_datasource/v0/schema.py | 48 +++++++ utils/interface-validator.py | 1 + 9 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 docs/json_schemas/grafana_datasource/v0/provider.json create mode 100644 docs/json_schemas/grafana_datasource/v0/requirer.json create mode 100644 interfaces/grafana_datasource/v0/README.md create mode 100644 interfaces/grafana_datasource/v0/interface.yaml create mode 100644 interfaces/grafana_datasource/v0/interface_tests/test_provider.py create mode 100644 interfaces/grafana_datasource/v0/interface_tests/test_requirer.py create mode 100644 interfaces/grafana_datasource/v0/schema.py diff --git a/README_INTERFACE_TESTS.md b/README_INTERFACE_TESTS.md index c37385b5..e3a3b5ae 100644 --- a/README_INTERFACE_TESTS.md +++ b/README_INTERFACE_TESTS.md @@ -301,7 +301,7 @@ providers: - name: traefik-k8s url: https://github.com/canonical/traefik-k8s-operator test_setup: - - location: foo/bar/baz.py # location of the identifier + location: foo/bar/baz.py # location of the identifier identifier: qux # name of a pytest fixture yielding a configured InterfaceTester requirers: [] diff --git a/docs/json_schemas/grafana_datasource/v0/provider.json b/docs/json_schemas/grafana_datasource/v0/provider.json new file mode 100644 index 00000000..b81bbbdb --- /dev/null +++ b/docs/json_schemas/grafana_datasource/v0/provider.json @@ -0,0 +1,133 @@ +{ + "$defs": { + "GrafanaSourceData": { + "properties": { + "model": { + "description": "Name of the Juju model where the source is deployed.", + "examples": [ + "cos" + ], + "title": "Model", + "type": "string" + }, + "model_uuid": { + "description": "UUID of the Juju model where the source is deployed.", + "examples": [ + "0000-0000-0000-0000" + ], + "title": "Model Uuid", + "type": "string" + }, + "application": { + "description": "Name of the Juju model where the source is deployed.", + "examples": [ + "tempo", + "loki", + "prometheus" + ], + "title": "Application", + "type": "string" + }, + "type": { + "description": "Type of the datasource.", + "examples": [ + "tempo", + "loki", + "prometheus" + ], + "title": "Type", + "type": "string" + }, + "extra_fields": { + "anyOf": [ + { + "contentMediaType": "application/json", + "contentSchema": {}, + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Any datasource-type-specific additional configuration.", + "title": "Extra Fields" + }, + "secure_extra_fields": { + "anyOf": [ + { + "contentMediaType": "application/json", + "contentSchema": {}, + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Any secure datasource-type-specific additional configuration.", + "title": "Secure Extra Fields" + } + }, + "required": [ + "model", + "model_uuid", + "application", + "type", + "extra_fields", + "secure_extra_fields" + ], + "title": "GrafanaSourceData", + "type": "object" + }, + "GrafanaSourceProviderAppData": { + "description": "Application databag model for the requirer side of this interface.", + "properties": { + "grafana_source_data": { + "contentMediaType": "application/json", + "contentSchema": { + "$ref": "#/$defs/GrafanaSourceData" + }, + "title": "Grafana Source Data", + "type": "string" + } + }, + "required": [ + "grafana_source_data" + ], + "title": "GrafanaSourceProviderAppData", + "type": "object" + }, + "GrafanaSourceProviderUnitData": { + "description": "Application databag model for the requirer side of this interface.", + "properties": { + "grafana_source_host": { + "description": "Hostname of a source server.", + "examples": [ + "localhost:80" + ], + "title": "Grafana Source Host", + "type": "string" + } + }, + "required": [ + "grafana_source_host" + ], + "title": "GrafanaSourceProviderUnitData", + "type": "object" + } + }, + "description": "The schemas for the requirer side of this interface.", + "properties": { + "unit": { + "$ref": "#/$defs/GrafanaSourceProviderUnitData" + }, + "app": { + "$ref": "#/$defs/GrafanaSourceProviderAppData" + } + }, + "required": [ + "unit", + "app" + ], + "title": "ProviderSchema", + "type": "object" +} \ No newline at end of file diff --git a/docs/json_schemas/grafana_datasource/v0/requirer.json b/docs/json_schemas/grafana_datasource/v0/requirer.json new file mode 100644 index 00000000..076cde74 --- /dev/null +++ b/docs/json_schemas/grafana_datasource/v0/requirer.json @@ -0,0 +1,52 @@ +{ + "$defs": { + "BaseModel": { + "properties": {}, + "title": "BaseModel", + "type": "object" + }, + "GrafanaSourceRequirerAppData": { + "description": "Application databag model for the requirer side of this interface.", + "properties": { + "datasource_uids": { + "contentMediaType": "application/json", + "contentSchema": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "title": "Datasource Uids", + "type": "string" + } + }, + "required": [ + "datasource_uids" + ], + "title": "GrafanaSourceRequirerAppData", + "type": "object" + } + }, + "description": "The schema for the provider side of this interface.", + "properties": { + "unit": { + "anyOf": [ + { + "$ref": "#/$defs/BaseModel" + }, + { + "type": "null" + } + ], + "default": null + }, + "app": { + "$ref": "#/$defs/GrafanaSourceRequirerAppData" + } + }, + "required": [ + "app" + ], + "title": "RequirerSchema", + "type": "object" +} \ No newline at end of file diff --git a/interfaces/grafana_datasource/v0/README.md b/interfaces/grafana_datasource/v0/README.md new file mode 100644 index 00000000..25a00186 --- /dev/null +++ b/interfaces/grafana_datasource/v0/README.md @@ -0,0 +1,85 @@ +# `grafana_datasource` + +## Usage + +This relation interface describes the expected behavior of any charm claiming to be able to provide a grafana datasource. + +In most cases, this will be accomplished using the [grafana_source library](https://github.com/canonical/grafana-k8s-operator/blob/main/lib/charms/grafana_k8s/v0/grafana_source.py), although charm developers are free to provide alternative libraries as long as they fulfill the behavioral and schematic requirements described in this document. + +## Direction +The `grafana_datasource` interface implements a provider/requirer pattern. +The provider is a charm that implements a grafana datasource-compatible endpoint, and the requirer is a charm that is able to use such an endpoint to query the data. + +The requirer is furthermore expected to share back to the provider a unique identifier assigned to the source. This can be used by the provider to share with other charms for data correlation and cross-referencing purposes. + +```mermaid +flowchart TD + Provider -- DatasourceEndpoint --> Requirer + Requirer -- DatasourceUID --> Provider +``` + +## Behavior + +The requirer and the provider need to adhere to a certain set of criteria to be considered compatible with the interface. + +### Provider + +- Is expected to expose a server implementing [the grafana source HTTP API](https://grafana.com/docs/grafana/latest/developers/http_api/data_source/). +- Is expected to communicate said endpoint URL over unit data, as each unit will expose its own server. + +### Requirer + +- Is expected to share back via application data a mapping from provider unit names to unique datasource IDs. + +## Relation Data + +[\[Pydantic model\]](./schema.py) + + +### Requirer + + +Additionally to a subset of the the (mandatory) [juju topology fields](https://discourse.charmhub.io/t/juju-topology-labels/8874), +the requirer is expected to share the following fields: +- `type`: the grafana datasource type. For the possible values see [the upstream docs](https://grafana.com/docs/grafana/latest/datasources/#built-in-core-data-sources). Required. +- `extra_fields`: used to configure certain datasources. Maps to the `jsonData` field. Optional. +- `secure_extra_fields`: used to configure certain datasources. Maps to the `secureJsonData` field. Optional. + +The whole configuration is expected to be json-encoded and nested under a `grafana_source_data` +toplevel field. + +#### Example + +```yaml +application_data: { + grafana_source_data: + { + model: cos, + model_uuid: 0000-0000-0000-0000, + application: tempo, + type: tempo, + extra_fields: { + some: value + }, + secure_extra_fields: { + some: password + }, + + } +} +``` + +### Provider + +The provider is expected to share back a unique identifier for each unit of the requirer, as a mapping. +This will be encoded as a json dict and nested under the `datasource_uids` field in the application databag. + +#### Example +```yaml +application-data: { + datasource_uids: { + "tempo/0": 0000-0000-0000-0001, + "tempo/1": 0000-0000-0000-0002, + } +} +``` diff --git a/interfaces/grafana_datasource/v0/interface.yaml b/interfaces/grafana_datasource/v0/interface.yaml new file mode 100644 index 00000000..47f2a6e8 --- /dev/null +++ b/interfaces/grafana_datasource/v0/interface.yaml @@ -0,0 +1,24 @@ +name: grafana_datasource + +internal: true + +version: 0 + +status: published + +providers: + - name: tempo-coordinator-k8s + url: https://github.com/canonical/tempo-coordinator-k8s-operator + test_setup: + location: tests/interface/conftest.py + identifier: grafana_datasource_tester + +requirers: + - name: grafana-k8s + url: https://github.com/canonical/grafana-k8s-operator + test_setup: + location: tests/interface/conftest.py + identifier: grafana_source_tester + + +maintainer: observability diff --git a/interfaces/grafana_datasource/v0/interface_tests/test_provider.py b/interfaces/grafana_datasource/v0/interface_tests/test_provider.py new file mode 100644 index 00000000..049c1bfa --- /dev/null +++ b/interfaces/grafana_datasource/v0/interface_tests/test_provider.py @@ -0,0 +1,24 @@ +from interface_tester import Tester +from scenario import State, Relation + + +def test_share_datasource_on_remote_joined(): + # GIVEN the remote side hasn't sent anything + tester = Tester(state_in=State( + relations=[ + Relation( + endpoint='grafana-source', + interface='grafana_datasource', + remote_app_name='foo', + remote_app_data={}, + remote_units_data={ + 0: {} + } + ) + ] + )) + # WHEN the provider processes a relation-joined event + tester.run('grafana-source-relation-joined') + # THEN the provider publishes valid datasource data + tester.assert_schema_valid() + diff --git a/interfaces/grafana_datasource/v0/interface_tests/test_requirer.py b/interfaces/grafana_datasource/v0/interface_tests/test_requirer.py new file mode 100644 index 00000000..67c06c23 --- /dev/null +++ b/interfaces/grafana_datasource/v0/interface_tests/test_requirer.py @@ -0,0 +1,80 @@ +from interface_tester import Tester +from scenario import State, Relation +import json + + +def test_nothing_happens_on_no_remote_data(): + # GIVEN the remote side hasn't shared their datasource endpoint yet + tester = Tester(state_in=State( + relations=[ + Relation( + endpoint='grafana-source', + interface='grafana_datasource', + remote_app_name='foo', + remote_app_data={}, + remote_units_data={ + 0: {} + } + ) + ] + )) + # WHEN the requirer processes a relation-joined event + tester.run('grafana-source-relation-joined') + # THEN nothing is written to the databags + tester.assert_relation_data_empty() + + +def test_nothing_happens_on_invalid_remote_data(): + # GIVEN the remote side has shared gibberish + tester = Tester(state_in=State( + relations=[ + Relation( + endpoint='grafana-source', + interface='grafana_datasource', + remote_app_name='foo', + remote_app_data={"foo": "bar"}, + remote_units_data={ + 0: {"baz": "qux"} + } + ) + ] + )) + # WHEN the requirer processes a relation-changed event + tester.run('grafana-source-relation-changed') + # THEN nothing is written to the databags + tester.assert_relation_data_empty() + + +def test_datasource_uid_shared_if_remote_data_valid(): + # GIVEN the remote side has shared a valid datasource endpoint + relation_in = Relation(endpoint='grafana-source', + interface='grafana_datasource', + remote_app_name='foo', + remote_app_data={"grafana_source_data": json.dumps( + {"model": "somemodel", "model_uuid": "0000-0000-0000-0042", "application": "myapp", + "type": "prometheus", })}, + remote_units_data={ + 0: {"grafana_source_host": "somehost:80"}, + 42: {"grafana_source_host": "someotherhost:80"}, + }) + tester = Tester(state_in=State( + relations=[ + relation_in + ] + )) + + # WHEN the requirer processes a relation-changed event + state_out = tester.run('grafana-source-relation-changed') + + # THEN the schema is valid + tester.assert_schema_valid() + + # AND THEN the requirer has shared a datasource UID + rel_out = [r for r in state_out.relations if r.id == relation_in.id][0] + ds_uids = json.loads(rel_out.local_app_data['datasource_uids']) + + # each requirer unit has received a datasource uid + assert ds_uids['foo/0'] + assert ds_uids['foo/42'] + + diff --git a/interfaces/grafana_datasource/v0/schema.py b/interfaces/grafana_datasource/v0/schema.py new file mode 100644 index 00000000..b40b883b --- /dev/null +++ b/interfaces/grafana_datasource/v0/schema.py @@ -0,0 +1,48 @@ +from typing import Dict, Any, Optional + +from interface_tester.schema_base import DataBagSchema +from pydantic import Json, BaseModel, Field + + +class GrafanaSourceData(BaseModel): + model: str = Field(description="Name of the Juju model where the source is deployed.", + examples=['cos']) + model_uuid: str = Field(description="UUID of the Juju model where the source is deployed.", + examples=["0000-0000-0000-0000"]) + application: str = Field(description="Name of the Juju model where the source is deployed.", + examples=['tempo', 'loki', 'prometheus']) + type: str = Field(description="Type of the datasource.", + examples=['tempo', 'loki', 'prometheus']) + extra_fields: Optional[Json[Any]] = Field( + description="Any datasource-type-specific additional configuration.") + secure_extra_fields: Optional[Json[Any]] = Field( + description="Any secure datasource-type-specific additional configuration.") + + +class GrafanaSourceProviderAppData(BaseModel): + """Application databag model for the requirer side of this interface.""" + grafana_source_data: Json[GrafanaSourceData] + + +class GrafanaSourceProviderUnitData(BaseModel): + """Application databag model for the requirer side of this interface.""" + grafana_source_host: str = Field( + description="Hostname of a source server.", + examples=['localhost:80'] + ) + + +class ProviderSchema(DataBagSchema): + """The schemas for the requirer side of this interface.""" + app: GrafanaSourceProviderAppData + unit: GrafanaSourceProviderUnitData + + +class GrafanaSourceRequirerAppData(BaseModel): + """Application databag model for the requirer side of this interface.""" + datasource_uids: Json[Dict[str, str]] + + +class RequirerSchema(DataBagSchema): + """The schema for the provider side of this interface.""" + app: GrafanaSourceRequirerAppData diff --git a/utils/interface-validator.py b/utils/interface-validator.py index b1485756..f32ad590 100644 --- a/utils/interface-validator.py +++ b/utils/interface-validator.py @@ -33,6 +33,7 @@ class CharmEntry(BaseModel): name: str url: AnyHttpUrl + branch: Optional[str] = None test_setup: Optional[TestSetup] = None class InterfaceModel(BaseModel):