diff --git a/acto/deploy.py b/acto/deploy.py index 355e536440..cfd9067986 100644 --- a/acto/deploy.py +++ b/acto/deploy.py @@ -53,6 +53,13 @@ def __init__(self, deploy_config: DeployConfig) -> None: break else: raise Exception("No operator yaml found in deploy config") + + # Extract the operator_container_name from config + self._operator_container_name = None + for step in self._deploy_config.steps: + if step.apply and step.apply.operator: + self._operator_container_name = step.apply.operator_container_name + break @property def operator_yaml(self) -> str: @@ -136,3 +143,7 @@ def operator_name(self) -> str: if yaml_["kind"] == "Deployment": return yaml_["metadata"]["name"] return None + + @property + def operator_container_name(self) -> str: + return self._operator_container_name diff --git a/acto/engine.py b/acto/engine.py index e30f37bf58..c8a16eac3a 100644 --- a/acto/engine.py +++ b/acto/engine.py @@ -426,6 +426,7 @@ def run_trial( self.kubeconfig, self.context_name, wait_time=self.wait_time, + operator_container_name=self.deploy.operator_container_name ) checker: CheckerSet = self.checker_t( self.context, @@ -1070,7 +1071,7 @@ def __learn(self, context_file, helper_crd, analysis_only=False): break apiclient = kubernetes_client(learn_kubeconfig, learn_context_name) runner = Runner( - self.context, "learn", learn_kubeconfig, learn_context_name + self.context, "learn", learn_kubeconfig, learn_context_name, self.deploy.operator_container_name ) runner.run_without_collect( self.operator_config.seed_custom_resource @@ -1183,7 +1184,7 @@ def run( self.is_reproduce, self.apply_testcase_f, self.acto_namespace, - self.operator_config.diff_ignore_fields, + self.operator_config.diff_ignore_fields ) runners.append(runner) diff --git a/acto/lib/operator_config.py b/acto/lib/operator_config.py index b4f38f3724..2ac0511426 100644 --- a/acto/lib/operator_config.py +++ b/acto/lib/operator_config.py @@ -12,6 +12,9 @@ class ApplyStep(BaseModel, extra="forbid"): operator: bool = Field( description="If the file contains the operator deployment", default=False) + operator_container_name: Optional[str] = Field( + description="The container name of the operator in the operator pod", + default=None) namespace: Optional[str] = Field( description="Namespace for applying the file. If not specified, " + "use the namespace in the file or Acto namespace. " + diff --git a/acto/post_process/post_diff_test.py b/acto/post_process/post_diff_test.py index 5d9f2d4a1e..90d4afb3c3 100644 --- a/acto/post_process/post_diff_test.py +++ b/acto/post_process/post_diff_test.py @@ -325,7 +325,7 @@ def run_cr(self, cr, trial, gen): add_acto_label(apiclient, self._context['namespace']) trial_dir = os.path.join(self._workdir, 'trial-%02d' % self._worker_id) os.makedirs(trial_dir, exist_ok=True) - runner = Runner(self._context, trial_dir, self._kubeconfig, self._context_name) + runner = Runner(self._context, trial_dir, self._kubeconfig, self._context_name, operator_container_name=self._deploy.operator_container_name) snapshot, err = runner.run(cr, generation=self._generation) difftest_result = { 'input_digest': hashlib.md5(json.dumps(cr, sort_keys=True).encode("utf-8")).hexdigest(), @@ -378,7 +378,7 @@ def run(self): trial_dir = os.path.join(self._workdir, 'trial-%02d' % self._worker_id) os.makedirs(trial_dir, exist_ok=True) - runner = Runner(self._context, trial_dir, self._kubeconfig, self._context_name) + runner = Runner(self._context, trial_dir, self._kubeconfig, self._context_name, operator_container_name=self._deploy.operator_container_name) while True: after_k8s_bootstrap_time = time.time() try: @@ -418,7 +418,7 @@ def run(self): kubectl_client=kubectl_client, namespace=self._context['namespace']) after_operator_deploy_time = time.time() - runner = Runner(self._context, trial_dir, self._kubeconfig, self._context_name) + runner = Runner(self._context, trial_dir, self._kubeconfig, self._context_name, operator_container_name=self._deploy.operator_container_name) generation += 1 diff --git a/acto/runner/runner.py b/acto/runner/runner.py index 1c94ef4c17..05d6c7da19 100644 --- a/acto/runner/runner.py +++ b/acto/runner/runner.py @@ -36,12 +36,14 @@ def __init__( context_name: str, custom_system_state_f: Optional[Callable[..., dict]] = None, wait_time: int = 45, + operator_container_name: str = None ): self.namespace = context["namespace"] self.crd_metainfo: dict = context["crd"] self.trial_dir = trial_dir self.kubeconfig = kubeconfig self.context_name = context_name + self.operator_container_name = operator_container_name self.wait_time = wait_time self.log_length = 0 @@ -311,10 +313,17 @@ def collect_operator_log(self) -> list: ) else: logger.error("Failed to find operator pod") - - log = self.core_v1_api.read_namespaced_pod_log( - name=operator_pod_list[0].metadata.name, namespace=self.namespace - ) + if self.operator_container_name != None: + log = self.core_v1_api.read_namespaced_pod_log( + name=operator_pod_list[0].metadata.name, + namespace=self.namespace, + container=self.operator_container_name + ) + else: + log = self.core_v1_api.read_namespaced_pod_log( + name=operator_pod_list[0].metadata.name, + namespace=self.namespace, + ) # only get the new log since previous result new_log = log.split("\n") diff --git a/data/clickhouse-operator/config.json b/data/clickhouse-operator/config.json new file mode 100644 index 0000000000..36a5caf166 --- /dev/null +++ b/data/clickhouse-operator/config.json @@ -0,0 +1,21 @@ +{ + "deploy": { + "steps": [ + { + "apply": { + "file": "data/clickhouse-operator/zookeeper.yaml", + "namespace": "zoo3ns" + } + }, + { + "apply": { + "file": "data/clickhouse-operator/operator.yaml", + "operator_container_name": "clickhouse-operator", + "operator": true + } + } + ] + }, + "crd_name": "clickhouseinstallations.clickhouse.altinity.com", + "seed_custom_resource": "data/clickhouse-operator/cr.yaml" +} \ No newline at end of file diff --git a/data/clickhouse-operator/context.json b/data/clickhouse-operator/context.json new file mode 100644 index 0000000000..e8ae3ddfb4 --- /dev/null +++ b/data/clickhouse-operator/context.json @@ -0,0 +1,1974 @@ +{ + "crd": { + "body": { + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": { + "creationTimestamp": "2024-01-07T16:33:27Z", + "generation": 1, + "labels": { + "clickhouse.altinity.com/chop": "0.22.2" + }, + "name": "clickhouseinstallations.clickhouse.altinity.com", + "resourceVersion": "1140", + "uid": "acc03d8f-dcd8-400c-9bb8-4d5d75f00d61" + }, + "spec": { + "conversion": { + "strategy": "None" + }, + "group": "clickhouse.altinity.com", + "names": { + "kind": "ClickHouseInstallation", + "listKind": "ClickHouseInstallationList", + "plural": "clickhouseinstallations", + "shortNames": [ + "chi" + ], + "singular": "clickhouseinstallation" + }, + "scope": "Namespaced", + "versions": [ + { + "additionalPrinterColumns": [ + { + "description": "Operator version", + "jsonPath": ".status.chop-version", + "name": "version", + "priority": 1, + "type": "string" + }, + { + "description": "Clusters count", + "jsonPath": ".status.clusters", + "name": "clusters", + "type": "integer" + }, + { + "description": "Shards count", + "jsonPath": ".status.shards", + "name": "shards", + "priority": 1, + "type": "integer" + }, + { + "description": "Hosts count", + "jsonPath": ".status.hosts", + "name": "hosts", + "type": "integer" + }, + { + "description": "TaskID", + "jsonPath": ".status.taskID", + "name": "taskID", + "priority": 1, + "type": "string" + }, + { + "description": "CHI status", + "jsonPath": ".status.status", + "name": "status", + "type": "string" + }, + { + "description": "Updated hosts count", + "jsonPath": ".status.hostsUpdated", + "name": "hosts-updated", + "priority": 1, + "type": "integer" + }, + { + "description": "Added hosts count", + "jsonPath": ".status.hostsAdded", + "name": "hosts-added", + "priority": 1, + "type": "integer" + }, + { + "description": "Completed hosts count", + "jsonPath": ".status.hostsCompleted", + "name": "hosts-completed", + "type": "integer" + }, + { + "description": "Hosts deleted count", + "jsonPath": ".status.hostsDeleted", + "name": "hosts-deleted", + "priority": 1, + "type": "integer" + }, + { + "description": "Hosts to be deleted count", + "jsonPath": ".status.hostsDelete", + "name": "hosts-delete", + "priority": 1, + "type": "integer" + }, + { + "description": "Client access endpoint", + "jsonPath": ".status.endpoint", + "name": "endpoint", + "priority": 1, + "type": "string" + }, + { + "description": "Age of the resource", + "jsonPath": ".metadata.creationTimestamp", + "name": "age", + "type": "date" + } + ], + "name": "v1", + "schema": { + "openAPIV3Schema": { + "description": "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "description": "Specification of the desired behavior of one or more ClickHouse clusters\nMore info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md\n", + "properties": { + "configuration": { + "description": "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource", + "properties": { + "clusters": { + "description": "describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level\nevery cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server`\nall Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml`\nClusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/\nIf `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables\n", + "items": { + "properties": { + "files": { + "description": "optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\noverride top-level `chi.spec.configuration.files`\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "insecure": { + "description": "optional, open insecure ports for cluster, defaults to \"yes\"", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "layout": { + "description": "describe current cluster layout, how much shards in cluster, how much replica in shard\nallows override settings on each shard and replica separatelly\n", + "properties": { + "replicas": { + "description": "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do", + "items": { + "properties": { + "files": { + "description": "optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\noverride top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "name": { + "description": "optional, by default replica name is generated, but you can override it and setup custom name", + "maxLength": 15, + "minLength": 1, + "pattern": "^[a-zA-Z0-9-]{0,15}$", + "type": "string" + }, + "settings": { + "description": "optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`\noverride top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "shards": { + "description": "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents", + "items": { + "properties": { + "files": { + "description": "optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\noverride top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "httpPort": { + "description": "optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort`\nallows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service`\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "httpsPort": { + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "insecure": { + "description": "optional, open insecure ports for cluster, defaults to \"yes\"\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "interserverHTTPPort": { + "description": "optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort`\nallows connect between replicas inside same shard during fetch replicated data parts HTTP protocol\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "name": { + "description": "optional, by default shard name is generated, but you can override it and setup custom name", + "maxLength": 15, + "minLength": 1, + "pattern": "^[a-zA-Z0-9-]{0,15}$", + "type": "string" + }, + "secure": { + "description": "optional, open secure ports\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "settings": { + "description": "optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`\noverride top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings`\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "tcpPort": { + "description": "optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort`\nallows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service`\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "templates": { + "description": "optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica\noverride top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates`\n", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + }, + "tlsPort": { + "maximum": 65535, + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "shardsCount": { + "description": "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`", + "minimum": 1, + "type": "integer" + }, + "templates": { + "description": "optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica\noverride top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`\n", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "replicasCount": { + "description": "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default", + "type": "integer" + }, + "shards": { + "description": "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do", + "items": { + "properties": { + "definitionType": { + "description": "DEPRECATED - to be removed soon", + "type": "string" + }, + "files": { + "description": "optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\noverride top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "internalReplication": { + "description": "optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise\nallows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication,\nwill apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml\nMore details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "name": { + "description": "optional, by default shard name is generated, but you can override it and setup custom name", + "maxLength": 15, + "minLength": 1, + "pattern": "^[a-zA-Z0-9-]{0,15}$", + "type": "string" + }, + "replicas": { + "description": "optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards`\n", + "items": { + "properties": { + "files": { + "description": "optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\noverride top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files`\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "httpPort": { + "description": "optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort`\nallows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service`\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "httpsPort": { + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "insecure": { + "description": "optional, open insecure ports for cluster, defaults to \"yes\"\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "interserverHTTPPort": { + "description": "optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort`\nallows connect between replicas inside same shard during fetch replicated data parts HTTP protocol\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "name": { + "description": "optional, by default replica name is generated, but you can override it and setup custom name", + "maxLength": 15, + "minLength": 1, + "pattern": "^[a-zA-Z0-9-]{0,15}$", + "type": "string" + }, + "secure": { + "description": "optional, open secure ports\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "settings": { + "description": "optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`\noverride top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings`\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "tcpPort": { + "description": "optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort`\nallows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service`\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "templates": { + "description": "optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica\noverride top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates`\n", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + }, + "tlsPort": { + "maximum": 65535, + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "replicasCount": { + "description": "optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance,\nshard contains 1 replica by default\noverride cluster-level `chi.spec.configuration.clusters.layout.replicasCount`\n", + "minimum": 1, + "type": "integer" + }, + "settings": { + "description": "optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/`\noverride top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings`\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "templates": { + "description": "optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard\noverride top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates`\n", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + }, + "weight": { + "description": "optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine,\nwill apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml\nMore details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/\n", + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "shardsCount": { + "description": "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default", + "type": "integer" + }, + "type": { + "description": "DEPRECATED - to be removed soon", + "type": "string" + } + }, + "type": "object" + }, + "name": { + "description": "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources", + "maxLength": 15, + "minLength": 1, + "pattern": "^[a-zA-Z0-9-]{0,15}$", + "type": "string" + }, + "schemaPolicy": { + "description": "describes how schema is propagated within replicas and shards\n", + "properties": { + "replica": { + "description": "how schema is propagated within a replica", + "enum": [ + "", + "None", + "All" + ], + "type": "string" + }, + "shard": { + "description": "how schema is propagated between shards", + "enum": [ + "", + "None", + "All", + "DistributedTablesOnly" + ], + "type": "string" + } + }, + "type": "object" + }, + "secret": { + "description": "optional, shared secret value to secure cluster communications", + "properties": { + "auto": { + "description": "Auto-generate shared secret value to secure cluster communications", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "value": { + "description": "Cluster shared secret value in plain text", + "type": "string" + }, + "valueFrom": { + "description": "Cluster shared secret source", + "properties": { + "secretKeyRef": { + "description": "Selects a key of a secret in the clickhouse installation namespace.\nShould not be used if value is not empty.\n", + "properties": { + "key": { + "description": "The key of the secret to select from. Must be a valid secret key.", + "type": "string" + }, + "name": { + "description": "Name of the referent. More info:\nhttps://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names\n", + "type": "string" + }, + "optional": { + "description": "Specify whether the Secret or its key must be defined", + "type": "boolean" + } + }, + "required": [ + "name", + "key" + ], + "type": "object" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "secure": { + "description": "optional, open secure ports for cluster", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "settings": { + "description": "optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/`\noverride top-level `chi.spec.configuration.settings`\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "templates": { + "description": "optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster\noverride top-level `chi.spec.configuration.templates`\n", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + }, + "zookeeper": { + "description": "optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/`\noverride top-level `chi.spec.configuration.zookeeper` settings\n", + "properties": { + "identity": { + "description": "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper", + "type": "string" + }, + "nodes": { + "description": "describe every available zookeeper cluster node for interaction", + "items": { + "properties": { + "host": { + "description": "dns name or ip address for Zookeeper node", + "type": "string" + }, + "port": { + "description": "TCP port which used to connect to Zookeeper node", + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "secure": { + "description": "if a secure connection to Zookeeper is required", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "operation_timeout_ms": { + "description": "one operation timeout during Zookeeper transactions", + "type": "integer" + }, + "root": { + "description": "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)", + "type": "string" + }, + "session_timeout_ms": { + "description": "session timeout during connect to Zookeeper", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "files": { + "description": "allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\nevery key in this object is the file name\nevery value in this object is the file content\nyou can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html\neach key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored\nMore details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "profiles": { + "description": "allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/`\nyou can configure any aspect of settings profile\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/\nYour yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "quotas": { + "description": "allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/`\nyou can configure any aspect of resource quotas\nMore details: https://clickhouse.tech/docs/en/operations/quotas/\nYour yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "settings": { + "description": "allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/`\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\nYour yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "users": { + "description": "allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/`\nyou can configure password hashed, authorization restrictions, database level security row filters etc.\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings-users/\nYour yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "zookeeper": { + "description": "allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/`\n`clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/\ncurrently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl`\nMore details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper\n", + "properties": { + "identity": { + "description": "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper", + "type": "string" + }, + "nodes": { + "description": "describe every available zookeeper cluster node for interaction", + "items": { + "properties": { + "host": { + "description": "dns name or ip address for Zookeeper node", + "type": "string" + }, + "port": { + "description": "TCP port which used to connect to Zookeeper node", + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "secure": { + "description": "if a secure connection to Zookeeper is required", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "operation_timeout_ms": { + "description": "one operation timeout during Zookeeper transactions", + "type": "integer" + }, + "root": { + "description": "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)", + "type": "string" + }, + "session_timeout_ms": { + "description": "session timeout during connect to Zookeeper", + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "defaults": { + "description": "define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level\nMore info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults\n", + "properties": { + "distributedDDL": { + "description": "allows change `` settings\nMore info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl\n", + "properties": { + "profile": { + "description": "Settings from this profile will be used to execute DDL queries", + "type": "string" + } + }, + "type": "object" + }, + "replicasUseFQDN": { + "description": "define should replicas be specified by FQDN in ``.\nIn case of \"no\" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup\n\"yes\" by default\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "storageManagement": { + "description": "default storage management options", + "properties": { + "provisioner": { + "description": "defines `PVC` provisioner - be it StatefulSet or the Operator", + "enum": [ + "", + "StatefulSet", + "Operator" + ], + "type": "string" + }, + "reclaimPolicy": { + "description": "defines behavior of `PVC` deletion.\n`Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet\n", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" + }, + "templates": { + "description": "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "namespaceDomainPattern": { + "description": "Custom domain pattern which will be used for DNS names of `Service` or `Pod`.\nTypical use scenario - custom cluster domain in Kubernetes cluster\nExample: %s.svc.my.test\n", + "type": "string" + }, + "reconciling": { + "description": "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side", + "properties": { + "cleanup": { + "description": "optional, define behavior for cleanup Kubernetes resources during reconcile cycle", + "properties": { + "reconcileFailedObjects": { + "description": "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`", + "properties": { + "configMap": { + "description": "behavior policy for failed ConfigMap reconciling, Retain by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "pvc": { + "description": "behavior policy for failed PVC reconciling, Retain by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "service": { + "description": "behavior policy for failed Service reconciling, Retain by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "statefulSet": { + "description": "behavior policy for failed StatefulSet reconciling, Retain by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" + }, + "unknownObjects": { + "description": "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`", + "properties": { + "configMap": { + "description": "behavior policy for unknown ConfigMap, Delete by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "pvc": { + "description": "behavior policy for unknown PVC, Delete by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "service": { + "description": "behavior policy for unknown Service, Delete by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "statefulSet": { + "description": "behavior policy for unknown StatefulSet, Delete by default", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "configMapPropagationTimeout": { + "description": "Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod`\nMore details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically\n", + "maximum": 3600, + "minimum": 0, + "type": "integer" + }, + "policy": { + "description": "DEPRECATED", + "type": "string" + } + }, + "type": "object" + }, + "restart": { + "description": "In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile.\nThis options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts.\n", + "enum": [ + "", + "RollingUpdate" + ], + "type": "string" + }, + "stop": { + "description": "Allows to stop all ClickHouse clusters defined in a CHI.\nWorks as the following:\n - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact.\n - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s.\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "taskID": { + "description": "Allows to define custom taskID for CHI update and watch status of this update execution.\nDisplayed in all .status.taskID* fields.\nBy default (if not filled) every update of CHI manifest will generate random taskID\n", + "type": "string" + }, + "templates": { + "description": "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it", + "properties": { + "hostTemplates": { + "description": "hostTemplate will use during apply to generate `clickhose-server` config files", + "items": { + "properties": { + "name": { + "description": "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`", + "type": "string" + }, + "portDistribution": { + "description": "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs", + "items": { + "properties": { + "type": { + "description": "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network", + "enum": [ + "", + "Unspecified", + "ClusterScopeIndex" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "spec": { + "properties": { + "files": { + "description": "optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/`\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "httpPort": { + "description": "optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply\nif specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]`\nMore info: https://clickhouse.tech/docs/en/interfaces/http/\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "httpsPort": { + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "insecure": { + "description": "optional, open insecure ports for cluster, defaults to \"yes\"\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "interserverHTTPPort": { + "description": "optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply\nif specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]`\nMore info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "name": { + "description": "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`", + "maxLength": 15, + "minLength": 1, + "pattern": "^[a-zA-Z0-9-]{0,15}$", + "type": "string" + }, + "secure": { + "description": "optional, open secure ports\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "settings": { + "description": "optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/`\nMore details: https://clickhouse.tech/docs/en/operations/settings/settings/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "tcpPort": { + "description": "optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply\nif specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]`\nMore info: https://clickhouse.tech/docs/en/interfaces/tcp/\n", + "maximum": 65535, + "minimum": 1, + "type": "integer" + }, + "templates": { + "description": "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do", + "properties": { + "clusterServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "dataVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "hostTemplate": { + "description": "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`", + "type": "string" + }, + "logVolumeClaimTemplate": { + "description": "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "podTemplate": { + "description": "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "replicaServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "serviceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource", + "type": "string" + }, + "shardServiceTemplate": { + "description": "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`", + "type": "string" + }, + "volumeClaimTemplate": { + "description": "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate", + "type": "string" + } + }, + "type": "object" + }, + "tlsPort": { + "maximum": 65535, + "minimum": 1, + "type": "integer" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "podTemplates": { + "description": "podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone\nMore information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates\n", + "items": { + "properties": { + "distribution": { + "description": "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`", + "enum": [ + "", + "Unspecified", + "OnePerHost" + ], + "type": "string" + }, + "generateName": { + "description": "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables", + "type": "string" + }, + "metadata": { + "description": "allows pass standard object's metadata from template to Pod\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "name": { + "description": "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`", + "type": "string" + }, + "podDistribution": { + "description": "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster", + "items": { + "properties": { + "number": { + "description": "define, how much ClickHouse Pods could be inside selected scope with selected distribution type", + "maximum": 65535, + "minimum": 0, + "type": "integer" + }, + "scope": { + "description": "scope for apply each podDistribution", + "enum": [ + "", + "Unspecified", + "Shard", + "Replica", + "Cluster", + "ClickHouseInstallation", + "Namespace" + ], + "type": "string" + }, + "topologyKey": { + "description": "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity", + "type": "string" + }, + "type": { + "description": "you can define multiple affinity policy types", + "enum": [ + "", + "Unspecified", + "ClickHouseAntiAffinity", + "ShardAntiAffinity", + "ReplicaAntiAffinity", + "AnotherNamespaceAntiAffinity", + "AnotherClickHouseInstallationAntiAffinity", + "AnotherClusterAntiAffinity", + "MaxNumberPerNode", + "NamespaceAffinity", + "ClickHouseInstallationAffinity", + "ClusterAffinity", + "ShardAffinity", + "ReplicaAffinity", + "PreviousTailAffinity", + "CircularReplication" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "spec": { + "description": "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "zone": { + "description": "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`", + "properties": { + "key": { + "description": "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`", + "type": "string" + }, + "values": { + "description": "optional, if defined, allows select kubernetes nodes by label with `value` in `values`", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "serviceTemplates": { + "description": "allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level\n", + "items": { + "properties": { + "generateName": { + "description": "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables", + "type": "string" + }, + "metadata": { + "description": "allows pass standard object's metadata from template to Service\nCould be use for define specificly for Cloud Provider metadata which impact to behavior of service\nMore info: https://kubernetes.io/docs/concepts/services-networking/service/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "name": { + "description": "template name, could use to link inside\nchi-level `chi.spec.defaults.templates.serviceTemplate`\ncluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate`\nshard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate`\nreplica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate`\n", + "type": "string" + }, + "spec": { + "description": "describe behavior of generated Service\nMore info: https://kubernetes.io/docs/concepts/services-networking/service/\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + } + }, + "type": "object" + }, + "type": "array" + }, + "volumeClaimTemplates": { + "description": "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else", + "items": { + "properties": { + "metadata": { + "description": "allows to pass standard object's metadata from template to PVC\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "name": { + "description": "template name, could use to link inside\ntop-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`,\ncluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`,\nshard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate`\nreplica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate`\n", + "type": "string" + }, + "provisioner": { + "description": "defines `PVC` provisioner - be it StatefulSet or the Operator", + "enum": [ + "", + "StatefulSet", + "Operator" + ], + "type": "string" + }, + "reclaimPolicy": { + "description": "defines behavior of `PVC` deletion.\n`Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet\n", + "enum": [ + "", + "Retain", + "Delete" + ], + "type": "string" + }, + "spec": { + "description": "allows define all aspects of `PVC` resource\nMore info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims\n", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "templating": { + "description": "Optional, defines policy for applying current ClickHouseInstallationTemplate to ClickHouseInstallation(s)", + "properties": { + "chiSelector": { + "description": "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "policy": { + "description": "When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate\nwill be auto-added into ClickHouseInstallation, selectable by `chiSelector`.\nDefault value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly.\n", + "enum": [ + "", + "auto", + "manual" + ], + "type": "string" + } + }, + "type": "object" + }, + "troubleshoot": { + "description": "Allows to troubleshoot Pods during CrashLoopBack state.\nThis may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start.\nCommand within ClickHouse container is modified with `sleep` in order to avoid quick restarts\nand give time to troubleshoot via CLI.\nLiveness and Readiness probes are disabled as well.\n", + "enum": [ + "", + "0", + "1", + "False", + "false", + "True", + "true", + "No", + "no", + "Yes", + "yes", + "Off", + "off", + "On", + "on", + "Disable", + "disable", + "Enable", + "enable", + "Disabled", + "disabled", + "Enabled", + "enabled" + ], + "type": "string" + }, + "useTemplates": { + "description": "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters", + "items": { + "properties": { + "name": { + "description": "name of `ClickHouseInstallationTemplate` (chit) resource", + "type": "string" + }, + "namespace": { + "description": "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`", + "type": "string" + }, + "useType": { + "description": "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`", + "enum": [ + "", + "merge" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "status": { + "description": "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other", + "properties": { + "action": { + "description": "Action", + "type": "string" + }, + "actions": { + "description": "Actions", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "chop-commit": { + "description": "ClickHouse operator git commit SHA", + "type": "string" + }, + "chop-date": { + "description": "ClickHouse operator build date", + "type": "string" + }, + "chop-ip": { + "description": "IP address of the operator's pod which managed this CHI", + "type": "string" + }, + "chop-version": { + "description": "ClickHouse operator version", + "type": "string" + }, + "clusters": { + "description": "Clusters count", + "minimum": 0, + "type": "integer" + }, + "endpoint": { + "description": "Endpoint", + "type": "string" + }, + "error": { + "description": "Last error", + "type": "string" + }, + "errors": { + "description": "Errors", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "fqdns": { + "description": "Pods FQDNs", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "generation": { + "description": "Generation", + "minimum": 0, + "type": "integer" + }, + "hosts": { + "description": "Hosts count", + "minimum": 0, + "type": "integer" + }, + "hostsAdded": { + "description": "Added Hosts count", + "minimum": 0, + "type": "integer" + }, + "hostsCompleted": { + "description": "Completed Hosts count", + "minimum": 0, + "type": "integer" + }, + "hostsDelete": { + "description": "About to delete Hosts count", + "minimum": 0, + "type": "integer" + }, + "hostsDeleted": { + "description": "Deleted Hosts count", + "minimum": 0, + "type": "integer" + }, + "hostsUpdated": { + "description": "Updated Hosts count", + "minimum": 0, + "type": "integer" + }, + "hostsWithTablesCreated": { + "description": "List of hosts with tables created by the operator", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "normalized": { + "description": "Normalized CHI requested", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "normalizedCompleted": { + "description": "Normalized CHI completed", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "pod-ips": { + "description": "Pod IPs", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "pods": { + "description": "Pods", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "replicas": { + "description": "Replicas count", + "minimum": 0, + "type": "integer" + }, + "shards": { + "description": "Shards count", + "minimum": 0, + "type": "integer" + }, + "status": { + "description": "Status", + "type": "string" + }, + "taskID": { + "description": "Current task id", + "type": "string" + }, + "taskIDsCompleted": { + "description": "Completed task ids", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "taskIDsStarted": { + "description": "Started task ids", + "items": { + "type": "string" + }, + "nullable": true, + "type": "array" + }, + "usedTemplates": { + "description": "List of templates used to build this CHI", + "items": { + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + }, + "nullable": true, + "type": "array", + "x-kubernetes-preserve-unknown-fields": true + } + }, + "type": "object" + } + }, + "required": [ + "spec" + ], + "type": "object" + } + }, + "served": true, + "storage": true, + "subresources": { + "status": {} + } + } + ] + }, + "status": { + "acceptedNames": { + "kind": "ClickHouseInstallation", + "listKind": "ClickHouseInstallationList", + "plural": "clickhouseinstallations", + "shortNames": [ + "chi" + ], + "singular": "clickhouseinstallation" + }, + "conditions": [ + { + "lastTransitionTime": "2024-01-07T16:33:27Z", + "message": "no conflicts found", + "reason": "NoConflicts", + "status": "True", + "type": "NamesAccepted" + }, + { + "lastTransitionTime": "2024-01-07T16:33:27Z", + "message": "the initial names have been accepted", + "reason": "InitialNamesAccepted", + "status": "True", + "type": "Established" + } + ], + "storedVersions": [ + "v1" + ] + } + }, + "group": "clickhouse.altinity.com", + "plural": "clickhouseinstallations", + "version": "v1" + }, + "learnrun_time": 461.00436758995056, + "namespace": "acto-clickhouse", + "preload_images": [ + "docker.io/altinity/clickhouse-operator:0.22.2", + "docker.io/clickhouse/clickhouse-server:22.3", + "docker.io/library/zookeeper:3.8.1", + "docker.io/altinity/metrics-exporter:0.22.2" + ], + "static_analysis_time": 2.7418136596679688e-05 +} \ No newline at end of file diff --git a/data/clickhouse-operator/cr.yaml b/data/clickhouse-operator/cr.yaml new file mode 100644 index 0000000000..10213ce642 --- /dev/null +++ b/data/clickhouse-operator/cr.yaml @@ -0,0 +1,37 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" + +metadata: + name: "repl-05" + +spec: + defaults: + templates: + dataVolumeClaimTemplate: default + podTemplate: clickhouse:19.6 + + configuration: + zookeeper: + nodes: + - host: zookeeper.zoo3.ns + clusters: + - name: replicated + layout: + shardsCount: 2 + replicasCount: 2 + + templates: + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Mi + podTemplates: + - name: clickhouse:19.6 + spec: + containers: + - name: clickhouse-pod + image: clickhouse/clickhouse-server:22.3 \ No newline at end of file diff --git a/data/clickhouse-operator/operator.yaml b/data/clickhouse-operator/operator.yaml new file mode 100644 index 0000000000..b2bad09194 --- /dev/null +++ b/data/clickhouse-operator/operator.yaml @@ -0,0 +1,3878 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallation +# SINGULAR=clickhouseinstallation +# PLURAL=clickhouseinstallations +# SHORT=chi +# OPERATOR_VERSION=0.22.2 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.22.2 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallation + singular: clickhouseinstallation + plural: clickhouseinstallations + shortNames: + - chi + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + jsonPath: .status.status + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHI completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: "Optional, defines policy for applying current ClickHouseInstallationTemplate to ClickHouseInstallation(s)" + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: DEPRECATED + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "optional, define behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "behavior policy for unknown StatefulSet, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + <<: *TypeObjectsCleanup + description: "behavior policy for unknown PVC, Delete by default" + configMap: + <<: *TypeObjectsCleanup + description: "behavior policy for unknown ConfigMap, Delete by default" + service: + <<: *TypeObjectsCleanup + description: "behavior policy for unknown Service, Delete by default" + reconcileFailedObjects: + type: object + description: "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + <<: *TypeObjectsCleanup + description: "behavior policy for failed StatefulSet reconciling, Retain by default" + pvc: + <<: *TypeObjectsCleanup + description: "behavior policy for failed PVC reconciling, Retain by default" + configMap: + <<: *TypeObjectsCleanup + description: "behavior policy for failed ConfigMap reconciling, Retain by default" + service: + <<: *TypeObjectsCleanup + description: "behavior policy for failed Service reconciling, Retain by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" +--- +# Template Parameters: +# +# KIND=ClickHouseInstallationTemplate +# SINGULAR=clickhouseinstallationtemplate +# PLURAL=clickhouseinstallationtemplates +# SHORT=chit +# OPERATOR_VERSION=0.22.2 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallationtemplates.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.22.2 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallationTemplate + singular: clickhouseinstallationtemplate + plural: clickhouseinstallationtemplates + shortNames: + - chit + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + jsonPath: .status.status + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHI completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: "Optional, defines policy for applying current ClickHouseInstallationTemplate to ClickHouseInstallation(s)" + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: DEPRECATED + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "optional, define behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: "what clickhouse-operator shall do when found Kubernetes resources which should be managed with clickhouse-operator, but not have `ownerReference` to any currently managed `ClickHouseInstallation` resource, default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "behavior policy for unknown StatefulSet, Delete by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + <<: *TypeObjectsCleanup + description: "behavior policy for unknown PVC, Delete by default" + configMap: + <<: *TypeObjectsCleanup + description: "behavior policy for unknown ConfigMap, Delete by default" + service: + <<: *TypeObjectsCleanup + description: "behavior policy for unknown Service, Delete by default" + reconcileFailedObjects: + type: object + description: "what clickhouse-operator shall do when reconciling Kubernetes resources are failed, default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + <<: *TypeObjectsCleanup + description: "behavior policy for failed StatefulSet reconciling, Retain by default" + pvc: + <<: *TypeObjectsCleanup + description: "behavior policy for failed PVC reconciling, Retain by default" + configMap: + <<: *TypeObjectsCleanup + description: "behavior policy for failed ConfigMap reconciling, Retain by default" + service: + <<: *TypeObjectsCleanup + description: "behavior policy for failed Service reconciling, Retain by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" +--- +# Template Parameters: +# +# NONE +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.22.2 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseOperatorConfiguration + singular: clickhouseoperatorconfiguration + plural: clickhouseoperatorconfigurations + shortNames: + - chopconf + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: namespaces + type: string + description: Watch namespaces + jsonPath: .status + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" + x-kubernetes-preserve-unknown-fields: true + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + Allows to define settings of the clickhouse-operator. + More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml + Check into etc-clickhouse-operator* ConfigMaps if you need more control + x-kubernetes-preserve-unknown-fields: true + properties: + watch: + type: object + description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" + properties: + namespaces: + type: array + description: "List of namespaces where clickhouse-operator watches for events." + items: + type: string + clickhouse: + type: object + description: "Clickhouse related parameters used by clickhouse-operator" + properties: + configuration: + type: object + properties: + file: + type: object + properties: + path: + type: object + properties: + common: + type: string + description: "Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. Default - config.d" + host: + type: string + description: "Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. Default - conf.d" + user: + type: string + description: "Path to the folder where ClickHouse configuration files with users settings are located. Files are common for all instances within a CHI. Default - users.d" + user: + type: object + description: "Default parameters for any user which will create" + properties: + default: + type: object + properties: + profile: + type: string + description: "ClickHouse server configuration `...` for any " + quota: + type: string + description: "ClickHouse server configuration `...` for any " + networksIP: + type: array + description: "ClickHouse server configuration `...` for any " + items: + type: string + password: + type: string + description: "ClickHouse server configuration `...` for any " + network: + type: object + description: "Default network parameters for any user which will create" + properties: + hostRegexpTemplate: + type: string + description: "ClickHouse server configuration `...` for any " + configurationRestartPolicy: + type: object + description: "Configuration restart policy describes what configuration changes require ClickHouse restart" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + rules: + type: array + description: "Set of configuration rules for specified ClickHouse version" + items: + type: object + description: "setting: value pairs for configuration restart policy" + access: + type: object + description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" + properties: + scheme: + type: string + description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" + username: + type: string + description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + password: + type: string + description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + rootCA: + type: string + description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + secret: + type: object + properties: + namespace: + type: string + description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + name: + type: string + description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + port: + type: integer + minimum: 1 + maximum: 65535 + description: "Port to be used by operator to connect to ClickHouse instances" + timeouts: + type: object + description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" + properties: + connect: + type: integer + minimum: 1 + maximum: 10 + description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." + query: + type: integer + minimum: 1 + maximum: 600 + description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." + metrics: + type: object + description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" + properties: + timeouts: + type: object + description: | + Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances + Specified in seconds. + properties: + collect: + type: integer + minimum: 1 + maximum: 600 + description: | + Timeout used to limit metrics collection request. In seconds. + Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. + All collected metrics are returned. + template: + type: object + description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" + properties: + chi: + type: object + properties: + policy: + type: string + description: | + CHI template updates handling policy + Possible policy values: + - ReadOnStart. Accept CHIT updates on the operators start only. + - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + enum: + - "" + - "ReadOnStart" + - "ApplyOnNextReconcile" + path: + type: string + description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileCHIsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: + type: object + description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. delete - delete newly created problematic StatefulSet. + 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for created/updated StatefulSet to be Ready" + pollInterval: + type: integer + description: "How many seconds to wait between checks for created/updated StatefulSet status" + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. + 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. + host: + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + + properties: + wait: + type: object + properties: + exclude: &TypeStringBool + type: string + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + queries: + <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + annotation: + type: object + description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + include annotations with names from the following list + items: + type: string + exclude: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + exclude annotations with names from the following list + items: + type: string + label: + type: object + description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + include labels from the following list + items: + type: string + exclude: + type: array + items: + type: string + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + exclude labels from the following list + appendScope: + <<: *TypeStringBool + description: | + Whether to append *Scope* labels to StatefulSet and Pod + - "LabelShardScopeIndex" + - "LabelReplicaScopeIndex" + - "LabelCHIScopeIndex" + - "LabelCHIScopeCycleSize" + - "LabelCHIScopeCycleIndex" + - "LabelCHIScopeCycleOffset" + - "LabelClusterScopeIndex" + - "LabelClusterScopeCycleSize" + - "LabelClusterScopeCycleIndex" + - "LabelClusterScopeCycleOffset" + statefulSet: + type: object + description: "define StatefulSet-specific parameters" + properties: + revisionHistoryLimit: + type: integer + description: | + revisionHistoryLimit is the maximum number of revisions that will be + maintained in the StatefulSet's revision history. + Look details in `statefulset.spec.revisionHistoryLimit` + pod: + type: object + description: "define pod specific parameters" + properties: + terminationGracePeriod: + type: integer + description: | + Optional duration in seconds the pod needs to terminate gracefully. + Look details in `pod.spec.terminationGracePeriodSeconds` + logger: + type: object + description: "allow setup clickhouse-operator logger behavior" + properties: + logtostderr: + type: string + description: "boolean, allows logs to stderr" + alsologtostderr: + type: string + description: "boolean allows logs to stderr and files both" + v: + type: string + description: "verbosity level of clickhouse-operator log, default - 1 max - 9" + stderrthreshold: + type: string + vmodule: + type: string + description: | + Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. + Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. + log_backtrace_at: + type: string + description: | + It can be set to a file and line number with a logging line. + Ex.: file.go:123 + Each time when this line is being executed, a stack trace will be written to the Info log. +--- +# Template Parameters: +# +# COMMENT= +# NAMESPACE=kube-system +# NAME=clickhouse-operator +# +# Setup ServiceAccount +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clickhouse-operator + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 +--- +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT=# +# ROLE_KIND=ClusterRole +# ROLE_NAME=clickhouse-operator-kube-system +# ROLE_BINDING_KIND=ClusterRoleBinding +# ROLE_BINDING_NAME=clickhouse-operator-kube-system +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clickhouse-operator-kube-system + #namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 +rules: +- apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete +- apiGroups: + - apps + resourceNames: + - clickhouse-operator + resources: + - deployments + verbs: + - get + - patch + - update + - delete +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations + verbs: + - get + - patch + - update + - delete +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update +- apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list +--- +# Setup ClusterRoleBinding between ClusterRole and ServiceAccount. +# ClusterRoleBinding is namespace-less and must have unique name +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clickhouse-operator-kube-system + #namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clickhouse-operator-kube-system +subjects: +- kind: ServiceAccount + name: clickhouse-operator + namespace: acto-clickhouse +--- +# Template Parameters: +# +# NAME=etc-clickhouse-operator-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: etc-clickhouse-operator-files + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +data: + config.yaml: | + # IMPORTANT + # This file is auto-generated + # Do not edit this file - all changes would be lost + # Edit appropriate template in the following folder: + # deploy/builder/templates-config + # IMPORTANT + # + # Template parameters available: + # WATCH_NAMESPACES= + # CH_USERNAME_PLAIN= + # CH_PASSWORD_PLAIN= + # CH_CREDENTIALS_SECRET_NAMESPACE= + # CH_CREDENTIALS_SECRET_NAME=clickhouse-operator + + ################################################ + ## + ## Watch section + ## + ################################################ + watch: + # List of namespaces where clickhouse-operator watches for events. + # Concurrently running operators should watch on different namespaces. + # IMPORTANT + # Regexp is applicable. + #namespaces: ["dev", "test"] + namespaces: [] + + clickhouse: + configuration: + ################################################ + ## + ## Configuration files section + ## + ################################################ + file: + path: + # Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. + common: config.d + # Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. + host: conf.d + # Path to the folder where ClickHouse configuration files with users' settings are located. + # Files are common for all instances within a CHI. + user: users.d + ################################################ + ## + ## Configuration users section + ## + ################################################ + user: + # Default settings for user accounts, created by the operator. + # IMPORTANT. These are not access credentials or settings for 'default' user account, + # it is a template for filling out missing fields for all user accounts to be created by the operator, + # with the following EXCEPTIONS: + # 1. 'default' user account DOES NOT use provided password, but uses all the rest of the fields. + # Password for 'default' user account has to be provided explicitly, if to be used. + # 2. CHOP user account DOES NOT use: + # - profile setting. It uses predefined profile called 'clickhouse_operator' + # - quota setting. It uses empty quota name. + # - networks IP setting. Operator specifies 'networks/ip' user setting to match operators' pod IP only. + # - password setting. Password for CHOP account is used from 'clickhouse.access.*' section + default: + # Default values for ClickHouse user account(s) created by the operator + # 1. user/profile - string + # 2. user/quota - string + # 3. user/networks/ip - multiple strings + # 4. user/password - string + # These values can be overwritten on per-user basis. + profile: "default" + quota: "default" + networksIP: + - "::1" + - "127.0.0.1" + password: "default" + ################################################ + ## + ## Configuration network section + ## + ################################################ + network: + # Default host_regexp to limit network connectivity from outside + hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" + + ################################################ + ## + ## Configuration restart policy section + ## Configuration restart policy describes what configuration changes require ClickHouse restart + ## + ################################################ + configurationRestartPolicy: + rules: + # IMPORTANT! + # Default version will also be used in case ClickHouse version is unknown. + # ClickHouse version may be unknown due to host being down - for example, because of incorrect "settings" section. + # ClickHouse is not willing to start in case incorrect/unknown settings are provided in config file. + - version: "*" + rules: + - settings/*: "yes" + - settings/dictionaries_config: "no" + - settings/logger: "no" + - settings/macros/*: "no" + - settings/max_server_memory_*: "no" + - settings/max_*_to_drop: "no" + - settings/max_concurrent_queries: "no" + - settings/models_config: "no" + - settings/user_defined_executable_functions_config: "no" + + - zookeeper/*: "yes" + + - files/*.xml: "yes" + - files/config.d/*.xml: "yes" + - files/config.d/*dict*.xml: "no" + + - profiles/default/background_*_pool_size: "yes" + - profiles/default/max_*_for_server: "yes" + - version: "21.*" + rules: + - settings/logger: "yes" + + ################################################# + ## + ## Access to ClickHouse instances + ## + ################################################ + access: + # Possible values for 'scheme' are: + # 1. http - force http to be used to connect to ClickHouse instances + # 2. https - force https to be used to connect to ClickHouse instances + # 3. auto - either http or https is selected based on open ports + scheme: "auto" + # ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances. + # These credentials are used for: + # 1. Metrics requests + # 2. Schema maintenance + # 3. DROP DNS CACHE + # User with these credentials can be specified in additional ClickHouse .xml config files, + # located in 'clickhouse.configuration.file.path.user' folder + username: "" + password: "" + rootCA: "" + + # Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances. + # Can be used instead of explicitly specified username and password available in sections: + # - clickhouse.access.username + # - clickhouse.access.password + # Secret should have two keys: + # 1. username + # 2. password + secret: + # Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running. + namespace: "" + # Empty `name` means no k8s Secret would be looked for + name: "clickhouse-operator" + # Port where to connect to ClickHouse instances to + port: 8123 + + # Timeouts used to limit connection and queries from the operator to ClickHouse instances + # Specified in seconds. + timeouts: + # Timout to setup connection from the operator to ClickHouse instances. In seconds. + connect: 1 + # Timout to perform SQL query from the operator to ClickHouse instances. In seconds. + query: 4 + + ################################################# + ## + ## Metrics collection + ## + ################################################ + + metrics: + # Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances + # Specified in seconds. + timeouts: + # Timeout used to limit metrics collection request. In seconds. + # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. + # All collected metrics are returned. + collect: 9 + + ################################################ + ## + ## Template(s) management section + ## + ################################################ + template: + chi: + # CHI template updates handling policy + # Possible policy values: + # - ReadOnStart. Accept CHIT updates on the operators start only. + # - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + policy: ApplyOnNextReconcile + + # Path to the folder where ClickHouseInstallation templates .yaml manifests are located. + # Templates are added to the list of all templates and used when CHI is reconciled. + # Templates are applied in sorted alpha-numeric order. + path: templates.d + + ################################################ + ## + ## Reconcile section + ## + ################################################ + reconcile: + # Reconcile runtime settings + runtime: + # Max number of concurrent CHI reconciles in progress + reconcileCHIsThreadsNumber: 10 + + # The operator reconciles shards concurrently in each CHI with the following limitations: + # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently + # can not be greater than 'reconcileShardsThreadsNumber'. + # 2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently + # can not be greater than 'reconcileShardsMaxConcurrencyPercent'. + # 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward. + # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage + + # Max number of concurrent shard reconciles within one CHI in progress + reconcileShardsThreadsNumber: 5 + # Max percentage of concurrent shard reconciles within one CHI in progress + reconcileShardsMaxConcurrencyPercent: 50 + + # Reconcile StatefulSet scenario + statefulSet: + # Create StatefulSet scenario + create: + # What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds + # Possible options: + # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is, + # do not try to fix or delete or update it, just abort reconcile cycle. + # Do not proceed to the next StatefulSet(s) and wait for an admin to assist. + # 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards. + # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. + onFailure: ignore + + # Update StatefulSet scenario + update: + # How many seconds to wait for created/updated StatefulSet to be 'Ready' + timeout: 300 + # How many seconds to wait between checks/polls for created/updated StatefulSet status + pollInterval: 5 + # What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds + # Possible options: + # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is, + # do not try to fix or delete or update it, just abort reconcile cycle. + # Do not proceed to the next StatefulSet(s) and wait for an admin to assist. + # 2. rollback - delete Pod and rollback StatefulSet to previous Generation. + # Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration. + # Follow 'abort' path afterwards. + # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet. + onFailure: abort + + # Reconcile Host scenario + host: + # Whether the operator during reconcile procedure should wait for a ClickHouse host: + # - to be excluded from a ClickHouse cluster + # - to complete all running queries + # - to be included into a ClickHouse cluster + # respectfully before moving forward + wait: + exclude: true + queries: true + include: false + + ################################################ + ## + ## Annotations management section + ## + ################################################ + annotation: + # Applied when: + # 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`, + # 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`, + # Include annotations from the following list: + # Applied only when not empty. Empty list means "include all, no selection" + include: [] + # Exclude annotations from the following list: + exclude: [] + + ################################################ + ## + ## Labels management section + ## + ################################################ + label: + # Applied when: + # 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`, + # 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`, + # Include labels from the following list: + # Applied only when not empty. Empty list means "include all, no selection" + include: [] + # Exclude labels from the following list: + # Applied only when not empty. Empty list means "nothing to exclude, no selection" + exclude: [] + # Whether to append *Scope* labels to StatefulSet and Pod. + # Full list of available *scope* labels check in 'labeler.go' + # LabelShardScopeIndex + # LabelReplicaScopeIndex + # LabelCHIScopeIndex + # LabelCHIScopeCycleSize + # LabelCHIScopeCycleIndex + # LabelCHIScopeCycleOffset + # LabelClusterScopeIndex + # LabelClusterScopeCycleSize + # LabelClusterScopeCycleIndex + # LabelClusterScopeCycleOffset + appendScope: "no" + + ################################################ + ## + ## StatefulSet management section + ## + ################################################ + statefulSet: + revisionHistoryLimit: 0 + + ################################################ + ## + ## Pod management section + ## + ################################################ + pod: + # Grace period for Pod termination. + # How many seconds to wait between sending + # SIGTERM and SIGKILL during Pod termination process. + # Increase this number is case of slow shutdown. + terminationGracePeriod: 30 + + ################################################ + ## + ## Log parameters section + ## + ################################################ + logger: + logtostderr: "true" + alsologtostderr: "false" + v: "1" + stderrthreshold: "" + vmodule: "" + log_backtrace_at: "" + +--- +# Template Parameters: +# +# NAME=etc-clickhouse-operator-confd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: etc-clickhouse-operator-confd-files + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +data: +--- +# Template Parameters: +# +# NAME=etc-clickhouse-operator-configd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: etc-clickhouse-operator-configd-files + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +data: + 01-clickhouse-01-listen.xml: | + + + + + + + + + :: + 0.0.0.0 + 1 + + + 01-clickhouse-02-logger.xml: | + + + + + + + + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + 1 + + + + 01-clickhouse-03-query_log.xml: | + + + + + + + + + system + query_log
+ Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day + 7500 + + + + + 01-clickhouse-04-part_log.xml: | + + + + + + + + + system + part_log
+ Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day + 7500 + + + + 01-clickhouse-05-trace_log.xml: | + + + + + + + + + system + trace_log
+ Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day + 7500 + + + +--- +# Template Parameters: +# +# NAME=etc-clickhouse-operator-templatesd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: etc-clickhouse-operator-templatesd-files + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +data: + 001-templates.json.example: | + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallationTemplate", + "metadata": { + "name": "01-default-volumeclaimtemplate" + }, + "spec": { + "templates": { + "volumeClaimTemplates": [ + { + "name": "chi-default-volume-claim-template", + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "2Gi" + } + } + } + } + ], + "podTemplates": [ + { + "name": "chi-default-oneperhost-pod-template", + "distribution": "OnePerHost", + "spec": { + "containers" : [ + { + "name": "clickhouse", + "image": "clickhouse/clickhouse-server:22.3", + "ports": [ + { + "name": "http", + "containerPort": 8123 + }, + { + "name": "client", + "containerPort": 9000 + }, + { + "name": "interserver", + "containerPort": 9009 + } + ] + } + ] + } + } + ] + } + } + } + + default-pod-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-oneperhost-pod-template" + spec: + templates: + podTemplates: + - name: default-oneperhost-pod-template + distribution: "OnePerHost" + default-storage-template.yaml.example: | + apiVersion: "clickhouse.altinity.com/v1" + kind: "ClickHouseInstallationTemplate" + metadata: + name: "default-storage-template-2Gi" + spec: + templates: + volumeClaimTemplates: + - name: default-storage-template-2Gi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + + readme: | + Templates in this folder are packaged with an operator and available via 'useTemplate' +--- +# Template Parameters: +# +# NAME=etc-clickhouse-operator-usersd-files +# NAMESPACE=kube-system +# COMMENT= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: etc-clickhouse-operator-usersd-files + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +data: + 01-clickhouse-operator-profile.xml: | + + + + + + + + + + + + 0 + 1 + 10 + 0 + 0 + + + + + 02-clickhouse-default-profile.xml: | + + + + + + + + + + 2 + 1 + 1000 + 1 + 1 + 1 + nearest_hostname + 0 + + + + +--- +# +# Template parameters available: +# NAMESPACE=kube-system +# COMMENT= +# OPERATOR_VERSION=0.22.2 +# CH_USERNAME_SECRET_PLAIN=clickhouse_operator +# CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password +# +apiVersion: v1 +kind: Secret +metadata: + name: clickhouse-operator + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +type: Opaque +stringData: + username: clickhouse_operator + password: clickhouse_operator_password +--- +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT= +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.22.2 +# OPERATOR_IMAGE_PULL_POLICY=Always +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.22.2 +# METRICS_EXPORTER_IMAGE_PULL_POLICY=Always +# +# Setup Deployment for clickhouse-operator +# Deployment would be created in kubectl-specified namespace +kind: Deployment +apiVersion: apps/v1 +metadata: + name: clickhouse-operator + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse-operator + template: + metadata: + labels: + app: clickhouse-operator + annotations: + prometheus.io/port: '8888' + prometheus.io/scrape: 'true' + clickhouse-operator-metrics/port: '9999' + clickhouse-operator-metrics/scrape: 'true' + spec: + serviceAccountName: clickhouse-operator + volumes: + - name: etc-clickhouse-operator-folder + configMap: + name: etc-clickhouse-operator-files + - name: etc-clickhouse-operator-confd-folder + configMap: + name: etc-clickhouse-operator-confd-files + - name: etc-clickhouse-operator-configd-folder + configMap: + name: etc-clickhouse-operator-configd-files + - name: etc-clickhouse-operator-templatesd-folder + configMap: + name: etc-clickhouse-operator-templatesd-files + - name: etc-clickhouse-operator-usersd-folder + configMap: + name: etc-clickhouse-operator-usersd-files + containers: + - name: clickhouse-operator + image: altinity/clickhouse-operator:0.22.2 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etc-clickhouse-operator-folder + mountPath: /etc/clickhouse-operator + - name: etc-clickhouse-operator-confd-folder + mountPath: /etc/clickhouse-operator/conf.d + - name: etc-clickhouse-operator-configd-folder + mountPath: /etc/clickhouse-operator/config.d + - name: etc-clickhouse-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/templates.d + - name: etc-clickhouse-operator-usersd-folder + mountPath: /etc/clickhouse-operator/users.d + env: + # Pod-specific + # spec.nodeName: ip-172-20-52-62.ec2.internal + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # metadata.name: clickhouse-operator-6f87589dbb-ftcsf + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # metadata.namespace: acto-clickhouse + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # status.podIP: 100.96.3.2 + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # spec.serviceAccount: clickhouse-operator + # spec.serviceAccountName: clickhouse-operator + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + + # Container-specific + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.memory + ports: + - containerPort: 9999 + name: metrics + + - name: metrics-exporter + image: altinity/metrics-exporter:0.22.2 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etc-clickhouse-operator-folder + mountPath: /etc/clickhouse-operator + - name: etc-clickhouse-operator-confd-folder + mountPath: /etc/clickhouse-operator/conf.d + - name: etc-clickhouse-operator-configd-folder + mountPath: /etc/clickhouse-operator/config.d + - name: etc-clickhouse-operator-templatesd-folder + mountPath: /etc/clickhouse-operator/templates.d + - name: etc-clickhouse-operator-usersd-folder + mountPath: /etc/clickhouse-operator/users.d + env: + # Pod-specific + # spec.nodeName: ip-172-20-52-62.ec2.internal + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # metadata.name: clickhouse-operator-6f87589dbb-ftcsf + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # metadata.namespace: acto-clickhouse + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # status.podIP: 100.96.3.2 + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # spec.serviceAccount: clickhouse-operator + # spec.serviceAccountName: clickhouse-operator + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + + # Container-specific + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.memory + ports: + - containerPort: 8888 + name: metrics +--- +# Template Parameters: +# +# NAMESPACE=kube-system +# COMMENT= +# +# Setup ClusterIP Service to provide monitoring metrics for Prometheus +# Service would be created in kubectl-specified namespace +# In order to get access outside of k8s it should be exposed as: +# kubectl --namespace prometheus port-forward service/prometheus 9090 +# and point browser to localhost:9090 +kind: Service +apiVersion: v1 +metadata: + name: clickhouse-operator-metrics + namespace: acto-clickhouse + labels: + clickhouse.altinity.com/chop: 0.22.2 + app: clickhouse-operator +spec: + ports: + - port: 8888 + name: clickhouse-metrics + - port: 9999 + name: operator-metrics + selector: + app: clickhouse-operator diff --git a/data/clickhouse-operator/zookeeper.yaml b/data/clickhouse-operator/zookeeper.yaml new file mode 100644 index 0000000000..741e7c7a12 --- /dev/null +++ b/data/clickhouse-operator/zookeeper.yaml @@ -0,0 +1,289 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/name: zoo3ns + name: zoo3ns +--- +# Setup Service to provide access to Zookeeper for clients +apiVersion: v1 +kind: Service +metadata: + # DNS would be like zookeeper.zoons + name: zookeeper + labels: + app: zookeeper +spec: + ports: + - port: 2181 + name: client + - port: 7000 + name: prometheus + selector: + app: zookeeper + what: node +--- +# Setup Service to provide access to Zookeeper for clients +apiVersion: v1 +kind: Service +metadata: + # DNS would be like zookeeper.zoons + name: zookeeper + namespace: zoo3ns + labels: + app: zookeeper +spec: + ports: + - port: 2181 + name: client + - port: 7000 + name: prometheus + selector: + app: zookeeper + what: node +--- +# Setup Headless Service for StatefulSet +apiVersion: v1 +kind: Service +metadata: + # DNS would be like zookeeper-0.zookeepers.etc + name: zookeepers + namespace: zoo3ns + labels: + app: zookeeper +spec: + ports: + - port: 2888 + name: server + - port: 3888 + name: leader-election + clusterIP: None + selector: + app: zookeeper + what: node +--- +# Setup max number of unavailable pods in StatefulSet +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: zookeeper-pod-disruption-budget + namespace: zoo3ns +spec: + selector: + matchLabels: + app: zookeeper + maxUnavailable: 1 +--- +# Setup Zookeeper StatefulSet +# Possible params: +# 1. replicas +# 2. memory +# 3. cpu +# 4. storage +# 5. storageClassName +# 6. user to run app +apiVersion: apps/v1 +kind: StatefulSet +metadata: + # nodes would be named as zookeeper-0, zookeeper-1, zookeeper-2 + name: zookeeper + namespace: zoo3ns + labels: + app: zookeeper +spec: + selector: + matchLabels: + app: zookeeper + serviceName: zookeepers + replicas: 3 + updateStrategy: + type: RollingUpdate + podManagementPolicy: OrderedReady + template: + metadata: + labels: + app: zookeeper + what: node + annotations: + prometheus.io/port: '7000' + prometheus.io/scrape: 'true' + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - zookeeper + # TODO think about multi-AZ EKS + # topologyKey: topology.kubernetes.io/zone + topologyKey: "kubernetes.io/hostname" + containers: + - name: kubernetes-zookeeper + imagePullPolicy: IfNotPresent + image: "docker.io/zookeeper:3.8.1" + resources: + requests: + memory: "512M" + cpu: "1" + limits: + memory: "4Gi" + cpu: "2" + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + - containerPort: 7000 + name: prometheus + env: + - name: SERVERS + value: "3" + +# See those links for proper startup settings: +# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper +# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper +# https://github.com/ClickHouse/ClickHouse/issues/11781 + command: + - bash + - -x + - -c + - | + HOST=`hostname -s` && + DOMAIN=`hostname -d` && + CLIENT_PORT=2181 && + SERVER_PORT=2888 && + ELECTION_PORT=3888 && + PROMETHEUS_PORT=7000 && + ZOO_DATA_DIR=/var/lib/zookeeper/data && + ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog && + { + echo "clientPort=${CLIENT_PORT}" + echo 'tickTime=2000' + echo 'initLimit=300' + echo 'syncLimit=10' + echo 'maxClientCnxns=2000' + echo 'maxTimeToWaitForEpoch=2000' + echo 'maxSessionTimeout=60000000' + echo "dataDir=${ZOO_DATA_DIR}" + echo "dataLogDir=${ZOO_DATA_LOG_DIR}" + echo 'autopurge.snapRetainCount=10' + echo 'autopurge.purgeInterval=1' + echo 'preAllocSize=131072' + echo 'snapCount=3000000' + echo 'leaderServes=yes' + echo 'standaloneEnabled=false' + echo '4lw.commands.whitelist=*' + echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider' + echo "metricsProvider.httpPort=${PROMETHEUS_PORT}" + echo "skipACL=true" + echo "fastleader.maxNotificationInterval=10000" + } > /conf/zoo.cfg && + { + echo "zookeeper.root.logger=CONSOLE" + echo "zookeeper.console.threshold=INFO" + echo "log4j.rootLogger=\${zookeeper.root.logger}" + echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" + echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" + echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" + echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" + } > /conf/log4j.properties && + echo 'JVMFLAGS="-Xms128M -Xmx4G -XX:ActiveProcessorCount=8 -XX:+AlwaysPreTouch -Djute.maxbuffer=8388608 -XX:MaxGCPauseMillis=50"' > /conf/java.env && + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} && + ORD=${BASH_REMATCH[2]}; + else + echo "Failed to parse name and ordinal of Pod" && + exit 1; + fi && + mkdir -pv ${ZOO_DATA_DIR} && + mkdir -pv ${ZOO_DATA_LOG_DIR} && + whoami && + chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" && + export MY_ID=$((ORD+1)) && + echo $MY_ID > $ZOO_DATA_DIR/myid && + for (( i=1; i<=$SERVERS; i++ )); do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg; + done && + if [[ $SERVERS -eq 1 ]]; then + echo "group.1=1" >> /conf/zoo.cfg; + else + echo "group.1=1:2:3" >> /conf/zoo.cfg; + fi && + for (( i=1; i<=$SERVERS; i++ )); do + WEIGHT=1 + if [[ $i == 1 ]]; then + WEIGHT=10 + fi + echo "weight.$i=$WEIGHT" >> /conf/zoo.cfg; + done && + zkServer.sh start-foreground + readinessProbe: + exec: + command: + - bash + - -c + - ' + IFS=; + MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;); + while [[ "$MNTR" == "This ZooKeeper instance is not currently serving requests" ]]; + do + echo "wait mntr works"; + sleep 1; + MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;); + done; + STATE=$(echo -e $MNTR | grep zk_server_state | cut -d " " -f 2); + if [[ "$STATE" =~ "leader" ]]; then + echo "check leader state"; + SYNCED_FOLLOWERS=$(echo -e $MNTR | grep zk_synced_followers | awk -F"[[:space:]]+" "{print \$2}" | cut -d "." -f 1); + if [[ "$SYNCED_FOLLOWERS" != "0" ]]; then + ./bin/zkCli.sh ls /; + exit $?; + else + exit 0; + fi; + elif [[ "$STATE" =~ "follower" ]]; then + echo "check follower state"; + PEER_STATE=$(echo -e $MNTR | grep zk_peer_state); + if [[ "$PEER_STATE" =~ "following - broadcast" ]]; then + ./bin/zkCli.sh ls /; + exit $?; + else + exit 1; + fi; + else + exit 1; + fi + ' + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 60 + livenessProbe: + exec: + command: + - bash + - -xc + - 'date && OK=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi' + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + volumeMounts: + - name: datadir-volume + mountPath: /var/lib/zookeeper + # Run as a non-privileged user + securityContext: + runAsUser: 1000 + fsGroup: 1000 + volumeClaimTemplates: + - metadata: + name: datadir-volume + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 25Gi \ No newline at end of file