diff --git a/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java b/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java index 43f3b6ca6..6978bf089 100644 --- a/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java +++ b/computer-api/src/main/java/org/apache/hugegraph/computer/core/config/ComputerOptions.java @@ -235,7 +235,7 @@ public static synchronized ComputerOptions instance() { "snapshot.minio_access_key", "The access key of MinIO.", null, - "" + "minioadmin" ); public static final ConfigOption SNAPSHOT_MINIO_SECRET_KEY = @@ -243,7 +243,7 @@ public static synchronized ComputerOptions instance() { "snapshot.minio_secret_key", "The secret key of MinIO.", null, - "" + "minioadmin" ); public static final ConfigOption SNAPSHOT_MINIO_BUCKET_NAME = diff --git a/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java b/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java index 4734ee5aa..674d95d72 100644 --- a/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java +++ b/computer-core/src/main/java/org/apache/hugegraph/computer/core/snapshot/SnapshotManager.java @@ -17,8 +17,10 @@ package org.apache.hugegraph.computer.core.snapshot; +import io.minio.BucketExistsArgs; import io.minio.DownloadObjectArgs; import io.minio.ListObjectsArgs; +import io.minio.MakeBucketArgs; import io.minio.MinioClient; import io.minio.RemoveObjectsArgs; import io.minio.Result; @@ -89,14 +91,29 @@ public String name() { @Override public void init(Config config) { String endpoint = config.get(ComputerOptions.SNAPSHOT_MINIO_ENDPOINT); - String accessKey = config.get(ComputerOptions.SNAPSHOT_MINIO_ACCESS_KEY); - String secretKey = config.get(ComputerOptions.SNAPSHOT_MINIO_SECRET_KEY); this.bucketName = config.get(ComputerOptions.SNAPSHOT_MINIO_BUCKET_NAME); - if (StringUtils.isNotEmpty(endpoint)) { + + if (StringUtils.isNotEmpty(endpoint) && StringUtils.isNotEmpty(this.bucketName)) { + String accessKey = config.get(ComputerOptions.SNAPSHOT_MINIO_ACCESS_KEY); + String secretKey = config.get(ComputerOptions.SNAPSHOT_MINIO_SECRET_KEY); this.minioClient = MinioClient.builder() .endpoint(endpoint) .credentials(accessKey, secretKey) .build(); + + try { + boolean bucketExist = this.minioClient.bucketExists( + BucketExistsArgs.builder() + .bucket(this.bucketName) + .build()); + if (!bucketExist) { + this.minioClient.makeBucket(MakeBucketArgs.builder() + .bucket(this.bucketName) + .build()); + } + } catch (Exception e) { + throw new ComputerException("Failed to initialize bucket %s", this.bucketName, e); + } } } diff --git a/computer-k8s-operator/crd-generate/config/default/kustomization.yaml b/computer-k8s-operator/crd-generate/config/default/kustomization.yaml index aa3037f2f..cb4e14596 100644 --- a/computer-k8s-operator/crd-generate/config/default/kustomization.yaml +++ b/computer-k8s-operator/crd-generate/config/default/kustomization.yaml @@ -35,6 +35,7 @@ resources: - ../rbac - ../etcd - ../manager +- ../minio # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - ../webhook diff --git a/computer-k8s-operator/crd-generate/config/manager/manager.yaml b/computer-k8s-operator/crd-generate/config/manager/manager.yaml index ec67232f4..3d5cc1aed 100644 --- a/computer-k8s-operator/crd-generate/config/manager/manager.yaml +++ b/computer-k8s-operator/crd-generate/config/manager/manager.yaml @@ -60,6 +60,8 @@ spec: value: "6" - name: INTERNAL_ETCD_URL value: "http://hugegraph-computer-operator-etcd.hugegraph-computer-operator-system:2379" + - name: INTERNAL_MINIO_URL + value: "http://hugegraph-computer-operator-minio.hugegraph-computer-operator-system:9000" - name: LOG_LEVEL value: "INFO" - name: AUTO_DESTROY_POD diff --git a/computer-k8s-operator/crd-generate/config/minio/kustomization.yaml b/computer-k8s-operator/crd-generate/config/minio/kustomization.yaml new file mode 100644 index 000000000..6776474a3 --- /dev/null +++ b/computer-k8s-operator/crd-generate/config/minio/kustomization.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +resources: + - minio_server.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: + - name: minio + newName: quay.io/minio/minio + newTag: RELEASE.2023-10-16T04-13-43Z +commonLabels: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 diff --git a/computer-k8s-operator/crd-generate/config/minio/minio_server.yaml b/computer-k8s-operator/crd-generate/config/minio/minio_server.yaml new file mode 100644 index 000000000..02bc642d8 --- /dev/null +++ b/computer-k8s-operator/crd-generate/config/minio/minio_server.yaml @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage + namespace: system +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + service.app: hugegraph-computer-operator-minio + name: minio + namespace: system + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: 'true' +spec: + ports: + - name: client + port: 9000 + protocol: TCP + targetPort: 9090 + selector: + service.app: hugegraph-computer-operator-minio + clusterIP: None + type: ClusterIP + sessionAffinity: None +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: system +spec: + replicas: 1 + selector: + matchLabels: + service.app: hugegraph-computer-operator-minio + template: + metadata: + labels: + service.app: hugegraph-computer-operator-minio + spec: + hostname: hugegraph-computer-operator-minio-0 + subdomain: hugegraph-computer-operator-minio + containers: + - name: hugegraph-computer-operator-minio + image: minio:latest + command: + - /bin/bash + - -c + args: + - minio server /data --console-address :9090 + env: + # MinIO access key and secret key + - name: MINIO_ACCESS_KEY + value: "minioadmin" + - name: MINIO_SECRET_KEY + value: "minioadmin" + ports: + - name: client + containerPort: 9090 + volumeMounts: + - mountPath: /data + name: localvolume # Corresponds to the `spec.volumes` Persistent Volume + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /minio/health/ready + # TODO: check the port + port: 9000 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + volumes: + - name: localvolume + hostPath: # MinIO generally recommends using locally-attached volumes + path: /mnt/disk1/data # Specify a path to a local drive or volume on the Kubernetes worker node + type: DirectoryOrCreate # The path to the last directory must exist diff --git a/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml b/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml index aebc27beb..9949da76f 100644 --- a/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml +++ b/computer-k8s-operator/manifest/hugegraph-computer-operator.yaml @@ -23,6 +23,17 @@ metadata: control-plane: controller-manager name: hugegraph-computer-operator-system --- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 + name: hugegraph-computer-operator-local-storage + namespace: system +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -436,6 +447,31 @@ spec: --- apiVersion: v1 kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + labels: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 + service.app: hugegraph-computer-operator-minio + name: hugegraph-computer-operator-minio + namespace: hugegraph-computer-operator-system +spec: + clusterIP: None + ports: + - name: client + port: 9000 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 + service.app: hugegraph-computer-operator-minio + sessionAffinity: None + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: name: hugegraph-computer-operator-webhook-service namespace: hugegraph-computer-operator-system @@ -525,6 +561,8 @@ spec: value: "6" - name: INTERNAL_ETCD_URL value: http://hugegraph-computer-operator-etcd.hugegraph-computer-operator-system:2379 + - name: INTERNAL_MINIO_URL + value: http://hugegraph-computer-operator-minio.hugegraph-computer-operator-system:9000 - name: LOG_LEVEL value: INFO - name: AUTO_DESTROY_POD @@ -620,6 +658,75 @@ spec: hostname: hugegraph-computer-operator-etcd-0 subdomain: hugegraph-computer-operator-etcd --- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 + name: hugegraph-computer-operator-minio + namespace: hugegraph-computer-operator-system +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 + service.app: hugegraph-computer-operator-minio + template: + metadata: + labels: + app.kubernetes.io/name: hugegraph-computer-operator-minio + app.kubernetes.io/version: v1 + service.app: hugegraph-computer-operator-minio + spec: + containers: + - args: + - minio server /data --console-address :9090 + command: + - /bin/bash + - -c + env: + - name: MINIO_ACCESS_KEY + value: minioadmin + - name: MINIO_SECRET_KEY + value: minioadmin + image: quay.io/minio/minio:RELEASE.2023-10-16T04-13-43Z + livenessProbe: + failureThreshold: 3 + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 10 + name: hugegraph-computer-operator-minio + ports: + - containerPort: 9090 + name: client + readinessProbe: + failureThreshold: 3 + httpGet: + path: /minio/health/ready + port: 9000 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 15 + successThreshold: 1 + timeoutSeconds: 10 + volumeMounts: + - mountPath: /data + name: localvolume + hostname: hugegraph-computer-operator-minio-0 + subdomain: hugegraph-computer-operator-minio + volumes: + - hostPath: + path: /mnt/disk1/data + type: DirectoryOrCreate + name: localvolume +--- apiVersion: cert-manager.io/v1 kind: Certificate metadata: diff --git a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java index 677c7cc16..f9ac66103 100644 --- a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java +++ b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/config/OperatorOptions.java @@ -134,6 +134,14 @@ public static synchronized OperatorOptions instance() { "http://127.0.0.1:2379" ); + public static final ConfigOption INTERNAL_MINIO_URL = + new ConfigOption<>( + "k8s.internal_minio_url", + "The internal minio url for operator system.", + disallowEmpty(), + "http://127.0.0.1:9000" + ); + public static final ConfigOption AUTO_DESTROY_POD = new ConfigOption<>( "k8s.auto_destroy_pod", diff --git a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java index 9d3aa61e4..08775088f 100644 --- a/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java +++ b/computer-k8s-operator/src/main/java/org/apache/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java @@ -101,12 +101,14 @@ public class ComputerJobDeployer { private static final Long TERMINATION_GRACE_PERIOD = 180L; private final String internalEtcdUrl; + private final String internalMinIOUrl; private final String timezone; public ComputerJobDeployer(NamespacedKubernetesClient kubeClient, HugeConfig config) { this.kubeClient = kubeClient; this.internalEtcdUrl = config.get(OperatorOptions.INTERNAL_ETCD_URL); + this.internalMinIOUrl = config.get(OperatorOptions.INTERNAL_MINIO_URL); this.timezone = config.get(OperatorOptions.TIMEZONE); } @@ -186,6 +188,8 @@ private Set handleConfig(ComputerJobSpec spec) { config.put(ComputerOptions.RPC_SERVER_HOST_NAME, ip); config.putIfAbsent(ComputerOptions.BSP_ETCD_ENDPOINTS.name(), this.internalEtcdUrl); + config.putIfAbsent(ComputerOptions.SNAPSHOT_MINIO_ENDPOINT.name(), + this.internalMinIOUrl); String transportPort = config.get( ComputerOptions.TRANSPORT_SERVER_PORT.name());