Merge pull request #4026 from chaosi-zju/localup

remove insecureSkipTLSVerify in local-up-karmada script
karmada-io · Sep 7, 2023 · b498f0f · b498f0f
2 parents 6bae737 + 57726b3
commit b498f0f
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 10 deletions.
diff --git a/artifacts/deploy/karmada-aggregated-apiserver-apiservice.yaml b/artifacts/deploy/karmada-aggregated-apiserver-apiservice.yaml
@@ -6,7 +6,7 @@ metadata:
     app: karmada-aggregated-apiserver
     apiserver: "true"
 spec:
-  insecureSkipTLSVerify: true
+  caBundle: {{caBundle}}
   group: cluster.karmada.io
   groupPriorityMinimum: 2000
   service:

diff --git a/artifacts/deploy/karmada-metrics-adapter-apiservice.yaml b/artifacts/deploy/karmada-metrics-adapter-apiservice.yaml
@@ -6,7 +6,7 @@ metadata:
     app: karmada-metrics-adapter
     apiserver: "true"
 spec:
-  insecureSkipTLSVerify: true
+  caBundle: {{caBundle}}
   group: metrics.k8s.io
   groupPriorityMinimum: 2000
   service:
@@ -25,7 +25,7 @@ spec:
     namespace:  karmada-system
   group: custom.metrics.k8s.io
   version: v1beta2
-  insecureSkipTLSVerify: true
+  caBundle: {{caBundle}}
   groupPriorityMinimum: 100
   versionPriority: 200
 ---
@@ -39,7 +39,7 @@ spec:
     namespace:  karmada-system
   group: custom.metrics.k8s.io
   version: v1beta1
-  insecureSkipTLSVerify: true
+  caBundle: {{caBundle}}
   groupPriorityMinimum: 100
   versionPriority: 200
 ---

diff --git a/artifacts/deploy/karmada-metrics-adapter.yaml b/artifacts/deploy/karmada-metrics-adapter.yaml
@@ -37,6 +37,8 @@ spec:
             - --authentication-kubeconfig=/etc/kubeconfig
             - --authorization-kubeconfig=/etc/kubeconfig
             - --client-ca-file=/etc/karmada/pki/ca.crt
+            - --tls-cert-file=/etc/karmada/pki/karmada.crt
+            - --tls-private-key-file=/etc/karmada/pki/karmada.key
             - --audit-log-path=-
             - --audit-log-maxage=0
             - --audit-log-maxbackup=0

diff --git a/artifacts/deploy/karmada-search-apiservice.yaml b/artifacts/deploy/karmada-search-apiservice.yaml
@@ -6,7 +6,7 @@ metadata:
     app: karmada-search
     apiserver: "true"
 spec:
-  insecureSkipTLSVerify: true
+  caBundle: {{caBundle}}
   group: search.karmada.io
   groupPriorityMinimum: 2000
   service:

diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh
@@ -250,21 +250,31 @@ util::fill_cabundle "${ROOT_CA_FILE}" "${TEMP_PATH_CRDS}/_crds/patches/webhook_i
 util::fill_cabundle "${ROOT_CA_FILE}" "${TEMP_PATH_CRDS}/_crds/patches/webhook_in_clusterresourcebindings.yaml"
 installCRDs "karmada-apiserver" "${TEMP_PATH_CRDS}"
 
+# render the caBundle in these apiservice with root ca, then karmada-apiserver can use caBundle to verify corresponding AA's server-cert
+TEMP_PATH_APISERVICE=$(mktemp -d)
+trap '{ rm -rf ${TEMP_PATH_APISERVICE}; }' EXIT
+cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-aggregated-apiserver-apiservice.yaml "${TEMP_PATH_APISERVICE}"/karmada-aggregated-apiserver-apiservice.yaml
+cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-metrics-adapter-apiservice.yaml "${TEMP_PATH_APISERVICE}"/karmada-metrics-adapter-apiservice.yaml
+cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-search-apiservice.yaml "${TEMP_PATH_APISERVICE}"/karmada-search-apiservice.yaml
+util::fill_cabundle "${ROOT_CA_FILE}" "${TEMP_PATH_APISERVICE}"/karmada-aggregated-apiserver-apiservice.yaml
+util::fill_cabundle "${ROOT_CA_FILE}" "${TEMP_PATH_APISERVICE}"/karmada-metrics-adapter-apiservice.yaml
+util::fill_cabundle "${ROOT_CA_FILE}" "${TEMP_PATH_APISERVICE}"/karmada-search-apiservice.yaml
+
 # deploy webhook configurations on karmada apiserver
 util::deploy_webhook_configuration "karmada-apiserver" "${ROOT_CA_FILE}" "${REPO_ROOT}/artifacts/deploy/webhook-configuration.yaml"
 
 # deploy APIService on karmada apiserver for karmada-aggregated-apiserver
-kubectl --context="karmada-apiserver" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-aggregated-apiserver-apiservice.yaml"
+kubectl --context="karmada-apiserver" apply -f "${TEMP_PATH_APISERVICE}"/karmada-aggregated-apiserver-apiservice.yaml
 # make sure apiservice for v1alpha1.cluster.karmada.io is Available
 util::wait_apiservice_ready "karmada-apiserver" "${KARMADA_AGGREGATION_APISERVER_LABEL}"
 
 # deploy APIService on karmada apiserver for karmada-search
-kubectl --context="karmada-apiserver" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-search-apiservice.yaml"
+kubectl --context="karmada-apiserver" apply -f "${TEMP_PATH_APISERVICE}"/karmada-search-apiservice.yaml
 # make sure apiservice for v1alpha1.search.karmada.io is Available
 util::wait_apiservice_ready "karmada-apiserver" "${KARMADA_SEARCH_LABEL}"
 
 # deploy APIService on karmada apiserver for karmada-metrics-adapter
-kubectl --context="karmada-apiserver" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-metrics-adapter-apiservice.yaml"
+kubectl --context="karmada-apiserver" apply -f "${TEMP_PATH_APISERVICE}"/karmada-metrics-adapter-apiservice.yaml
 # make sure apiservice for karmada metrics adapter is Available
 util::wait_apiservice_ready "karmada-apiserver" "${KARMADA_METRICS_ADAPTER_LABEL}"
 

diff --git a/hack/deploy-metrics-adapter.sh b/hack/deploy-metrics-adapter.sh
@@ -7,7 +7,7 @@ REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${REPO_ROOT}"/hack/util.sh
 function usage() {
   echo "This script will deploy karmada-metrics-adapter on host cluster"
-  echo "Usage: hack/deploy-metrics-adapter.sh  <HOST_CLUSTER_KUBECONFIG> <HOST_CONTEXT_NAME> <KARMADA_APISERVER_KUBECONFIG> <KARMADA_APISERVER_CONTEXT_NAME>"
+  echo "Usage: hack/deploy-metrics-adapter.sh <HOST_CLUSTER_KUBECONFIG> <HOST_CONTEXT_NAME> <KARMADA_APISERVER_KUBECONFIG> <KARMADA_APISERVER_CONTEXT_NAME>"
   echo "Example: hack/deploy-metrics-adapter.sh ~/.kube/karmada.config karmada-host ~/.kube/karmada.config karmada-apiserver"
 }
 
@@ -66,8 +66,17 @@ util::wait_pod_ready "${HOST_CONTEXT_NAME}" "${KARMADA_METRICS_ADAPTER_LABEL}" "
 
 export KUBECONFIG=$KARMADA_APISERVER_KUBECONFIG
 
+# get karmada CA from configmap cluster-info, which generated in karmada-apiserver context when installing karmada.
+karmada_ca=$(kubectl --context="${KARMADA_APISERVER_CONTEXT_NAME}" get cm cluster-info -n kube-public -o jsonpath='{.data.kubeconfig}' | grep 'certificate-authority-data' | awk -F ': ' '{print $2}')
+
+# render the caBundle in apiservice with root ca, then karmada-apiserver can use caBundle to verify karmada-metrics-adapter's server-cert
+TEMP_PATH_APISERVICE=$(mktemp -d)
+trap '{ rm -rf ${TEMP_PATH_APISERVICE}; }' EXIT
+cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-metrics-adapter-apiservice.yaml "${TEMP_PATH_APISERVICE}"/karmada-metrics-adapter-apiservice.yaml
+sed -i'' -e "s/{{caBundle}}/${karmada_ca}/g" "${TEMP_PATH_APISERVICE}"/karmada-metrics-adapter-apiservice.yaml
+
 # deploy karmada-metrics-adapter-apiservice
-kubectl --context="${KARMADA_APISERVER_CONTEXT_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-metrics-adapter-apiservice.yaml"
+kubectl --context="${KARMADA_APISERVER_CONTEXT_NAME}" apply -f "${TEMP_PATH_APISERVICE}"/karmada-metrics-adapter-apiservice.yaml
 
 # make sure that karmada-metrics-adapter-apiservice is ready
 util::wait_apiservice_ready "${KARMADA_APISERVER_CONTEXT_NAME}" "${KARMADA_METRICS_ADAPTER_LABEL}"