diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0177d621c..2b6201cb6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -197,22 +197,58 @@ jobs: run: | export GOBIN="$(go env GOPATH)/bin" cd "${GITHUB_WORKSPACE}/kubernetes" && go install ./cmd/{kube-apiserver,kube-controller-manager,kube-scheduler} - mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF + mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >>"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF kind: KwokctlConfiguration apiVersion: config.kwok.x-k8s.io/v1alpha1 options: kubeBinaryPrefix: "${GOBIN//\\//}" + --- EOF - name: Build Kubernetes Binary for MacOS if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'macos-latest' }} shell: bash run: | cd "${GITHUB_WORKSPACE}/kubernetes" && make WHAT="cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler" - mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF + mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >>"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF kind: KwokctlConfiguration apiVersion: config.kwok.x-k8s.io/v1alpha1 options: kubeBinaryPrefix: "${GITHUB_WORKSPACE}/kubernetes/_output/bin" + --- + EOF + + # Build metrics-server binary + - name: Download Metrics Server Source Code + uses: actions/checkout@v3 + if: ${{ matrix.kwokctl-runtime == 'binary' }} + with: + repository: kubernetes-sigs/metrics-server + path: metrics-server + ref: v0.6.3 + - name: Build Metrics Server Binary for Linux and MacOS + if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os != 'windows-latest' }} + shell: bash + run: | + cd "${GITHUB_WORKSPACE}/metrics-server" && make metrics-server + mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >>"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF + kind: KwokctlConfiguration + apiVersion: config.kwok.x-k8s.io/v1alpha1 + options: + metricsServerBinary: "${GITHUB_WORKSPACE}/metrics-server/metrics-server" + --- + EOF + - name: Build Metrics Server Binary for Windows + if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'windows-latest' }} + shell: bash + run: | + export GOBIN="$(go env GOPATH)/bin" + cd "${GITHUB_WORKSPACE}/metrics-server" && go install ./cmd/metrics-server + mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >>"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF + kind: KwokctlConfiguration + apiVersion: config.kwok.x-k8s.io/v1alpha1 + options: + metricsServerBinary: "${GOBIN//\\//}/metrics-server.exe" + --- EOF # TODO: workaround for https://github.com/actions/runner-images/issues/7753 (caused by https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394). diff --git a/kustomize/kwokctl/resource/node.yaml b/kustomize/kwokctl/resource/node.yaml index b31157d90..ff4ed7860 100644 --- a/kustomize/kwokctl/resource/node.yaml +++ b/kustomize/kwokctl/resource/node.yaml @@ -20,6 +20,7 @@ template: |- annotations: kwok.x-k8s.io/node: fake node.alpha.kubernetes.io/ttl: "0" + metrics.k8s.io/resource-metrics-path: "/metrics/nodes/{{ Name }}/metrics/resource" labels: beta.kubernetes.io/arch: {{ .nodeInfo.architecture }} beta.kubernetes.io/os: {{ .nodeInfo.operatingSystem }} diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index e155baace..311cea9d8 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -127,6 +127,9 @@ type KwokctlConfigurationOptions struct { // is the default value for env KWOK_JAEGER_VERSION JaegerVersion string `json:"jaegerVersion,omitempty"` + // MetricsServerVersion is the version of metrics-server to use. + MetricsServerVersion string `json:"metricsServerVersion,omitempty"` + // DockerComposeVersion is the version of docker-compose to use. // is the default value for env KWOK_DOCKER_COMPOSE_VERSION // Deprecated: docker compose will be removed in a future release @@ -160,6 +163,10 @@ type KwokctlConfigurationOptions struct { // +default=false DisableKubeControllerManager *bool `json:"disableKubeControllerManager,omitempty"` + // EnableMetricsServer is the flag to enable metrics-server. + // +default=false + EnableMetricsServer *bool `json:"enableMetricsServer,omitempty"` + // KubeImagePrefix is the prefix of the kubernetes image. // is the default value for env KWOK_KUBE_IMAGE_PREFIX //+k8s:conversion-gen=false @@ -189,6 +196,10 @@ type KwokctlConfigurationOptions struct { //+k8s:conversion-gen=false JaegerImagePrefix string `json:"jaegerImagePrefix,omitempty"` + // MetricsServerImagePrefix is the prefix of the metrics-server image. + //+k8s:conversion-gen=false + MetricsServerImagePrefix string `json:"metricsServerImagePrefix,omitempty"` + // EtcdImage is the image of etcd. // is the default value for flag --etcd-image and env KWOK_ETCD_IMAGE EtcdImage string `json:"etcdImage,omitempty"` @@ -220,6 +231,9 @@ type KwokctlConfigurationOptions struct { // is the default value for flag --jaeger-image and env KWOK_JAEGER_IMAGE JaegerImage string `json:"jaegerImage,omitempty"` + // MetricsServerImage is the image of metrics-server. + MetricsServerImage string `json:"metricsServerImage,omitempty"` + // KindNodeImagePrefix is the prefix of the kind node image. // is the default value for env KWOK_KIND_NODE_IMAGE_PREFIX //+k8s:conversion-gen=false @@ -310,6 +324,13 @@ type KwokctlConfigurationOptions struct { // is the default value for env KWOK_JAEGER_TAR JaegerBinaryTar string `json:"jaegerBinaryTar,omitempty"` + // MetricsServerBinaryPrefix is the prefix of the metrics-server binary. + //+k8s:conversion-gen=false + MetricsServerBinaryPrefix string `json:"metricsServerBinaryPrefix,omitempty"` + + // MetricsServerBinary is the binary of metrics-server. + MetricsServerBinary string `json:"metricsServerBinary,omitempty"` + // DockerComposeBinaryPrefix is the binary of docker-compose. // is the default value for env KWOK_DOCKER_COMPOSE_BINARY_PREFIX // Deprecated: docker compose will be removed in a future release @@ -373,6 +394,9 @@ type KwokctlConfigurationOptions struct { // is the default value for flag --controller-port and env KWOK_CONTROLLER_PORT KwokControllerPort uint32 `json:"kwokControllerPort,omitempty"` + // MetricsServerPort is metrics-server port that is exposed to the host. + MetricsServerPort uint32 `json:"metricsServerPort,omitempty"` + // CacheDir is the directory of the cache. CacheDir string `json:"cacheDir,omitempty"` diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 443fc8af2..ab7301727 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -288,6 +288,11 @@ func (in *KwokctlConfigurationOptions) DeepCopyInto(out *KwokctlConfigurationOpt *out = new(bool) **out = **in } + if in.EnableMetricsServer != nil { + in, out := &in.EnableMetricsServer, &out.EnableMetricsServer + *out = new(bool) + **out = **in + } if in.KubeAuthorization != nil { in, out := &in.KubeAuthorization, &out.KubeAuthorization *out = new(bool) diff --git a/pkg/apis/config/v1alpha1/zz_generated.defaults.go b/pkg/apis/config/v1alpha1/zz_generated.defaults.go index 0799c266a..3f8a759c0 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/config/v1alpha1/zz_generated.defaults.go @@ -82,6 +82,10 @@ func SetObjectDefaults_KwokctlConfiguration(in *KwokctlConfiguration) { var ptrVar1 bool = false in.Options.DisableKubeControllerManager = &ptrVar1 } + if in.Options.EnableMetricsServer == nil { + var ptrVar1 bool = false + in.Options.EnableMetricsServer = &ptrVar1 + } if in.Options.KubeControllerManagerNodeMonitorPeriodMilliseconds == 0 { in.Options.KubeControllerManagerNodeMonitorPeriodMilliseconds = 600000 } diff --git a/pkg/apis/internalversion/kwokctl_configuration_types.go b/pkg/apis/internalversion/kwokctl_configuration_types.go index 1fb2e9247..2e8a9412d 100644 --- a/pkg/apis/internalversion/kwokctl_configuration_types.go +++ b/pkg/apis/internalversion/kwokctl_configuration_types.go @@ -104,6 +104,9 @@ type KwokctlConfigurationOptions struct { // JaegerVersion is the version of Jaeger to use. JaegerVersion string + // MetricsServerVersion is the version of metrics-server to use. + MetricsServerVersion string + // DockerComposeVersion is the version of docker-compose to use. DockerComposeVersion string @@ -126,6 +129,9 @@ type KwokctlConfigurationOptions struct { // DisableKubeControllerManager is the flag to disable kube-controller-manager. DisableKubeControllerManager bool + // EnableMetricsServer is the flag to enable metrics-server. + EnableMetricsServer bool + // EtcdImage is the image of etcd. EtcdImage string @@ -150,6 +156,9 @@ type KwokctlConfigurationOptions struct { // JaegerImage is the image of Jaeger JaegerImage string + // MetricsServerImage is the image of metrics-server. + MetricsServerImage string + // KindNodeImage is the image of kind node. KindNodeImage string @@ -194,6 +203,9 @@ type KwokctlConfigurationOptions struct { // JaegerBinaryTar is the tar of binary of Jaeger. JaegerBinaryTar string + // MetricsServerBinary is the binary of metrics-server. + MetricsServerBinary string + // DockerComposeBinary is the binary of Docker compose. DockerComposeBinary string @@ -236,6 +248,9 @@ type KwokctlConfigurationOptions struct { // KwokControllerPort is kwok-controller port that is exposed to the host. KwokControllerPort uint32 + // MetricsServerPort is metrics-server port that is exposed to the host. + MetricsServerPort uint32 + // CacheDir is the directory of the cache. CacheDir string diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 0503f15fb..50ccded57 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -1460,6 +1460,7 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl out.DashboardVersion = in.DashboardVersion out.PrometheusVersion = in.PrometheusVersion out.JaegerVersion = in.JaegerVersion + out.MetricsServerVersion = in.MetricsServerVersion out.DockerComposeVersion = in.DockerComposeVersion out.KindVersion = in.KindVersion if err := v1.Convert_bool_To_Pointer_bool(&in.SecurePort, &out.SecurePort, s); err != nil { @@ -1475,6 +1476,9 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl if err := v1.Convert_bool_To_Pointer_bool(&in.DisableKubeControllerManager, &out.DisableKubeControllerManager, s); err != nil { return err } + if err := v1.Convert_bool_To_Pointer_bool(&in.EnableMetricsServer, &out.EnableMetricsServer, s); err != nil { + return err + } out.EtcdImage = in.EtcdImage out.KubeApiserverImage = in.KubeApiserverImage out.KubeControllerManagerImage = in.KubeControllerManagerImage @@ -1483,6 +1487,7 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl out.DashboardImage = in.DashboardImage out.PrometheusImage = in.PrometheusImage out.JaegerImage = in.JaegerImage + out.MetricsServerImage = in.MetricsServerImage out.KindNodeImage = in.KindNodeImage out.BinSuffix = in.BinSuffix out.KubeApiserverBinary = in.KubeApiserverBinary @@ -1496,6 +1501,7 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl out.PrometheusBinaryTar = in.PrometheusBinaryTar out.JaegerBinary = in.JaegerBinary out.JaegerBinaryTar = in.JaegerBinaryTar + out.MetricsServerBinary = in.MetricsServerBinary out.DockerComposeBinary = in.DockerComposeBinary out.KindBinary = in.KindBinary out.Mode = in.Mode @@ -1514,6 +1520,7 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl out.KubeSchedulerPort = in.KubeSchedulerPort out.DashboardPort = in.DashboardPort out.KwokControllerPort = in.KwokControllerPort + out.MetricsServerPort = in.MetricsServerPort out.CacheDir = in.CacheDir out.KubeControllerManagerNodeMonitorPeriodMilliseconds = in.KubeControllerManagerNodeMonitorPeriodMilliseconds out.KubeControllerManagerNodeMonitorGracePeriodMilliseconds = in.KubeControllerManagerNodeMonitorGracePeriodMilliseconds @@ -1546,6 +1553,7 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl out.DashboardVersion = in.DashboardVersion out.PrometheusVersion = in.PrometheusVersion out.JaegerVersion = in.JaegerVersion + out.MetricsServerVersion = in.MetricsServerVersion out.DockerComposeVersion = in.DockerComposeVersion out.KindVersion = in.KindVersion if err := v1.Convert_Pointer_bool_To_bool(&in.SecurePort, &out.SecurePort, s); err != nil { @@ -1561,12 +1569,16 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl if err := v1.Convert_Pointer_bool_To_bool(&in.DisableKubeControllerManager, &out.DisableKubeControllerManager, s); err != nil { return err } + if err := v1.Convert_Pointer_bool_To_bool(&in.EnableMetricsServer, &out.EnableMetricsServer, s); err != nil { + return err + } // INFO: in.KubeImagePrefix opted out of conversion generation // INFO: in.EtcdImagePrefix opted out of conversion generation // INFO: in.KwokImagePrefix opted out of conversion generation // INFO: in.DashboardImagePrefix opted out of conversion generation // INFO: in.PrometheusImagePrefix opted out of conversion generation // INFO: in.JaegerImagePrefix opted out of conversion generation + // INFO: in.MetricsServerImagePrefix opted out of conversion generation out.EtcdImage = in.EtcdImage out.KubeApiserverImage = in.KubeApiserverImage out.KubeControllerManagerImage = in.KubeControllerManagerImage @@ -1575,6 +1587,7 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl out.DashboardImage = in.DashboardImage out.PrometheusImage = in.PrometheusImage out.JaegerImage = in.JaegerImage + out.MetricsServerImage = in.MetricsServerImage // INFO: in.KindNodeImagePrefix opted out of conversion generation out.KindNodeImage = in.KindNodeImage out.BinSuffix = in.BinSuffix @@ -1594,6 +1607,8 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl // INFO: in.JaegerBinaryPrefix opted out of conversion generation out.JaegerBinary = in.JaegerBinary out.JaegerBinaryTar = in.JaegerBinaryTar + // INFO: in.MetricsServerBinaryPrefix opted out of conversion generation + out.MetricsServerBinary = in.MetricsServerBinary // INFO: in.DockerComposeBinaryPrefix opted out of conversion generation out.DockerComposeBinary = in.DockerComposeBinary // INFO: in.KindBinaryPrefix opted out of conversion generation @@ -1614,6 +1629,7 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl out.KubeSchedulerPort = in.KubeSchedulerPort out.DashboardPort = in.DashboardPort out.KwokControllerPort = in.KwokControllerPort + out.MetricsServerPort = in.MetricsServerPort out.CacheDir = in.CacheDir out.KubeControllerManagerNodeMonitorPeriodMilliseconds = in.KubeControllerManagerNodeMonitorPeriodMilliseconds out.KubeControllerManagerNodeMonitorGracePeriodMilliseconds = in.KubeControllerManagerNodeMonitorGracePeriodMilliseconds diff --git a/pkg/config/vars.go b/pkg/config/vars.go index b8273f85d..7a24d4227 100644 --- a/pkg/config/vars.go +++ b/pkg/config/vars.go @@ -246,6 +246,8 @@ func setKwokctlConfigurationDefaults(config *configv1alpha1.KwokctlConfiguration setKwokctlJaegerConfig(conf) + setMetricsServerConfig(conf) + return config } @@ -527,6 +529,34 @@ func setKwokctlJaegerConfig(conf *configv1alpha1.KwokctlConfigurationOptions) { conf.JaegerBinaryTar = envs.GetEnvWithPrefix("JAEGER_BINARY_TAR", conf.JaegerBinaryTar) } +func setMetricsServerConfig(conf *configv1alpha1.KwokctlConfigurationOptions) { + if conf.MetricsServerVersion == "" { + conf.MetricsServerVersion = consts.MetricsServerVersion + } + conf.MetricsServerVersion = version.AddPrefixV(envs.GetEnvWithPrefix("METRICS_SERVER_VERSION", conf.MetricsServerVersion)) + + if conf.MetricsServerImagePrefix == "" { + conf.MetricsServerImagePrefix = consts.MetricsServerImagePrefix + } + conf.MetricsServerImagePrefix = envs.GetEnvWithPrefix("METRICS_SERVER_IMAGE_PREFIX", conf.MetricsServerImagePrefix) + + if conf.MetricsServerImage == "" { + conf.MetricsServerImage = joinImageURI(conf.MetricsServerImagePrefix, "metrics-server", version.AddPrefixV(conf.MetricsServerVersion)) + } + conf.MetricsServerImage = envs.GetEnvWithPrefix("METRICS_SERVER_IMAGE", conf.MetricsServerImage) + + if conf.MetricsServerBinaryPrefix == "" { + conf.MetricsServerBinaryPrefix = consts.MetricsServerBinaryPrefix + "/" + conf.MetricsServerVersion + } + conf.MetricsServerBinaryPrefix = envs.GetEnvWithPrefix("METRICS_SERVER_BINARY_PREFIX", conf.MetricsServerBinaryPrefix) + + if conf.MetricsServerBinaryPrefix != "" && + conf.MetricsServerBinary == "" { + conf.MetricsServerBinary = conf.MetricsServerBinaryPrefix + "/metrics-server-" + GOOS + "-" + GOARCH + conf.BinSuffix + } + conf.MetricsServerBinary = envs.GetEnvWithPrefix("METRICS_SERVER_BINARY", conf.MetricsServerBinary) +} + // joinImageURI joins the image URI. func joinImageURI(prefix, name, version string) string { return prefix + "/" + name + ":" + version diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index 85d13ddb2..225668ab6 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -61,6 +61,10 @@ var ( JaegerBinaryPrefix = "https://github.com/jaegertracing/jaeger/releases/download" JaegerImagePrefix = "docker.io/jaegertracing" + MetricsServerVersion = "0.6.3" + MetricsServerBinaryPrefix = "https://github.com/kubernetes-sigs/metrics-server/releases/download" + MetricsServerImagePrefix = "registry.k8s.io/metrics-server" + DefaultUnlimitedQPS = 5000.0 DefaultUnlimitedBurst = 10000 ) diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index b9c548f1c..e1850eba2 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -68,6 +68,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Options.KubeSchedulerConfig, "kube-scheduler-config", flags.Options.KubeSchedulerConfig, `Path to a kube-scheduler configuration file`) cmd.Flags().BoolVar(&flags.Options.DisableKubeScheduler, "disable-kube-scheduler", flags.Options.DisableKubeScheduler, `Disable the kube-scheduler`) cmd.Flags().BoolVar(&flags.Options.DisableKubeControllerManager, "disable-kube-controller-manager", flags.Options.DisableKubeControllerManager, `Disable the kube-controller-manager`) + cmd.Flags().BoolVar(&flags.Options.EnableMetricsServer, "enable-metrics-server", flags.Options.EnableMetricsServer, `Enable the metrics-server`) cmd.Flags().StringVar(&flags.Options.EtcdImage, "etcd-image", flags.Options.EtcdImage, `Image of etcd, only for docker/podman/nerdctl runtime '${KWOK_KUBE_IMAGE_PREFIX}/etcd:${KWOK_ETCD_VERSION}' `) @@ -85,6 +86,9 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().Uint32Var(&flags.Options.KubeSchedulerPort, "kube-scheduler-port", flags.Options.KubeSchedulerPort, `Port of kube-scheduler given to the host, only for binary and docker/podman/nerdctl runtime`) cmd.Flags().StringVar(&flags.Options.KwokControllerImage, "kwok-controller-image", flags.Options.KwokControllerImage, `Image of kwok-controller, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_IMAGE_PREFIX}/kwok:${KWOK_VERSION}' +`) + cmd.Flags().StringVar(&flags.Options.MetricsServerImage, "metrics-server-image", flags.Options.MetricsServerImage, `Image of metrics-server, only for docker/podman/nerdctl/kind/kind-podman runtime +'${KWOK_METRICS_SERVER_IMAGE_PREFIX}/metrics-server:${KWOK_METRICS_SERVER_VERSION}' `) cmd.Flags().StringVar(&flags.Options.PrometheusImage, "prometheus-image", flags.Options.PrometheusImage, `Image of Prometheus, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_PROMETHEUS_IMAGE_PREFIX}/prometheus:${KWOK_PROMETHEUS_VERSION}' @@ -111,6 +115,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Options.EtcdBinary, "etcd-binary", flags.Options.EtcdBinary, `Binary of etcd, only for binary runtime`) cmd.Flags().StringVar(&flags.Options.EtcdBinaryTar, "etcd-binary-tar", flags.Options.EtcdBinaryTar, `Tar of etcd, if --etcd-binary is set, this is ignored, only for binary runtime `) + cmd.Flags().StringVar(&flags.Options.MetricsServerBinary, "metrics-server-binary", flags.Options.MetricsServerBinary, `Binary of metrics-server, only for binary runtime`) cmd.Flags().StringVar(&flags.Options.PrometheusBinary, "prometheus-binary", flags.Options.PrometheusBinary, `Binary of Prometheus, only for binary runtime`) cmd.Flags().StringVar(&flags.Options.PrometheusBinaryTar, "prometheus-binary-tar", flags.Options.PrometheusBinaryTar, `Tar of Prometheus, if --prometheus-binary is set, this is ignored, only for binary runtime `) @@ -283,6 +288,10 @@ func runE(ctx context.Context, flags *flagpole) error { if err != nil { return fmt.Errorf("failed to init crds %q: %w", name, err) } + err = rt.InitCRs(ctx) + if err != nil { + return fmt.Errorf("failed to init crs %q: %w", name, err) + } // Wait for cluster to be ready if flags.Wait > 0 { diff --git a/pkg/kwokctl/cmd/logs/logs.go b/pkg/kwokctl/cmd/logs/logs.go index 5d7797d24..d415e2bd7 100644 --- a/pkg/kwokctl/cmd/logs/logs.go +++ b/pkg/kwokctl/cmd/logs/logs.go @@ -41,7 +41,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd := &cobra.Command{ Use: "logs [command]", - Short: "Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, prometheus, jaeger]", + Short: "Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger]", RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { return cmd.Help() diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index 3bacb5c68..6a23fdaf3 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -172,6 +172,8 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon "--service-account-key-file=/etc/kubernetes/pki/admin.key", "--service-account-signing-key-file=/etc/kubernetes/pki/admin.key", "--service-account-issuer=https://kubernetes.default.svc.cluster.local", + "--proxy-client-key-file=/etc/kubernetes/pki/admin.key", + "--proxy-client-cert-file=/etc/kubernetes/pki/admin.crt", ) } else { kubeApiserverArgs = append(kubeApiserverArgs, @@ -183,6 +185,8 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon "--service-account-key-file="+conf.AdminKeyPath, "--service-account-signing-key-file="+conf.AdminKeyPath, "--service-account-issuer=https://kubernetes.default.svc.cluster.local", + "--proxy-client-key-file="+conf.AdminKeyPath, + "--proxy-client-cert-file="+conf.AdminCertPath, ) } } else { diff --git a/pkg/kwokctl/components/metric-server.go b/pkg/kwokctl/components/metric-server.go new file mode 100644 index 000000000..f57c6098a --- /dev/null +++ b/pkg/kwokctl/components/metric-server.go @@ -0,0 +1,135 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/version" +) + +// BuildMetricsServerComponentConfig is the configuration for building a metrics server component. +type BuildMetricsServerComponentConfig struct { + Binary string + Image string + Version version.Version + Workdir string + BindAddress string + Port uint32 + CaCertPath string + AdminCertPath string + AdminKeyPath string + KubeconfigPath string + Verbosity log.Level + ExtraArgs []internalversion.ExtraArgs + ExtraVolumes []internalversion.Volume + ExtraEnvs []internalversion.Env +} + +// BuildMetricsServerComponent builds a metrics server component. +func BuildMetricsServerComponent(conf BuildMetricsServerComponentConfig) (component internalversion.Component, err error) { + metricsServerArgs := []string{ + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--kubelet-use-node-status-port", + "--metric-resolution=15s", + } + metricsServerArgs = append(metricsServerArgs, extraArgsToStrings(conf.ExtraArgs)...) + + inContainer := conf.Image != "" + user := "" + var volumes []internalversion.Volume + volumes = append(volumes, conf.ExtraVolumes...) + var ports []internalversion.Port + if inContainer { + volumes = append(volumes, + internalversion.Volume{ + HostPath: conf.KubeconfigPath, + MountPath: "/root/.kube/config", + ReadOnly: true, + }, + internalversion.Volume{ + HostPath: conf.CaCertPath, + MountPath: "/etc/kubernetes/pki/ca.crt", + ReadOnly: true, + }, + internalversion.Volume{ + HostPath: conf.AdminCertPath, + MountPath: "/etc/kubernetes/pki/admin.crt", + ReadOnly: true, + }, + internalversion.Volume{ + HostPath: conf.AdminKeyPath, + MountPath: "/etc/kubernetes/pki/admin.key", + ReadOnly: true, + }, + ) + + metricsServerArgs = append(metricsServerArgs, + "--bind-address="+conf.BindAddress, + "--secure-port=4443", + "--kubeconfig=/root/.kube/config", + "--authentication-kubeconfig=/root/.kube/config", + "--authorization-kubeconfig=/root/.kube/config", + "--tls-cert-file=/etc/kubernetes/pki/admin.crt", + "--tls-private-key-file=/etc/kubernetes/pki/admin.key", + ) + if conf.Port != 0 { + ports = []internalversion.Port{ + { + HostPort: conf.Port, + Port: 4443, + }, + } + } + user = "root" + } else { + metricsServerArgs = append(metricsServerArgs, + "--bind-address="+conf.BindAddress, + "--secure-port="+format.String(conf.Port), + "--kubeconfig="+conf.KubeconfigPath, + "--authentication-kubeconfig="+conf.KubeconfigPath, + "--authorization-kubeconfig="+conf.KubeconfigPath, + "--tls-cert-file="+conf.AdminCertPath, + "--tls-private-key-file="+conf.AdminKeyPath, + ) + } + + if conf.Verbosity != log.LevelInfo { + metricsServerArgs = append(metricsServerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity))) + } + + envs := []internalversion.Env{} + envs = append(envs, conf.ExtraEnvs...) + + return internalversion.Component{ + Name: "metrics-server", + Version: conf.Version.String(), + Links: []string{ + "kwok-controller", + }, + Command: []string{"/metrics-server"}, + User: user, + Ports: ports, + Volumes: volumes, + Args: metricsServerArgs, + Binary: conf.Binary, + Image: conf.Image, + WorkDir: conf.Workdir, + Envs: envs, + }, nil +} diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 441912d92..c704b6cc6 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -17,6 +17,7 @@ limitations under the License. package binary import ( + "bytes" "context" "fmt" "io" @@ -34,6 +35,7 @@ import ( "sigs.k8s.io/kwok/pkg/kwokctl/dryrun" "sigs.k8s.io/kwok/pkg/kwokctl/k8s" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/kwokctl/snapshot" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/exec" "sigs.k8s.io/kwok/pkg/utils/file" @@ -102,6 +104,14 @@ func (c *Cluster) download(ctx context.Context, env *env) error { return err } + if conf.EnableMetricsServer { + metricsServerPath := c.GetBinPath("metrics-server" + conf.BinSuffix) + err = c.DownloadWithCache(ctx, conf.CacheDir, conf.MetricsServerBinary, metricsServerPath, 0750, conf.QuietPull) + if err != nil { + return err + } + } + etcdPath := c.GetBinPath(consts.ComponentEtcd + conf.BinSuffix) if conf.EtcdBinary == "" { err = c.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.EtcdBinaryTar, etcdPath, consts.ComponentEtcd+conf.BinSuffix, 0750, conf.QuietPull, true) @@ -355,6 +365,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.addMetricsServer(ctx, env) + if err != nil { + return err + } + err = c.addPrometheus(ctx, env) if err != nil { return err @@ -615,9 +630,49 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { +func (c *Cluster) addMetricsServer(ctx context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options + if conf.EnableMetricsServer { + metricsServerPath := c.GetBinPath("metrics-server" + conf.BinSuffix) + metricsServerVersion, err := c.ParseVersionFromBinary(ctx, metricsServerPath) + if err != nil { + return err + } + + err = c.setupPorts(ctx, + &conf.MetricsServerPort, + ) + if err != nil { + return err + } + + metricsServerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, "metrics-server") + metricsServerComponent, err := components.BuildMetricsServerComponent(components.BuildMetricsServerComponentConfig{ + Workdir: env.workdir, + Binary: metricsServerPath, + Version: metricsServerVersion, + BindAddress: conf.BindAddress, + Port: conf.MetricsServerPort, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, + KubeconfigPath: env.kubeconfigPath, + Verbosity: env.verbosity, + ExtraArgs: metricsServerComponentPatches.ExtraArgs, + ExtraVolumes: metricsServerComponentPatches.ExtraVolumes, + ExtraEnvs: metricsServerComponentPatches.ExtraEnvs, + }) + if err != nil { + return err + } + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, metricsServerComponent) + } + return nil +} + +func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options // Configure the prometheus if conf.PrometheusPort != 0 { prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) @@ -1078,7 +1133,7 @@ func (c *Cluster) ListBinaries(ctx context.Context) ([]string, error) { } conf := &config.Options - return []string{ + list := []string{ conf.EtcdBinaryTar, conf.KubeApiserverBinary, conf.KubeControllerManagerBinary, @@ -1086,7 +1141,11 @@ func (c *Cluster) ListBinaries(ctx context.Context) ([]string, error) { conf.KwokControllerBinary, conf.PrometheusBinaryTar, conf.KubectlBinary, - }, nil + } + if conf.MetricsServerBinary != "" { + list = append(list, conf.MetricsServerBinary) + } + return list, nil } // ListImages list images in the cluster @@ -1153,3 +1212,43 @@ func (c *Cluster) WaitReady(ctx context.Context, timeout time.Duration) error { } return nil } + +// InitCRs initializes the CRs. +func (c *Cluster) InitCRs(ctx context.Context) error { + config, err := c.Config(ctx) + if err != nil { + return err + } + conf := config.Options + + if c.IsDryRun() { + if conf.EnableMetricsServer { + dryrun.PrintMessage("# Set up apiservice for metrics server") + } + + return nil + } + + buf := bytes.NewBuffer(nil) + if conf.EnableMetricsServer { + apiserve, err := BuildMetricsServerAPIService(BuildMetricsServerAPIServiceConfig{ + Port: conf.MetricsServerPort, + }) + if err != nil { + return err + } + _, _ = buf.WriteString(apiserve) + _, _ = buf.WriteString("---\n") + } + + if buf.Len() == 0 { + return nil + } + + clientset, err := c.GetClientset(ctx) + if err != nil { + return err + } + + return snapshot.Load(ctx, clientset, bytes.NewBuffer(buf.Bytes()), nil) +} diff --git a/pkg/kwokctl/runtime/binary/metrics_server_apiservice.go b/pkg/kwokctl/runtime/binary/metrics_server_apiservice.go new file mode 100644 index 000000000..39b66d1a1 --- /dev/null +++ b/pkg/kwokctl/runtime/binary/metrics_server_apiservice.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binary + +import ( + "bytes" + "fmt" + "text/template" + + _ "embed" +) + +//go:embed metrics_server_apiservice.yaml.tpl +var metricsServerAPIServiceYamlTpl string + +var metricsServerAPIServiceYamlTemplate = template.Must(template.New("_").Parse(metricsServerAPIServiceYamlTpl)) + +// BuildMetricsServerAPIService builds the metrics server apiservice yaml content. +func BuildMetricsServerAPIService(conf BuildMetricsServerAPIServiceConfig) (string, error) { + buf := bytes.NewBuffer(nil) + err := metricsServerAPIServiceYamlTemplate.Execute(buf, conf) + if err != nil { + return "", fmt.Errorf("failed to execute metrics server apiservice yaml template: %w", err) + } + return buf.String(), nil +} + +// BuildMetricsServerAPIServiceConfig is the config for BuildMetricsServerAPIService. +type BuildMetricsServerAPIServiceConfig struct { + Port uint32 +} diff --git a/pkg/kwokctl/runtime/binary/metrics_server_apiservice.yaml.tpl b/pkg/kwokctl/runtime/binary/metrics_server_apiservice.yaml.tpl new file mode 100644 index 000000000..768657bdf --- /dev/null +++ b/pkg/kwokctl/runtime/binary/metrics_server_apiservice.yaml.tpl @@ -0,0 +1,27 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + port: {{ .Port }} + version: v1beta1 + versionPriority: 100 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + externalName: localhost + type: ExternalName diff --git a/pkg/kwokctl/runtime/cluster.go b/pkg/kwokctl/runtime/cluster.go index 22b434bb4..ddef0f708 100644 --- a/pkg/kwokctl/runtime/cluster.go +++ b/pkg/kwokctl/runtime/cluster.go @@ -58,6 +58,7 @@ var ( KindName = "kind.yaml" KwokPod = "kwok-controller-pod.yaml" DashboardDeploy = "dashboard-deployment.yaml" + MetricsServerDeploy = "metrics-server-deployment.yaml" PrometheusDeploy = "prometheus-deployment.yaml" JaegerDeploy = "jaeger-deployment.yaml" AuditPolicyName = "audit.yaml" @@ -543,6 +544,11 @@ func (c *Cluster) InitCRDs(ctx context.Context) error { return nil } + if c.IsDryRun() { + dryrun.PrintMessage("# Init CRDs %s", strings.Join(crds, ",")) + return nil + } + clientset, err := c.GetClientset(ctx) if err != nil { return err diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 32a3630ef..97d40a12d 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -17,6 +17,7 @@ limitations under the License. package compose import ( + "bytes" "context" "fmt" "io" @@ -31,6 +32,7 @@ import ( "sigs.k8s.io/kwok/pkg/kwokctl/dryrun" "sigs.k8s.io/kwok/pkg/kwokctl/k8s" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/kwokctl/snapshot" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/envs" "sigs.k8s.io/kwok/pkg/utils/exec" @@ -153,6 +155,9 @@ func (c *Cluster) pullAllImages(ctx context.Context, env *env) error { if conf.JaegerPort != 0 { images = append(images, conf.JaegerImage) } + if conf.EnableMetricsServer { + images = append(images, conf.MetricsServerImage) + } err := c.PullImages(ctx, c.runtime, images, conf.QuietPull) if err != nil { return err @@ -165,6 +170,7 @@ func (c *Cluster) setup(ctx context.Context, env *env) error { if !file.Exists(env.pkiPath) { sans := []string{ c.Name() + "-kube-apiserver", + c.Name() + "-kwok-controller", } ips, err := net.GetAllIPs() if err != nil { @@ -363,6 +369,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.addMetricsServer(ctx, env) + if err != nil { + return err + } + err = c.addPrometheus(ctx, env) if err != nil { return err @@ -579,7 +590,6 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options - // Configure the kwok-controller kwokControllerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KwokControllerImage, "kwok") if err != nil { @@ -619,6 +629,39 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return nil } +func (c *Cluster) addMetricsServer(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options + + if conf.EnableMetricsServer { + metricsServerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.MetricsServerImage, "metrics-server") + if err != nil { + return err + } + + metricsServerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, "metrics-server") + metricsServerComponent, err := components.BuildMetricsServerComponent(components.BuildMetricsServerComponentConfig{ + Workdir: env.workdir, + Image: conf.MetricsServerImage, + Version: metricsServerVersion, + BindAddress: conf.BindAddress, + Port: conf.MetricsServerPort, + CaCertPath: env.caCertPath, + AdminCertPath: env.adminCertPath, + AdminKeyPath: env.adminKeyPath, + KubeconfigPath: env.inClusterOnHostKubeconfigPath, + Verbosity: env.verbosity, + ExtraArgs: metricsServerComponentPatches.ExtraArgs, + ExtraVolumes: metricsServerComponentPatches.ExtraVolumes, + ExtraEnvs: metricsServerComponentPatches.ExtraEnvs, + }) + if err != nil { + return err + } + env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, metricsServerComponent) + } + return nil +} + func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options @@ -1105,6 +1148,7 @@ func (c *Cluster) ListImages(ctx context.Context) ([]string, error) { conf.KubeSchedulerImage, conf.KwokControllerImage, conf.PrometheusImage, + conf.MetricsServerImage, }, nil } @@ -1165,3 +1209,43 @@ func (c *Cluster) WaitReady(ctx context.Context, timeout time.Duration) error { } return nil } + +// InitCRs initializes the CRs. +func (c *Cluster) InitCRs(ctx context.Context) error { + config, err := c.Config(ctx) + if err != nil { + return err + } + conf := config.Options + + if c.IsDryRun() { + if conf.EnableMetricsServer { + dryrun.PrintMessage("# Set up apiservice for metrics server") + } + + return nil + } + + buf := bytes.NewBuffer(nil) + if conf.EnableMetricsServer { + apiserve, err := BuildMetricsServerAPIService(BuildMetricsServerAPIServiceConfig{ + Name: c.Name(), + }) + if err != nil { + return err + } + _, _ = buf.WriteString(apiserve) + _, _ = buf.WriteString("---\n") + } + + if buf.Len() == 0 { + return nil + } + + clientset, err := c.GetClientset(ctx) + if err != nil { + return err + } + + return snapshot.Load(ctx, clientset, bytes.NewBuffer(buf.Bytes()), nil) +} diff --git a/pkg/kwokctl/runtime/compose/metrics_server_apiservice.go b/pkg/kwokctl/runtime/compose/metrics_server_apiservice.go new file mode 100644 index 000000000..2ef2b5cd0 --- /dev/null +++ b/pkg/kwokctl/runtime/compose/metrics_server_apiservice.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compose + +import ( + "bytes" + "fmt" + "text/template" + + _ "embed" +) + +//go:embed metrics_server_apiservice.yaml.tpl +var metricsServerAPIServiceYamlTpl string + +var metricsServerAPIServiceYamlTemplate = template.Must(template.New("_").Parse(metricsServerAPIServiceYamlTpl)) + +// BuildMetricsServerAPIService builds the metrics server apiservice yaml content. +func BuildMetricsServerAPIService(conf BuildMetricsServerAPIServiceConfig) (string, error) { + buf := bytes.NewBuffer(nil) + err := metricsServerAPIServiceYamlTemplate.Execute(buf, conf) + if err != nil { + return "", fmt.Errorf("failed to execute metrics server apiservice yaml template: %w", err) + } + return buf.String(), nil +} + +// BuildMetricsServerAPIServiceConfig is the config for BuildMetricsServerAPIService. +type BuildMetricsServerAPIServiceConfig struct { + Name string +} diff --git a/pkg/kwokctl/runtime/compose/metrics_server_apiservice.yaml.tpl b/pkg/kwokctl/runtime/compose/metrics_server_apiservice.yaml.tpl new file mode 100644 index 000000000..b1a2a0563 --- /dev/null +++ b/pkg/kwokctl/runtime/compose/metrics_server_apiservice.yaml.tpl @@ -0,0 +1,27 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + port: 4443 + version: v1beta1 + versionPriority: 100 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + externalName: {{ .Name }}-metrics-server + type: ExternalName diff --git a/pkg/kwokctl/runtime/config.go b/pkg/kwokctl/runtime/config.go index 3c3b72d0d..8c65e9043 100644 --- a/pkg/kwokctl/runtime/config.go +++ b/pkg/kwokctl/runtime/config.go @@ -125,6 +125,9 @@ type Runtime interface { // InitCRDs init the crds of cluster InitCRDs(ctx context.Context) error + // InitCRs init the crs of cluster + InitCRs(ctx context.Context) error + // IsDryRun returns true if the runtime is in dry-run mode IsDryRun() bool } diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index ff10fc51d..30ca80b00 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -137,6 +137,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.addMetricsServer(ctx, env) + if err != nil { + return err + } + err = c.addPrometheus(ctx, env) if err != nil { return err @@ -312,9 +317,32 @@ func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) { return nil } -func (c *Cluster) addPrometheus(_ context.Context, env *env) (err error) { +func (c *Cluster) addMetricsServer(_ context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options + if conf.EnableMetricsServer { + metricsServerPatches := runtime.GetComponentPatches(env.kwokctlConfig, "metrics-server") + metricsServerConf := BuildMetricsServerDeploymentConfig{ + MetricsServerImage: conf.MetricsServerImage, + Name: c.Name(), + Verbosity: log.ToKlogLevel(env.verbosity), + ExtraArgs: metricsServerPatches.ExtraArgs, + ExtraVolumes: metricsServerPatches.ExtraVolumes, + ExtraEnvs: metricsServerPatches.ExtraEnvs, + } + metricsServerDeploy, err := BuildMetricsServerDeployment(metricsServerConf) + if err != nil { + return err + } + err = c.WriteFile(c.GetWorkdirPath(runtime.MetricsServerDeploy), []byte(metricsServerDeploy)) + if err != nil { + return fmt.Errorf("failed to write %s: %w", runtime.MetricsServerDeploy, err) + } + } + return nil +} +func (c *Cluster) addPrometheus(_ context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options if conf.PrometheusPort != 0 { prometheusPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) prometheusConf := BuildPrometheusDeploymentConfig{ @@ -410,6 +438,14 @@ func (c *Cluster) Up(ctx context.Context) error { ) } + if conf.EnableMetricsServer { + config.Components = append(config.Components, + internalversion.Component{ + Name: "metrics-server", + }, + ) + } + if !conf.DisableKubeScheduler { config.Components = append(config.Components, internalversion.Component{ @@ -519,6 +555,13 @@ func (c *Cluster) Up(ctx context.Context) error { } } + if conf.EnableMetricsServer { + err = c.Kubectl(exec.WithAllWriteToErrOut(ctx), "apply", "-f", c.GetWorkdirPath(runtime.MetricsServerDeploy)) + if err != nil { + return err + } + } + if conf.PrometheusPort != 0 { err = c.Kubectl(exec.WithAllWriteToErrOut(ctx), "apply", "-f", c.GetWorkdirPath(runtime.PrometheusDeploy)) if err != nil { @@ -564,6 +607,9 @@ func (c *Cluster) pullAllImages(ctx context.Context, env *env) error { if conf.DashboardPort != 0 { images = append(images, conf.DashboardImage) } + if conf.EnableMetricsServer { + images = append(images, conf.MetricsServerImage) + } if conf.PrometheusPort != 0 { images = append(images, conf.PrometheusImage) } @@ -592,6 +638,9 @@ func (c *Cluster) loadAllImages(ctx context.Context) error { if conf.DashboardPort != 0 { images = append(images, conf.DashboardImage) } + if conf.EnableMetricsServer { + images = append(images, conf.MetricsServerImage) + } if conf.PrometheusPort != 0 { images = append(images, conf.PrometheusImage) } @@ -1053,6 +1102,7 @@ func (c *Cluster) ListImages(ctx context.Context) ([]string, error) { conf.KindNodeImage, conf.KwokControllerImage, conf.PrometheusImage, + conf.MetricsServerImage, }, nil } @@ -1095,3 +1145,8 @@ func (c *Cluster) preDownloadKind(ctx context.Context) (string, error) { return "kind", nil } + +// InitCRs initializes the CRs. +func (c *Cluster) InitCRs(ctx context.Context) error { + return nil +} diff --git a/pkg/kwokctl/runtime/kind/metrics_server_deployment.go b/pkg/kwokctl/runtime/kind/metrics_server_deployment.go new file mode 100644 index 000000000..2e320017b --- /dev/null +++ b/pkg/kwokctl/runtime/kind/metrics_server_deployment.go @@ -0,0 +1,60 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "bytes" + "fmt" + "text/template" + + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + + _ "embed" +) + +//go:embed metrics_server_deployment.yaml.tpl +var metricsServerDeploymentYamlTpl string + +var metricsServerDeploymentYamlTemplate = template.Must(template.New("_").Parse(metricsServerDeploymentYamlTpl)) + +// BuildMetricsServerDeployment builds the metrics server deployment yaml content. +func BuildMetricsServerDeployment(conf BuildMetricsServerDeploymentConfig) (string, error) { + buf := bytes.NewBuffer(nil) + + var err error + conf.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(conf.ExtraVolumes) + if err != nil { + return "", fmt.Errorf("failed to expand host volume paths: %w", err) + } + + err = metricsServerDeploymentYamlTemplate.Execute(buf, conf) + if err != nil { + return "", fmt.Errorf("failed to execute metrics server deployment yaml template: %w", err) + } + return buf.String(), nil +} + +// BuildMetricsServerDeploymentConfig is the configuration for building the metrics server deployment +type BuildMetricsServerDeploymentConfig struct { + MetricsServerImage string + Name string + Verbosity int + ExtraArgs []internalversion.ExtraArgs + ExtraVolumes []internalversion.Volume + ExtraEnvs []internalversion.Env +} diff --git a/pkg/kwokctl/runtime/kind/metrics_server_deployment.yaml.tpl b/pkg/kwokctl/runtime/kind/metrics_server_deployment.yaml.tpl new file mode 100644 index 000000000..ed100206f --- /dev/null +++ b/pkg/kwokctl/runtime/kind/metrics_server_deployment.yaml.tpl @@ -0,0 +1,196 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + - --v={{ .Verbosity }} + image: {{ .MetricsServerImage }} + imagePullPolicy: IfNotPresent + {{ with .ExtraEnvs }} + env: + {{ range . }} + - name: {{ .Name }} + value: {{ .Value }} + {{ end }} + {{ end }} + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + {{ range .ExtraVolumes }} + - mountPath: {{ .MountPath }} + name: {{ .Name }} + readOnly: {{ .ReadOnly }} + {{ end }} + hostNetwork: true + restartPolicy: Always + serviceAccountName: metrics-server + nodeName: {{ .Name }}-control-plane + volumes: + - emptyDir: {} + name: tmp-dir + {{ range .ExtraVolumes }} + - hostPath: + path: /var/components/controller{{ .MountPath }} + type: {{ .PathType }} + name: {{ .Name }} + {{ end }} +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index c4eb9fa80..5b76527e2 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -2222,6 +2222,17 @@ is the default value for env KWOK_JAEGER_VERSION

+metricsServerVersion + +string + + + +

MetricsServerVersion is the version of metrics-server to use.

+ + + + dockerComposeVersion string @@ -2308,6 +2319,17 @@ is the default value for flag –disable-kube-controller-manager and env KWO +enableMetricsServer + +bool + + + +

EnableMetricsServer is the flag to enable metrics-server.

+ + + + kubeImagePrefix string @@ -2379,6 +2401,17 @@ is the default value for env KWOK_JAEGER_IMAGE_PREFIX

+metricsServerImagePrefix + +string + + + +

MetricsServerImagePrefix is the prefix of the metrics-server image.

+ + + + etcdImage string @@ -2474,6 +2507,17 @@ is the default value for flag –jaeger-image and env KWOK_JAEGER_IMAGE

+metricsServerImage + +string + + + +

MetricsServerImage is the image of metrics-server.

+ + + + kindNodeImagePrefix string @@ -2702,6 +2746,28 @@ is the default value for env KWOK_JAEGER_TAR

+metricsServerBinaryPrefix + +string + + + +

MetricsServerBinaryPrefix is the prefix of the metrics-server binary.

+ + + + +metricsServerBinary + +string + + + +

MetricsServerBinary is the binary of metrics-server.

+ + + + dockerComposeBinaryPrefix string @@ -2891,6 +2957,17 @@ is the default value for flag –controller-port and env KWOK_CONTROLLER_POR +metricsServerPort + +uint32 + + + +

MetricsServerPort is metrics-server port that is exposed to the host.

+ + + + cacheDir string diff --git a/site/content/en/docs/generated/kwokctl.md b/site/content/en/docs/generated/kwokctl.md index da3f33e1e..b46e4004f 100644 --- a/site/content/en/docs/generated/kwokctl.md +++ b/site/content/en/docs/generated/kwokctl.md @@ -25,7 +25,7 @@ kwokctl [command] [flags] * [kwokctl export](kwokctl_export.md) - Exports one of [logs] * [kwokctl get](kwokctl_get.md) - Gets one of [artifacts, clusters, kubeconfig] * [kwokctl kubectl](kwokctl_kubectl.md) - kubectl in cluster -* [kwokctl logs](kwokctl_logs.md) - Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, prometheus, jaeger] +* [kwokctl logs](kwokctl_logs.md) - Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger] * [kwokctl scale](kwokctl_scale.md) - Scale a resource in cluster * [kwokctl snapshot](kwokctl_snapshot.md) - Snapshot [save, restore, export] one of cluster * [kwokctl start](kwokctl_start.md) - Start one of [cluster] diff --git a/site/content/en/docs/generated/kwokctl_create_cluster.md b/site/content/en/docs/generated/kwokctl_create_cluster.md index 8ee3947f2..2ddad824b 100644 --- a/site/content/en/docs/generated/kwokctl_create_cluster.md +++ b/site/content/en/docs/generated/kwokctl_create_cluster.md @@ -18,6 +18,7 @@ kwokctl create cluster [flags] --disable-kube-scheduler Disable the kube-scheduler --disable-qps-limits Disable QPS limits for components --enable-crds strings List of CRDs to enable + --enable-metrics-server Enable the metrics-server --etcd-binary string Binary of etcd, only for binary runtime --etcd-binary-tar string Tar of etcd, if --etcd-binary is set, this is ignored, only for binary runtime (default "https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9-linux-amd64.tar.gz") @@ -68,6 +69,10 @@ kwokctl create cluster [flags] --kwok-controller-image string Image of kwok-controller, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_IMAGE_PREFIX}/kwok:${KWOK_VERSION}' (default "registry.k8s.io/kwok/kwok:v0.4.0") + --metrics-server-binary string Binary of metrics-server, only for binary runtime (default "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.3/metrics-server-linux-amd64") + --metrics-server-image string Image of metrics-server, only for docker/podman/nerdctl/kind/kind-podman runtime + '${KWOK_METRICS_SERVER_IMAGE_PREFIX}/metrics-server:${KWOK_METRICS_SERVER_VERSION}' + (default "registry.k8s.io/metrics-server/metrics-server:v0.6.3") --prometheus-binary string Binary of Prometheus, only for binary runtime --prometheus-binary-tar string Tar of Prometheus, if --prometheus-binary is set, this is ignored, only for binary runtime (default "https://github.com/prometheus/prometheus/releases/download/v2.44.0/prometheus-2.44.0.linux-amd64.tar.gz") diff --git a/site/content/en/docs/generated/kwokctl_logs.md b/site/content/en/docs/generated/kwokctl_logs.md index f65e29797..2901a8fec 100644 --- a/site/content/en/docs/generated/kwokctl_logs.md +++ b/site/content/en/docs/generated/kwokctl_logs.md @@ -1,6 +1,6 @@ ## kwokctl logs -Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, prometheus, jaeger] +Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger] ``` kwokctl logs [command] [flags] diff --git a/site/content/en/docs/user/kwokctl-metrics-server-binaries.md b/site/content/en/docs/user/kwokctl-metrics-server-binaries.md new file mode 100644 index 000000000..f2d3a9611 --- /dev/null +++ b/site/content/en/docs/user/kwokctl-metrics-server-binaries.md @@ -0,0 +1,33 @@ +--- +title: "Metrics Server Binaries" +--- + +# Metrics Server Binary + +{{< hint "info" >}} + +This document provides details on how to build Kubernetes binaries and run `kwokctl` locally. + +{{< /hint >}} + +## Build + +Building Kubernetes Binaries and setting up `kwokctl` defaults to use them + +``` bash +METRICS_SERVER_VERSION="0.6.3" +SRC_DIR="${HOME}/.kwok/cache/metrics-server/v${METRICS_SERVER_VERSION}" +mkdir -p "${SRC_DIR}" && cd "${SRC_DIR}" && +wget "https://github.com/kubernetes-sigs/metrics-server/archive/refs/tags/v${METRICS_SERVER_VERSION}.tar.gz" -O - | tar xz && +cd metrics-server-${METRICS_SERVER_VERSION} && make GIT_TAG="v${METRICS_SERVER_VERSION}" metrics-server && +cat <> ~/.kwok/kwok.yaml +--- +kind: KwokctlConfiguration +apiVersion: config.kwok.x-k8s.io/v1alpha1 +options: + metricsServerBinary: $(pwd)/metrics-server +--- +EOF +``` + +The binaries will be located in `~/.kwok/cache/metrics-server/${METRICS_SERVER_VERSION}/metrics-server`. diff --git a/test/kwokctl/kwokctl_with_dryrun.test.sh b/test/kwokctl/kwokctl_with_dryrun.test.sh index 032bd5cf1..97ee76ac0 100755 --- a/test/kwokctl/kwokctl_with_dryrun.test.sh +++ b/test/kwokctl/kwokctl_with_dryrun.test.sh @@ -64,7 +64,7 @@ function main() { fi echo "------------------------------" echo "Testing dryrun with verbosity on runtime ${runtime}" - got="$(KWOK_RUNTIME="${runtime}" create_cluster "${name}" "" -v=debug --prometheus-port 9090 --jaeger-port 16686 --dashboard-port 8000 --dry-run | clear_testdata "${name}")" + got="$(KWOK_RUNTIME="${runtime}" create_cluster "${name}" "" -v=debug --enable-metrics-server --prometheus-port 9090 --jaeger-port 16686 --dashboard-port 8000 --dry-run | clear_testdata "${name}")" want="$(<"${DIR}/testdata/${runtime}/create_cluster_with_verbosity.txt")" if [[ "${got}" != "${want}" ]]; then echo "------------------------------" diff --git a/test/kwokctl/kwokctl_workable_test.sh b/test/kwokctl/kwokctl_workable_test.sh index df1b52dca..b1ed3eb2f 100755 --- a/test/kwokctl/kwokctl_workable_test.sh +++ b/test/kwokctl/kwokctl_workable_test.sh @@ -206,9 +206,9 @@ function main() { minor="${minor%.*}" if [[ $minor -lt 22 ]]; then - create_cluster "${name}" "${release}" -v=debug --prometheus-port 9090 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port 8000 + create_cluster "${name}" "${release}" -v=debug --enable-metrics-server --prometheus-port 9090 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port 8000 else - create_cluster "${name}" "${release}" -v=debug --prometheus-port 9090 --jaeger-port 16686 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port 8000 + create_cluster "${name}" "${release}" -v=debug --enable-metrics-server --prometheus-port 9090 --jaeger-port 16686 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port 8000 fi test_workable "${name}" || failed+=("workable_${name}") diff --git a/test/kwokctl/metric-node-resource.yaml b/test/kwokctl/metric-node-resource.yaml new file mode 100644 index 000000000..c3df72b97 --- /dev/null +++ b/test/kwokctl/metric-node-resource.yaml @@ -0,0 +1,86 @@ +kind: Metric +apiVersion: kwok.x-k8s.io/v1alpha1 +metadata: + name: resource-default +spec: + path: "/metrics/nodes/{nodeName}/metrics/resource" + metrics: + - name: scrape_error + dimension: node + help: "[ALPHA] 1 if there was an error while getting metrics from the node, 0 otherwise" + kind: gauge + value: '0' + - name: container_start_time_seconds + dimension: container + help: "[ALPHA] Start time of the container since unix epoch in seconds" + kind: gauge + labels: + - name: container + value: 'container.name' + - name: namespace + value: 'pod.metadata.namespace' + - name: pod + value: 'pod.metadata.name' + value: 'pod.SinceSecond()' + + # CPU + - name: container_cpu_usage_seconds_total + dimension: container + help: "[ALPHA] Cumulative cpu time consumed by the container in core-seconds" + kind: counter + labels: + - name: container + value: 'container.name' + - name: namespace + value: 'pod.metadata.namespace' + - name: pod + value: 'pod.metadata.name' + value: 'pod.SinceSecond() / 10.0' + - name: pod_cpu_usage_seconds_total + dimension: pod + help: "[ALPHA] Cumulative cpu time consumed by the pod in core-seconds" + kind: counter + labels: + - name: namespace + value: 'pod.metadata.namespace' + - name: pod + value: 'pod.metadata.name' + # TODO: sum(container_cpu_usage_seconds_total) by (pod) + value: 'pod.SinceSecond()' + - name: node_cpu_usage_seconds_total + dimension: node + help: "[ALPHA] Cumulative cpu time consumed by the node in core-seconds" + kind: counter + # TODO: sum(pod_cpu_usage_seconds_total) by (node) + value: '10.0 * node.SinceSecond()' + + # Memory + - name: container_memory_working_set_bytes + dimension: container + help: "[ALPHA] Current working set of the container in bytes" + kind: gauge + labels: + - name: container + value: 'container.name' + - name: namespace + value: 'pod.metadata.namespace' + - name: pod + value: 'pod.metadata.name' + value: '1024.0 * 1024.0 * 1024.0' + - name: pod_memory_working_set_bytes + dimension: pod + help: "[ALPHA] Current working set of the pod in bytes" + kind: gauge + labels: + - name: namespace + value: 'pod.metadata.namespace' + - name: pod + value: 'pod.metadata.name' + # TODO: sum(container_memory_working_set_bytes) by (pod) + value: '10.0 * 1024.0 * 1024.0 * 1024.0' + - name: node_memory_working_set_bytes + dimension: node + help: "[ALPHA] Current working set of the node in bytes" + kind: gauge + # TODO: sum(pod_memory_working_set_bytes) by (node) + value: '100.0 * 1024.0 * 1024.0 * 1024.0' diff --git a/test/kwokctl/testdata/binary/create_cluster.txt b/test/kwokctl/testdata/binary/create_cluster.txt index 98bda5717..317260107 100644 --- a/test/kwokctl/testdata/binary/create_cluster.txt +++ b/test/kwokctl/testdata/binary/create_cluster.txt @@ -35,7 +35,7 @@ EOF # Add context kwok- to ~/.kube/config cd /workdir/clusters/ && etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 >/workdir/clusters//logs/etcd.log 2>&1 & echo $! >/workdir/clusters//pids/etcd.pid -cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid diff --git a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt index 955ebff8f..b213078d6 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt @@ -121,7 +121,7 @@ EOF # Add context kwok- to ~/.kube/config cd /workdir/clusters/ && TEST_KEY=TEST_VALUE etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 >/workdir/clusters//logs/etcd.log 2>&1 & echo $! >/workdir/clusters//pids/etcd.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-controller-manager --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid diff --git a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt index 5cb30c042..ec4800369 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt @@ -130,7 +130,7 @@ cd /workdir/clusters/ && etcd --name=node0 --auto-compac echo $! >/workdir/clusters//pids/etcd.pid cd /workdir/clusters/ && jaeger-all-in-one --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:32762 --log-level=debug >/workdir/clusters//logs/jaeger-all-in-one.log 2>&1 & echo $! >/workdir/clusters//pids/jaeger-all-in-one.pid -cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --tracing-config-file=/workdir/clusters//apiserver-tracing-config.yaml --v=4 >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt --tracing-config-file=/workdir/clusters//apiserver-tracing-config.yaml --v=4 >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid diff --git a/test/kwokctl/testdata/binary/start_cluster.txt b/test/kwokctl/testdata/binary/start_cluster.txt index 4025a35ab..0d9d46638 100644 --- a/test/kwokctl/testdata/binary/start_cluster.txt +++ b/test/kwokctl/testdata/binary/start_cluster.txt @@ -1,6 +1,6 @@ cd /workdir/clusters/ && etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 >/workdir/clusters//logs/etcd.log 2>&1 & echo $! >/workdir/clusters//pids/etcd.pid -cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid diff --git a/test/kwokctl/testdata/docker/create_cluster.txt b/test/kwokctl/testdata/docker/create_cluster.txt index 2f961a6c9..710bdbf9c 100644 --- a/test/kwokctl/testdata/docker/create_cluster.txt +++ b/test/kwokctl/testdata/docker/create_cluster.txt @@ -53,7 +53,7 @@ EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index b5e3695ff..c0f0ae3a9 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -140,7 +140,7 @@ EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 diff --git a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt index 20dc4a7b8..36e35a708 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt @@ -11,6 +11,7 @@ docker pull registry.k8s.io/kube-scheduler:v1.28.0 docker pull docker.io/kubernetesui/dashboard:v2.7.0 docker pull docker.io/prom/prometheus:v2.44.0 docker pull docker.io/jaegertracing/all-in-one:1.45.0 +docker pull registry.k8s.io/metrics-server/metrics-server:v0.6.3 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -149,11 +150,12 @@ EOF docker network create kwok- --label=com.docker.compose.project=kwok- docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug docker create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug -docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 +docker create --name=kwok--metrics-server --pull=never --entrypoint=/metrics-server --network=kwok- --user=root --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.6.3 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=~/.kube/config --authentication-kubeconfig=~/.kube/config --authorization-kubeconfig=~/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --v=4 docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config docker start kwok--etcd @@ -163,4 +165,6 @@ docker start kwok--kube-controller-manager docker start kwok--kube-scheduler docker start kwok--kwok-controller docker start kwok--dashboard +docker start kwok--metrics-server docker start kwok--prometheus +# Set up apiservice for metrics server diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt index a2aa7c48b..12a3c2045 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -192,6 +192,186 @@ spec: type: DirectoryOrCreate name: k8s-certs EOF +cat </workdir/clusters//metrics-server-deployment.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + - --v=4 + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + hostNetwork: true + restartPolicy: Always + serviceAccountName: metrics-server + nodeName: kwok--control-plane + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +EOF cat </workdir/clusters//prometheus-deployment.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -369,6 +549,7 @@ EOF podman pull docker.io/kindest/node:v1.28.0 podman pull localhost/kwok:test podman pull docker.io/kubernetesui/dashboard:v2.7.0 +podman pull registry.k8s.io/metrics-server/metrics-server:v0.6.3 podman pull docker.io/prom/prometheus:v2.44.0 podman pull docker.io/jaegertracing/all-in-one:1.45.0 # Save cluster config to /workdir/clusters//kwok.yaml @@ -381,6 +562,10 @@ mkdir -p /workdir/cache/image-archive/docker.io/kubernetesui/dashboard podman save docker.io/kubernetesui/dashboard:v2.7.0 -o /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar --name kwok- rm /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar +mkdir -p /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server +podman save registry.k8s.io/metrics-server/metrics-server:v0.6.3 -o /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server/v0.6.3.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server/v0.6.3.tar --name kwok- +rm /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server/v0.6.3.tar mkdir -p /workdir/cache/image-archive/docker.io/prom/prometheus podman save docker.io/prom/prometheus:v2.44.0 -o /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar --name kwok- @@ -397,6 +582,7 @@ mkdir -p /workdir/clusters//pki podman cp kwok--control-plane:/etc/kubernetes/pki/ca.crt /workdir/clusters//pki/ca.crt podman cp kwok--control-plane:/etc/kubernetes/pki/ca.key /workdir/clusters//pki/ca.key kubectl apply -f /workdir/clusters//dashboard-deployment.yaml +kubectl apply -f /workdir/clusters//metrics-server-deployment.yaml kubectl apply -f /workdir/clusters//prometheus-deployment.yaml kubectl apply -f /workdir/clusters//jaeger-deployment.yaml kubectl cordon kwok--control-plane diff --git a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt index 2cc7f5230..290691100 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt @@ -192,6 +192,186 @@ spec: type: DirectoryOrCreate name: k8s-certs EOF +cat </workdir/clusters//metrics-server-deployment.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + - --v=4 + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + hostNetwork: true + restartPolicy: Always + serviceAccountName: metrics-server + nodeName: kwok--control-plane + volumes: + - emptyDir: {} + name: tmp-dir +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +EOF cat </workdir/clusters//prometheus-deployment.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -369,12 +549,14 @@ EOF docker pull docker.io/kindest/node:v1.28.0 docker pull localhost/kwok:test docker pull docker.io/kubernetesui/dashboard:v2.7.0 +docker pull registry.k8s.io/metrics-server/metrics-server:v0.6.3 docker pull docker.io/prom/prometheus:v2.44.0 docker pull docker.io/jaegertracing/all-in-one:1.45.0 # Save cluster config to /workdir/clusters//kwok.yaml KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.28.0 --wait 29m KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image localhost/kwok:test --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/dashboard:v2.7.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/metrics-server/metrics-server:v0.6.3 --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.44.0 --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/jaegertracing/all-in-one:1.45.0 --name kwok- kubectl config view --minify=true --raw=true @@ -385,6 +567,7 @@ mkdir -p /workdir/clusters//pki docker cp kwok--control-plane:/etc/kubernetes/pki/ca.crt /workdir/clusters//pki/ca.crt docker cp kwok--control-plane:/etc/kubernetes/pki/ca.key /workdir/clusters//pki/ca.key kubectl apply -f /workdir/clusters//dashboard-deployment.yaml +kubectl apply -f /workdir/clusters//metrics-server-deployment.yaml kubectl apply -f /workdir/clusters//prometheus-deployment.yaml kubectl apply -f /workdir/clusters//jaeger-deployment.yaml kubectl cordon kwok--control-plane diff --git a/test/kwokctl/testdata/nerdctl/create_cluster.txt b/test/kwokctl/testdata/nerdctl/create_cluster.txt index c82e87010..33395e5b3 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster.txt @@ -53,7 +53,7 @@ EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index 5c5b4879d..cd8f2205b 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -140,7 +140,7 @@ EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt index 16bb44a35..2be67e096 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -11,6 +11,7 @@ nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 nerdctl pull docker.io/prom/prometheus:v2.44.0 nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 +nerdctl pull registry.k8s.io/metrics-server/metrics-server:v0.6.3 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -149,11 +150,12 @@ EOF nerdctl network create kwok- --label=com.docker.compose.project=kwok- nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug nerdctl create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug -nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 +nerdctl create --name=kwok--metrics-server --pull=never --entrypoint=/metrics-server --network=kwok- --user=root --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.6.3 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=~/.kube/config --authentication-kubeconfig=~/.kube/config --authorization-kubeconfig=~/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --v=4 nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd @@ -163,4 +165,6 @@ nerdctl start kwok--kube-controller-manager nerdctl start kwok--kube-scheduler nerdctl start kwok--kwok-controller nerdctl start kwok--dashboard +nerdctl start kwok--metrics-server nerdctl start kwok--prometheus +# Set up apiservice for metrics server diff --git a/test/kwokctl/testdata/podman/create_cluster.txt b/test/kwokctl/testdata/podman/create_cluster.txt index cf8ffb752..568868c52 100644 --- a/test/kwokctl/testdata/podman/create_cluster.txt +++ b/test/kwokctl/testdata/podman/create_cluster.txt @@ -53,7 +53,7 @@ EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index 6cbc65984..86de3538c 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -140,7 +140,7 @@ EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 diff --git a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt index 67855e287..b9bc03769 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt @@ -11,6 +11,7 @@ podman pull registry.k8s.io/kube-scheduler:v1.28.0 podman pull docker.io/kubernetesui/dashboard:v2.7.0 podman pull docker.io/prom/prometheus:v2.44.0 podman pull docker.io/jaegertracing/all-in-one:1.45.0 +podman pull registry.k8s.io/metrics-server/metrics-server:v0.6.3 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -149,11 +150,12 @@ EOF podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug podman create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug -podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 +podman create --name=kwok--metrics-server --pull=never --entrypoint=/metrics-server --network=kwok- --user=root --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.6.3 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=~/.kube/config --authentication-kubeconfig=~/.kube/config --authorization-kubeconfig=~/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --v=4 podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config podman start kwok--etcd @@ -163,4 +165,6 @@ podman start kwok--kube-controller-manager podman start kwok--kube-scheduler podman start kwok--kwok-controller podman start kwok--dashboard +podman start kwok--metrics-server podman start kwok--prometheus +# Set up apiservice for metrics server