From 5bf44c5aaa46810cd21ce1a20ee8f362f37e7b16 Mon Sep 17 00:00:00 2001 From: Karen Almog Date: Thu, 30 Dec 2021 13:06:03 +0000 Subject: [PATCH] config refactoring: use runtime config Signed-off-by: Karen Almog --- cmd/airgap/listimages.go | 6 +- cmd/api/api.go | 6 - cmd/backup/backup.go | 16 +- cmd/config/validate.go | 4 +- cmd/controller/controller.go | 37 +-- cmd/etcd/etcd.go | 6 - cmd/etcd/leave.go | 11 +- cmd/etcd/list.go | 9 +- cmd/install/install.go | 13 +- cmd/kubeconfig/admin.go | 5 +- cmd/kubeconfig/create.go | 18 +- cmd/kubeconfig/kubeconfig_test.go | 21 +- cmd/reset/reset.go | 6 +- cmd/restore/restore.go | 6 +- cmd/token/create.go | 10 +- docs/airgap-install.md | 4 +- docs/configuration.md | 98 +++---- docs/k0s-in-docker.md | 2 +- examples/footloose-ha-controllers/k0s.yaml | 2 +- .../footloose-mysql/{mke.yaml => k0s.yaml} | 5 +- internal/testutil/runtime_config.go | 156 ++++++++++++ inttest/backup/backup_test.go | 2 +- inttest/common/footloosesuite.go | 4 +- inttest/customports/customports_test.go | 6 +- inttest/externaletcd/external_etcd_test.go | 9 +- .../v1beta1/clusterconfig_types.go | 12 +- .../k0s.k0sproject.io/v1beta1/network_test.go | 2 +- pkg/backup/config.go | 22 +- pkg/backup/manager.go | 32 +-- pkg/cleanup/users.go | 4 +- pkg/component/controller/clusterConfig.go | 7 +- pkg/component/controller/k0scontrolapi.go | 1 - pkg/component/controller/kubeproxy.go | 9 +- pkg/config/api_config.go | 99 ++++++++ pkg/config/cli.go | 35 +++ pkg/config/config.go | 181 ++++++------- pkg/config/config_test.go | 239 ++++++++++++++++++ pkg/config/file_config.go | 148 +++++++++++ 38 files changed, 939 insertions(+), 314 deletions(-) rename examples/footloose-mysql/{mke.yaml => k0s.yaml} (92%) create mode 100644 internal/testutil/runtime_config.go create mode 100644 pkg/config/api_config.go create mode 100644 pkg/config/config_test.go create mode 100644 pkg/config/file_config.go diff --git a/cmd/airgap/listimages.go b/cmd/airgap/listimages.go index fad76fe51ee4..b669905729a5 100644 --- a/cmd/airgap/listimages.go +++ b/cmd/airgap/listimages.go @@ -33,11 +33,7 @@ func NewAirgapListImagesCmd() *cobra.Command { Example: `k0s airgap list-images`, RunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetYamlFromFile(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - uris := airgap.GetImageURIs(cfg.Spec.Images) + uris := airgap.GetImageURIs(c.ClusterConfig.Spec.Images) for _, uri := range uris { fmt.Println(uri) } diff --git a/cmd/api/api.go b/cmd/api/api.go index e8cb993f402a..6c1c021729e2 100644 --- a/cmd/api/api.go +++ b/cmd/api/api.go @@ -63,16 +63,10 @@ func NewAPICmd() *cobra.Command { logrus.SetOutput(os.Stdout) c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - c.NodeConfig = cfg return c.startAPI() }, } cmd.SilenceUsage = true - cmd.Flags().AddFlagSet(config.FileInputFlag()) cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) return cmd } diff --git a/cmd/backup/backup.go b/cmd/backup/backup.go index 3b4bbb470431..acd1d33db1f8 100644 --- a/cmd/backup/backup.go +++ b/cmd/backup/backup.go @@ -43,12 +43,7 @@ func NewBackupCmd() *cobra.Command { Short: "Back-Up k0s configuration. Must be run as root (or with sudo)", RunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetYamlFromFile(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - c.ClusterConfig = cfg - if c.ClusterConfig.Spec.Storage.Etcd.IsExternalClusterUsed() { + if c.NodeConfig.Spec.Storage.Etcd.IsExternalClusterUsed() { return fmt.Errorf("command 'k0s backup' does not support external etcd cluster") } return c.backup() @@ -57,7 +52,6 @@ func NewBackupCmd() *cobra.Command { } cmd.Flags().StringVar(&savePath, "save-path", "", "destination directory path for backup assets") cmd.SilenceUsage = true - cmd.Flags().AddFlagSet(config.FileInputFlag()) cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) return cmd } @@ -86,16 +80,18 @@ func (c *CmdOpts) backup() error { if err != nil { return err } - return mgr.RunBackup(c.CfgFile, c.ClusterConfig.Spec, c.K0sVars, savePath) + return mgr.RunBackup(c.CfgFile, c.NodeConfig.Spec, c.K0sVars, savePath) } return fmt.Errorf("backup command must be run on the controller node, have `%s`", status.Role) } func preRunValidateConfig(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - _, err := config.GetConfigFromYAML(c.CfgFile, c.K0sVars) + + loadingRules := config.ClientConfigLoadingRules{K0sVars: c.K0sVars} + _, err := loadingRules.ParseRuntimeConfig() if err != nil { - return err + return fmt.Errorf("failed to get config: %v", err) } return nil } diff --git a/cmd/config/validate.go b/cmd/config/validate.go index 3c47b5bceab1..2cafc80b6656 100644 --- a/cmd/config/validate.go +++ b/cmd/config/validate.go @@ -28,7 +28,9 @@ func NewValidateCmd() *cobra.Command { k0s config validate --config path_to_config.yaml`, RunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - _, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) + + loadingRules := config.ClientConfigLoadingRules{K0sVars: c.K0sVars} + _, err := loadingRules.ParseRuntimeConfig() return err }, SilenceUsage: true, diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index ba0cec4a5312..1fe41da8ad91 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -91,10 +91,6 @@ func NewControllerCmd() *cobra.Command { } c.TokenArg = string(bytes) } - if c.SingleNode { - c.EnableWorker = true - c.K0sVars.DefaultStorageType = "kine" - } c.Logging = stringmap.Merge(c.CmdLogLevels, c.DefaultLogLevels) cmd.SilenceUsage = true @@ -124,15 +120,25 @@ func (c *CmdOpts) startController(ctx context.Context) error { if err := dir.Init(c.K0sVars.CertRootDir, constant.CertRootDirMode); err != nil { return err } + // let's make sure run-dir exists + if err := dir.Init(c.K0sVars.RunDir, constant.RunDirMode); err != nil { + logrus.Fatalf("failed to initialize dir: %v", err) + } - nodeConfig, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err + // initialize runtime config + loadingRules := config.ClientConfigLoadingRules{Nodeconfig: true} + if err := loadingRules.InitRuntimeConfig(); err != nil { + logrus.Fatalf("failed to initialize k0s runtime config: %s", err.Error()) } - c.NodeConfig = nodeConfig + + // from now on, we only refer to the runtime config + c.CfgFile = loadingRules.RuntimeConfigPath + certificateManager := certificate.Manager{K0sVars: c.K0sVars} var joinClient *token.JoinClient + var err error + if c.TokenArg != "" && c.needToJoin() { joinClient, err = joinController(ctx, c.TokenArg, c.K0sVars.CertRootDir) if err != nil { @@ -282,8 +288,9 @@ func (c *CmdOpts) startController(ctx context.Context) error { // Stop components if stopErr := c.NodeComponents.Stop(); stopErr != nil { logrus.Errorf("error while stopping node component %s", stopErr) + } else { + logrus.Info("all node components stopped") } - logrus.Info("all node components stopped") }() // in-cluster component reconcilers @@ -335,11 +342,7 @@ func (c *CmdOpts) startController(ctx context.Context) error { return err } } else { - fullCfg, err := config.GetYamlFromFile(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - cfgSource, err = clusterconfig.NewStaticSource(fullCfg) + cfgSource, err = clusterconfig.NewStaticSource(c.ClusterConfig) if err != nil { return err } @@ -402,7 +405,7 @@ func (c *CmdOpts) startController(ctx context.Context) error { logrus.Info("Shutting down k0s controller") perfTimer.Output() - return err + return os.Remove(c.CfgFile) } func (c *CmdOpts) startClusterComponents(ctx context.Context) error { @@ -420,7 +423,7 @@ func (c *CmdOpts) startBootstrapReconcilers(ctx context.Context, cf kubernetes.C return err } - cfgReconciler, err := controller.NewClusterConfigReconciler(c.CfgFile, leaderElector, c.K0sVars, c.ClusterComponents, manifestSaver, cf, configSource) + cfgReconciler, err := controller.NewClusterConfigReconciler(leaderElector, c.K0sVars, c.ClusterComponents, manifestSaver, cf, configSource) if err != nil { logrus.Warnf("failed to initialize cluster-config reconciler: %s", err.Error()) return err @@ -472,7 +475,7 @@ func (c *CmdOpts) createClusterReconcilers(ctx context.Context, cf kubernetes.Cl } if !stringslice.Contains(c.DisableComponents, constant.KubeProxyComponentName) { - proxy, err := controller.NewKubeProxy(c.CfgFile, c.K0sVars) + proxy, err := controller.NewKubeProxy(c.CfgFile, c.K0sVars, c.NodeConfig) if err != nil { return fmt.Errorf("failed to initialize kube-proxy reconciler: %s", err.Error()) diff --git a/cmd/etcd/etcd.go b/cmd/etcd/etcd.go index 2fd3e6b6a086..1bc5db505ff7 100644 --- a/cmd/etcd/etcd.go +++ b/cmd/etcd/etcd.go @@ -32,11 +32,6 @@ func NewEtcdCmd() *cobra.Command { Short: "Manage etcd cluster", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - c.ClusterConfig = cfg if c.ClusterConfig.Spec.Storage.Type != v1beta1.EtcdStorageType { return fmt.Errorf("wrong storage type: %s", c.ClusterConfig.Spec.Storage.Type) } @@ -49,7 +44,6 @@ func NewEtcdCmd() *cobra.Command { cmd.SilenceUsage = true cmd.AddCommand(etcdLeaveCmd()) cmd.AddCommand(etcdListCmd()) - cmd.Flags().AddFlagSet(config.FileInputFlag()) cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) return cmd } diff --git a/cmd/etcd/leave.go b/cmd/etcd/leave.go index b2f89377aa01..11b9e8700889 100644 --- a/cmd/etcd/leave.go +++ b/cmd/etcd/leave.go @@ -33,22 +33,16 @@ func etcdLeaveCmd() *cobra.Command { Short: "Sign off a given etc node from etcd cluster", RunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - c.ClusterConfig = cfg - ctx := context.Background() if etcdPeerAddress == "" { - etcdPeerAddress = c.ClusterConfig.Spec.Storage.Etcd.PeerAddress + etcdPeerAddress = c.NodeConfig.Spec.Storage.Etcd.PeerAddress } if etcdPeerAddress == "" { return fmt.Errorf("can't leave etcd cluster: peer address is empty, check the config file or use cli argument") } peerURL := fmt.Sprintf("https://%s:2380", etcdPeerAddress) - etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.ClusterConfig.Spec.Storage.Etcd) + etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.NodeConfig.Spec.Storage.Etcd) if err != nil { return fmt.Errorf("can't connect to the etcd: %v", err) } @@ -76,6 +70,5 @@ func etcdLeaveCmd() *cobra.Command { cmd.Flags().StringVar(&etcdPeerAddress, "peer-address", "", "etcd peer address") cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) - cmd.Flags().AddFlagSet(config.FileInputFlag()) return cmd } diff --git a/cmd/etcd/list.go b/cmd/etcd/list.go index a9859173b7f8..95de0d916fa5 100644 --- a/cmd/etcd/list.go +++ b/cmd/etcd/list.go @@ -33,14 +33,8 @@ func etcdListCmd() *cobra.Command { Short: "Returns etcd cluster members list", RunE: func(cmd *cobra.Command, args []string) error { c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - c.ClusterConfig = cfg - ctx := context.Background() - etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.ClusterConfig.Spec.Storage.Etcd) + etcdClient, err := etcd.NewClient(c.K0sVars.CertRootDir, c.K0sVars.EtcdCertDir, c.NodeConfig.Spec.Storage.Etcd) if err != nil { return fmt.Errorf("can't list etcd cluster members: %v", err) } @@ -51,7 +45,6 @@ func etcdListCmd() *cobra.Command { return json.NewEncoder(os.Stdout).Encode(map[string]interface{}{"members": members}) }, } - cmd.Flags().AddFlagSet(config.FileInputFlag()) cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) return cmd } diff --git a/cmd/install/install.go b/cmd/install/install.go index 695beb95ccef..b6637dab775b 100644 --- a/cmd/install/install.go +++ b/cmd/install/install.go @@ -50,12 +50,7 @@ func (c *CmdOpts) setup(role string, args []string) error { } if role == "controller" { - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err - } - c.ClusterConfig = cfg - if err := install.CreateControllerUsers(c.ClusterConfig, c.K0sVars); err != nil { + if err := install.CreateControllerUsers(c.NodeConfig, c.K0sVars); err != nil { return fmt.Errorf("failed to create controller users: %v", err) } } @@ -97,9 +92,11 @@ func (c *CmdOpts) convertFileParamsToAbsolute() (err error) { func preRunValidateConfig(_ *cobra.Command, _ []string) error { c := CmdOpts(config.GetCmdOpts()) - _, err := config.GetConfigFromYAML(c.CfgFile, c.K0sVars) + + loadingRules := config.ClientConfigLoadingRules{K0sVars: c.K0sVars} + _, err := loadingRules.ParseRuntimeConfig() if err != nil { - return err + return fmt.Errorf("failed to get config: %v", err) } return nil } diff --git a/cmd/kubeconfig/admin.go b/cmd/kubeconfig/admin.go index 9ba4b54db61f..b0751e79285d 100644 --- a/cmd/kubeconfig/admin.go +++ b/cmd/kubeconfig/admin.go @@ -42,10 +42,7 @@ func kubeConfigAdminCmd() *cobra.Command { log.Fatal(err) } - clusterAPIURL, err := c.getAPIURL() - if err != nil { - return fmt.Errorf("failed to fetch cluster's API Address: %w", err) - } + clusterAPIURL := c.NodeConfig.Spec.API.APIAddressURL() newContent := strings.Replace(string(content), "https://localhost:6443", clusterAPIURL, -1) os.Stdout.Write([]byte(newContent)) } else { diff --git a/cmd/kubeconfig/create.go b/cmd/kubeconfig/create.go index f9514fd805e3..94ed4dbea033 100644 --- a/cmd/kubeconfig/create.go +++ b/cmd/kubeconfig/create.go @@ -24,7 +24,6 @@ import ( "path" "github.com/cloudflare/cfssl/log" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/k0sproject/k0s/pkg/certificate" @@ -78,10 +77,8 @@ Note: A certificate once signed cannot be revoked for a particular user`, } username := args[0] c := CmdOpts(config.GetCmdOpts()) - clusterAPIURL, err := c.getAPIURL() - if err != nil { - return fmt.Errorf("failed to fetch cluster's API Address: %w", err) - } + clusterAPIURL := c.NodeConfig.Spec.API.APIAddressURL() + caCert, err := os.ReadFile(path.Join(c.K0sVars.CertRootDir, "ca.crt")) if err != nil { return fmt.Errorf("failed to read cluster ca certificate: %w, check if the control plane is initialized on this node", err) @@ -130,17 +127,6 @@ Note: A certificate once signed cannot be revoked for a particular user`, }, } cmd.Flags().StringVar(&groups, "groups", "", "Specify groups") - cmd.Flags().AddFlagSet(config.FileInputFlag()) cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) return cmd } - -func (c *CmdOpts) getAPIURL() (string, error) { - // Disable logrus - logrus.SetLevel(logrus.WarnLevel) - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return "", err - } - return cfg.Spec.API.APIAddressURL(), nil -} diff --git a/cmd/kubeconfig/kubeconfig_test.go b/cmd/kubeconfig/kubeconfig_test.go index b5e388555d8b..94529866e0fa 100644 --- a/cmd/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeconfig/kubeconfig_test.go @@ -26,9 +26,8 @@ import ( "k8s.io/client-go/tools/clientcmd" "github.com/k0sproject/k0s/internal/pkg/file" - "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/internal/testutil" "github.com/k0sproject/k0s/pkg/certificate" - "github.com/k0sproject/k0s/pkg/config" "github.com/k0sproject/k0s/pkg/constant" ) @@ -47,12 +46,11 @@ spec: api: externalAddress: 10.0.0.86 ` - cfgFilePath, err := file.WriteTmpFile(yamlData, "k0s-config") + configGetter := testutil.NewConfigGetter(yamlData, false, constant.GetConfig("")) + cfg, err := configGetter.FakeConfigFromFile() s.NoError(err) - c := CmdOpts(config.GetCmdOpts()) - c.CfgFile = cfgFilePath - + defer os.Remove(testutil.RuntimeFakePath) caCert := ` -----BEGIN CERTIFICATE----- MIIDADCCAeigAwIBAgIUW+2hawM8HgHrfxmDRV51wOq95icwDQYJKoZIhvcNAQEL @@ -121,12 +119,15 @@ yJm2KSue0toWmkBFK8WMTjAvmAw3Z/qUhJRKoqCu3k6Mf8DNl6t+Uw== certManager := certificate.Manager{ K0sVars: k0sVars, } - err = os.Mkdir(path.Join(k0sVars.CertRootDir), 0755) - userCert, err := certManager.EnsureCertificate(userReq, "root") + pkiPath := path.Join(k0sVars.CertRootDir) + err = os.Mkdir(pkiPath, 0o755) s.NoError(err) - clusterAPIURL, err := c.getAPIURL() + + defer os.RemoveAll(pkiPath) + userCert, err := certManager.EnsureCertificate(userReq, "root") s.NoError(err) + clusterAPIURL := cfg.Spec.API.APIAddressURL() data := struct { CACert string @@ -151,8 +152,6 @@ yJm2KSue0toWmkBFK8WMTjAvmAw3Z/qUhJRKoqCu3k6Mf8DNl6t+Uw== config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) s.NoError(err) s.Equal("https://10.0.0.86:6443", config.Host) - _, err = v1beta1.ConfigFromString(yamlData) - s.NoError(err) } func TestCLITestSuite(t *testing.T) { diff --git a/cmd/reset/reset.go b/cmd/reset/reset.go index 0d33da6a2034..e25f18828a78 100644 --- a/cmd/reset/reset.go +++ b/cmd/reset/reset.go @@ -75,9 +75,11 @@ func (c *CmdOpts) reset() error { func preRunValidateConfig(_ *cobra.Command, _ []string) error { c := CmdOpts(config.GetCmdOpts()) - _, err := config.GetConfigFromYAML(c.CfgFile, c.K0sVars) + + loadingRules := config.ClientConfigLoadingRules{K0sVars: c.K0sVars} + _, err := loadingRules.ParseRuntimeConfig() if err != nil { - return err + return fmt.Errorf("failed to get config: %v", err) } return nil } diff --git a/cmd/restore/restore.go b/cmd/restore/restore.go index e0364437c998..7f10cf9eea2d 100644 --- a/cmd/restore/restore.go +++ b/cmd/restore/restore.go @@ -97,9 +97,11 @@ func (c *CmdOpts) restore(path string) error { // TODO Need to move to some common place, now it's defined in restore and backup commands func preRunValidateConfig(_ *cobra.Command, _ []string) error { c := CmdOpts(config.GetCmdOpts()) - _, err := config.GetConfigFromYAML(c.CfgFile, c.K0sVars) + + loadingRules := config.ClientConfigLoadingRules{K0sVars: c.K0sVars} + _, err := loadingRules.ParseRuntimeConfig() if err != nil { - return err + return fmt.Errorf("failed to get config: %v", err) } return nil } diff --git a/cmd/token/create.go b/cmd/token/create.go index c2c6cc24ceec..1993d249cca8 100644 --- a/cmd/token/create.go +++ b/cmd/token/create.go @@ -42,10 +42,6 @@ k0s token create --role worker --expiry 10m //sets expiration time to 10 minute // Disable logrus for token commands logrus.SetLevel(logrus.WarnLevel) c := CmdOpts(config.GetCmdOpts()) - cfg, err := config.GetNodeConfig(c.CfgFile, c.K0sVars) - if err != nil { - return err - } expiry, err := time.ParseDuration(tokenExpiry) if err != nil { return err @@ -61,22 +57,18 @@ k0s token create --role worker --expiry 10m //sets expiration time to 10 minute }, func(err error) bool { return waitCreate }, func() error { - bootstrapConfig, err = token.CreateKubeletBootstrapConfig(cmd.Context(), cfg, c.K0sVars, createTokenRole, expiry) - + bootstrapConfig, err = token.CreateKubeletBootstrapConfig(cmd.Context(), c.NodeConfig, c.K0sVars, createTokenRole, expiry) return err }) if err != nil { return err } - fmt.Println(bootstrapConfig) - return nil }, } // append flags cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) - cmd.Flags().AddFlagSet(config.FileInputFlag()) cmd.Flags().StringVar(&tokenExpiry, "expiry", "0s", "Expiration time of the token. Format 1.5h, 2h45m or 300ms.") cmd.Flags().StringVar(&createTokenRole, "role", "worker", "Either worker or controller") cmd.Flags().BoolVar(&waitCreate, "wait", false, "wait forever (default false)") diff --git a/docs/airgap-install.md b/docs/airgap-install.md index 6c179b6da7da..f89551dac6a0 100644 --- a/docs/airgap-install.md +++ b/docs/airgap-install.md @@ -59,7 +59,7 @@ As an alternative to the previous step, you can use k0sctl to upload the bundle ```YAML apiVersion: k0sctl.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: k0s-cluster spec: @@ -93,7 +93,7 @@ Use the following `k0s.yaml` to ensure that containerd does not pull images for ```yaml apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: k0s spec: diff --git a/docs/configuration.md b/docs/configuration.md index d4425048de42..39198910283b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,7 +34,7 @@ A YAML config file follows, with defaults as generated by the `k0s config create ```yaml apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: k0s spec: @@ -110,54 +110,54 @@ spec: ### `spec.api` -| Element | Description | -|-----------|---------------------------| -| `externalAddress` | The loadbalancer address (for k0s controllers running behind a loadbalancer). Configures all cluster components to connect to this address and also configures this address for use when joining new nodes to the cluster.| -| `address` | Local address on wihich to bind an API. Also serves as one of the addresses pushed on the k0s create service certificate on the API. Defaults to first non-local address found on the node.| -| `sans` | List of additional addresses to push to API servers serving the certificate.| -| `extraArgs` | Map of key-values (strings) for any extra arguments to pass down to Kubernetes api-server process.| -| `port`¹ | Custom port for kube-api server to listen on (default: 6443)| -| `k0sApiPort`¹ | Custom port for k0s-api server to listen on (default: 9443)| +| Element | Description | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `externalAddress` | The loadbalancer address (for k0s controllers running behind a loadbalancer). Configures all cluster components to connect to this address and also configures this address for use when joining new nodes to the cluster. | +| `address` | Local address on wihich to bind an API. Also serves as one of the addresses pushed on the k0s create service certificate on the API. Defaults to first non-local address found on the node. | +| `sans` | List of additional addresses to push to API servers serving the certificate. | +| `extraArgs` | Map of key-values (strings) for any extra arguments to pass down to Kubernetes api-server process. | +| `port`¹ | Custom port for kube-api server to listen on (default: 6443) | +| `k0sApiPort`¹ | Custom port for k0s-api server to listen on (default: 9443) | ¹ If `port` and `k0sApiPort` are used with the `externalAddress` element, the loadbalancer serving at `externalAddress` must listen on the same ports. ### `spec.storage` -| Element | Description | -|-----------|---------------------------| -| `type` | Type of the data store (valid values:`etcd` or `kine`). **Note**: Type `etcd` will cause k0s to create and manage an elastic etcd cluster within the controller nodes.| -| `etcd.peerAddress` | Node address used for etcd cluster peering.| -| `kine.dataSource` | [kine](https://github.com/rancher/kine/) datasource URL.| +| Element | Description | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `type` | Type of the data store (valid values:`etcd` or `kine`). **Note**: Type `etcd` will cause k0s to create and manage an elastic etcd cluster within the controller nodes. | +| `etcd.peerAddress` | Node address used for etcd cluster peering. | +| `kine.dataSource` | [kine](https://github.com/rancher/kine/) datasource URL. | ### `spec.network` -| Element | Description | -|-----------|---------------------------| -| `provider` | Network provider (valid values: `calico`, `kuberouter`, or `custom`). For `custom`, you can push any network provider (default: `kuberouter`). Be aware that it is your responsibility to configure all of the CNI-related setups, including the CNI provider itself and all necessary host levels setups (for example, CNI binaries). **Note:** Once you initialize the cluster with a network provider the only way to change providers is through a full cluster redeployment.| -| `podCIDR` | Pod network CIDR to use in the cluster.| -| `serviceCIDR` | Network CIDR to use for cluster VIP services.| +| Element | Description | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `provider` | Network provider (valid values: `calico`, `kuberouter`, or `custom`). For `custom`, you can push any network provider (default: `kuberouter`). Be aware that it is your responsibility to configure all of the CNI-related setups, including the CNI provider itself and all necessary host levels setups (for example, CNI binaries). **Note:** Once you initialize the cluster with a network provider the only way to change providers is through a full cluster redeployment. | +| `podCIDR` | Pod network CIDR to use in the cluster. | +| `serviceCIDR` | Network CIDR to use for cluster VIP services. | #### `spec.network.calico` -| Element | Description | -|-----------|---------------------------| -| `mode` | `vxlan` (default) or `ipip`| -| `overlay` | Overlay mode: `Always` (default), `CrossSubnet` or `Never` (requires `mode=vxlan` to disable calico overlay-network). -| `vxlanPort` | The UDP port for VXLAN (default: `4789`).| -| `vxlanVNI` | The virtual network ID for VXLAN (default: `4096`).| -| `mtu` | MTU for overlay network (default: `0`, which causes Calico to detect optimal MTU during bootstrap).| -| `wireguard` | Enable wireguard-based encryption (default: `false`). Your host system must be wireguard ready (refer to the [Calico documentation](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic) for details).| -| `flexVolumeDriverPath` | The host path for Calicos flex-volume-driver(default: `/usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds`). Change this path only if the default path is unwriteable (refer to [Project Calico Issue #2712](https://github.com/projectcalico/calico/issues/2712) for details). Ideally, you will pair this option with a custom ``volumePluginDir`` in the profile you use for your worker nodes.| -| `ipAutodetectionMethod` | Use to force Calico to pick up the interface for pod network inter-node routing (default: `""`, meaning not set, so that Calico will instead use its defaults). For more information, refer to the [Calico documentation](https://docs.projectcalico.org/reference/node/configuration#ip-autodetection-methods).| +| Element | Description | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `mode` | `vxlan` (default) or `ipip` | +| `overlay` | Overlay mode: `Always` (default), `CrossSubnet` or `Never` (requires `mode=vxlan` to disable calico overlay-network). | +| `vxlanPort` | The UDP port for VXLAN (default: `4789`). | +| `vxlanVNI` | The virtual network ID for VXLAN (default: `4096`). | +| `mtu` | MTU for overlay network (default: `0`, which causes Calico to detect optimal MTU during bootstrap). | +| `wireguard` | Enable wireguard-based encryption (default: `false`). Your host system must be wireguard ready (refer to the [Calico documentation](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic) for details). | +| `flexVolumeDriverPath` | The host path for Calicos flex-volume-driver(default: `/usr/libexec/k0s/kubelet-plugins/volume/exec/nodeagent~uds`). Change this path only if the default path is unwriteable (refer to [Project Calico Issue #2712](https://github.com/projectcalico/calico/issues/2712) for details). Ideally, you will pair this option with a custom ``volumePluginDir`` in the profile you use for your worker nodes. | +| `ipAutodetectionMethod` | Use to force Calico to pick up the interface for pod network inter-node routing (default: `""`, meaning not set, so that Calico will instead use its defaults). For more information, refer to the [Calico documentation](https://docs.projectcalico.org/reference/node/configuration#ip-autodetection-methods). | #### `spec.network.kuberouter` -| Element | Description | -|-----------|---------------------------| -| `autoMTU` | Autodetection of used MTU (default: `true`).| -| `mtu` | Override MTU setting, if `autoMTU` must be set to `false`).| -| `peerRouterIPs` | Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).| -| `peerRouterASNs` | Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).| +| Element | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `autoMTU` | Autodetection of used MTU (default: `true`). | +| `mtu` | Override MTU setting, if `autoMTU` must be set to `false`). | +| `peerRouterIPs` | Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers). | +| `peerRouterASNs` | Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers). | **Note**: Kube-router allows many networking aspects to be configured per node, service, and pod (for more information, refer to the [Kube-router user guide](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md)). @@ -167,33 +167,33 @@ Use the `spec.podSecurityPolicy` key to configure the default [PSP](https://kube k0s creates two PSPs out-of-the-box: -| PSP | Description | -|-----------|---------------------------| -| `00-k0s-privileged` | Default; no restrictions; used also for Kubernetes/k0s level system pods.| -| `99-k0s-restricted` | Does not allow any host namespaces or root users, nor any bind mounts from the host| +| PSP | Description | +| ------------------- | ----------------------------------------------------------------------------------- | +| `00-k0s-privileged` | Default; no restrictions; used also for Kubernetes/k0s level system pods. | +| `99-k0s-restricted` | Does not allow any host namespaces or root users, nor any bind mounts from the host | **Note**: Users can create supplemental PSPs and bind them to users / access accounts as necessary. ### `spec.controllerManager` -| Element | Description | -|-----------|---------------------------| -| `extraArgs` | Map of key-values (strings) for any extra arguments you want to pass down to the Kubernetes controller manager process.| +| Element | Description | +| ----------- | ----------------------------------------------------------------------------------------------------------------------- | +| `extraArgs` | Map of key-values (strings) for any extra arguments you want to pass down to the Kubernetes controller manager process. | ### `spec.scheduler` -| Element | Description | -|-----------|---------------------------| -| `extraArgs` | Map of key-values (strings) for any extra arguments you want to pass down to Kubernetes scheduler process.| +| Element | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------- | +| `extraArgs` | Map of key-values (strings) for any extra arguments you want to pass down to Kubernetes scheduler process. | ### `spec.workerProfiles` Array of `spec.workerProfiles.workerProfile`. Each element has following properties: -| Property | Description | -|-----------|---------------------------| -| `name` | String; name to use as profile selector for the worker process| -| `values` | Mapping object| +| Property | Description | +| -------- | -------------------------------------------------------------- | +| `name` | String; name to use as profile selector for the worker process | +| `values` | Mapping object | For each profile, the control plane creates a separate ConfigMap with `kubelet-config yaml`. Based on the `--profile` argument given to the `k0s worker`, the corresponding ConfigMap is used to extract the `kubelet-config.yaml` file. `values` are recursively merged with default `kubelet-config.yaml` diff --git a/docs/k0s-in-docker.md b/docs/k0s-in-docker.md index 321bb565f203..1e01b585f0cf 100644 --- a/docs/k0s-in-docker.md +++ b/docs/k0s-in-docker.md @@ -75,7 +75,7 @@ services: environment: K0S_CONFIG: |- apiVersion: k0s.k0sproject.io/v1beta1 - kind: Cluster + kind: ClusterConfig metadata: name: k0s # Any additional configuration goes here ... diff --git a/examples/footloose-ha-controllers/k0s.yaml b/examples/footloose-ha-controllers/k0s.yaml index ea0e061a71c3..f2528536941e 100644 --- a/examples/footloose-ha-controllers/k0s.yaml +++ b/examples/footloose-ha-controllers/k0s.yaml @@ -1,5 +1,5 @@ apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: k0s-etcd spec: diff --git a/examples/footloose-mysql/mke.yaml b/examples/footloose-mysql/k0s.yaml similarity index 92% rename from examples/footloose-mysql/mke.yaml rename to examples/footloose-mysql/k0s.yaml index 8dc220ce8d7d..0d9652e07ce6 100644 --- a/examples/footloose-mysql/mke.yaml +++ b/examples/footloose-mysql/k0s.yaml @@ -1,6 +1,5 @@ - apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: foobar spec: @@ -10,6 +9,4 @@ spec: dataSource: mysql://root:kine@tcp(172.17.0.4)/kine api: address: 172.17.0.2 # Address where the k8s API is accessed at (nodes public IP or LB) - - diff --git a/internal/testutil/runtime_config.go b/internal/testutil/runtime_config.go new file mode 100644 index 000000000000..30ce5e7bdf0a --- /dev/null +++ b/internal/testutil/runtime_config.go @@ -0,0 +1,156 @@ +/* +Copyright 2022 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package testutil + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/sirupsen/logrus" + "sigs.k8s.io/yaml" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/k0sproject/k0s/internal/pkg/file" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/fake" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/typed/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/config" + "github.com/k0sproject/k0s/pkg/constant" +) + +const RuntimeFakePath = "/tmp/k0s.yaml" + +var resourceType = v1.TypeMeta{APIVersion: "k0s.k0sproject.io/v1beta1", Kind: "clusterconfigs"} + +type ConfigGetter struct { + NodeConfig bool + YamlData string + + k0sVars constant.CfgVars + cfgFilePath string +} + +// NewConfigGetter sets the parameters required to fetch a fake config for testing +func NewConfigGetter(yamlData string, isNodeConfig bool, k0sVars constant.CfgVars) *ConfigGetter { + return &ConfigGetter{ + YamlData: yamlData, + NodeConfig: isNodeConfig, + k0sVars: k0sVars, + } +} + +// FakeRuntimeConfig takes a yaml construct and returns a config object from a fake runtime config path +func (c *ConfigGetter) FakeConfigFromFile() (*v1beta1.ClusterConfig, error) { + err := c.initRuntimeConfig() + if err != nil { + return nil, err + } + loadingRules := config.ClientConfigLoadingRules{ + RuntimeConfigPath: RuntimeFakePath, + Nodeconfig: c.NodeConfig, + CfgFileOverride: c.cfgFilePath, + K0sVars: c.k0sVars, + } + return loadingRules.Load() +} + +func (c *ConfigGetter) FakeAPIConfig() (*v1beta1.ClusterConfig, error) { + err := c.initRuntimeConfig() + if err != nil { + return nil, err + } + + // create the API config using a fake client + client := fake.NewSimpleClientset() + + err = c.createFakeAPIConfig(client.K0sV1beta1()) + if err != nil { + return nil, fmt.Errorf("failed to get fake API client: %v", err) + } + + loadingRules := config.ClientConfigLoadingRules{ + RuntimeConfigPath: RuntimeFakePath, + Nodeconfig: c.NodeConfig, + CfgFileOverride: c.cfgFilePath, + APIClient: client.K0sV1beta1(), + K0sVars: c.k0sVars, + } + + return loadingRules.Load() +} + +func (c *ConfigGetter) initRuntimeConfig() error { + // write the yaml string into a temporary config file path + cfgFilePath, err := file.WriteTmpFile(c.YamlData, "k0s-config") + if err != nil { + return fmt.Errorf("error creating tempfile: %v", err) + } + + c.cfgFilePath = cfgFilePath + + logrus.Infof("using config path: %s", cfgFilePath) + f, err := os.Open(c.cfgFilePath) + if err != nil { + return err + } + defer f.Close() + + mergedConfig, err := v1beta1.ConfigFromReader(f, c.getStorageSpec()) + if err != nil { + return fmt.Errorf("unable to parse config from %s: %v", cfgFilePath, err) + } + data, err := yaml.Marshal(&mergedConfig) + if err != nil { + return fmt.Errorf("failed to marshal config: %v", err) + } + err = os.WriteFile(RuntimeFakePath, data, 0755) + if err != nil { + return fmt.Errorf("failed to write runtime config to %s: %v", RuntimeFakePath, err) + } + return nil +} + +func (c *ConfigGetter) createFakeAPIConfig(client k0sv1beta1.K0sV1beta1Interface) error { + clusterConfigs := client.ClusterConfigs(constant.ClusterConfigNamespace) + ctxWithTimeout, cancelFunction := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancelFunction() + + cfg, err := v1beta1.ConfigFromString(c.YamlData, c.getStorageSpec()) + if err != nil { + return fmt.Errorf("failed to parse config yaml: %s", err.Error()) + } + + _, err = clusterConfigs.Create(ctxWithTimeout, cfg.GetClusterWideConfig().StripDefaults(), v1.CreateOptions{TypeMeta: resourceType}) + if err != nil { + return fmt.Errorf("failed to create clusterConfig in the API: %s", err.Error()) + } + return nil +} + +func (c *ConfigGetter) getStorageSpec() *v1beta1.StorageSpec { + var storage *v1beta1.StorageSpec + + if c.k0sVars.DefaultStorageType == "kine" { + storage = &v1beta1.StorageSpec{ + Type: v1beta1.KineStorageType, + Kine: v1beta1.DefaultKineConfig(c.k0sVars.DataDir), + } + } + return storage +} diff --git a/inttest/backup/backup_test.go b/inttest/backup/backup_test.go index 15a63ab4a381..b3fe28acc755 100644 --- a/inttest/backup/backup_test.go +++ b/inttest/backup/backup_test.go @@ -32,7 +32,7 @@ import ( const configWithExternaladdress = ` apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: k0s spec: diff --git a/inttest/common/footloosesuite.go b/inttest/common/footloosesuite.go index 0895e8da3df2..82e33a2bf5c8 100644 --- a/inttest/common/footloosesuite.go +++ b/inttest/common/footloosesuite.go @@ -443,7 +443,9 @@ func (s *FootlooseSuite) GetJoinToken(role string, extraArgs ...string) (string, return "", err } defer ssh.Disconnect() - token, err := ssh.ExecWithOutput(fmt.Sprintf("%s token create --role=%s %s 2>/dev/null", s.K0sFullPath, role, strings.Join(extraArgs, " "))) + + tokenCmd := fmt.Sprintf("%s token create --role=%s %s 2>/dev/null", s.K0sFullPath, role, strings.Join(extraArgs, " ")) + token, err := ssh.ExecWithOutput(tokenCmd) if err != nil { return "", fmt.Errorf("can't get join token: %v", err) } diff --git a/inttest/customports/customports_test.go b/inttest/customports/customports_test.go index b5b37b80def9..867926012ad2 100644 --- a/inttest/customports/customports_test.go +++ b/inttest/customports/customports_test.go @@ -36,7 +36,7 @@ type Suite struct { const configWithExternaladdress = ` apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: k0s spec: @@ -102,7 +102,7 @@ func (ds *Suite) TestControllerJoinsWithCustomPort() { ds.Require().NoError(ds.InitController(0, "--config=/tmp/k0s.yaml")) - workerToken, err := ds.GetJoinToken("worker", "", "--config=/tmp/k0s.yaml") + workerToken, err := ds.GetJoinToken("worker", "") ds.Require().NoError(err) ds.Require().NoError(ds.RunWorkersWithToken("/var/lib/k0s", workerToken)) @@ -112,7 +112,7 @@ func (ds *Suite) TestControllerJoinsWithCustomPort() { err = ds.WaitForNodeReady("worker0", kc) ds.Require().NoError(err) - controllerToken, err := ds.GetJoinToken("controller", "", "--config=/tmp/k0s.yaml") + controllerToken, err := ds.GetJoinToken("controller", "") ds.Require().NoError(err) ds.Require().NoError(ds.InitController(1, controllerToken, "", "--config=/tmp/k0s.yaml")) ds.Require().NoError(ds.InitController(2, controllerToken, "", "--config=/tmp/k0s.yaml")) diff --git a/inttest/externaletcd/external_etcd_test.go b/inttest/externaletcd/external_etcd_test.go index 53efbd4cdc79..e4c15c1b3a1c 100644 --- a/inttest/externaletcd/external_etcd_test.go +++ b/inttest/externaletcd/external_etcd_test.go @@ -18,11 +18,12 @@ package externaletcd import ( "context" "fmt" + "testing" + "github.com/avast/retry-go" "github.com/k0sproject/k0s/inttest/common" "github.com/stretchr/testify/suite" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" ) const k0sConfig = ` @@ -95,15 +96,15 @@ func (s *ExternalEtcdSuite) TestK0sWithExternalEtcdCluster() { s.Require().NoError(err) s.Require().Contains(output, "/k0s-tenant/services/specs/kube-system/kube-dns") - etcdLeaveOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd leave --config=/tmp/k0s.yaml") + etcdLeaveOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd leave") s.Require().Error(err) s.Require().Contains(etcdLeaveOutput, "command 'k0s etcd' does not support external etcd cluster") - etcdListOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd member-list --config=/tmp/k0s.yaml") + etcdListOutput, err := k0sControllerSSH.ExecWithOutput("k0s etcd member-list") s.Require().Error(err) s.Require().Contains(etcdListOutput, "command 'k0s etcd' does not support external etcd cluster") - backupOutput, err := k0sControllerSSH.ExecWithOutput("k0s backup --config=/tmp/k0s.yaml") + backupOutput, err := k0sControllerSSH.ExecWithOutput("k0s backup") s.Require().Error(err) s.Require().Contains(backupOutput, "command 'k0s backup' does not support external etcd cluster") } diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/clusterconfig_types.go b/pkg/apis/k0s.k0sproject.io/v1beta1/clusterconfig_types.go index 5a0e1d9266df..db1bd5e45791 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/clusterconfig_types.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/clusterconfig_types.go @@ -279,13 +279,21 @@ func (c *ClusterConfig) Validate() []error { } // GetBootstrappingConfig returns a ClusterConfig object stripped of Cluster-Wide Settings -func (c *ClusterConfig) GetBootstrappingConfig(storageSpec *StorageSpec) *ClusterConfig { +func (c *ClusterConfig) GetBootstrappingConfig() *ClusterConfig { + var etcdConfig *EtcdConfig + if c.Spec.Storage.Type == EtcdStorageType { + etcdConfig = &EtcdConfig{ + ExternalCluster: c.Spec.Storage.Etcd.ExternalCluster, + PeerAddress: c.Spec.Storage.Etcd.PeerAddress, + } + c.Spec.Storage.Etcd = etcdConfig + } return &ClusterConfig{ ObjectMeta: c.ObjectMeta, TypeMeta: c.TypeMeta, Spec: &ClusterSpec{ API: c.Spec.API, - Storage: storageSpec, + Storage: c.Spec.Storage, Network: &Network{ ServiceCIDR: c.Spec.Network.ServiceCIDR, DualStack: c.Spec.Network.DualStack, diff --git a/pkg/apis/k0s.k0sproject.io/v1beta1/network_test.go b/pkg/apis/k0s.k0sproject.io/v1beta1/network_test.go index 4ee7a8f82f89..704a372c56af 100644 --- a/pkg/apis/k0s.k0sproject.io/v1beta1/network_test.go +++ b/pkg/apis/k0s.k0sproject.io/v1beta1/network_test.go @@ -93,7 +93,7 @@ func (s *NetworkSuite) TestNetworkDefaults() { func (s *NetworkSuite) TestCalicoDefaultsAfterMashaling() { yamlData := ` apiVersion: k0s.k0sproject.io/v1beta1 -kind: Cluster +kind: ClusterConfig metadata: name: foobar spec: diff --git a/pkg/backup/config.go b/pkg/backup/config.go index a51b61ce4463..e689f634a255 100644 --- a/pkg/backup/config.go +++ b/pkg/backup/config.go @@ -19,8 +19,6 @@ limitations under the License. package backup import ( - "fmt" - "os" "path" "github.com/sirupsen/logrus" @@ -29,38 +27,30 @@ import ( ) type configurationStep struct { - path string + cfgPath string restoredConfigPath string } -func newConfigurationStep(path string, restoredConfigPath string) *configurationStep { +func newConfigurationStep(cfgPath string, restoredConfigPath string) *configurationStep { return &configurationStep{ - path: path, + cfgPath: cfgPath, restoredConfigPath: restoredConfigPath, } } func (c configurationStep) Name() string { - return c.path + return "k0s-config" } func (c configurationStep) Backup() (StepResult, error) { - _, err := os.Stat(c.path) - if os.IsNotExist(err) { - logrus.Warn("default k0s.yaml is used, do not back it up") - return StepResult{}, nil - } - if err != nil { - return StepResult{}, fmt.Errorf("can't backup `%s`: %v", c.path, err) - } - return StepResult{filesForBackup: []string{c.path}}, nil + return StepResult{filesForBackup: []string{c.cfgPath}}, nil } func (c configurationStep) Restore(restoreFrom, restoreTo string) error { objectPathInArchive := path.Join(restoreFrom, "k0s.yaml") if !file.Exists(objectPathInArchive) { - logrus.Debugf("%s does not exist in the backup file", objectPathInArchive) + logrus.Infof("%s does not exist in the backup file", objectPathInArchive) return nil } logrus.Infof("Previously used k0s.yaml saved under the data directory `%s`", restoreTo) diff --git a/pkg/backup/manager.go b/pkg/backup/manager.go index 92deecac82c3..405d3cfd125f 100644 --- a/pkg/backup/manager.go +++ b/pkg/backup/manager.go @@ -43,8 +43,8 @@ type Manager struct { } // RunBackup backups cluster -func (bm *Manager) RunBackup(cfgPath string, clusterSpec *v1beta1.ClusterSpec, vars constant.CfgVars, savePathDir string) error { - bm.discoverSteps(cfgPath, clusterSpec, vars, "backup", "") +func (bm *Manager) RunBackup(cfgPath string, nodeSpec *v1beta1.ClusterSpec, vars constant.CfgVars, savePathDir string) error { + bm.discoverSteps(cfgPath, nodeSpec, vars, "backup", "") defer os.RemoveAll(bm.tmpDir) assets := make([]string, 0, len(bm.steps)) @@ -70,11 +70,11 @@ func (bm *Manager) RunBackup(cfgPath string, clusterSpec *v1beta1.ClusterSpec, v return nil } -func (bm *Manager) discoverSteps(cfgPath string, clusterSpec *v1beta1.ClusterSpec, vars constant.CfgVars, action string, restoredConfigPath string) { - if clusterSpec.Storage.Type == v1beta1.EtcdStorageType && !clusterSpec.Storage.Etcd.IsExternalClusterUsed() { - bm.Add(newEtcdStep(bm.tmpDir, vars.CertRootDir, vars.EtcdCertDir, clusterSpec.Storage.Etcd.PeerAddress, vars.EtcdDataDir)) - } else if clusterSpec.Storage.Type == v1beta1.KineStorageType && strings.HasPrefix(clusterSpec.Storage.Kine.DataSource, "sqlite://") { - bm.Add(newSqliteStep(bm.tmpDir, clusterSpec.Storage.Kine.DataSource, vars.DataDir)) +func (bm *Manager) discoverSteps(configFilePath string, nodeSpec *v1beta1.ClusterSpec, vars constant.CfgVars, action string, restoredConfigPath string) { + if nodeSpec.Storage.Type == v1beta1.EtcdStorageType && !nodeSpec.Storage.Etcd.IsExternalClusterUsed() { + bm.Add(newEtcdStep(bm.tmpDir, vars.CertRootDir, vars.EtcdCertDir, nodeSpec.Storage.Etcd.PeerAddress, vars.EtcdDataDir)) + } else if nodeSpec.Storage.Type == v1beta1.KineStorageType && strings.HasPrefix(nodeSpec.Storage.Kine.DataSource, "sqlite://") { + bm.Add(newSqliteStep(bm.tmpDir, nodeSpec.Storage.Kine.DataSource, vars.DataDir)) } else { logrus.Warnf("only internal etcd and sqlite %s are supported. Other storage backends must be backed-up/restored manually.", action) } @@ -91,7 +91,7 @@ func (bm *Manager) discoverSteps(cfgPath string, clusterSpec *v1beta1.ClusterSpe } bm.Add(NewFilesystemStep(path)) } - bm.Add(newConfigurationStep(cfgPath, restoredConfigPath)) + bm.Add(newConfigurationStep(configFilePath, restoredConfigPath)) } // Add adds backup step @@ -126,7 +126,7 @@ func (bm Manager) save(backupFileName string, assets []string) error { } // RunRestore restores cluster -func (bm *Manager) RunRestore(archivePath string, k0sVars constant.CfgVars, restoredConfigPath string) error { +func (bm *Manager) RunRestore(archivePath string, k0sVars constant.CfgVars, desiredRestoredConfigPath string) error { if err := archive.Extract(archivePath, bm.tmpDir); err != nil { return fmt.Errorf("failed to unpack backup archive `%s`: %v", archivePath, err) } @@ -135,7 +135,7 @@ func (bm *Manager) RunRestore(archivePath string, k0sVars constant.CfgVars, rest if err != nil { return fmt.Errorf("failed to parse backed-up configuration file, check the backup archive: %v", err) } - bm.discoverSteps(fmt.Sprintf("%s/k0s.yaml", bm.tmpDir), cfg.Spec, k0sVars, "restore", restoredConfigPath) + bm.discoverSteps(fmt.Sprintf("%s/k0s.yaml", bm.tmpDir), cfg.Spec, k0sVars, "restore", desiredRestoredConfigPath) logrus.Info("Starting restore") for _, step := range bm.steps { @@ -149,13 +149,13 @@ func (bm *Manager) RunRestore(archivePath string, k0sVars constant.CfgVars, rest func (bm Manager) getConfigForRestore(k0sVars constant.CfgVars) (*v1beta1.ClusterConfig, error) { configFromBackup := path.Join(bm.tmpDir, "k0s.yaml") - _, err := os.Stat(configFromBackup) - if os.IsNotExist(err) { - return v1beta1.DefaultClusterConfig(), nil - } - logrus.Infof("Using k0s.yaml from: %s", configFromBackup) + logrus.Debugf("Using k0s.yaml from: %s", configFromBackup) - cfg, err := config.GetNodeConfig(configFromBackup, k0sVars) + loadingRules := config.ClientConfigLoadingRules{ + RuntimeConfigPath: configFromBackup, + K0sVars: k0sVars, + } + cfg, err := loadingRules.Load() if err != nil { return nil, err } diff --git a/pkg/cleanup/users.go b/pkg/cleanup/users.go index ac15b5713cd5..1db893d4d716 100644 --- a/pkg/cleanup/users.go +++ b/pkg/cleanup/users.go @@ -32,10 +32,10 @@ func (u *users) Name() string { // Run removes all controller users that are present on the host func (u *users) Run() error { - cfg, err := config.GetNodeConfig(u.Config.cfgFile, u.Config.k0sVars) + loadingRules := config.ClientConfigLoadingRules{Nodeconfig: true, K0sVars: u.Config.k0sVars} + cfg, err := loadingRules.Load() if err != nil { logrus.Errorf("failed to get cluster setup: %v", err) - return nil } if err := install.DeleteControllerUsers(cfg); err != nil { // don't fail, just notify on delete error diff --git a/pkg/component/controller/clusterConfig.go b/pkg/component/controller/clusterConfig.go index b0644da5a6d4..7cfd8e583b6e 100644 --- a/pkg/component/controller/clusterConfig.go +++ b/pkg/component/controller/clusterConfig.go @@ -61,10 +61,11 @@ type ClusterConfigReconciler struct { } // NewClusterConfigReconciler creates a new clusterConfig reconciler -func NewClusterConfigReconciler(cfgFile string, leaderElector LeaderElector, k0sVars constant.CfgVars, mgr *component.Manager, s manifestsSaver, kubeClientFactory kubeutil.ClientFactoryInterface, configSource clusterconfig.ConfigSource) (*ClusterConfigReconciler, error) { - cfg, err := config.GetYamlFromFile(cfgFile, k0sVars) +func NewClusterConfigReconciler(leaderElector LeaderElector, k0sVars constant.CfgVars, mgr *component.Manager, s manifestsSaver, kubeClientFactory kubeutil.ClientFactoryInterface, configSource clusterconfig.ConfigSource) (*ClusterConfigReconciler, error) { + loadingRules := config.ClientConfigLoadingRules{K0sVars: k0sVars} + cfg, err := loadingRules.ParseRuntimeConfig() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get config: %v", err) } configClient, err := kubeClientFactory.GetConfigClient() diff --git a/pkg/component/controller/k0scontrolapi.go b/pkg/component/controller/k0scontrolapi.go index e4811428d0bd..ea4966eb821e 100644 --- a/pkg/component/controller/k0scontrolapi.go +++ b/pkg/component/controller/k0scontrolapi.go @@ -54,7 +54,6 @@ func (m *K0SControlAPI) Run(_ context.Context) error { DataDir: m.K0sVars.DataDir, Args: []string{ "api", - fmt.Sprintf("--config=%s", m.ConfigPath), fmt.Sprintf("--data-dir=%s", m.K0sVars.DataDir), }, } diff --git a/pkg/component/controller/kubeproxy.go b/pkg/component/controller/kubeproxy.go index d0998cb182b2..4d68e58b7481 100644 --- a/pkg/component/controller/kubeproxy.go +++ b/pkg/component/controller/kubeproxy.go @@ -22,7 +22,6 @@ import ( "path/filepath" "github.com/k0sproject/k0s/pkg/component" - "github.com/k0sproject/k0s/pkg/config" "github.com/sirupsen/logrus" @@ -45,16 +44,12 @@ var _ component.Component = &KubeProxy{} var _ component.ReconcilerComponent = &KubeProxy{} // NewKubeProxy creates new KubeProxy component -func NewKubeProxy(configFile string, k0sVars constant.CfgVars) (*KubeProxy, error) { +func NewKubeProxy(configFile string, k0sVars constant.CfgVars, nodeConfig *v1beta1.ClusterConfig) (*KubeProxy, error) { log := logrus.WithFields(logrus.Fields{"component": "kubeproxy"}) - cfg, err := config.GetNodeConfig(configFile, k0sVars) - if err != nil { - return nil, err - } proxyDir := path.Join(k0sVars.ManifestsDir, "kubeproxy") return &KubeProxy{ log: log, - nodeConf: cfg, + nodeConf: nodeConfig, K0sVars: k0sVars, previousConfig: proxyConfig{}, manifestDir: proxyDir, diff --git a/pkg/config/api_config.go b/pkg/config/api_config.go new file mode 100644 index 000000000000..218804b614c5 --- /dev/null +++ b/pkg/config/api_config.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package config + +import ( + "context" + "fmt" + "time" + + "github.com/imdario/mergo" + "github.com/sirupsen/logrus" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/typed/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/constant" +) + +var ( + resourceType = v1.TypeMeta{APIVersion: "k0s.k0sproject.io/v1beta1", Kind: "clusterconfigs"} + getOpts = v1.GetOptions{TypeMeta: resourceType} +) + +// run a config-request from the API and wait until the API is up +func (rules *ClientConfigLoadingRules) getConfigFromAPI(client k0sv1beta1.K0sV1beta1Interface) (*v1beta1.ClusterConfig, error) { + timeout := time.After(120 * time.Second) + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + // Keep trying until we're timed out or got a result or got an error + for { + select { + // Got a timeout! fail with a timeout error + case <-timeout: + return nil, fmt.Errorf("timed out waiting for API to return cluster-config") + // Got a tick, we should check on doSomething() + case <-ticker.C: + logrus.Debug("fetching cluster-config from API...") + cfg, err := rules.configRequest(client) + if err != nil { + continue + } + return cfg, nil + } + } +} + +// when API config is enabled, but only node config is needed (for bootstrapping commands) +func (rules *ClientConfigLoadingRules) fetchNodeConfig() (*v1beta1.ClusterConfig, error) { + cfg, err := rules.readRuntimeConfig() + if err != nil { + logrus.Errorf("failed to read config from file: %v", err) + return nil, err + } + return cfg.GetBootstrappingConfig(), nil +} + +// when API config is enabled, but only node config is needed (for bootstrapping commands) +func (rules *ClientConfigLoadingRules) mergeNodeAndClusterconfig(nodeConfig *v1beta1.ClusterConfig, apiConfig *v1beta1.ClusterConfig) (*v1beta1.ClusterConfig, error) { + clusterConfig := &v1beta1.ClusterConfig{} + + // API config takes precedence over Node config. This is why we are merging it first + err := mergo.Merge(clusterConfig, apiConfig) + if err != nil { + return nil, err + } + + err = mergo.Merge(clusterConfig, nodeConfig.GetBootstrappingConfig(), mergo.WithOverride) + if err != nil { + return nil, err + } + + return clusterConfig, nil +} + +// fetch cluster-config from API +func (rules *ClientConfigLoadingRules) configRequest(client k0sv1beta1.K0sV1beta1Interface) (clusterConfig *v1beta1.ClusterConfig, err error) { + clusterConfigs := client.ClusterConfigs(constant.ClusterConfigNamespace) + ctxWithTimeout, cancelFunction := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancelFunction() + + cfg, err := clusterConfigs.Get(ctxWithTimeout, "k0s", getOpts) + if err != nil { + return nil, fmt.Errorf("failed to fetch cluster-config from API: %v", err) + } + return cfg, nil +} diff --git a/pkg/config/cli.go b/pkg/config/cli.go index 46ee057c86a4..d813e3594a92 100644 --- a/pkg/config/cli.go +++ b/pkg/config/cli.go @@ -16,10 +16,12 @@ limitations under the License. package config import ( + "os" "path/filepath" "strings" "time" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" k8s "k8s.io/client-go/kubernetes" cloudprovider "k8s.io/cloud-provider" @@ -195,11 +197,26 @@ func FileInputFlag() *pflag.FlagSet { func GetCmdOpts() CLIOptions { K0sVars = constant.GetConfig(DataDir) + if controllerOpts.SingleNode { + controllerOpts.EnableWorker = true + K0sVars.DefaultStorageType = "kine" + } + + // when a custom config file is used, verify that it can be opened + if CfgFile != constant.K0sConfigPathDefault { + _, err := os.Open(CfgFile) + if err != nil { + logrus.Fatalf("failed to load config file (%s): %v", CfgFile, err) + } + } + opts := CLIOptions{ ControllerOptions: controllerOpts, WorkerOptions: workerOpts, CfgFile: CfgFile, + ClusterConfig: getClusterConfig(K0sVars), + NodeConfig: getNodeConfig(K0sVars), Debug: Debug, Verbose: Verbose, DefaultLogLevels: DefaultLogLevels(), @@ -208,3 +225,21 @@ func GetCmdOpts() CLIOptions { } return opts } + +func getNodeConfig(k0sVars constant.CfgVars) *v1beta1.ClusterConfig { + loadingRules := ClientConfigLoadingRules{Nodeconfig: true, K0sVars: k0sVars} + cfg, err := loadingRules.Load() + if err != nil { + return nil + } + return cfg +} + +func getClusterConfig(k0sVars constant.CfgVars) *v1beta1.ClusterConfig { + loadingRules := ClientConfigLoadingRules{K0sVars: k0sVars} + cfg, err := loadingRules.Load() + if err != nil { + return nil + } + return cfg +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 8a4cb2f59f95..1e3729613241 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,115 +17,124 @@ package config import ( "fmt" - "os" - "strings" + "github.com/k0sproject/k0s/internal/pkg/file" + cfgClient "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/typed/k0s.k0sproject.io/v1beta1" "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "github.com/k0sproject/k0s/pkg/constant" - "github.com/sirupsen/logrus" + "k8s.io/client-go/tools/clientcmd" ) -// GetYamlFromFile parses a yaml file into a ClusterConfig object -func GetYamlFromFile(cfgPath string, k0sVars constant.CfgVars) (clusterConfig *v1beta1.ClusterConfig, err error) { - if cfgPath == "" { - // no config file exists, using defaults - logrus.Warn("no config file given, using defaults") - } - cfg, err := GetConfigFromYAML(cfgPath, k0sVars) - if err != nil { - return nil, err - } - return cfg, nil +// general interface for config related methods +type Loader interface { + BootstrapConfig() (*v1beta1.ClusterConfig, error) + ClusterConfig() (*v1beta1.ClusterConfig, error) + IsAPIConfig() bool + IsDefaultConfig() bool + Load() (*v1beta1.ClusterConfig, error) } -// GetConfigFromYAML will attempt to read a config yaml, validate it and return a clusterConfig object -func GetConfigFromYAML(cfgPath string, k0sVars constant.CfgVars) (clusterConfig *v1beta1.ClusterConfig, err error) { - var storage *v1beta1.StorageSpec - var cfg *v1beta1.ClusterConfig +type K0sConfigGetter struct { + k0sConfigGetter Getter +} - CfgFile = cfgPath +func (g *K0sConfigGetter) IsAPIConfig() bool { + return false +} - // first, let's set the default storage type - if k0sVars.DefaultStorageType == "kine" { - storage = &v1beta1.StorageSpec{ - Type: v1beta1.KineStorageType, - Kine: v1beta1.DefaultKineConfig(k0sVars.DataDir), - } - } +func (g *K0sConfigGetter) IsDefaultConfig() bool { + return false +} - switch CfgFile { - // read config file flag - default: - f, err := os.Open(CfgFile) - if err != nil { - return nil, err - } - defer f.Close() +func (g *K0sConfigGetter) BootstrapConfig() (*v1beta1.ClusterConfig, error) { + return g.k0sConfigGetter() +} - cfg, err = v1beta1.ConfigFromReader(f, storage) - if err != nil { - return nil, err - } +func (g *K0sConfigGetter) Load() (*v1beta1.ClusterConfig, error) { + return g.k0sConfigGetter() +} + +type Getter func() (*v1beta1.ClusterConfig, error) + +var _ Loader = &ClientConfigLoadingRules{} + +type ClientConfigLoadingRules struct { + // APIClient is an optional field for passing a kubernetes API client, to fetch the API config + // mostly used by tests, to pass a fake client + APIClient k0sv1beta1.K0sV1beta1Interface + + // Nodeconfig is an optional field indicating if provided config-file is a node-config or a standard cluster-config file. + Nodeconfig bool + + // RuntimeConfigPath is an optional field indicating the location of the runtime config file (default: /run/k0s/k0s.yaml) + // this parameter is mainly used for testing purposes, to override the default location on local dev system + RuntimeConfigPath string - // stdin input - case "-": - cfg, err = v1beta1.ConfigFromReader(os.Stdin, storage) + // CfgFileOverride is an optional field for overriding the CfgFile parameter from cobra. Used mainly for testing purposes. + CfgFileOverride string - // config file not provided: try to read config from default location. - // if not exists, generate default config - case constant.K0sConfigPathDefault: - f, err := os.Open(constant.K0sConfigPathDefault) + // K0sVars is needed for fetching the right config from the API + K0sVars constant.CfgVars +} + +func (rules *ClientConfigLoadingRules) BootstrapConfig() (*v1beta1.ClusterConfig, error) { + return rules.fetchNodeConfig() +} + +// ClusterConfig generates a client and queries the API for the cluster config +func (rules *ClientConfigLoadingRules) ClusterConfig() (*v1beta1.ClusterConfig, error) { + if rules.APIClient == nil { + // generate a kubernetes client from AdminKubeConfigPath + config, err := clientcmd.BuildConfigFromFlags("", K0sVars.AdminKubeConfigPath) if err != nil { - if os.IsNotExist(err) { - logrus.Debugf("could not find config in %s, using defaults", constant.K0sConfigPathDefault) - cfg = v1beta1.DefaultClusterConfig(storage) - } else { - return nil, err - } + return nil, fmt.Errorf("can't read kubeconfig: %v", err) } - if err == nil { - logrus.Debugf("found config file in %s", constant.K0sConfigPathDefault) - cfg, err = v1beta1.ConfigFromReader(f, storage) - if err != nil { - return nil, err - } - defer f.Close() + client, err := cfgClient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("can't create kubernetes typed client for cluster config: %v", err) } - } - if cfg.Spec.Storage.Type == v1beta1.KineStorageType && cfg.Spec.Storage.Kine == nil { - logrus.Warn("storage type is kine but no config given, setting up defaults") - cfg.Spec.Storage.Kine = v1beta1.DefaultKineConfig(k0sVars.DataDir) - } - if cfg.Spec.Install == nil { - cfg.Spec.Install = v1beta1.DefaultInstallSpec() + rules.APIClient = client.K0sV1beta1() } + return rules.getConfigFromAPI(rules.APIClient) +} - errors := cfg.Validate() - if len(errors) > 0 { - messages := make([]string, len(errors)) - for _, e := range errors { - messages = append(messages, e.Error()) - } - return nil, fmt.Errorf(strings.Join(messages, "\n")) +func (rules *ClientConfigLoadingRules) IsAPIConfig() bool { + return controllerOpts.EnableDynamicConfig +} + +func (rules *ClientConfigLoadingRules) IsDefaultConfig() bool { + // if no custom-value is provided as a config file, and no config-file exists in the default location + // we assume we need to generate configuration defaults + if CfgFile == constant.K0sConfigPathDefault && !file.Exists(constant.K0sConfigPathDefault) { + return true } - return cfg, nil + return false } -// GetNodeConfig takes a config-file parameter and returns a ClusterConfig stripped of Cluster-Wide Settings -func GetNodeConfig(cfgPath string, k0sVars constant.CfgVars) (*v1beta1.ClusterConfig, error) { - cfg, err := GetYamlFromFile(cfgPath, k0sVars) - if err != nil { - return nil, err +func (rules *ClientConfigLoadingRules) Load() (*v1beta1.ClusterConfig, error) { + if rules.CfgFileOverride != "" { + CfgFile = rules.CfgFileOverride + } + + if rules.Nodeconfig { + return rules.fetchNodeConfig() } - nodeConfig := cfg.GetBootstrappingConfig(cfg.Spec.Storage) - var etcdConfig *v1beta1.EtcdConfig - if cfg.Spec.Storage.Type == v1beta1.EtcdStorageType { - etcdConfig = &v1beta1.EtcdConfig{ - ExternalCluster: cfg.Spec.Storage.Etcd.ExternalCluster, - PeerAddress: cfg.Spec.Storage.Etcd.PeerAddress, + if !rules.IsAPIConfig() { + return rules.readRuntimeConfig() + } + if rules.IsAPIConfig() { + nodeConfig, err := rules.BootstrapConfig() + if err != nil { + return nil, err + } + apiConfig, err := rules.ClusterConfig() + if err != nil { + return nil, err } - nodeConfig.Spec.Storage.Etcd = etcdConfig + // get node config from the config-file and cluster-wide settings from the API and return a combined result + return rules.mergeNodeAndClusterconfig(nodeConfig, apiConfig) } - return nodeConfig, nil + return nil, nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 000000000000..8a63b39d8e50 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,239 @@ +/* +Copyright 2021 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package config + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/k0sproject/k0s/internal/pkg/file" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/fake" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/typed/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/k0sproject/k0s/pkg/constant" + "github.com/sirupsen/logrus" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + configPathRuntimeTest = "/tmp/k0s.yaml" + cOpts = v1.CreateOptions{TypeMeta: resourceType} + fileYaml = ` +apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + externalAddress: file_external_address + network: + serviceCIDR: 12.12.12.12/12 + podCIDR: 13.13.13.13/13 + kubeProxy: + mode: ipvs +` + apiYaml = ` +apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + externalAddress: api_external_address + network: + serviceCIDR: api_cidr +` +) + +// Test using config from a yaml file +func TestGetConfigFromFile(t *testing.T) { + cfgFilePath := writeConfigFile(fileYaml) + CfgFile = cfgFilePath + defer os.Remove(configPathRuntimeTest) + + loadingRules := ClientConfigLoadingRules{RuntimeConfigPath: configPathRuntimeTest} + err := loadingRules.InitRuntimeConfig() + if err != nil { + t.Fatalf("failed to initialize k0s config: %s", err.Error()) + } + + cfg, err := loadingRules.Load() + if err != nil { + t.Fatalf("failed to load config: %s", err.Error()) + } + if cfg == nil { + t.Fatal("received an empty config! failing") + } + testCases := []struct { + name string + got string + expected string + }{ + {"API_external_address", cfg.Spec.API.ExternalAddress, "file_external_address"}, + {"Network_ServiceCIDR", cfg.Spec.Network.ServiceCIDR, "12.12.12.12/12"}, + {"Network_KubeProxy_Mode", cfg.Spec.Network.KubeProxy.Mode, "ipvs"}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%s eq %s", tc.name, tc.expected), func(t *testing.T) { + if tc.got != tc.expected { + t.Fatalf("expected to read '%s' for the %s test value. Got: %s", tc.expected, tc.name, tc.got) + } + }) + } +} + +// Test using config from a yaml file +func TestConfigFromDefaults(t *testing.T) { + CfgFile = constant.K0sConfigPathDefault // this path doesn't exist, so default values should be generated + defer os.Remove(configPathRuntimeTest) + + loadingRules := ClientConfigLoadingRules{RuntimeConfigPath: configPathRuntimeTest} + cfg, err := loadingRules.Load() + if err != nil { + t.Fatalf("failed to load config: %s", err.Error()) + } + if cfg == nil { + t.Fatal("received an empty config! failing") + } + testCases := []struct { + name string + got string + expected string + }{ + {"API_external_address", cfg.Spec.API.ExternalAddress, ""}, + {"Network_ServiceCIDR", cfg.Spec.Network.ServiceCIDR, "10.96.0.0/12"}, + {"Network_KubeProxy_Mode", cfg.Spec.Network.KubeProxy.Mode, "iptables"}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%s eq %s", tc.name, tc.expected), func(t *testing.T) { + if tc.got != tc.expected { + t.Fatalf("expected to read '%s' for the %s test value. Got: %s", tc.expected, tc.name, tc.got) + } + }) + } +} + +// Test using node-config from a file when API config is enabled +func TestNodeConfigWithAPIConfig(t *testing.T) { + cfgFilePath := writeConfigFile(fileYaml) + CfgFile = cfgFilePath + + // if API config is enabled, Nodeconfig will be stripped of any cluster-wide-config settings + controllerOpts.EnableDynamicConfig = true + defer os.Remove(configPathRuntimeTest) + + loadingRules := ClientConfigLoadingRules{Nodeconfig: true, RuntimeConfigPath: configPathRuntimeTest} + + err := loadingRules.InitRuntimeConfig() + if err != nil { + t.Fatalf("failed to initialize k0s config: %s", err.Error()) + } + + cfg, err := loadingRules.Load() + if err != nil { + t.Fatalf("failed to fetch Node Config: %s", err.Error()) + } + testCases := []struct { + name string + got string + expected string + }{ + {"API_external_address", cfg.Spec.API.ExternalAddress, "file_external_address"}, + // PodCIDR is a cluster-wide setting. It shouldn't exist in Node config + {"Network_PodCIDR", cfg.Spec.Network.PodCIDR, ""}, + {"Network_ServiceCIDR", cfg.Spec.Network.ServiceCIDR, "12.12.12.12/12"}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%s eq %s", tc.name, tc.expected), func(t *testing.T) { + if tc.got != tc.expected { + t.Fatalf("expected to read '%s' for the %s test value. Got: %s", tc.expected, tc.name, tc.got) + } + }) + } +} + +// when a component requests an API config, +// the merged node and cluster config should be returned +func TestAPIConfig(t *testing.T) { + CfgFile = writeConfigFile(fileYaml) + + controllerOpts.EnableDynamicConfig = true + // create the API config using a fake client + client := fake.NewSimpleClientset() + + err := createFakeAPIConfig(client.K0sV1beta1()) + if err != nil { + t.Fatalf("failed to create API config: %s", err.Error()) + } + defer os.Remove(configPathRuntimeTest) + + loadingRules := ClientConfigLoadingRules{RuntimeConfigPath: configPathRuntimeTest, APIClient: client.K0sV1beta1()} + err = loadingRules.InitRuntimeConfig() + if err != nil { + t.Fatalf("failed to initialize k0s config: %s", err.Error()) + } + + cfg, err := loadingRules.Load() + if err != nil { + t.Fatalf("failed to fetch Node Config: %s", err.Error()) + } + + testCases := []struct { + name string + got string + expected string + }{ + {"API_external_address", cfg.Spec.API.ExternalAddress, "file_external_address"}, + {"Network_PodCIDR", cfg.Spec.Network.PodCIDR, "10.244.0.0/16"}, + {"Network_ServiceCIDR", cfg.Spec.Network.ServiceCIDR, "12.12.12.12/12"}, + {"Network_KubeProxy_Mode", cfg.Spec.Network.KubeProxy.Mode, "iptables"}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("%s eq %s", tc.name, tc.expected), func(t *testing.T) { + if tc.got != tc.expected { + t.Fatalf("expected to read '%s' for the %s test value. Got: %s", tc.expected, tc.name, tc.got) + } + }) + } +} + +func writeConfigFile(yamlData string) (filePath string) { + cfgFilePath, err := file.WriteTmpFile(yamlData, "k0s-config") + if err != nil { + logrus.Fatalf("Error creating tempfile: %v", err) + } + return cfgFilePath +} + +func createFakeAPIConfig(client k0sv1beta1.K0sV1beta1Interface) error { + clusterConfigs := client.ClusterConfigs(constant.ClusterConfigNamespace) + ctxWithTimeout, cancelFunction := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) + defer cancelFunction() + + config, err := v1beta1.ConfigFromString(apiYaml, v1beta1.DefaultStorageSpec()) + if err != nil { + return fmt.Errorf("failed to parse config yaml: %s", err.Error()) + } + + _, err = clusterConfigs.Create(ctxWithTimeout, config.GetClusterWideConfig().StripDefaults(), cOpts) + if err != nil { + return fmt.Errorf("failed to create clusterConfig in the API: %s", err.Error()) + } + return nil +} diff --git a/pkg/config/file_config.go b/pkg/config/file_config.go new file mode 100644 index 000000000000..958d6d205e00 --- /dev/null +++ b/pkg/config/file_config.go @@ -0,0 +1,148 @@ +/* +Copyright 2022 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package config + +import ( + "fmt" + "os" + "strings" + + "github.com/k0sproject/k0s/internal/pkg/file" + "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" + "github.com/sirupsen/logrus" + "sigs.k8s.io/yaml" +) + +var ( + runtimeConfigPathDefault = "/run/k0s/k0s.yaml" +) + +// InitRuntimeConfig generates the runtime /run/k0s/k0s.yaml +func (rules *ClientConfigLoadingRules) InitRuntimeConfig() error { + cfg, err := rules.ParseRuntimeConfig() + if err != nil { + return err + } + // this is used for Singlenode, where we set + // kine as default using k0sVars + cfg.Spec.Storage = rules.getStorageSpec() + + yamlData, err := yaml.Marshal(cfg) + if err != nil { + return err + } + return rules.writeConfig(yamlData) +} + +// readRuntimeConfig returns the configuration from the runtime configuration file +func (rules *ClientConfigLoadingRules) readRuntimeConfig() (clusterConfig *v1beta1.ClusterConfig, err error) { + if rules.RuntimeConfigPath == "" { + rules.RuntimeConfigPath = runtimeConfigPathDefault + } + return rules.ParseRuntimeConfig() +} + +// generic function that reads a config file, and returns a ClusterConfig object + +// ParseRuntimeConfig parses the `--config` flag and generates a config object +// it searches for the default config path. if it does not exist, and no other custom config-file is given, it will generate default config +func (rules *ClientConfigLoadingRules) ParseRuntimeConfig() (*v1beta1.ClusterConfig, error) { + var cfg *v1beta1.ClusterConfig + + if rules.RuntimeConfigPath == "" { + rules.RuntimeConfigPath = runtimeConfigPathDefault + } + + // don't create the runtime config file, if it already exists + if file.Exists(rules.RuntimeConfigPath) { + logrus.Debugf("runtime config found: using %s", rules.RuntimeConfigPath) + CfgFile = rules.RuntimeConfigPath + } + + switch CfgFile { + // stdin input + case "-": + return v1beta1.ConfigFromReader(os.Stdin, rules.getStorageSpec()) + default: + f, err := os.Open(CfgFile) + if err != nil { + if os.IsNotExist(err) { + return rules.generateDefaults(), nil + } + return nil, err + } + defer f.Close() + + cfg, err = v1beta1.ConfigFromReader(f, rules.getStorageSpec()) + if err != nil { + return nil, err + } + } + + if cfg.Spec.Storage.Type == v1beta1.KineStorageType && cfg.Spec.Storage.Kine == nil { + logrus.Warn("storage type is kine but no config given, setting up defaults") + cfg.Spec.Storage.Kine = v1beta1.DefaultKineConfig(rules.K0sVars.DataDir) + } + + if cfg.Spec.Install == nil { + cfg.Spec.Install = v1beta1.DefaultInstallSpec() + } + + errors := cfg.Validate() + if len(errors) > 0 { + messages := make([]string, len(errors)) + for _, e := range errors { + messages = append(messages, e.Error()) + } + return nil, fmt.Errorf(strings.Join(messages, "\n")) + } + return cfg, nil + +} + +// generate default config and return the config object +func (rules *ClientConfigLoadingRules) generateDefaults() (config *v1beta1.ClusterConfig) { + logrus.Debugf("no config file given, using defaults") + return v1beta1.DefaultClusterConfig(rules.getStorageSpec()) +} + +func (rules *ClientConfigLoadingRules) writeConfig(yamlData []byte) error { + mergedConfig, err := v1beta1.ConfigFromString(string(yamlData), rules.getStorageSpec()) + if err != nil { + return fmt.Errorf("unable to parse config: %v", err) + } + data, err := yaml.Marshal(&mergedConfig) + if err != nil { + return fmt.Errorf("failed to marshal config: %v", err) + } + + err = os.WriteFile(rules.RuntimeConfigPath, data, 0755) + if err != nil { + return fmt.Errorf("failed to write runtime config to %s (%v): %v", rules.K0sVars.RunDir, rules.RuntimeConfigPath, err) + } + return nil +} + +func (rules *ClientConfigLoadingRules) getStorageSpec() *v1beta1.StorageSpec { + var storage v1beta1.StorageSpec + if rules.K0sVars.DefaultStorageType == "kine" { + storage = v1beta1.StorageSpec{ + Type: v1beta1.KineStorageType, + Kine: v1beta1.DefaultKineConfig(rules.K0sVars.DataDir), + } + } + return &storage +}