From 0359c8537c1b3b01e94394604e16fd817b986f9e Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Wed, 29 May 2024 19:29:31 +0400 Subject: [PATCH] chore: unify toml packages being used Drop BurntSushi one, and use /v2 of pelletier package. There is indirect use of v1 which should hopefully go away once we move away from sonobouy. Signed-off-by: Andrey Smirnov --- go.mod | 5 ++-- go.sum | 2 -- .../containers/cri/containerd/containerd.go | 4 ++-- .../pkg/containers/cri/containerd/hosts.go | 18 ++++++++------ .../containers/cri/containerd/hosts_test.go | 22 ++++++++--------- .../cri/containerd/testdata/cri.toml | 21 ++++++++-------- internal/pkg/toml/merge.go | 24 ++++++++++++++----- internal/pkg/toml/testdata/expected.toml | 23 ++++++++++-------- 8 files changed, 68 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 8ea956be7a..1c6e716167 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 - github.com/BurntSushi/toml v1.3.2 github.com/alexflint/go-filemutex v1.3.0 github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 @@ -113,7 +112,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/opencontainers/runtime-spec v1.2.0 github.com/packethost/packngo v0.31.0 - github.com/pelletier/go-toml v1.9.5 + github.com/pelletier/go-toml/v2 v2.2.2 github.com/pin/tftp/v3 v3.1.0 github.com/pmorjan/kmod v1.1.1 github.com/prometheus/procfs v0.14.0 @@ -302,7 +301,7 @@ require ( github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect diff --git a/go.sum b/go.sum index d05685e657..3c6fcdfc0c 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= diff --git a/internal/pkg/containers/cri/containerd/containerd.go b/internal/pkg/containers/cri/containerd/containerd.go index ae76a0c86c..370b8fbc29 100644 --- a/internal/pkg/containers/cri/containerd/containerd.go +++ b/internal/pkg/containers/cri/containerd/containerd.go @@ -9,7 +9,7 @@ import ( "bytes" "path/filepath" - "github.com/BurntSushi/toml" + "github.com/pelletier/go-toml/v2" "github.com/siderolabs/talos/pkg/machinery/config/config" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -39,7 +39,7 @@ func GenerateCRIConfig(r config.Registries) ([]byte, error) { var buf bytes.Buffer - if err := toml.NewEncoder(&buf).Encode(&ctrdCfg); err != nil { + if err := toml.NewEncoder(&buf).SetIndentTables(true).Encode(&ctrdCfg); err != nil { return nil, err } diff --git a/internal/pkg/containers/cri/containerd/hosts.go b/internal/pkg/containers/cri/containerd/hosts.go index 399e3c2e85..ae5fb4ab7c 100644 --- a/internal/pkg/containers/cri/containerd/hosts.go +++ b/internal/pkg/containers/cri/containerd/hosts.go @@ -14,7 +14,7 @@ import ( "strings" "github.com/containerd/containerd/v2/core/remotes/docker" - "github.com/pelletier/go-toml" + "github.com/pelletier/go-toml/v2" "github.com/siderolabs/talos/pkg/machinery/config/config" ) @@ -128,11 +128,14 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error) configureEndpoint(u.Host, directoryName, hostsToml.HostConfigs[endpoint], directory) - tomlBytes, err := toml.Marshal(hostsToml) - if err != nil { + var tomlBuf bytes.Buffer + + if err := toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostsToml); err != nil { return nil, err } + tomlBytes := tomlBuf.Bytes() + // this is an ugly hack, and neither TOML format nor go-toml library make it easier // // we need to marshal each endpoint in the order they are specified in the config, but go-toml defines @@ -146,7 +149,7 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error) // [host."bar.foo"] // // but this is invalid TOML, as `[host]' is repeated, so we do an ugly hack and remove it below - const hostPrefix = "\n[host]\n" + const hostPrefix = "[host]\n" if i > 0 { if bytes.HasPrefix(tomlBytes, []byte(hostPrefix)) { @@ -206,8 +209,9 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error) configureEndpoint(hostname, directoryName, hostsToml.HostConfigs[defaultHost], directory) - marshaled, err := toml.Marshal(hostsToml) - if err != nil { + var tomlBuf bytes.Buffer + + if err = toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostsToml); err != nil { return nil, err } @@ -215,7 +219,7 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error) &HostsFile{ Name: "hosts.toml", Mode: 0o600, - Contents: marshaled, + Contents: tomlBuf.Bytes(), }, ) diff --git a/internal/pkg/containers/cri/containerd/hosts_test.go b/internal/pkg/containers/cri/containerd/hosts_test.go index ede9431cdc..b712c91a2b 100644 --- a/internal/pkg/containers/cri/containerd/hosts_test.go +++ b/internal/pkg/containers/cri/containerd/hosts_test.go @@ -59,7 +59,7 @@ func TestGenerateHostsWithTLS(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://registry-1.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n\n [host.\"https://registry-2.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n"), //nolint:lll + Contents: []byte("[host]\n [host.'https://registry-1.docker.io']\n capabilities = ['pull', 'resolve']\n [host.'https://registry-2.docker.io']\n capabilities = ['pull', 'resolve']\n skip_verify = true\n"), //nolint:lll }, }, }, @@ -83,7 +83,7 @@ func TestGenerateHostsWithTLS(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://some.host:123\"]\n ca = \"/etc/cri/conf.d/hosts/some.host_123_/some.host:123-ca.crt\"\n client = [[\"/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.crt\", \"/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.key\"]]\n skip_verify = true\n"), //nolint:lll + Contents: []byte("[host]\n [host.'https://some.host:123']\n ca = '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-ca.crt'\n client = [['/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.crt', '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.key']]\n skip_verify = true\n"), //nolint:lll }, }, }, @@ -92,7 +92,7 @@ func TestGenerateHostsWithTLS(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://registry-2.docker.io\"]\n skip_verify = true\n"), + Contents: []byte("[host]\n [host.'https://registry-2.docker.io']\n skip_verify = true\n"), }, }, }, @@ -132,7 +132,7 @@ func TestGenerateHostsWithoutTLS(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://registry-1.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n\n [host.\"https://registry-2.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n"), //nolint:lll + Contents: []byte("[host]\n [host.'https://registry-1.docker.io']\n capabilities = ['pull', 'resolve']\n [host.'https://registry-2.docker.io']\n capabilities = ['pull', 'resolve']\n"), //nolint:lll }, }, }, @@ -141,7 +141,7 @@ func TestGenerateHostsWithoutTLS(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://some.host:123\"]\n"), + Contents: []byte("[host]\n [host.'https://some.host:123']\n"), }, }, }, @@ -150,7 +150,7 @@ func TestGenerateHostsWithoutTLS(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://my-registry\"]\n capabilities = [\"pull\", \"resolve\"]\n"), + Contents: []byte("[host]\n [host.'https://my-registry']\n capabilities = ['pull', 'resolve']\n"), }, }, }, @@ -205,7 +205,7 @@ func TestGenerateHostsTLSWildcard(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://my-registry1\"]\n ca = \"/etc/cri/conf.d/hosts/_default/my-registry1-ca.crt\"\n capabilities = [\"pull\", \"resolve\"]\n\n [host.\"https://my-registry2\"]\n capabilities = [\"pull\", \"resolve\"]\n"), //nolint:lll + Contents: []byte("[host]\n [host.'https://my-registry1']\n capabilities = ['pull', 'resolve']\n ca = '/etc/cri/conf.d/hosts/_default/my-registry1-ca.crt'\n [host.'https://my-registry2']\n capabilities = ['pull', 'resolve']\n"), //nolint:lll }, }, }, @@ -219,7 +219,7 @@ func TestGenerateHostsTLSWildcard(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://my-registry1\"]\n ca = \"/etc/cri/conf.d/hosts/my-registry1/my-registry1-ca.crt\"\n"), + Contents: []byte("[host]\n [host.'https://my-registry1']\n ca = '/etc/cri/conf.d/hosts/my-registry1/my-registry1-ca.crt'\n"), }, }, }, @@ -269,7 +269,7 @@ func TestGenerateHostsWithHarbor(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://harbor/v2/mirrors/proxy.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n override_path = true\n skip_verify = true\n"), + Contents: []byte("[host]\n [host.'https://harbor/v2/mirrors/proxy.docker.io']\n capabilities = ['pull', 'resolve']\n override_path = true\n skip_verify = true\n"), }, }, }, @@ -278,7 +278,7 @@ func TestGenerateHostsWithHarbor(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://harbor/v2/mirrors/proxy.ghcr.io\"]\n capabilities = [\"pull\", \"resolve\"]\n override_path = true\n skip_verify = true\n"), + Contents: []byte("[host]\n [host.'https://harbor/v2/mirrors/proxy.ghcr.io']\n capabilities = ['pull', 'resolve']\n override_path = true\n skip_verify = true\n"), }, }, }, @@ -287,7 +287,7 @@ func TestGenerateHostsWithHarbor(t *testing.T) { { Name: "hosts.toml", Mode: 0o600, - Contents: []byte("\n[host]\n\n [host.\"https://harbor\"]\n skip_verify = true\n"), + Contents: []byte("[host]\n [host.'https://harbor']\n skip_verify = true\n"), }, }, }, diff --git a/internal/pkg/containers/cri/containerd/testdata/cri.toml b/internal/pkg/containers/cri/containerd/testdata/cri.toml index 3f60b82468..3edb5da6e5 100644 --- a/internal/pkg/containers/cri/containerd/testdata/cri.toml +++ b/internal/pkg/containers/cri/containerd/testdata/cri.toml @@ -1,11 +1,12 @@ [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/cri/conf.d/hosts" - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."some.host:123"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."some.host:123".auth] - username = "root" - password = "secret" - auth = "auth" - identitytoken = "token" + [plugins.'io.containerd.grpc.v1.cri'] + [plugins.'io.containerd.grpc.v1.cri'.registry] + config_path = '/etc/cri/conf.d/hosts' + + [plugins.'io.containerd.grpc.v1.cri'.registry.configs] + [plugins.'io.containerd.grpc.v1.cri'.registry.configs.'some.host:123'] + [plugins.'io.containerd.grpc.v1.cri'.registry.configs.'some.host:123'.auth] + username = 'root' + password = 'secret' + auth = 'auth' + identitytoken = 'token' diff --git a/internal/pkg/toml/merge.go b/internal/pkg/toml/merge.go index eb5781a57e..cc4023de3e 100644 --- a/internal/pkg/toml/merge.go +++ b/internal/pkg/toml/merge.go @@ -7,24 +7,36 @@ package toml import ( "bytes" "fmt" + "os" - "github.com/BurntSushi/toml" + "github.com/pelletier/go-toml/v2" "github.com/siderolabs/talos/pkg/machinery/config/merge" ) +func tomlDecodeFile(path string, dest any) error { + f, err := os.Open(path) + if err != nil { + return err + } + + defer f.Close() //nolint:errcheck + + return toml.NewDecoder(f).Decode(dest) +} + // Merge several TOML documents in files into one. // -// Merge process relies on generic map[string]interface{} merge which might not always be correct. +// Merge process relies on generic map[string]any merge which might not always be correct. func Merge(parts []string) ([]byte, error) { - merged := map[string]interface{}{} + merged := map[string]any{} var header []byte for _, part := range parts { - partial := map[string]interface{}{} + partial := map[string]any{} - if _, err := toml.DecodeFile(part, &partial); err != nil { + if err := tomlDecodeFile(part, &partial); err != nil { return nil, fmt.Errorf("error decoding %q: %w", part, err) } @@ -40,7 +52,7 @@ func Merge(parts []string) ([]byte, error) { _, _ = out.Write(header) _ = out.WriteByte('\n') - if err := toml.NewEncoder(&out).Encode(merged); err != nil { + if err := toml.NewEncoder(&out).SetIndentTables(true).Encode(merged); err != nil { return nil, fmt.Errorf("error encoding merged config: %w", err) } diff --git a/internal/pkg/toml/testdata/expected.toml b/internal/pkg/toml/testdata/expected.toml index 7748750c4d..9bb7f81479 100644 --- a/internal/pkg/toml/testdata/expected.toml +++ b/internal/pkg/toml/testdata/expected.toml @@ -5,16 +5,19 @@ version = 2 [metrics] - address = "0.0.0.0:11234" + address = '0.0.0.0:11234' [plugins] - [plugins."io.containerd.grpc.v1.cri"] - sandbox_image = "registry.k8s.io/pause:3.8" - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + [plugins.'io.containerd.grpc.v1.cri'] + sandbox_image = 'registry.k8s.io/pause:3.8' + + [plugins.'io.containerd.grpc.v1.cri'.containerd] + [plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] + [plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] discard_unpacked_layers = true - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "/etc/cri/conf.d/hosts" - [plugins."io.containerd.grpc.v1.cri".registry.configs] + runtime_type = 'io.containerd.runc.v2' + + [plugins.'io.containerd.grpc.v1.cri'.registry] + config_path = '/etc/cri/conf.d/hosts' + + [plugins.'io.containerd.grpc.v1.cri'.registry.configs]