diff --git a/test/infrastructure/container/docker.go b/test/infrastructure/container/docker.go index a790f881b782..e0da3f9e7a4e 100644 --- a/test/infrastructure/container/docker.go +++ b/test/infrastructure/container/docker.go @@ -21,6 +21,7 @@ package container import ( "bytes" "context" + "encoding/csv" "fmt" "io" "os" @@ -35,6 +36,7 @@ import ( "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-connections/nat" "github.com/pkg/errors" + "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -43,6 +45,10 @@ const ( httpProxy = "HTTP_PROXY" httpsProxy = "HTTPS_PROXY" noProxy = "NO_PROXY" + + btrfsStorage = "btrfs" + zfsStorage = "zfs" + xfsStorage = "xfs" ) type dockerRuntime struct { @@ -372,8 +378,10 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine } restartPolicy := runConfig.RestartPolicy + restartMaximumRetryCount := 0 if restartPolicy == "" { - restartPolicy = "unless-stopped" + restartPolicy = "on-failure" + restartMaximumRetryCount = 1 } hostConfig := dockercontainer.HostConfig{ @@ -383,11 +391,12 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine // including some ones docker would otherwise do by default. // for now this is what we want. in the future we may revisit this. Privileged: true, - SecurityOpt: []string{"seccomp=unconfined"}, // ignore seccomp + SecurityOpt: []string{"seccomp=unconfined", "apparmor=unconfined"}, // ignore seccomp NetworkMode: dockercontainer.NetworkMode(runConfig.Network), Tmpfs: runConfig.Tmpfs, PortBindings: nat.PortMap{}, - RestartPolicy: dockercontainer.RestartPolicy{Name: restartPolicy}, + RestartPolicy: dockercontainer.RestartPolicy{Name: restartPolicy, MaximumRetryCount: restartMaximumRetryCount}, + Init: pointer.Bool(false), } networkConfig := network.NetworkingConfig{} @@ -398,21 +407,21 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine } } - // mount /dev/mapper if docker storage driver if Btrfs or ZFS - // https://github.com/kubernetes-sigs/kind/pull/1464 - needed, err := d.needsDevMapper(ctx) + info, err := d.dockerClient.Info(ctx) if err != nil { return errors.Wrapf(err, "unable to get Docker engine info, failed to create container %q", runConfig.Name) } - if needed { + // mount /dev/mapper if docker storage driver if Btrfs or ZFS + // https://github.com/kubernetes-sigs/kind/pull/1464 + if d.needsDevMapper(info) { hostConfig.Binds = append(hostConfig.Binds, "/dev/mapper:/dev/mapper:ro") } envVars := environmentVariables(runConfig) // pass proxy environment variables to be used by node's docker daemon - proxyDetails, err := d.getProxyDetails(ctx, runConfig.Network) + proxyDetails, err := d.getProxyDetails(ctx, runConfig.Network, runConfig.Name) if err != nil { return errors.Wrapf(err, "error getting subnets for %q", runConfig.Network) } @@ -421,15 +430,30 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine } containerConfig.Env = envVars + // handle Docker on Btrfs or ZFS + // https://github.com/kubernetes-sigs/kind/issues/1416#issuecomment-606514724 + if d.mountDevMapper(info) { + runConfig.Mounts = append(runConfig.Mounts, Mount{ + Source: "/dev/mapper", + Target: "/dev/mapper", + }) + } + configureVolumes(runConfig, &containerConfig, &hostConfig) configurePortMappings(runConfig.PortMappings, &containerConfig, &hostConfig) - if d.usernsRemap(ctx) { + if d.usernsRemap(info) { // We need this argument in order to make this command work // in systems that have userns-remap enabled on the docker daemon hostConfig.UsernsMode = "host" } + // enable /dev/fuse explicitly for fuse-overlayfs + // (Rootless Docker does not automatically mount /dev/fuse with --privileged) + if d.mountFuse(info) { + hostConfig.Devices = append(hostConfig.Devices, dockercontainer.DeviceMapping{PathOnHost: "/dev/fuse"}) + } + // Make sure we have the image if err := d.PullContainerImageIfNotExists(ctx, runConfig.Image); err != nil { return errors.Wrapf(err, "error pulling container image %s", runConfig.Image) @@ -511,13 +535,8 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine // needsDevMapper checks whether we need to mount /dev/mapper. // This is required when the docker storage driver is Btrfs or ZFS. // https://github.com/kubernetes-sigs/kind/pull/1464 -func (d *dockerRuntime) needsDevMapper(ctx context.Context) (bool, error) { - info, err := d.dockerClient.Info(ctx) - if err != nil { - return false, err - } - - return info.Driver == "btrfs" || info.Driver == "zfs", nil +func (d *dockerRuntime) needsDevMapper(info types.Info) bool { + return info.Driver == btrfsStorage || info.Driver == zfsStorage } // ownerAndGroup gets the user configuration for the container (user:group). @@ -601,7 +620,7 @@ type proxyDetails struct { // getProxyDetails returns a struct with the host environment proxy settings // that should be passed to the nodes. -func (d *dockerRuntime) getProxyDetails(ctx context.Context, network string) (*proxyDetails, error) { +func (d *dockerRuntime) getProxyDetails(ctx context.Context, network string, nodeNames ...string) (*proxyDetails, error) { var val string details := proxyDetails{Envs: make(map[string]string)} proxyEnvs := []string{httpProxy, httpsProxy, noProxy} @@ -626,21 +645,24 @@ func (d *dockerRuntime) getProxyDetails(ctx context.Context, network string) (*p if err != nil { return &details, err } - noProxyList := strings.Join(append(subnets, details.Envs[noProxy]), ",") - details.Envs[noProxy] = noProxyList - details.Envs[strings.ToLower(noProxy)] = noProxyList + noProxyList := append(subnets, details.Envs[noProxy]) + noProxyList = append(noProxyList, nodeNames...) + // Add pod and service dns names to no_proxy to allow in cluster + // Note: this is best effort based on the default CoreDNS spec + // https://github.com/kubernetes/dns/blob/master/docs/specification.md + // Any user created pod/service hostnames, namespaces, custom DNS services + // are expected to be no-proxied by the user explicitly. + noProxyList = append(noProxyList, ".svc", ".svc.cluster", ".svc.cluster.local") + noProxyJoined := strings.Join(noProxyList, ",") + details.Envs[noProxy] = noProxyJoined + details.Envs[strings.ToLower(noProxy)] = noProxyJoined } return &details, nil } // usernsRemap checks if userns-remap is enabled in dockerd. -func (d *dockerRuntime) usernsRemap(ctx context.Context) bool { - info, err := d.dockerClient.Info(ctx) - if err != nil { - return false - } - +func (d *dockerRuntime) usernsRemap(info types.Info) bool { for _, secOpt := range info.SecurityOptions { if strings.Contains(secOpt, "name=userns") { return true @@ -649,6 +671,49 @@ func (d *dockerRuntime) usernsRemap(ctx context.Context) bool { return false } +// mountDevMapper checks if the Docker storage driver is Btrfs or ZFS +// or if the backing filesystem is Btrfs. +func (d *dockerRuntime) mountDevMapper(info types.Info) bool { + storage := "" + storage = strings.ToLower(strings.TrimSpace(info.Driver)) + if storage == btrfsStorage || storage == zfsStorage || storage == "devicemapper" { + return true + } + + // check the backing file system + // docker info -f '{{json .DriverStatus }}' + // [["Backing Filesystem","extfs"],["Supports d_type","true"],["Native Overlay Diff","true"]] + for _, item := range info.DriverStatus { + if item[0] == "Backing Filesystem" { + storage = strings.ToLower(item[1]) + break + } + } + + return storage == btrfsStorage || storage == zfsStorage || storage == xfsStorage +} + +// rootless: use fuse-overlayfs by default +// https://github.com/kubernetes-sigs/kind/issues/2275 +func (d *dockerRuntime) mountFuse(info types.Info) bool { + for _, o := range info.SecurityOptions { + // o is like "name=seccomp,profile=default", or "name=rootless", + csvReader := csv.NewReader(strings.NewReader(o)) + sliceSlice, err := csvReader.ReadAll() + if err != nil { + return false + } + for _, f := range sliceSlice { + for _, ff := range f { + if ff == "name=rootless" { + return true + } + } + } + } + return false +} + func isSELinuxEnforcing() bool { dat, err := os.ReadFile("/sys/fs/selinux/enforce") if err != nil { diff --git a/test/infrastructure/docker/internal/docker/kind_manager.go b/test/infrastructure/docker/internal/docker/kind_manager.go index 9d3cd01a1097..2eec6a137b2f 100644 --- a/test/infrastructure/docker/internal/docker/kind_manager.go +++ b/test/infrastructure/docker/internal/docker/kind_manager.go @@ -170,6 +170,12 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error) }, IPFamily: opts.IPFamily, } + if opts.Role == constants.ControlPlaneNodeRoleValue { + runOptions.EnvironmentVars = map[string]string{ + "KUBECONFIG": "/etc/kubernetes/admin.conf", + } + } + log.V(6).Info("Container run options: %+v", runOptions) containerRuntime, err := container.RuntimeFrom(ctx)