From adca3e460774a147aa3189e373e68044ebaa6be5 Mon Sep 17 00:00:00 2001 From: yuchanns Date: Wed, 5 Oct 2022 18:13:24 +0800 Subject: [PATCH] Support argument `--mac-address` for nerdctl run command Signed-off-by: Hanchin Hsieh --- README.md | 3 + cmd/nerdctl/create_linux_test.go | 73 +++++++++++++++++++++++++ cmd/nerdctl/run.go | 11 +++- cmd/nerdctl/run_network.go | 73 ++++++++++++++++++------- cmd/nerdctl/run_network_linux_test.go | 46 ++++++++++++++++ pkg/labels/labels.go | 2 + pkg/ocihook/ocihook.go | 29 ++++++++++ pkg/testutil/nettestutil/nettestutil.go | 12 ++++ 8 files changed, 227 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9d8846a4e46..83d309d2e76 100644 --- a/README.md +++ b/README.md @@ -397,6 +397,9 @@ Network flags: - :whale: `-h, --hostname`: Container host name - :whale: `--add-host`: Add a custom host-to-IP mapping (host:ip) - :whale: `--ip`: Specific static IP address(es) to use +- :whale: `--mac-address`: Specific MAC address to use. Be aware that it does not + check if manually specified MAC addresses are unique. Supports network + type `bridge` and `macvlan` Resource flags: - :whale: `--cpus`: Number of CPUs diff --git a/cmd/nerdctl/create_linux_test.go b/cmd/nerdctl/create_linux_test.go index 57cda609281..693788729da 100644 --- a/cmd/nerdctl/create_linux_test.go +++ b/cmd/nerdctl/create_linux_test.go @@ -17,9 +17,11 @@ package main import ( + "fmt" "testing" "github.com/containerd/nerdctl/pkg/testutil" + "github.com/containerd/nerdctl/pkg/testutil/nettestutil" ) func TestCreate(t *testing.T) { @@ -33,3 +35,74 @@ func TestCreate(t *testing.T) { base.Cmd("start", tID).AssertOK() base.Cmd("logs", tID).AssertOutContains("foo") } + +func TestCreateWithMACAddress(t *testing.T) { + base := testutil.NewBase(t) + tID := testutil.Identifier(t) + networkBridge := "testNetworkBridge" + tID + networkMACvlan := "testNetworkMACvlan" + tID + networkIPvlan := "testNetworkIPvlan" + tID + base.Cmd("network", "create", networkBridge, "--driver", "bridge").AssertOK() + base.Cmd("network", "create", networkMACvlan, "--driver", "macvlan").AssertOK() + base.Cmd("network", "create", networkIPvlan, "--driver", "ipvlan").AssertOK() + t.Cleanup(func() { + base.Cmd("network", "rm", networkBridge).Run() + base.Cmd("network", "rm", networkMACvlan).Run() + base.Cmd("network", "rm", networkIPvlan).Run() + }) + tests := []struct { + Network string + WantErr bool + Expect string + }{ + {"host", true, "conflicting options"}, + {"none", true, "can't open '/sys/class/net/eth0/address'"}, + {"container:whatever" + tID, true, "conflicting options"}, + {"bridge", false, ""}, + {networkBridge, false, ""}, + {networkMACvlan, false, ""}, + {networkIPvlan, true, "not support"}, + } + for i, test := range tests { + containerName := fmt.Sprintf("%s_%d", tID, i) + macAddress, err := nettestutil.GenerateMACAddress() + if err != nil { + t.Errorf("failed to generate MAC address: %s", err) + } + if test.Expect == "" && !test.WantErr { + test.Expect = macAddress + } + t.Cleanup(func() { + base.Cmd("rm", "-f", containerName).Run() + }) + cmd := base.Cmd("create", "--network", test.Network, "--mac-address", macAddress, "--name", containerName, testutil.CommonImage, "cat", "/sys/class/net/eth0/address") + if !test.WantErr { + cmd.AssertOK() + base.Cmd("start", containerName).AssertOK() + cmd = base.Cmd("logs", containerName) + cmd.AssertOK() + cmd.AssertOutContains(test.Expect) + } else { + if (testutil.GetTarget() == testutil.Docker && test.Network == networkIPvlan) || test.Network == "none" { + // 1. unlike nerdctl + // when using network ipvlan in Docker + // it delays fail on executing start command + // 2. start on network none will success in both + // nerdctl and Docker + cmd.AssertOK() + cmd = base.Cmd("start", containerName) + if test.Network == "none" { + // we check the result on logs command + cmd.AssertOK() + cmd = base.Cmd("logs", containerName) + } + } + cmd.AssertCombinedOutContains(test.Expect) + if test.Network == "none" { + cmd.AssertOK() + } else { + cmd.AssertFail() + } + } + } +} diff --git a/cmd/nerdctl/run.go b/cmd/nerdctl/run.go index 616c867e5a7..1e399a6ec50 100644 --- a/cmd/nerdctl/run.go +++ b/cmd/nerdctl/run.go @@ -142,6 +142,7 @@ func setCreateFlags(cmd *cobra.Command) { // FIXME: not support IPV6 yet cmd.Flags().String("ip", "", "IPv4 address to assign to the container") cmd.Flags().StringP("hostname", "h", "", "Container host name") + cmd.Flags().String("mac-address", "", "MAC address to assign to the container") // #endregion cmd.Flags().String("ipc", "", `IPC namespace to use ("host"|"private")`) @@ -573,7 +574,7 @@ func createContainer(cmd *cobra.Command, ctx context.Context, client *containerd opts = append(opts, withCustomEtcHostname(hostnamePath)) } - netOpts, netSlice, ipAddress, ports, err := generateNetOpts(cmd, dataStore, stateDir, ns, id) + netOpts, netSlice, ipAddress, ports, macAddress, err := generateNetOpts(cmd, dataStore, stateDir, ns, id) if err != nil { return nil, nil, err } @@ -656,7 +657,7 @@ func createContainer(cmd *cobra.Command, ctx context.Context, client *containerd return nil, nil, err } } - ilOpt, err := withInternalLabels(ns, name, hostname, stateDir, extraHosts, netSlice, ipAddress, ports, logURI, anonVolumes, pidFile, platform, mountPoints) + ilOpt, err := withInternalLabels(ns, name, hostname, stateDir, extraHosts, netSlice, ipAddress, ports, logURI, anonVolumes, pidFile, platform, mountPoints, macAddress) if err != nil { return nil, nil, err } @@ -998,7 +999,7 @@ func withStop(stopSignal string, stopTimeout int, ensuredImage *imgutil.EnsuredI } } -func withInternalLabels(ns, name, hostname, containerStateDir string, extraHosts, networks []string, ipAddress string, ports []gocni.PortMapping, logURI string, anonVolumes []string, pidFile, platform string, mountPoints []*mountutil.Processed) (containerd.NewContainerOpts, error) { +func withInternalLabels(ns, name, hostname, containerStateDir string, extraHosts, networks []string, ipAddress string, ports []gocni.PortMapping, logURI string, anonVolumes []string, pidFile, platform string, mountPoints []*mountutil.Processed, macAddress string) (containerd.NewContainerOpts, error) { m := make(map[string]string) m[labels.Namespace] = ns if name != "" { @@ -1056,6 +1057,10 @@ func withInternalLabels(ns, name, hostname, containerStateDir string, extraHosts m[labels.Mounts] = string(mountPointsJSON) } + if macAddress != "" { + m[labels.MACAddress] = macAddress + } + return containerd.WithAdditionalContainerLabels(m), nil } diff --git a/cmd/nerdctl/run_network.go b/cmd/nerdctl/run_network.go index a3cc8fb3394..a8588320353 100644 --- a/cmd/nerdctl/run_network.go +++ b/cmd/nerdctl/run_network.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io/fs" + "net" "path/filepath" "runtime" "strings" @@ -110,75 +111,88 @@ func withCustomHosts(src string) func(context.Context, oci.Client, *containers.C } } -func generateNetOpts(cmd *cobra.Command, dataStore, stateDir, ns, id string) ([]oci.SpecOpts, []string, string, []gocni.PortMapping, error) { +func generateNetOpts(cmd *cobra.Command, dataStore, stateDir, ns, id string) ([]oci.SpecOpts, []string, string, []gocni.PortMapping, string, error) { opts := []oci.SpecOpts{} portSlice, err := cmd.Flags().GetStringSlice("publish") if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } ipAddress, err := cmd.Flags().GetString("ip") if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } netSlice, err := getNetworkSlice(cmd) if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } if (len(netSlice) == 0) && (ipAddress != "") { logrus.Warnf("You have assign an IP address %s but no network, So we will use the default network", ipAddress) } + macAddress, err := getMACAddress(cmd, netSlice) + if err != nil { + return nil, nil, "", nil, "", err + } + ports := make([]gocni.PortMapping, 0) netType, err := nettype.Detect(netSlice) if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } switch netType { case nettype.None: // NOP + // Docker compatible: if macAddress is specified, set MAC address shall + // not work but run command will success case nettype.Host: + if macAddress != "" { + return nil, nil, "", nil, "", errors.New("conflicting options: mac-address and the network mode") + } opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace), oci.WithHostHostsFile, oci.WithHostResolvconf) case nettype.CNI: // We only verify flags and generate resolv.conf here. // The actual network is configured in the oci hook. - if err := verifyCNINetwork(cmd, netSlice); err != nil { - return nil, nil, "", nil, err + if err := verifyCNINetwork(cmd, netSlice, macAddress); err != nil { + return nil, nil, "", nil, "", err } if runtime.GOOS == "linux" { resolvConfPath := filepath.Join(stateDir, "resolv.conf") if err := buildResolvConf(cmd, resolvConfPath); err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } // the content of /etc/hosts is created in OCI Hook etcHostsPath, err := hostsstore.AllocHostsFile(dataStore, ns, id) if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } opts = append(opts, withCustomResolvConf(resolvConfPath), withCustomHosts(etcHostsPath)) for _, p := range portSlice { pm, err := portutil.ParseFlagP(p) if err != nil { - return nil, nil, "", pm, err + return nil, nil, "", pm, "", err } ports = append(ports, pm...) } } case nettype.Container: + if macAddress != "" { + return nil, nil, "", nil, "", errors.New("conflicting options: mac-address and the network mode") + } if err := verifyContainerNetwork(cmd, netSlice); err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } network := strings.Split(netSlice[0], ":") if len(network) != 2 { - return nil, nil, "", nil, fmt.Errorf("invalid network: %s, should be \"container:\"", netSlice[0]) + return nil, nil, "", nil, "", fmt.Errorf("invalid network: %s, should be \"container:\"", netSlice[0]) } containerName := network[1] client, ctx, cancel, err := newClient(cmd) if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } defer cancel() @@ -224,15 +238,15 @@ func generateNetOpts(cmd *cobra.Command, dataStore, stateDir, ns, id string) ([] } n, err := walker.Walk(ctx, containerName) if err != nil { - return nil, nil, "", nil, err + return nil, nil, "", nil, "", err } if n == 0 { - return nil, nil, "", nil, fmt.Errorf("no such container: %s", containerName) + return nil, nil, "", nil, "", fmt.Errorf("no such container: %s", containerName) } default: - return nil, nil, "", nil, fmt.Errorf("unexpected network type %v", netType) + return nil, nil, "", nil, "", fmt.Errorf("unexpected network type %v", netType) } - return opts, netSlice, ipAddress, ports, nil + return opts, netSlice, ipAddress, ports, macAddress, nil } func getContainerNetNSPath(ctx context.Context, c containerd.Container) (string, error) { @@ -250,7 +264,7 @@ func getContainerNetNSPath(ctx context.Context, c containerd.Container) (string, return fmt.Sprintf("/proc/%d/ns/net", task.Pid()), nil } -func verifyCNINetwork(cmd *cobra.Command, netSlice []string) error { +func verifyCNINetwork(cmd *cobra.Command, netSlice []string, macAddress string) error { cniPath, err := cmd.Flags().GetString("cni-path") if err != nil { return err @@ -263,12 +277,19 @@ func verifyCNINetwork(cmd *cobra.Command, netSlice []string) error { if err != nil { return err } + macValidNetworks := []string{"bridge", "macvlan"} netMap := e.NetworkMap() for _, netstr := range netSlice { - _, ok := netMap[netstr] + netConfig, ok := netMap[netstr] if !ok { return fmt.Errorf("network %s not found", netstr) } + // if MAC address is specified, the type of the network + // must be one of macValidNetworks + netType := netConfig.Plugins[0].Network.Type + if macAddress != "" && !strutil.InStringSlice(macValidNetworks, netType) { + return fmt.Errorf("%s interfaces on network %s do not support --mac-address", netType, netstr) + } } return nil } @@ -360,3 +381,17 @@ func buildResolvConf(cmd *cobra.Command, resolvConfPath string) error { } return nil } + +func getMACAddress(cmd *cobra.Command, netSlice []string) (string, error) { + macAddress, err := cmd.Flags().GetString("mac-address") + if err != nil { + return "", err + } + if macAddress == "" { + return "", nil + } + if _, err := net.ParseMAC(macAddress); err != nil { + return "", err + } + return macAddress, nil +} diff --git a/cmd/nerdctl/run_network_linux_test.go b/cmd/nerdctl/run_network_linux_test.go index 688613323e7..1efb93ad4cb 100644 --- a/cmd/nerdctl/run_network_linux_test.go +++ b/cmd/nerdctl/run_network_linux_test.go @@ -529,3 +529,49 @@ func TestSharedNetworkStack(t *testing.T) { base.Cmd("exec", containerNameJoin, "wget", "-qO-", "http://127.0.0.1:80"). AssertOutContains(testutil.NginxAlpineIndexHTMLSnippet) } + +func TestRunContainerWithMACAddress(t *testing.T) { + base := testutil.NewBase(t) + tID := testutil.Identifier(t) + networkBridge := "testNetworkBridge" + tID + networkMACvlan := "testNetworkMACvlan" + tID + networkIPvlan := "testNetworkIPvlan" + tID + base.Cmd("network", "create", networkBridge, "--driver", "bridge").AssertOK() + base.Cmd("network", "create", networkMACvlan, "--driver", "macvlan").AssertOK() + base.Cmd("network", "create", networkIPvlan, "--driver", "ipvlan").AssertOK() + t.Cleanup(func() { + base.Cmd("network", "rm", networkBridge).Run() + base.Cmd("network", "rm", networkMACvlan).Run() + base.Cmd("network", "rm", networkIPvlan).Run() + }) + tests := []struct { + Network string + WantErr bool + Expect string + }{ + {"host", true, "conflicting options"}, + {"none", true, "can't open '/sys/class/net/eth0/address'"}, + {"container:whatever" + tID, true, "conflicting options"}, + {"bridge", false, ""}, + {networkBridge, false, ""}, + {networkMACvlan, false, ""}, + {networkIPvlan, true, "not support"}, + } + for _, test := range tests { + macAddress, err := nettestutil.GenerateMACAddress() + if err != nil { + t.Errorf("failed to generate MAC address: %s", err) + } + if test.Expect == "" && !test.WantErr { + test.Expect = macAddress + } + cmd := base.Cmd("run", "--rm", "--network", test.Network, "--mac-address", macAddress, testutil.CommonImage, "cat", "/sys/class/net/eth0/address") + if test.WantErr { + cmd.AssertFail() + cmd.AssertCombinedOutContains(test.Expect) + } else { + cmd.AssertOK() + cmd.AssertOutContains(test.Expect) + } + } +} diff --git a/pkg/labels/labels.go b/pkg/labels/labels.go index 6a20246aa1a..d749f9fe828 100644 --- a/pkg/labels/labels.go +++ b/pkg/labels/labels.go @@ -84,6 +84,8 @@ const ( // StopTimeout is seconds to wait for stop a container. StopTimout = Prefix + "stop-timeout" + + MACAddress = Prefix + "mac-address" ) var ShellCompletions = []string{ diff --git a/pkg/ocihook/ocihook.go b/pkg/ocihook/ocihook.go index bb030c74c31..2c206679457 100644 --- a/pkg/ocihook/ocihook.go +++ b/pkg/ocihook/ocihook.go @@ -178,6 +178,10 @@ func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath strin o.containerIP = ipAddress } + if macAddress, ok := o.state.Annotations[labels.MACAddress]; ok { + o.contianerMAC = macAddress + } + if rootlessutil.IsRootlessChild() { o.rootlessKitClient, err = rootlessutil.NewRootlessKitClient() if err != nil { @@ -213,6 +217,7 @@ type handlerOpts struct { bypassClient b4nndclient.Client extraHosts map[string]string // host:ip containerIP string + contianerMAC string } // hookSpec is from https://github.com/containerd/containerd/blob/v1.4.3/cmd/containerd/command/oci-hook.go#L59-L64 @@ -340,6 +345,20 @@ func getIPAddressOpts(opts *handlerOpts) ([]gocni.NamespaceOpts, error) { return nil, nil } +func getMACAddressOpts(opts *handlerOpts) ([]gocni.NamespaceOpts, error) { + if opts.contianerMAC != "" { + return []gocni.NamespaceOpts{ + gocni.WithLabels(map[string]string{ + // allow loose CNI argument verification + // FYI: https://github.com/containernetworking/cni/issues/560 + "IgnoreUnknown": "1", + }), + gocni.WithArgs("MAC", opts.contianerMAC), + }, nil + } + return nil, nil +} + func onCreateRuntime(opts *handlerOpts) error { loadAppArmor() @@ -361,9 +380,14 @@ func onCreateRuntime(opts *handlerOpts) error { if err != nil { return err } + macAddressOpts, err := getMACAddressOpts(opts) + if err != nil { + return err + } var namespaceOpts []gocni.NamespaceOpts namespaceOpts = append(namespaceOpts, portMapOpts...) namespaceOpts = append(namespaceOpts, ipAddressOpts...) + namespaceOpts = append(namespaceOpts, macAddressOpts...) hsMeta := hostsstore.Meta{ Namespace: opts.state.Annotations[labels.Namespace], ID: opts.state.ID, @@ -454,9 +478,14 @@ func onPostStop(opts *handlerOpts) error { if err != nil { return err } + macAddressOpts, err := getMACAddressOpts(opts) + if err != nil { + return err + } var namespaceOpts []gocni.NamespaceOpts namespaceOpts = append(namespaceOpts, portMapOpts...) namespaceOpts = append(namespaceOpts, ipAddressOpts...) + namespaceOpts = append(namespaceOpts, macAddressOpts...) if err := opts.cni.Remove(ctx, opts.fullID, "", namespaceOpts...); err != nil { logrus.WithError(err).Errorf("failed to call cni.Remove") return err diff --git a/pkg/testutil/nettestutil/nettestutil.go b/pkg/testutil/nettestutil/nettestutil.go index 3662cd0c921..960d7738bca 100644 --- a/pkg/testutil/nettestutil/nettestutil.go +++ b/pkg/testutil/nettestutil/nettestutil.go @@ -17,6 +17,7 @@ package nettestutil import ( + "crypto/rand" "crypto/tls" "fmt" "net" @@ -73,3 +74,14 @@ func NonLoopbackIPv4() (net.IP, error) { } return nil, fmt.Errorf("non-loopback IPv4 address not found, attempted=%+v: %w", addrs, errdefs.ErrNotFound) } + +func GenerateMACAddress() (string, error) { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + return "", err + } + // make sure byte 0 (broadcast) of the first byte is not set + // and byte 1 (local) is set + buf[0] = buf[0]&254 | 2 + return fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]), nil +}