From 62773a6fe494e04e7d3e2939b1a9546cac8fddd9 Mon Sep 17 00:00:00 2001 From: YaoZengzeng Date: Fri, 16 Mar 2018 16:32:45 +0800 Subject: [PATCH] feature: dns configuration for cri manager Signed-off-by: YaoZengzeng --- daemon/mgr/cri.go | 40 ++++++++++++++++-- daemon/mgr/cri_utils.go | 85 +++++++++++++++++++++++++++++++++++++++ hack/cri-test/test-cri.sh | 2 +- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/daemon/mgr/cri.go b/daemon/mgr/cri.go index 711494214..899253872 100644 --- a/daemon/mgr/cri.go +++ b/daemon/mgr/cri.go @@ -3,6 +3,8 @@ package mgr import ( "bytes" "fmt" + "os" + "path" "time" apitypes "github.com/alibaba/pouch/apis/types" @@ -51,6 +53,9 @@ const ( namespaceModeHost = "host" namespaceModeNone = "none" + + // resolvConfPath is the abs path of resolv.conf on host or container. + resolvConfPath = "/etc/resolv.conf" ) var ( @@ -78,6 +83,9 @@ type CriManager struct { // StreamServer is the stream server of CRI serves container streaming request. StreamServer stream.Server + + // SandboxBaseDir is the directory used to store sandbox files like /etc/hosts, /etc/resolv.conf, etc. + SandboxBaseDir string } // NewCriManager creates a brand new cri manager. @@ -88,10 +96,11 @@ func NewCriManager(config *config.Config, ctrMgr ContainerMgr, imgMgr ImageMgr) } c := &CriManager{ - ContainerMgr: ctrMgr, - ImageMgr: imgMgr, - CniMgr: NewCniManager(&config.CriConfig), - StreamServer: streamServer, + ContainerMgr: ctrMgr, + ImageMgr: imgMgr, + CniMgr: NewCniManager(&config.CriConfig), + StreamServer: streamServer, + SandboxBaseDir: path.Join(config.HomeDir, "sandboxes"), } return NewCriWrapper(c), nil @@ -149,6 +158,18 @@ func (c *CriManager) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox return nil, fmt.Errorf("failed to start sandbox container for pod %q: %v", config.Metadata.Name, err) } + sandboxRootDir := path.Join(c.SandboxBaseDir, id) + err = os.MkdirAll(sandboxRootDir, 0755) + if err != nil { + return nil, fmt.Errorf("failed to create sandbox root directory: %v", err) + } + + // Setup sandbox file /etc/resolv.conf. + err = setupSandboxFiles(sandboxRootDir, config) + if err != nil { + return nil, fmt.Errorf("failed to setup sandbox files: %v", err) + } + // Step 4: Setup networking for the sandbox. container, err := c.ContainerMgr.Get(ctx, id) if err != nil { @@ -260,6 +281,13 @@ func (c *CriManager) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS return nil, fmt.Errorf("failed to remove sandbox %q: %v", podSandboxID, err) } + // Cleanup the sandbox root directory. + sandboxRootDir := path.Join(c.SandboxBaseDir, podSandboxID) + err = os.RemoveAll(sandboxRootDir) + if err != nil { + return nil, fmt.Errorf("failed to remove root directory %q: %v", sandboxRootDir, err) + } + return &runtime.RemovePodSandboxResponse{}, nil } @@ -380,6 +408,10 @@ func (c *CriManager) CreateContainer(ctx context.Context, r *runtime.CreateConta return nil, err } + // Bindings to overwrite the container's /etc/resolv.conf, /etc/hosts etc. + sandboxRootDir := path.Join(c.SandboxBaseDir, podSandboxID) + createConfig.HostConfig.Binds = append(createConfig.HostConfig.Binds, generateContainerMounts(sandboxRootDir)...) + // TODO: devices and security option configurations. containerName := makeContainerName(sandboxConfig, config) diff --git a/daemon/mgr/cri_utils.go b/daemon/mgr/cri_utils.go index 048d5138c..e02658eac 100644 --- a/daemon/mgr/cri_utils.go +++ b/daemon/mgr/cri_utils.go @@ -3,6 +3,10 @@ package mgr import ( "bytes" "fmt" + "io" + "io/ioutil" + "os" + "path" "strconv" "strings" "time" @@ -91,6 +95,18 @@ func extractLabels(input map[string]string) (map[string]string, map[string]strin return labels, annotations } +// generateContainerMounts sets up necessary container mounts including /dev/shm, /etc/hosts +// and /etc/resolv.conf. +func generateContainerMounts(sandboxRootDir string) []string { + // TODO: more attr and check whether these bindings is included in cri mounts. + result := []string{} + hostPath := path.Join(sandboxRootDir, "resolv.conf") + containerPath := resolvConfPath + result = append(result, fmt.Sprintf("%s:%s", hostPath, containerPath)) + + return result +} + func generateMountBindings(mounts []*runtime.Mount) []string { result := make([]string, 0, len(mounts)) for _, m := range mounts { @@ -300,6 +316,75 @@ func filterCRISandboxes(sandboxes []*runtime.PodSandbox, filter *runtime.PodSand return filtered } +// parseDNSOptions parse DNS options into resolv.conf format content, +// if none option is specified, will return empty with no error. +func parseDNSOptions(servers, searches, options []string) (string, error) { + resolvContent := "" + + if len(searches) > 0 { + resolvContent += fmt.Sprintf("search %s\n", strings.Join(searches, " ")) + } + + if len(servers) > 0 { + resolvContent += fmt.Sprintf("nameserver %s\n", strings.Join(servers, "\nnameserver ")) + } + + if len(options) > 0 { + resolvContent += fmt.Sprintf("options %s\n", strings.Join(options, " ")) + } + + return resolvContent, nil +} + +// copyFile copys src file to dest file +func copyFile(src, dest string, perm os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +// setupSandboxFiles sets up necessary sandbox files. +func setupSandboxFiles(sandboxRootDir string, config *runtime.PodSandboxConfig) error { + // Set DNS options. Maintain a resolv.conf for the sandbox. + var resolvContent string + resolvPath := path.Join(sandboxRootDir, "resolv.conf") + + var err error + dnsConfig := config.GetDnsConfig() + if dnsConfig != nil { + resolvContent, err = parseDNSOptions(dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options) + if err != nil { + return fmt.Errorf("failed to parse sandbox DNSConfig %+v: %v", dnsConfig, err) + } + } + + if resolvContent == "" { + // Copy host's resolv.conf to resolvPath. + err = copyFile(resolvConfPath, resolvPath, 0644) + if err != nil { + return fmt.Errorf("failed to copy host's resolv.conf to %q: %v", resolvPath, err) + } + } else { + err = ioutil.WriteFile(resolvPath, []byte(resolvContent), 0644) + if err != nil { + return fmt.Errorf("failed to write resolv content to %q: %v", resolvPath, err) + } + } + + return nil +} + // Container related tool functions. func makeContainerName(s *runtime.PodSandboxConfig, c *runtime.ContainerConfig) string { diff --git a/hack/cri-test/test-cri.sh b/hack/cri-test/test-cri.sh index 3046b1072..b4cb89a01 100755 --- a/hack/cri-test/test-cri.sh +++ b/hack/cri-test/test-cri.sh @@ -23,7 +23,7 @@ POUCH_SOCK="/var/run/pouchcri.sock" # CRI_FOCUS focuses the test to run. # With the CRI manager completes its function, we may need to expand this field. -CRI_FOCUS=${CRI_FOCUS:-"PodSandbox|AppArmor|Privileged is true|basic operations on container|Runtime info|mount propagation|volume and device|RunAsUser|container port|Streaming|NamespaceOption|SupplementalGroups"} +CRI_FOCUS=${CRI_FOCUS:-"PodSandbox|AppArmor|Privileged is true|basic operations on container|Runtime info|mount propagation|volume and device|RunAsUser|Networking|Streaming|NamespaceOption|SupplementalGroups"} # CRI_SKIP skips the test to skip. CRI_SKIP=${CRI_SKIP:-"RunAsUserName|HostNetwork"}