From 261af6c4b388747e17a8ecf16dbd6f2843d3cfde Mon Sep 17 00:00:00 2001 From: Jeffrey Nelson Date: Wed, 7 Dec 2022 14:46:29 -0600 Subject: [PATCH] VPC-CNI minimal image builds (#2146) * VPC-CNI minimal image builds * update dependencies for ginkgo when running integration tests * address review comments and break up init main function * review comments for sysctl * Simplify binary installation, fix review comments Since init container is required to always run, let binary installation for external plugins happen in init container. This simplifies the main container entrypoint and the dockerfile for each image. --- .gitignore | 6 + Makefile | 12 +- cmd/aws-vpc-cni-init/main.go | 195 ++++++++++++ cmd/aws-vpc-cni/main.go | 395 +++++++++++++++++++++++++ go.mod | 2 +- scripts/dockerfiles/Dockerfile.init | 17 +- scripts/dockerfiles/Dockerfile.release | 22 +- scripts/entrypoint.sh | 207 ------------- scripts/init.sh | 72 ----- scripts/run-cni-release-tests.sh | 1 - utils/cp/cp.go | 76 +++++ utils/imds/imds.go | 28 ++ utils/sysctl/sysctl.go | 49 +++ 13 files changed, 777 insertions(+), 305 deletions(-) create mode 100644 cmd/aws-vpc-cni-init/main.go create mode 100644 cmd/aws-vpc-cni/main.go delete mode 100755 scripts/entrypoint.sh delete mode 100755 scripts/init.sh create mode 100644 utils/cp/cp.go create mode 100644 utils/imds/imds.go create mode 100644 utils/sysctl/sysctl.go diff --git a/.gitignore b/.gitignore index e561e39793..0cbb73cc79 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ +# Ignore generated binaries aws-k8s-agent aws-cni +aws-vpc-cni +aws-vpc-cni-init +bandwidth +host-local +loopback verify-aws verify-network *~ diff --git a/Makefile b/Makefile index 3d488b79f8..6609ee1d92 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ LDFLAGS = -X pkg/version/info.Version=$(VERSION) -X pkg/awsutils/awssession.vers # ALLPKGS is the set of packages provided in source. ALLPKGS = $(shell go list $(VENDOR_OVERRIDE_FLAG) ./... | grep -v cmd/packet-verifier) # BINS is the set of built command executables. -BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper +BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper aws-vpc-cni aws-vpc-cni-init # Plugin binaries # Not copied: bridge dhcp firewall flannel host-device host-local ipvlan macvlan ptp sbr static tuning vlan # For gnu tar, the full path in the tar file is required @@ -119,6 +119,16 @@ build-linux: ## Build the VPC CNI plugin agent using the host's Go toolchain. go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o grpc-health-probe ./cmd/grpc-health-probe go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o egress-v4-cni ./cmd/egress-v4-cni-plugin +# Build VPC CNI init container entrypoint +build-aws-vpc-cni-init: BUILD_FLAGS = $(BUILD_MODE) -ldflags '-s -w $(LDFLAGS)' +build-aws-vpc-cni-init: ## Build the VPC CNI init container using the host's Go toolchain. + go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o aws-vpc-cni-init ./cmd/aws-vpc-cni-init + +# Build VPC CNI container entrypoint +build-aws-vpc-cni: BUILD_FLAGS = $(BUILD_MODE) -ldflags '-s -w $(LDFLAGS)' +build-aws-vpc-cni: ## Build the VPC CNI container using the host's Go toolchain. + go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o aws-vpc-cni ./cmd/aws-vpc-cni + # Build VPC CNI plugin & agent container image. docker: setup-ec2-sdk-override ## Build VPC CNI plugin & agent container image. docker build $(DOCKER_BUILD_FLAGS) \ diff --git a/cmd/aws-vpc-cni-init/main.go b/cmd/aws-vpc-cni-init/main.go new file mode 100644 index 0000000000..e9d46814f1 --- /dev/null +++ b/cmd/aws-vpc-cni-init/main.go @@ -0,0 +1,195 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// The aws-node initialization +package main + +import ( + "os" + + "github.com/aws/amazon-vpc-cni-k8s/utils/cp" + "github.com/aws/amazon-vpc-cni-k8s/utils/imds" + "github.com/aws/amazon-vpc-cni-k8s/utils/sysctl" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" +) + +const ( + defaultHostCNIBinPath = "/host/opt/cni/bin" + vpcCniInitDonePath = "/vpc-cni-init/done" + metadataLocalIP = "local-ipv4" + metadataMAC = "mac" + + envDisableIPv4TcpEarlyDemux = "DISABLE_TCP_EARLY_DEMUX" + envEnableIPv6 = "ENABLE_IPv6" + envHostCniBinPath = "HOST_CNI_BIN_PATH" +) + +func getEnv(env, defaultVal string) string { + if val, ok := os.LookupEnv(env); ok { + return val + } + return defaultVal +} + +func getNodePrimaryIF() (string, error) { + var primaryIF string + primaryMAC, err := imds.GetMetaData("mac") + if err != nil { + return primaryIF, errors.Wrap(err, "Failed to get primary MAC from IMDS") + } + log.Infof("Found primaryMAC %s", primaryMAC) + + links, err := netlink.LinkList() + if err != nil { + return primaryIF, errors.Wrap(err, "Failed to list links") + } + for _, link := range links { + if link.Attrs().HardwareAddr.String() == primaryMAC { + primaryIF = link.Attrs().Name + break + } + } + + if primaryIF == "" { + return primaryIF, errors.Wrap(err, "Failed to retrieve primary IF") + } + return primaryIF, nil +} + +func configureSystemParams(sysctlUtil sysctl.Interface, primaryIF string) error { + var err error + // Configure rp_filter in loose mode + entry := "net/ipv4/conf/" + primaryIF + "/rp_filter" + err = sysctlUtil.Set(entry, 2) + if err != nil { + return errors.Wrapf(err, "Failed to set rp_filter for %s", primaryIF) + } + val, _ := sysctlUtil.Get(entry) + log.Infof("Updated %s to %d", entry, val) + + // Enable or disable TCP early demux based on environment variable + // Note that older kernels may not support tcp_early_demux, so we must first check that it exists. + entry = "net/ipv4/tcp_early_demux" + if _, err := sysctlUtil.Get(entry); err == nil { + disableIPv4EarlyDemux := getEnv(envDisableIPv4TcpEarlyDemux, "false") + if disableIPv4EarlyDemux == "true" { + err = sysctlUtil.Set(entry, 0) + if err != nil { + return errors.Wrap(err, "Failed to disable tcp_early_demux") + } + } else { + err = sysctlUtil.Set(entry, 1) + if err != nil { + return errors.Wrap(err, "Failed to enable tcp_early_demux") + } + } + val, _ = sysctlUtil.Get(entry) + log.Infof("Updated %s to %d", entry, val) + } + return nil +} + +func configureIPv6Settings(sysctlUtil sysctl.Interface, primaryIF string) error { + var err error + // Enable IPv6 when environment variable is set + // Note that IPv6 is not disabled when environment variable is unset. This is omitted to preserve default host semantics. + enableIPv6 := getEnv(envEnableIPv6, "false") + if enableIPv6 == "true" { + entry := "net/ipv6/conf/all/disable_ipv6" + err = sysctlUtil.Set(entry, 0) + if err != nil { + return errors.Wrap(err, "Failed to set disable_ipv6 to 0") + } + val, _ := sysctlUtil.Get(entry) + log.Infof("Updated %s to %d", entry, val) + + entry = "net/ipv6/conf/all/forwarding" + err = sysctlUtil.Set(entry, 1) + if err != nil { + return errors.Wrap(err, "Failed to enable ipv6 forwarding") + } + val, _ = sysctlUtil.Get(entry) + log.Infof("Updated %s to %d", entry, val) + + entry = "net/ipv6/conf/" + primaryIF + "/accept_ra" + err = sysctlUtil.Set(entry, 2) + if err != nil { + return errors.Wrap(err, "Failed to enable ipv6 accept_ra") + } + val, _ = sysctlUtil.Get(entry) + log.Infof("Updated %s to %d", entry, val) + } + return nil +} + +func main() { + os.Exit(_main()) +} + +func _main() int { + log.Debug("Started Initialization") + pluginBins := []string{"loopback", "portmap", "bandwidth", "host-local", "aws-cni-support.sh"} + var err error + for _, plugin := range pluginBins { + if _, err = os.Stat(plugin); err != nil { + log.WithError(err).Fatalf("Required executable: %s not found", plugin) + return 1 + } + } + + log.Infof("Copying CNI plugin binaries ...") + hostCNIBinPath := getEnv(envHostCniBinPath, defaultHostCNIBinPath) + err = cp.InstallBinaries(pluginBins, hostCNIBinPath) + if err != nil { + log.WithError(err).Errorf("Failed to install binaries") + return 1 + } + log.Infof("Copied all CNI plugin binaries to %s", hostCNIBinPath) + + var primaryIF string + primaryIF, err = getNodePrimaryIF() + if err != nil { + log.WithError(err).Errorf("Failed to get primary IF") + return 1 + } + log.Infof("Found primaryIF %s", primaryIF) + + sysctlUtil := sysctl.New() + err = configureSystemParams(sysctlUtil, primaryIF) + if err != nil { + log.WithError(err).Errorf("Failed to configure system parameters") + return 1 + } + + err = configureIPv6Settings(sysctlUtil, primaryIF) + if err != nil { + log.WithError(err).Errorf("Failed to configure IPv6 settings") + return 1 + } + + // TODO: In order to speed up pod launch time, VPC CNI init container is not a Kubernetes init container. + // The VPC CNI container blocks on the existence of vpcCniInitDonePath + //err = cp.TouchFile(vpcCniInitDonePath) + //if err != nil { + // log.WithError(err).Errorf("Failed to set VPC CNI init done") + // return 1 + //} + + log.Infof("CNI init container done") + + // TODO: Since VPC CNI init container is a real container, it never exits + // time.Sleep(time.Duration(1<<63 - 1)) + return 0 +} diff --git a/cmd/aws-vpc-cni/main.go b/cmd/aws-vpc-cni/main.go new file mode 100644 index 0000000000..3c6777b4b2 --- /dev/null +++ b/cmd/aws-vpc-cni/main.go @@ -0,0 +1,395 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// NOTE(jaypipes): Normally, we would prefer *not* to have an entrypoint script +// and instead just start the agent daemon as the container's CMD. However, the +// design of CNI is such that Kubelet looks for the presence of binaries and CNI +// configuration files in specific directories, and the presence of those files +// is the trigger to Kubelet that that particular CNI plugin is "ready". +// +// In the case of the AWS VPC CNI plugin, we have two components to the plugin. +// The first component is the actual CNI binary that is execve'd from Kubelet +// when a container is started or destroyed. The second component is the +// aws-k8s-agent daemon which houses the IPAM controller. +// +// As mentioned above, Kubelet considers a CNI plugin "ready" when it sees the +// binary and configuration file for the plugin in a well-known directory. For +// the AWS VPC CNI plugin binary, we only want to copy the CNI plugin binary +// into that well-known directory AFTER we have successfully started the IPAM +// daemon and know that it can connect to Kubernetes and the local EC2 metadata +// service. This is why this entrypoint script exists; we start the IPAM daemon +// and wait until we know it is up and running successfully before copying the +// CNI plugin binary and its configuration file to the well-known directory that +// Kubelet looks in. + +// AWS VPC CNI entrypoint binary +package main + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net" + "os" + "os/exec" + "strings" + "time" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/aws/amazon-vpc-cni-k8s/utils/cp" + "github.com/aws/amazon-vpc-cni-k8s/utils/imds" + "github.com/containernetworking/cni/pkg/types" +) + +const ( + defaultHostCNIBinPath = "/host/opt/cni/bin" + defaultHostCNIConfDirPath = "/host/etc/cni/net.d" + defaultAWSconflistFile = "/app/10-aws.conflist" + tmpAWSconflistFile = "/tmp/10-aws.conflist" + defaultAgentLogPath = "aws-k8s-agent.log" + defaultVethPrefix = "eni" + defaultMTU = "9001" + defaultEnablePodEni = "false" + defaultPodSGEnforcingMode = "strict" + defaultPluginLogFile = "/var/log/aws-routed-eni/plugin.log" + defaultEgressV4PluginLogFile = "/var/log/aws-routed-eni/egress-v4-plugin.log" + defaultPluginLogLevel = "Debug" + defaultEnableIPv6 = "false" + defaultRandomizeSNAT = "prng" + awsConflistFile = "/10-aws.conflist" + vpcCniInitDonePath = "/vpc-cni-init/done" + + envAgentLogPath = "AGENT_LOG_PATH" + envHostCniBinPath = "HOST_CNI_BIN_PATH" + envHostCniConfDirPath = "HOST_CNI_CONFDIR_PATH" + envVethPrefix = "AWS_VPC_K8S_CNI_VETHPREFIX" + envEniMTU = "AWS_VPC_ENI_MTU" + envEnablePodEni = "ENABLE_POD_ENI" + envPodSGEnforcingMode = "POD_SECURITY_GROUP_ENFORCING_MODE" + envPluginLogFile = "AWS_VPC_K8S_PLUGIN_LOG_FILE" + envPluginLogLevel = "AWS_VPC_K8S_PLUGIN_LOG_LEVEL" + envEgressV4PluginLogFile = "AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE" + envEnPrefixDelegation = "ENABLE_PREFIX_DELEGATION" + envWarmIPTarget = "WARM_IP_TARGET" + envMinIPTarget = "MINIMUM_IP_TARGET" + envWarmPrefixTarget = "WARM_PREFIX_TARGET" + envEnBandwidthPlugin = "ENABLE_BANDWIDTH_PLUGIN" + envEnIPv6 = "ENABLE_IPv6" + envRandomizeSNAT = "AWS_VPC_K8S_CNI_RANDOMIZESNAT" +) + +func getEnv(env, defaultVal string) string { + if val, ok := os.LookupEnv(env); ok { + return val + } + return defaultVal +} + +// NetConfList describes an ordered list of networks. +type NetConfList struct { + CNIVersion string `json:"cniVersion,omitempty"` + + Name string `json:"name,omitempty"` + DisableCheck bool `json:"disableCheck,omitempty"` + Plugins []*NetConf `json:"plugins,omitempty"` +} + +// NetConf stores the common network config for the CNI plugin +type NetConf struct { + CNIVersion string `json:"cniVersion,omitempty"` + + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Capabilities map[string]bool `json:"capabilities,omitempty"` + IPAM *IPAMConfig `json:"ipam,omitempty"` + DNS *types.DNS `json:"dns,omitempty"` + + RawPrevResult map[string]interface{} `json:"prevResult,omitempty"` + PrevResult types.Result `json:"-"` + + // Interface inside container to create + IfName string `json:"ifName,omitempty"` + + Enabled string `json:"enabled,,omitempty"` + + // IP to use as SNAT target + NodeIP net.IP `json:"nodeIP,omitempty"` + + VethPrefix string `json:"vethPrefix,omitempty"` + + PodSGEnforcingMode string `json:"podSGEnforcingMode,omitempty"` + + RandomizeSNAT string `json:"randomizeSNAT,omitempty"` + + // MTU for eth0 + MTU string `json:"mtu,omitempty"` + + PluginLogFile string `json:"pluginLogFile,omitempty"` + + PluginLogLevel string `json:"pluginLogLevel,omitempty"` +} + +// IPAMConfig references containernetworking structure defined at https://github.com/containernetworking/plugins/blob/main/plugins/ipam/host-local/backend/allocator/config.go +type IPAMConfig struct { + *Range + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Routes []*types.Route `json:"routes,omitempty"` + DataDir string `json:"dataDir,omitempty"` + ResolvConf string `json:"resolvConf,omitempty"` + Ranges []RangeSet `json:"ranges"` + IPArgs []net.IP `json:"-"` // Requested IPs from CNI_ARGS and args +} + +// RangeSet references containernetworking structure +type RangeSet []Range + +// Range references containernetworking structure +type Range struct { + RangeStart net.IP `json:"rangeStart,omitempty"` // The first ip, inclusive + RangeEnd net.IP `json:"rangeEnd,omitempty"` // The last ip, inclusive + Subnet types.IPNet `json:"subnet"` + Gateway net.IP `json:"gateway,omitempty"` +} + +func waitForIPAM() bool { + for { + cmd := exec.Command("./grpc-health-probe", "-addr", "127.0.0.1:50051", ">", "/dev/null", "2>&1") + var outb bytes.Buffer + cmd.Stdout = &outb + cmd.Run() + if outb.String() == "" { + return true + } + } +} + +// Wait for vpcCniInitDonePath to exist (maximum wait time is 60 seconds) +func waitForInit() error { + start := time.Now() + maxEnd := start.Add(time.Minute) + for { + // Check for existence of vpcCniInitDonePath + if _, err := os.Stat(vpcCniInitDonePath); err == nil { + // Delete the done file in case of a reboot of the node or restart of the container (force init container to run again) + if err := os.Remove(vpcCniInitDonePath); err == nil { + return nil + } + // If file deletion fails, log and allow retry + log.Errorf("Failed to delete file: %s", vpcCniInitDonePath) + } + if time.Now().After(maxEnd) { + return errors.Errorf("time exceeded") + } + time.Sleep(1 * time.Second) + } +} + +func getNodePrimaryV4Address() (string, error) { + var hostIP string + var err error + for { + hostIP, err = imds.GetMetaData("local-ipv4") + if err != nil { + log.WithError(err).Fatalf("aws-vpc-cni failed") + return "", err + } + if hostIP != "" { + return hostIP, nil + } + + time.Sleep(1 * time.Second) + } +} + +func isValidJSON(inFile string) error { + var result map[string]interface{} + return json.Unmarshal([]byte(inFile), &result) +} + +func generateJSON(jsonFile string, outFile string) error { + byteValue, err := ioutil.ReadFile(jsonFile) + if err != nil { + return err + } + + var nodeIP string + nodeIP, err = getNodePrimaryV4Address() + if err != nil { + log.Errorf("Failed to get Node IP") + return err + } + + vethPrefix := getEnv(envVethPrefix, defaultVethPrefix) + mtu := getEnv(envEniMTU, defaultMTU) + podSGEnforcingMode := getEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode) + pluginLogFile := getEnv(envPluginLogFile, defaultPluginLogFile) + pluginLogLevel := getEnv(envPluginLogLevel, defaultPluginLogLevel) + egressV4pluginLogFile := getEnv(envEgressV4PluginLogFile, defaultEgressV4PluginLogFile) + enabledIPv6 := getEnv(envEnIPv6, defaultEnableIPv6) + randomizeSNAT := getEnv(envRandomizeSNAT, defaultRandomizeSNAT) + + netconf := string(byteValue) + netconf = strings.Replace(netconf, "__VETHPREFIX__", vethPrefix, -1) + netconf = strings.Replace(netconf, "__MTU__", mtu, -1) + netconf = strings.Replace(netconf, "__PODSGENFORCINGMODE__", podSGEnforcingMode, -1) + netconf = strings.Replace(netconf, "__PLUGINLOGFILE__", pluginLogFile, -1) + netconf = strings.Replace(netconf, "__PLUGINLOGLEVEL__", pluginLogLevel, -1) + netconf = strings.Replace(netconf, "__EGRESSV4PLUGINLOGFILE__", egressV4pluginLogFile, -1) + netconf = strings.Replace(netconf, "__EGRESSV4PLUGINENABLED__", enabledIPv6, -1) + netconf = strings.Replace(netconf, "__RANDOMIZESNAT__", randomizeSNAT, -1) + netconf = strings.Replace(netconf, "__NODEIP__", nodeIP, -1) + + byteValue = []byte(netconf) + + enBandwidthPlugin := getEnv(envEnBandwidthPlugin, "false") + if enBandwidthPlugin == "true" { + data := NetConfList{} + err = json.Unmarshal(byteValue, &data) + if err != nil { + return err + } + + bwPlugin := NetConf{ + Type: "bandwidth", + Capabilities: map[string]bool{"bandwidth": true}, + } + data.Plugins = append(data.Plugins, &bwPlugin) + byteValue, err = json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + } + + err = isValidJSON(string(byteValue)) + if err != nil { + log.Fatalf("%s is not a valid json object, error: %s", netconf, err) + } + + err = ioutil.WriteFile(outFile, byteValue, 0644) + return err +} + +func validateEnvVars() bool { + pluginLogFile := getEnv(envPluginLogFile, defaultPluginLogFile) + if pluginLogFile == "stdout" { + log.Errorf("AWS_VPC_K8S_PLUGIN_LOG_FILE cannot be set to stdout") + return false + } + + // Validate that veth prefix is less than or equal to four characters and not in reserved set: (eth, lo, vlan) + vethPrefix := getEnv(envVethPrefix, defaultVethPrefix) + if len(vethPrefix) > 4 { + log.Errorf("AWS_VPC_K8S_CNI_VETHPREFIX cannot be longer than 4 characters") + return false + } + + if vethPrefix == "eth" || vethPrefix == "lo" || vethPrefix == "vlan" { + log.Errorf("AWS_VPC_K8S_CNI_VETHPREFIX cannot be set to reserved values 'eth', 'vlan', or 'lo'") + return false + } + + // When ENABLE_POD_ENI is set, validate security group enforcing mode + enablePodEni := getEnv(envEnablePodEni, defaultEnablePodEni) + if enablePodEni == "true" { + podSGEnforcingMode := getEnv(envPodSGEnforcingMode, defaultPodSGEnforcingMode) + if podSGEnforcingMode != "strict" && podSGEnforcingMode != "standard" { + log.Errorf("%s must be set to either 'strict' or 'standard'", envPodSGEnforcingMode) + return false + } + } + + prefixDelegationEn := getEnv(envEnPrefixDelegation, "false") + warmIPTarget := getEnv(envWarmIPTarget, "0") + warmPrefixTarget := getEnv(envWarmPrefixTarget, "0") + minimumIPTarget := getEnv(envMinIPTarget, "0") + + // Note that these string values should probably be cast to integers, but the comparison for values greater than 0 works either way + if (prefixDelegationEn == "true") && (warmIPTarget <= "0" && warmPrefixTarget <= "0" && minimumIPTarget <= "0") { + log.Errorf("Setting WARM_PREFIX_TARGET = 0 is not supported while WARM_IP_TARGET/MINIMUM_IP_TARGET is not set. Please configure either one of the WARM_{PREFIX/IP}_TARGET or MINIMUM_IP_TARGET env variables") + return false + } + return true +} + +func main() { + os.Exit(_main()) +} + +func _main() int { + log.Debug("Started aws-node container") + if !validateEnvVars() { + return 1 + } + + pluginBins := []string{"aws-cni", "egress-v4-cni"} + hostCNIBinPath := getEnv(envHostCniBinPath, defaultHostCNIBinPath) + err := cp.InstallBinaries(pluginBins, hostCNIBinPath) + if err != nil { + log.WithError(err).Errorf("Failed to install CNI binaries") + return 1 + } + + log.Infof("Starting IPAM daemon... ") + agentLogPath := getEnv(envAgentLogPath, defaultAgentLogPath) + + cmd := "./aws-k8s-agent" + ipamdDaemon := exec.Command(cmd, "|", "tee", "-i", agentLogPath, "2>&1") + err = ipamdDaemon.Start() + if err != nil { + log.WithError(err).Errorf("Failed to execute command: %s", cmd) + return 1 + } + + log.Infof("Checking for IPAM connectivity... ") + if !waitForIPAM() { + log.Errorf("Timed out waiting for IPAM daemon to start") + + byteValue, err := ioutil.ReadFile(agentLogPath) + if err != nil { + log.WithError(err).Errorf("Failed to read %s", agentLogPath) + } + log.Infof("%s", string(byteValue)) + return 1 + } + + // Wait for init container to complete + //if err := waitForInit(); err != nil { + // log.WithError(err).Errorf("Init container failed to complete") + // return 1 + //} + + log.Infof("Copying config file... ") + err = generateJSON(defaultAWSconflistFile, tmpAWSconflistFile) + if err != nil { + log.WithError(err).Errorf("Failed to generate 10-awsconflist") + return 1 + } + + err = cp.CopyFile(tmpAWSconflistFile, defaultHostCNIConfDirPath+awsConflistFile) + if err != nil { + log.WithError(err).Errorf("Failed to copy 10-awsconflist") + return 1 + } + log.Infof("Successfully copied CNI plugin binary and config file.") + + err = ipamdDaemon.Wait() + if err != nil { + log.WithError(err).Errorf("Failed to wait for IPAM daemon to complete") + return 1 + } + log.Infof("IPAMD stopped hence exiting ...") + return 0 +} diff --git a/go.mod b/go.mod index 0adc32c18e..a8bbb78ba4 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/prometheus/client_golang v1.7.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.10.0 + github.com/sirupsen/logrus v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 @@ -127,7 +128,6 @@ require ( github.com/russross/blackfriday v1.5.2 // indirect github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 // indirect github.com/shopspring/decimal v1.2.0 // indirect - github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/cobra v1.1.1 // indirect github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect diff --git a/scripts/dockerfiles/Dockerfile.init b/scripts/dockerfiles/Dockerfile.init index 5ba768e6f1..793debd9b6 100644 --- a/scripts/dockerfiles/Dockerfile.init +++ b/scripts/dockerfiles/Dockerfile.init @@ -7,16 +7,18 @@ ARG TARGETARCH ENV GO111MODULE=on ENV GOPROXY=direct +# Copy modules in before the rest of the source to only expire cache on module changes: +COPY go.mod go.sum ./ +RUN go mod download + COPY Makefile ./ RUN make plugins && make debug-script COPY . ./ +RUN make build-aws-vpc-cni-init -# Build the architecture specific container image: -FROM public.ecr.aws/amazonlinux/amazonlinux:2 -RUN yum update -y && \ - yum install -y iproute procps-ng && \ - yum clean all +# Build from EKS minimal base + glibc +FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-glibc:latest.2 WORKDIR /init @@ -24,7 +26,8 @@ COPY --from=builder \ /go/src/github.com/aws/amazon-vpc-cni-k8s/loopback \ /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap \ /go/src/github.com/aws/amazon-vpc-cni-k8s/bandwidth \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/host-local \ /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni-support.sh \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/init.sh /init/ + /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-vpc-cni-init /init/ -ENTRYPOINT ["/init/init.sh"] +CMD ["/init/aws-vpc-cni-init"] diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 0e79a389c2..864b9c1f48 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -12,29 +12,19 @@ COPY go.mod go.sum ./ RUN go mod download COPY Makefile ./ -RUN make plugins && make debug-script - COPY . ./ -RUN make build-linux +RUN make build-aws-vpc-cni && make build-linux -# Build the architecture specific container image: -FROM public.ecr.aws/amazonlinux/amazonlinux:2 -RUN yum update -y && \ - yum install -y iptables iproute jq && \ - yum clean all +# Build from EKS minimal base + iptables +FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables:latest.2 WORKDIR /app COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni \ /go/src/github.com/aws/amazon-vpc-cni-k8s/misc/10-aws.conflist \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/loopback \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/bandwidth \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/host-local \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni-support.sh \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-k8s-agent \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-k8s-agent \ /go/src/github.com/aws/amazon-vpc-cni-k8s/grpc-health-probe \ /go/src/github.com/aws/amazon-vpc-cni-k8s/egress-v4-cni \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/entrypoint.sh /app/ + /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-vpc-cni /app/ -ENTRYPOINT ["/app/entrypoint.sh"] +ENTRYPOINT ["/app/aws-vpc-cni"] diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh deleted file mode 100755 index 59b12640b3..0000000000 --- a/scripts/entrypoint.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash - -# NOTE(jaypipes): Normally, we would prefer *not* to have an entrypoint script -# and instead just start the agent daemon as the container's CMD. However, the -# design of CNI is such that Kubelet looks for the presence of binaries and CNI -# configuration files in specific directories, and the presence of those files -# is the trigger to Kubelet that that particular CNI plugin is "ready". -# -# In the case of the AWS VPC CNI plugin, we have two components to the plugin. -# The first component is the actual CNI binary that is execve'd from Kubelet -# when a container is started or destroyed. The second component is the -# aws-k8s-agent daemon which houses the IPAM controller. -# -# As mentioned above, Kubelet considers a CNI plugin "ready" when it sees the -# binary and configuration file for the plugin in a well-known directory. For -# the AWS VPC CNI plugin binary, we only want to copy the CNI plugin binary -# into that well-known directory AFTER we have successfully started the IPAM -# daemon and know that it can connect to Kubernetes and the local EC2 metadata -# service. This is why this entrypoint script exists; we start the IPAM daemon -# and wait until we know it is up and running successfully before copying the -# CNI plugin binary and its configuration file to the well-known directory that -# Kubelet looks in. - -# turn on exit on subprocess error and exit on undefined variables -set -eu -# turn on bash's job control -set -m - -log_in_json() -{ - FILENAME="${0##*/}" - LOGTYPE=$1 - MSG=$2 - TIMESTAMP=$(date +%FT%T.%3NZ) - printf '{"level":"%s","ts":"%s","caller":"%s","msg":"%s"}\n' "$LOGTYPE" "$TIMESTAMP" "$FILENAME" "$MSG" -} - -unsupported_prefix_target_conf() -{ - if [ "${WARM_PREFIX_TARGET}" -le "0" ] && [ "${WARM_IP_TARGET}" -le "0" ] && [ "${MINIMUM_IP_TARGET}" -le "0" ];then - true - else - false - fi -} - -is_prefix_delegation_enabled() -{ - if [ "${ENABLE_PREFIX_DELEGATION}" == "true" ]; then - true - else - false - fi -} - -validate_env_var() -{ - log_in_json info "Validating env variables ..." - if [[ "${AWS_VPC_K8S_PLUGIN_LOG_FILE,,}" == "stdout" ]]; then - log_in_json error "AWS_VPC_K8S_PLUGIN_LOG_FILE cannot be set to stdout" - exit 1 - fi - - if [[ ${#AWS_VPC_K8S_CNI_VETHPREFIX} -gt 4 ]]; then - log_in_json error "AWS_VPC_K8S_CNI_VETHPREFIX cannot be longer than 4 characters" - exit 1 - fi - - case ${AWS_VPC_K8S_CNI_VETHPREFIX} in - eth|vlan|lo) - log_in_json error "AWS_VPC_K8S_CNI_VETHPREFIX cannot be set to reserved values eth or vlan or lo" - exit 1 - ;; - esac - - case ${POD_SECURITY_GROUP_ENFORCING_MODE} in - strict|standard) - ;; - *) - log_in_json error "POD_SECURITY_GROUP_ENFORCING_MODE must be set to either strict or standard" - exit 1 - ;; - esac - - if is_prefix_delegation_enabled && unsupported_prefix_target_conf ; then - log_in_json error "Setting WARM_PREFIX_TARGET = 0 is not supported while WARM_IP_TARGET/MINIMUM_IP_TARGET is not set. Please configure either one of the WARM_{PREFIX/IP}_TARGET or MINIMUM_IP_TARGET env variables" - exit 1 - fi -} - -# Check for all the required binaries before we go forward -if [ ! -f aws-k8s-agent ]; then - log_in_json error "Required aws-k8s-agent executable not found." - exit 1 -fi -if [ ! -f grpc-health-probe ]; then - log_in_json error "Required grpc-health-probe executable not found." - exit 1 -fi - -AGENT_LOG_PATH=${AGENT_LOG_PATH:-"aws-k8s-agent.log"} -HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-"/host/opt/cni/bin"} -HOST_CNI_CONFDIR_PATH=${HOST_CNI_CONFDIR_PATH:-"/host/etc/cni/net.d"} -AWS_VPC_K8S_CNI_VETHPREFIX=${AWS_VPC_K8S_CNI_VETHPREFIX:-"eni"} -AWS_VPC_K8S_CNI_RANDOMIZESNAT=${AWS_VPC_K8S_CNI_RANDOMIZESNAT:-"prng"} -AWS_VPC_ENI_MTU=${AWS_VPC_ENI_MTU:-"9001"} -POD_SECURITY_GROUP_ENFORCING_MODE=${POD_SECURITY_GROUP_ENFORCING_MODE:-"strict"} -AWS_VPC_K8S_PLUGIN_LOG_FILE=${AWS_VPC_K8S_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/plugin.log"} -AWS_VPC_K8S_PLUGIN_LOG_LEVEL=${AWS_VPC_K8S_PLUGIN_LOG_LEVEL:-"Debug"} -AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE=${AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/egress-v4-plugin.log"} -NODE_IP=${NODE_IP:=""} - -AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER=${AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:-"true"} -ENABLE_PREFIX_DELEGATION=${ENABLE_PREFIX_DELEGATION:-"false"} -WARM_IP_TARGET=${WARM_IP_TARGET:-"0"} -MINIMUM_IP_TARGET=${MINIMUM_IP_TARGET:-"0"} -WARM_PREFIX_TARGET=${WARM_PREFIX_TARGET:-"0"} -ENABLE_BANDWIDTH_PLUGIN=${ENABLE_BANDWIDTH_PLUGIN:-"false"} -TMP_AWS_CONFLIST_FILE="/tmp/10-aws.conflist" -TMP_AWS_BW_CONFLIST_FILE="/tmp/10-aws-bandwidth-plugin.conflist" - -validate_env_var - -# Check for ipamd connectivity on localhost port 50051 -wait_for_ipam() { - while : - do - if ./grpc-health-probe -addr 127.0.0.1:50051 >/dev/null 2>&1; then - return 0 - fi - log_in_json info "Retrying waiting for IPAM-D" - done -} - -#NodeIP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) -get_node_primary_v4_address() { - while : - do - token=$(curl -Ss -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") - NODE_IP=$(curl -H "X-aws-ec2-metadata-token: $token" -Ss http://169.254.169.254/latest/meta-data/local-ipv4) - if [[ "${NODE_IP}" != "" ]]; then - return 0 - fi - # We sleep for 1 second between each retry - sleep 1 - log_in_json info "Retrying fetching node-IP" - done -} - -# If there is no init container, copy the required files -if [[ "$AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" != "false" ]]; then - # Copy files - log_in_json info "Copying CNI plugin binaries ... " - PLUGIN_BINS="loopback portmap bandwidth host-local aws-cni-support.sh" - for b in $PLUGIN_BINS; do - # Install the binary - install "$b" "$HOST_CNI_BIN_PATH" - done -fi - -log_in_json info "Install CNI binaries.." -install aws-cni "$HOST_CNI_BIN_PATH" -install egress-v4-cni "$HOST_CNI_BIN_PATH" - -log_in_json info "Starting IPAM daemon in the background ... " -./aws-k8s-agent | tee -i "$AGENT_LOG_PATH" 2>&1 & - -log_in_json info "Checking for IPAM connectivity ... " - -if ! wait_for_ipam; then - log_in_json error "Timed out waiting for IPAM daemon to start:" - cat "$AGENT_LOG_PATH" >&2 - exit 1 -fi - -get_node_primary_v4_address -log_in_json info "Copying config file ... " - -# modify the static config to populate it with the env vars -sed \ - -e s~__VETHPREFIX__~"${AWS_VPC_K8S_CNI_VETHPREFIX}"~g \ - -e s~__MTU__~"${AWS_VPC_ENI_MTU}"~g \ - -e s~__PODSGENFORCINGMODE__~"${POD_SECURITY_GROUP_ENFORCING_MODE}"~g \ - -e s~__PLUGINLOGFILE__~"${AWS_VPC_K8S_PLUGIN_LOG_FILE}"~g \ - -e s~__PLUGINLOGLEVEL__~"${AWS_VPC_K8S_PLUGIN_LOG_LEVEL}"~g \ - -e s~__EGRESSV4PLUGINLOGFILE__~"${AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE}"~g \ - -e s~__EGRESSV4PLUGINENABLED__~"${ENABLE_IPv6}"~g \ - -e s~__RANDOMIZESNAT__~"${AWS_VPC_K8S_CNI_RANDOMIZESNAT}"~g \ - -e s~__NODEIP__~"${NODE_IP}"~g \ - 10-aws.conflist > "$TMP_AWS_CONFLIST_FILE" - -if [[ "$ENABLE_BANDWIDTH_PLUGIN" == "true" ]]; then - jq '.plugins += [{"type": "bandwidth","capabilities": {"bandwidth": true}}]' "$TMP_AWS_CONFLIST_FILE" > "$TMP_AWS_BW_CONFLIST_FILE" - mv "$TMP_AWS_BW_CONFLIST_FILE" "$TMP_AWS_CONFLIST_FILE" -fi - -mv "$TMP_AWS_CONFLIST_FILE" "$HOST_CNI_CONFDIR_PATH/10-aws.conflist" - -log_in_json info "Successfully copied CNI plugin binary and config file." - -if [[ -f "$HOST_CNI_CONFDIR_PATH/aws.conf" ]]; then - rm "$HOST_CNI_CONFDIR_PATH/aws.conf" -fi - -# Bring the aws-k8s-agent process back into the foreground -log_in_json info "Foregrounding IPAM daemon ..." -fg %1 >/dev/null 2>&1 || { log_in_json error "failed (process terminated)" && cat "$AGENT_LOG_PATH" && exit 1; } diff --git a/scripts/init.sh b/scripts/init.sh deleted file mode 100755 index ae734a1778..0000000000 --- a/scripts/init.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -get_metadata() -{ - TOKEN=$(curl -Ss -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") - attempts=60 - false - while [ "${?}" -gt 0 ]; do - if [ "${attempts}" -eq 0 ]; then - echo "Failed to get metdata" - exit 1 - fi - meta=$(curl -Ss -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/${1}) - if [ "${?}" -gt 0 ]; then - let attempts-- - sleep 0.5 - false - fi - done - echo "$meta" -} - -PLUGIN_BINS="loopback portmap bandwidth aws-cni-support.sh" - -for b in $PLUGIN_BINS; do - if [ ! -f "$b" ]; then - echo "Required $b executable not found." - exit 1 - fi -done - -HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-"/host/opt/cni/bin"} - -# Copy files -echo "Copying CNI plugin binaries ... " - -for b in $PLUGIN_BINS; do - # Install the binary - install "$b" "$HOST_CNI_BIN_PATH" -done - -# Configure rp_filter -echo "Configure rp_filter loose... " - -HOST_IP=$(get_metadata 'local-ipv4') -PRIMARY_MAC=$(get_metadata 'mac') -PRIMARY_IF=$(ip -o link show | grep -F "link/ether $PRIMARY_MAC" | awk -F'[ :]+' '{print $2}') -sysctl -w "net.ipv4.conf.$PRIMARY_IF.rp_filter=2" -cat "/proc/sys/net/ipv4/conf/$PRIMARY_IF/rp_filter" - -# Set DISABLE_TCP_EARLY_DEMUX to true to enable kubelet to pod-eni TCP communication -# https://lwn.net/Articles/503420/ and https://github.com/aws/amazon-vpc-cni-k8s/pull/1212 for background -if [ "${DISABLE_TCP_EARLY_DEMUX:-false}" == "true" ]; then - sysctl -w "net.ipv4.tcp_early_demux=0" -else - sysctl -e -w "net.ipv4.tcp_early_demux=1" -fi - -# If IPv6 is enabled,set `disable_ipv6` to `0` and ipv6 `forwarding` to `1` -# We also set `accept_ra` to `2` on primary interface to allow it to honor RA packets. -if [ "${ENABLE_IPv6:-false}" == "true" ]; then - sysctl -w "net.ipv6.conf.all.disable_ipv6=0" - sysctl -w "net.ipv6.conf.all.forwarding=1" - sysctl -w "net.ipv6.conf.$PRIMARY_IF.accept_ra=2" - cat "/proc/sys/net/ipv6/conf/all/disable_ipv6" - cat "/proc/sys/net/ipv6/conf/all/forwarding" - cat "/proc/sys/net/ipv6/conf/$PRIMARY_IF/accept_ra" -fi - -echo "CNI init container done" diff --git a/scripts/run-cni-release-tests.sh b/scripts/run-cni-release-tests.sh index 444bcc1f64..3a0366be31 100755 --- a/scripts/run-cni-release-tests.sh +++ b/scripts/run-cni-release-tests.sh @@ -16,7 +16,6 @@ set -e SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration" -CALICO_TEST_DIR="$SCRIPT_DIR/../test/e2e/calico" source "$SCRIPT_DIR"/lib/cluster.sh source "$SCRIPT_DIR"/lib/integration.sh diff --git a/utils/cp/cp.go b/utils/cp/cp.go new file mode 100644 index 0000000000..338ad103d6 --- /dev/null +++ b/utils/cp/cp.go @@ -0,0 +1,76 @@ +package cp + +import ( + "fmt" + "io" + "os" +) + +func TouchFile(filePath string) error { + file, err := os.OpenFile(filePath, os.O_RDONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + return file.Close() +} + +func cp(src, dst string) error { + sourceFileStat, err := os.Stat(src) + if err != nil { + return err + } + + if !sourceFileStat.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + _, err = io.Copy(destination, source) + return err +} + +func CopyFile(src, dst string) (err error) { + dstTmp := fmt.Sprintf("%s.tmp", dst) + if err := cp(src, dstTmp); err != nil { + return fmt.Errorf("failed to copy file: %s", err) + } + + err = os.Rename(dstTmp, dst) + if err != nil { + return fmt.Errorf("failed to rename file: %s", err) + } + + si, err := os.Stat(src) + if err != nil { + return fmt.Errorf("failed to stat file: %s", err) + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return fmt.Errorf("failed to chmod file: %s", err) + } + + return nil +} + +func InstallBinaries(pluginBins []string, hostCNIBinPath string) error { + for _, plugin := range pluginBins { + target := fmt.Sprintf("%s/%s", hostCNIBinPath, plugin) + source := fmt.Sprintf("%s", plugin) + + if err := CopyFile(source, target); err != nil { + return fmt.Errorf("Failed to install %s: %s", target, err) + } + fmt.Printf("Installed %s\n", target) + } + return nil +} diff --git a/utils/imds/imds.go b/utils/imds/imds.go new file mode 100644 index 0000000000..92a77be604 --- /dev/null +++ b/utils/imds/imds.go @@ -0,0 +1,28 @@ +package imds + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + ec2metadatasvc "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/session" +) + +// EC2Metadata wraps the methods from the amazon-sdk-go's ec2metadata package +type EC2Metadata interface { + GetMetadata(path string) (string, error) + Region() (string, error) +} + +func GetMetaData(key string) (string, error) { + awsSession := session.Must(session.NewSession(aws.NewConfig(). + WithMaxRetries(10), + )) + var ec2Metadata EC2Metadata + ec2Metadata = ec2metadatasvc.New(awsSession) + requestedData, err := ec2Metadata.GetMetadata(key) + if err != nil { + return "", fmt.Errorf("get instance metadata: failed to retrieve %s - %s", key, err) + } + return requestedData, nil +} diff --git a/utils/sysctl/sysctl.go b/utils/sysctl/sysctl.go new file mode 100644 index 0000000000..234a611fd5 --- /dev/null +++ b/utils/sysctl/sysctl.go @@ -0,0 +1,49 @@ +// Ref: https://github.com/kubernetes/kubernetes/blob/cb2ea4bf7c029e595f44ee62013c982626fb5bd4/staging/src/k8s.io/component-helpers/node/utils/sysctl/sysctl.go + +package sysctl + +import ( + "io/ioutil" + "path" + "strconv" + "strings" +) + +const ( + sysctlBase = "/proc/sys" +) + +// Interface is an injectable interface for running sysctl commands. +type Interface interface { + // Get returns the value for the specified sysctl setting + Get(sysctl string) (int, error) + // Set modifies the specified sysctl flag to the new value + Set(sysctl string, newVal int) error +} + +// New returns a new Interface for accessing sysctl +func New() Interface { + return &procSysctl{} +} + +// procSysctl implements Interface by reading and writing files under /proc/sys +type procSysctl struct { +} + +// Get returns the value for the specified sysctl setting +func (*procSysctl) Get(sysctl string) (int, error) { + data, err := ioutil.ReadFile(path.Join(sysctlBase, sysctl)) + if err != nil { + return -1, err + } + val, err := strconv.Atoi(strings.Trim(string(data), " \n")) + if err != nil { + return -1, err + } + return val, nil +} + +// Set modifies the specified sysctl flag to the new value +func (*procSysctl) Set(sysctl string, newVal int) error { + return ioutil.WriteFile(path.Join(sysctlBase, sysctl), []byte(strconv.Itoa(newVal)), 0640) +}