Checking if required fault injection kernel modules are available for…

… agent capability

agent/app/agent_capability_unix.go

@@ -17,9 +17,11 @@
 package app
 
 import (
+	"context"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/aws/amazon-ecs-agent/agent/config"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
@@ -28,17 +30,21 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/taskresource/volume"
 	"github.com/aws/amazon-ecs-agent/agent/utils"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/cihub/seelog"
 )
 
 const (
-	AVX                   = "avx"
-	AVX2                  = "avx2"
-	SSE41                 = "sse4_1"
-	SSE42                 = "sse4_2"
-	CpuInfoPath           = "/proc/cpuinfo"
-	capabilityDepsRootDir = "/managed-agents"
+	AVX                         = "avx"
+	AVX2                        = "avx2"
+	SSE41                       = "sse4_1"
+	SSE42                       = "sse4_2"
+	CpuInfoPath                 = "/proc/cpuinfo"
+	capabilityDepsRootDir       = "/managed-agents"
+	modInfoCmd                  = "modinfo"
+	faultInjectionKernelModules = "sch_netem"
+	ctxTimeoutDuration          = 60 * time.Second
 )
 
 var (
@@ -243,6 +249,7 @@ var isFaultInjectionToolingAvailable = checkFaultInjectionTooling
 
 // wrapper around exec.LookPath
 var lookPathFunc = exec.LookPath
+var osExecWrapper = execwrapper.NewExec()
 
 // checkFaultInjectionTooling checks for the required network packages like iptables, tc
 // to be available on the host before ecs.capability.fault-injection can be advertised
@@ -256,5 +263,18 @@ func checkFaultInjectionTooling() bool {
 			return false
 		}
 	}
+	return checkFaultInjectionModules()
+}
+
+// checkFaultInjectionModules checks for the required kernel modules such as sch_netem to be installed
+// and avaialble on the host before ecs.capability.fault-injection can be advertised
+func checkFaultInjectionModules() bool {
+	ctxWithTimeout, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
+	defer cancel()
+	_, err := osExecWrapper.CommandContext(ctxWithTimeout, modInfoCmd, faultInjectionKernelModules).CombinedOutput()
+	if err != nil {
+		seelog.Warnf("Failed to find kernel module %s that is needed for fault-injection feature: %v", faultInjectionKernelModules, err)
+		return false
+	}
 	return true
 }

agent/app/agent_capability_unix_test.go

@@ -40,6 +40,8 @@ import (
 	mock_mobypkgwrapper "github.com/aws/amazon-ecs-agent/agent/utils/mobypkgwrapper/mocks"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs"
 	md "github.com/aws/amazon-ecs-agent/ecs-agent/manageddaemon"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper"
+	mock_execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks"
 	"github.com/aws/aws-sdk-go/aws"
 	aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/golang/mock/gomock"
@@ -979,16 +981,47 @@ func TestCheckFaultInjectionTooling(t *testing.T) {
 	defer func() {
 		lookPathFunc = originalLookPath
 	}()
+	originalOSExecWrapper := execwrapper.NewExec()
+	defer func() {
+		osExecWrapper = originalOSExecWrapper
+	}()
 
-	t.Run("all tools available", func(t *testing.T) {
+	t.Run("all tools and kernel modules available", func(t *testing.T) {
 		lookPathFunc = func(file string) (string, error) {
 			return "/usr/bin" + file, nil
 		}
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+		mockExec := mock_execwrapper.NewMockExec(ctrl)
+		cmdExec := mock_execwrapper.NewMockCmd(ctrl)
+		gomock.InOrder(
+			mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
+			cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
+		)
+		osExecWrapper = mockExec
 		assert.True(t,
 			checkFaultInjectionTooling(),
 			"Expected checkFaultInjectionTooling to return true when all tools are available")
 	})
 
+	t.Run("missing kernel modules", func(t *testing.T) {
+		lookPathFunc = func(file string) (string, error) {
+			return "/usr/bin" + file, nil
+		}
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+		mockExec := mock_execwrapper.NewMockExec(ctrl)
+		cmdExec := mock_execwrapper.NewMockCmd(ctrl)
+		gomock.InOrder(
+			mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
+			cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, errors.New("modinfo: ERROR: Module sch_netem not found.")),
+		)
+		osExecWrapper = mockExec
+		assert.False(t,
+			checkFaultInjectionTooling(),
+			"Expected checkFaultInjectionTooling to return false when kernel modules are not available")
+	})
+
 	tools := []string{"iptables", "tc", "nsenter"}
 	for _, tool := range tools {
 		t.Run(tool+" missing", func(t *testing.T) {

ecs-init/docker/docker_config.go

@@ -48,6 +48,7 @@ func createHostConfig(binds []string) *godocker.HostConfig {
 		"/usr/bin/lsblk:/usr/bin/lsblk",
 	)
 	binds = append(binds, getNsenterBinds(os.Stat)...)
+	binds = append(binds, getModInfoBinds(os.Stat)...)
 
 	logConfig := config.AgentDockerLogDriverConfiguration()
 
@@ -97,3 +98,24 @@ func getNsenterBinds(statFn func(string) (os.FileInfo, error)) []string {
 	}
 	return binds
 }
+
+// Returns modinfo bind as a slice if modinfo is available on the host.
+// Otherwise, it will return an empty slice.
+func getModInfoBinds(statFn func(string) (os.FileInfo, error)) []string {
+	binds := []string{}
+	modInfoPathLocations := []string{
+		"/sbin/modinfo",
+		"/usr/sbin/modinfo",
+	}
+	for _, path := range modInfoPathLocations {
+		if _, err := statFn(path); err == nil {
+			seelog.Debugf("modinfo found at %s", path)
+			binds = append(binds, path+":"+path)
+			break
+		} else {
+			seelog.Infof("modinfo not found at %s, skip binding it to Agent container: %v",
+				path, err)
+		}
+	}
+	return binds
+}

ecs-init/docker/docker_config_test.go

@@ -36,3 +36,17 @@ func TestGetNsenterBinds(t *testing.T) {
 		assert.Equal(t, "/usr/bin/nsenter:/usr/bin/nsenter", binds[0])
 	})
 }
+
+func TestGetModinfoBinds(t *testing.T) {
+	t.Run("modinfo not found", func(t *testing.T) {
+		binds := getModInfoBinds(
+			func(s string) (os.FileInfo, error) { return nil, errors.New("not found") })
+		assert.Empty(t, binds)
+	})
+	t.Run("modinfo is found", func(t *testing.T) {
+		binds := getModInfoBinds(
+			func(s string) (os.FileInfo, error) { return nil, nil })
+		require.Len(t, binds, 1)
+		assert.Equal(t, "/sbin/modinfo:/sbin/modinfo", binds[0])
+	})
+}

ecs-init/docker/docker_test.go

@@ -32,7 +32,7 @@ import (
 // Note: Change this value every time when a new bind mount is added to
 // agent for the tests to pass
 const (
-	defaultExpectedAgentBinds = 21
+	defaultExpectedAgentBinds = 22
 )
 
 func TestIsAgentImageLoadedListFailure(t *testing.T) {