Fault Injection Service Feature

agent/app/agent_capability.go

@@ -134,6 +134,7 @@ var (
 		attributePrefix + taskEIAWithOptimizedCPU,
 		attributePrefix + capabilityServiceConnect,
 		attributePrefix + capabilityEBSTaskAttach,
+		attributePrefix + capabilityFaultInjection,
 	}
 	// List of capabilities that are only supported on external capaciity. Currently only one but keep as a list
 	// for future proof and also align with externalUnsupportedCapabilities.
@@ -314,8 +315,7 @@ func (agent *ecsAgent) capabilities() ([]*ecs.Attribute, error) {
 		capabilities = removeAttributesByNames(capabilities, externalUnsupportedCapabilities)
 	}
 
-	// TODO add fault-injection capabilities if applicable
-	// capabilities = agent.appendFaultInjectionCapabilities(capabilities)
+	capabilities = agent.appendFaultInjectionCapabilities(capabilities)
 
 	return capabilities, nil
 }
@@ -543,9 +543,6 @@ func (agent *ecsAgent) appendEBSTaskAttachCapabilities(capabilities []*ecs.Attri
 	return capabilities
 }
 
-// TODO Remove linter directive below when the function becomes used
-//
-//lint:ignore U1000 as this method will be used in the future.
 func (agent *ecsAgent) appendFaultInjectionCapabilities(capabilities []*ecs.Attribute) []*ecs.Attribute {
 
 	// Check if the agent is running in EXTERNAL launch type

agent/app/agent_capability_unix.go

@@ -17,9 +17,11 @@
 package app
 
 import (
+	"context"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/aws/amazon-ecs-agent/agent/config"
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
@@ -28,17 +30,21 @@ import (
 	"github.com/aws/amazon-ecs-agent/agent/taskresource/volume"
 	"github.com/aws/amazon-ecs-agent/agent/utils"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/cihub/seelog"
 )
 
 const (
-	AVX                   = "avx"
-	AVX2                  = "avx2"
-	SSE41                 = "sse4_1"
-	SSE42                 = "sse4_2"
-	CpuInfoPath           = "/proc/cpuinfo"
-	capabilityDepsRootDir = "/managed-agents"
+	AVX                         = "avx"
+	AVX2                        = "avx2"
+	SSE41                       = "sse4_1"
+	SSE42                       = "sse4_2"
+	CpuInfoPath                 = "/proc/cpuinfo"
+	capabilityDepsRootDir       = "/managed-agents"
+	modInfoCmd                  = "modinfo"
+	faultInjectionKernelModules = "sch_netem"
+	ctxTimeoutDuration          = 60 * time.Second
 )
 
 var (
@@ -243,6 +249,7 @@ var isFaultInjectionToolingAvailable = checkFaultInjectionTooling
 
 // wrapper around exec.LookPath
 var lookPathFunc = exec.LookPath
+var osExecWrapper = execwrapper.NewExec()
 
 // checkFaultInjectionTooling checks for the required network packages like iptables, tc
 // to be available on the host before ecs.capability.fault-injection can be advertised
@@ -256,5 +263,18 @@ func checkFaultInjectionTooling() bool {
 			return false
 		}
 	}
+	return checkFaultInjectionModules()
+}
+
+// checkFaultInjectionModules checks for the required kernel modules such as sch_netem to be installed
+// and avaialble on the host before ecs.capability.fault-injection can be advertised
+func checkFaultInjectionModules() bool {
+	ctxWithTimeout, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
+	defer cancel()
+	_, err := osExecWrapper.CommandContext(ctxWithTimeout, modInfoCmd, faultInjectionKernelModules).CombinedOutput()
+	if err != nil {
+		seelog.Warnf("Failed to find kernel module %s that is needed for fault-injection feature: %v", faultInjectionKernelModules, err)
+		return false
+	}
 	return true
 }

agent/app/agent_capability_unix_test.go

@@ -40,6 +40,8 @@ import (
 	mock_mobypkgwrapper "github.com/aws/amazon-ecs-agent/agent/utils/mobypkgwrapper/mocks"
 	"github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs"
 	md "github.com/aws/amazon-ecs-agent/ecs-agent/manageddaemon"
+	"github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper"
+	mock_execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks"
 	"github.com/aws/aws-sdk-go/aws"
 	aws_credentials "github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/golang/mock/gomock"
@@ -979,16 +981,47 @@ func TestCheckFaultInjectionTooling(t *testing.T) {
 	defer func() {
 		lookPathFunc = originalLookPath
 	}()
+	originalOSExecWrapper := execwrapper.NewExec()
+	defer func() {
+		osExecWrapper = originalOSExecWrapper
+	}()
 
-	t.Run("all tools available", func(t *testing.T) {
+	t.Run("all tools and kernel modules available", func(t *testing.T) {
 		lookPathFunc = func(file string) (string, error) {
 			return "/usr/bin" + file, nil
 		}
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+		mockExec := mock_execwrapper.NewMockExec(ctrl)
+		cmdExec := mock_execwrapper.NewMockCmd(ctrl)
+		gomock.InOrder(
+			mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
+			cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil),
+		)
+		osExecWrapper = mockExec
 		assert.True(t,
 			checkFaultInjectionTooling(),
 			"Expected checkFaultInjectionTooling to return true when all tools are available")
 	})
 
+	t.Run("missing kernel modules", func(t *testing.T) {
+		lookPathFunc = func(file string) (string, error) {
+			return "/usr/bin" + file, nil
+		}
+		ctrl := gomock.NewController(t)
+		defer ctrl.Finish()
+		mockExec := mock_execwrapper.NewMockExec(ctrl)
+		cmdExec := mock_execwrapper.NewMockCmd(ctrl)
+		gomock.InOrder(
+			mockExec.EXPECT().CommandContext(gomock.Any(), modInfoCmd, faultInjectionKernelModules).Times(1).Return(cmdExec),
+			cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, errors.New("modinfo: ERROR: Module sch_netem not found.")),
+		)
+		osExecWrapper = mockExec
+		assert.False(t,
+			checkFaultInjectionTooling(),
+			"Expected checkFaultInjectionTooling to return false when kernel modules are not available")
+	})
+
 	tools := []string{"iptables", "tc", "nsenter"}
 	for _, tool := range tools {
 		t.Run(tool+" missing", func(t *testing.T) {

agent/handlers/task_server_setup.go

@@ -96,9 +96,8 @@ func taskServerSetup(
 	agentAPIV1HandlersSetup(muxRouter, state, credentialsManager, cluster, tmdsAgentState,
 		taskProtectionClientFactory, metricsFactory)
 
-	// TODO: Future PR to pass in TMDS server router once all of the handlers have been implemented.
 	execWrapper := execwrapper.NewExec()
-	registerFaultHandlers(nil, tmdsAgentState, metricsFactory, execWrapper)
+	registerFaultHandlers(muxRouter, tmdsAgentState, metricsFactory, execWrapper)
 
 	return tmds.NewServer(auditLogger,
 		tmds.WithHandler(muxRouter),

ecs-agent/tmds/handlers/v4/state/response.go

@@ -38,7 +38,7 @@ type TaskResponse struct {
 	EphemeralStorageMetrics *EphemeralStorageMetrics `json:"EphemeralStorageMetrics,omitempty"`
 	CredentialsID           string                   `json:"-"`
 	TaskNetworkConfig       *TaskNetworkConfig       `json:"-"`
-	FaultInjectionEnabled   bool                     `json:"FaultInjectionEnabled,omitempty"`
+	FaultInjectionEnabled   bool                     `json:"FaultInjectionEnabled"`
 }
 
 // TaskNetworkConfig contains required network configurations for network faults injection.

ecs-init/docker/docker_config.go

@@ -15,8 +15,10 @@ package docker
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/aws/amazon-ecs-agent/ecs-init/config"
+	"github.com/cihub/seelog"
 	ctrdapparmor "github.com/containerd/containerd/pkg/apparmor"
 	godocker "github.com/fsouza/go-dockerclient"
 )
@@ -45,6 +47,8 @@ func createHostConfig(binds []string) *godocker.HostConfig {
 		iptablesLegacyDir+":"+iptablesLegacyDir+readOnly,
 		"/usr/bin/lsblk:/usr/bin/lsblk",
 	)
+	binds = append(binds, getNsenterBinds(os.Stat)...)
+	binds = append(binds, getModInfoBinds(os.Stat)...)
 
 	logConfig := config.AgentDockerLogDriverConfiguration()
 
@@ -80,3 +84,38 @@ func createHostConfig(binds []string) *godocker.HostConfig {
 
 	return hostConfig
 }
+
+// Returns nsenter bind as a slice if nsenter is available on the host.
+// Returns an empty slice otherwise.
+func getNsenterBinds(statFn func(string) (os.FileInfo, error)) []string {
+	binds := []string{}
+	const nsenterPath = "/usr/bin/nsenter"
+	if _, err := statFn(nsenterPath); err == nil {
+		binds = append(binds, nsenterPath+":"+nsenterPath)
+	} else {
+		seelog.Warnf("nsenter not found at %s, skip binding it to Agent container: %v",
+			nsenterPath, err)
+	}
+	return binds
+}
+
+// Returns modinfo bind as a slice if modinfo is available on the host.
+// Otherwise, it will return an empty slice.
+func getModInfoBinds(statFn func(string) (os.FileInfo, error)) []string {
+	binds := []string{}
+	modInfoPathLocations := []string{
+		"/sbin/modinfo",
+		"/usr/sbin/modinfo",
+	}
+	for _, path := range modInfoPathLocations {
+		if _, err := statFn(path); err == nil {
+			seelog.Debugf("modinfo found at %s", path)
+			binds = append(binds, path+":"+path)
+			break
+		} else {
+			seelog.Infof("modinfo not found at %s, skip binding it to Agent container: %v",
+				path, err)
+		}
+	}
+	return binds
+}

ecs-init/docker/docker_config_test.go

@@ -0,0 +1,52 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package docker
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetNsenterBinds(t *testing.T) {
+	t.Run("nsenter not found", func(t *testing.T) {
+		binds := getNsenterBinds(
+			func(s string) (os.FileInfo, error) { return nil, errors.New("not found") })
+		assert.Empty(t, binds)
+	})
+
+	t.Run("nsenter is found", func(t *testing.T) {
+		binds := getNsenterBinds(
+			func(s string) (os.FileInfo, error) { return nil, nil })
+		require.Len(t, binds, 1)
+		assert.Equal(t, "/usr/bin/nsenter:/usr/bin/nsenter", binds[0])
+	})
+}
+
+func TestGetModinfoBinds(t *testing.T) {
+	t.Run("modinfo not found", func(t *testing.T) {
+		binds := getModInfoBinds(
+			func(s string) (os.FileInfo, error) { return nil, errors.New("not found") })
+		assert.Empty(t, binds)
+	})
+	t.Run("modinfo is found", func(t *testing.T) {
+		binds := getModInfoBinds(
+			func(s string) (os.FileInfo, error) { return nil, nil })
+		require.Len(t, binds, 1)
+		assert.Equal(t, "/sbin/modinfo:/sbin/modinfo", binds[0])
+	})
+}

ecs-init/docker/docker_test.go

@@ -32,7 +32,7 @@ import (
 // Note: Change this value every time when a new bind mount is added to
 // agent for the tests to pass
 const (
-	defaultExpectedAgentBinds = 20
+	defaultExpectedAgentBinds = 22
 )
 
 func TestIsAgentImageLoadedListFailure(t *testing.T) {