get container exit code from event if we can't get it from inspect.

agent/dockerclient/dockerapi/docker_client.go

@@ -64,6 +64,10 @@ const (
 	maxHealthCheckOutputLength = 1024
 	// VolumeDriverType is one of the plugin capabilities see https://docs.docker.com/engine/reference/commandline/plugin_ls/#filtering
 	VolumeDriverType = "volumedriver"
+	// dockerContainerDieEvent is the name of the event generated by Docker when a container died.
+	dockerContainerDieEvent = "die"
+	// dockerContainerEventExitCodeAttribute is the attribute name to get exit code from Docker event attribute.
+	dockerContainerEventExitCodeAttribute = "exitCode"
 )
 
 // Timelimits for docker operations enforced above docker
@@ -1015,6 +1019,10 @@ func (dg *dockerGoClient) handleContainerEvents(ctx context.Context,
 		}
 
 		metadata := dg.containerMetadata(ctx, containerID)
+		// In case when we received a container die event but was not able to inspect the container (e.g. due to timeout),
+		// we will use the exit code from the event, so that the exit code of the container is still reported and
+		// available for customer to see from describing task.
+		setExitCodeFromEvent(event, &metadata)
 
 		changedContainers <- DockerContainerChangeEvent{
 			Status:                  status,
@@ -1024,6 +1032,26 @@ func (dg *dockerGoClient) handleContainerEvents(ctx context.Context,
 	}
 }
 
+// setExitCodeFromEvent tries to get exit code from event and stores it in metadata, if metadata doesn't
+// contain the exit code already.
+func setExitCodeFromEvent(event *events.Message, metadata *DockerContainerMetadata) {
+	// exit code is only available from die event.
+	if metadata.ExitCode != nil || event.Status != dockerContainerDieEvent {
+		return
+	}
+	exitCode, ok := event.Actor.Attributes[dockerContainerEventExitCodeAttribute]
+	if !ok {
+		return
+	}
+	code, err := strconv.Atoi(exitCode)
+	if err != nil {
+		seelog.Errorf("Received invalid exit code from docker container event. exit code: %s, parse err: %v",
+			exitCode, err)
+		return
+	}
+	metadata.ExitCode = &code
+}
+
 // ListContainers returns a slice of container IDs.
 func (dg *dockerGoClient) ListContainers(ctx context.Context, all bool, timeout time.Duration) ListContainersResponse {
 	ctx, cancel := context.WithTimeout(ctx, timeout)

agent/dockerclient/dockerapi/docker_client_test.go

@@ -995,6 +995,74 @@ func TestContainerEventsError(t *testing.T) {
 	}
 }
 
+func TestSetExitCodeFromEvent(t *testing.T) {
+	var (
+		exitCodeInt    = 42
+		exitCodeStr    = "42"
+		altExitCodeInt = 1
+	)
+
+	defaultEvent := &events.Message{
+		Status: dockerContainerDieEvent,
+		Actor: events.Actor{
+			Attributes: map[string]string{
+				dockerContainerEventExitCodeAttribute: exitCodeStr,
+			},
+		},
+	}
+
+	testCases := []struct {
+		name             string
+		event            *events.Message
+		metadata         DockerContainerMetadata
+		expectedExitCode *int
+	}{
+		{
+			name:             "exit code set from event",
+			event:            defaultEvent,
+			metadata:         DockerContainerMetadata{},
+			expectedExitCode: &exitCodeInt,
+		},
+		{
+			name:  "exit code not set from event when metadata already has it",
+			event: defaultEvent,
+			metadata: DockerContainerMetadata{
+				ExitCode: &altExitCodeInt,
+			},
+			expectedExitCode: &altExitCodeInt,
+		},
+		{
+			name: "exit code not set from event when event does not has it",
+			event: &events.Message{
+				Status: dockerContainerDieEvent,
+				Actor:  events.Actor{},
+			},
+			metadata:         DockerContainerMetadata{},
+			expectedExitCode: nil,
+		},
+		{
+			name: "exit code not set from event when event has invalid exit code",
+			event: &events.Message{
+				Status: dockerContainerDieEvent,
+				Actor: events.Actor{
+					Attributes: map[string]string{
+						dockerContainerEventExitCodeAttribute: "invalid",
+					},
+				},
+			},
+			metadata:         DockerContainerMetadata{},
+			expectedExitCode: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			setExitCodeFromEvent(tc.event, &tc.metadata)
+			assert.Equal(t, tc.expectedExitCode, tc.metadata.ExitCode)
+		})
+	}
+}
+
 func TestDockerVersion(t *testing.T) {
 	mockDockerSDK, client, _, _, _, done := dockerClientSetup(t)
 	defer done()