diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy index 252100e98d..4ef895c188 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy @@ -169,28 +169,16 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo if ((status == ExecutionStatus.SUCCEEDED) || (status == ExecutionStatus.TERMINAL)) { if (stage.context.propertyFile) { - Map properties = [:] - try { - retrySupport.retry({ - properties = katoRestService.getFileContents(appName, account, location, name, stage.context.propertyFile) - }, - configProperties.getFileContentRetry().maxAttempts, - Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), - configProperties.getFileContentRetry().exponentialBackoffEnabled - ) - } catch (Exception e) { - if (status == ExecutionStatus.SUCCEEDED) { - throw new ConfigurationException("Property File: ${stage.context.propertyFile} contents could not be retrieved. Error: " + e) - } - log.warn("failed to get file contents for ${appName}, account: ${account}, namespace: ${location}, " + - "manifest: ${name} from propertyFile: ${stage.context.propertyFile}. Error: ", e) - } - - if (properties.size() == 0) { - if (status == ExecutionStatus.SUCCEEDED) { - throw new ConfigurationException("Expected properties file ${stage.context.propertyFile} but it was either missing, empty or contained invalid syntax") - } - } else if (properties.size() > 0) { + Map properties = getPropertyFileContents( + job, + appName, + status, + account, + location, + name, + stage.context.propertyFile as String) + + if (properties.size() > 0) { outputs << properties outputs.propertyFileContents = properties } @@ -259,4 +247,197 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo } throw new JobFailedException(errorMessage) } + + /** + *

this method attempts to get property file from clouddriver and then parses its contents. Depending + * on the job itself, it could be handled by any job provider in clouddriver. This method should only be + * called for jobs with ExecutionStatus as either SUCCEEDED or TERMINAL. + * + *

If property file contents could not be retrieved from clouddriver, then the error handling depends + * on the job's ExecutionStatus. If it is SUCCEEDED, then an exception is thrown. Otherwise, no exception + * is thrown since we don't want to mask the real reason behind the job failure. + * + *

If ExecutionStatus == SUCCEEDED, and especially for kubernetes run jobs, it can so happen that a user + * has configured the job spec to run 1 pod, have completions and parallelism == 1, and + * restartPolicy == Never. Despite that, kubernetes may end up running another pod as stated here: + * https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures + * In such a scenario, it may so happen that two pods are created for that job. The first pod may still be + * around, such as in a PodInitializing state and the second pod could complete before the first one is + * terminated. This leads to the getFileContents() call failing, since under the covers, kubernetes job + * provider runs kubectl logs job/ command, which picks one out of the two pods to obtain the + * logs as seen here: + * + *

kubectl -n test logs job/test-run-job-5j2vl -c parser + * Found 2 pods, using pod/test-run-job-5j2vl-fj8hd + * Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is PodInitializing + * + *

That means, even if kubernetes and clouddriver marked the job as successful, since number of + * succeeded pods >= number of completions, the kubectl command shown above could still end using + * the failed pod for obtaining the logs. + * + *

To handle this case, if we get an error while making the getFileContents() call or if we don't receive + * any properties, then for kubernetes jobs, we figure out if the job status has any pod with phase + * SUCCEEDED. If we find such a pod, then we directly get the logs from this succeeded pod. Otherwise, + * we throw an exception as before. + * + *

we aren't handling the above case for ExecutionStatus == TERMINAL, because at that point, we wouldn't + * know which pod to query for properties file contents. It could so happen that all the pods in such a job + * have failed, then we would have to loop over each pod and see what it generated. Then if say, two pods + * generated different property values for the same key, which one do we choose? Bearing this complexity + * in mind, and knowing that for succeeded jobs, this solution actually helps prevent a pipeline failure, + * we are limiting this logic to succeeded jobs only for now. + * + * @param job - job status returned by clouddriver + * @param appName - application name where the job is run + * @param status - Execution status of the job. Should either be SUCCEEDED or TERMINAL + * @param account - account under which this job is run + * @param location - where this job is run + * @param name - name of the job + * @param propertyFile - file name to query from the job + * @return map of property file contents + */ + private Map getPropertyFileContents( + Map job, + String appName, + ExecutionStatus status, + String account, + String location, + String name, + String propertyFile + ) { + Map properties = [:] + try { + retrySupport.retry({ + properties = katoRestService.getFileContents(appName, account, location, name, propertyFile) + }, + configProperties.getFileContentRetry().maxAttempts, + Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), + configProperties.getFileContentRetry().exponentialBackoffEnabled + ) + } catch (Exception e) { + log.warn("Error occurred while retrieving property file contents from job: ${name}" + + " in application: ${appName}, in account: ${account}, location: ${location}," + + " using propertyFile: ${propertyFile}. Error: ", e + ) + + // For succeeded kubernetes jobs, let's try one more time to get property file contents. + if (status == ExecutionStatus.SUCCEEDED) { + properties = getPropertyFileContentsForSucceededKubernetesJob( + job, + appName, + account, + location, + propertyFile + ) + if (properties.size() == 0) { + // since we didn't get any properties, we fail with this exception + throw new ConfigurationException("Expected properties file: ${propertyFile} in " + + "job: ${name}, application: ${appName}, location: ${location}, account: ${account} " + + "but it was either missing, empty or contained invalid syntax. Error: ${e}") + } + } + } + + if (properties.size() == 0) { + log.warn("Could not parse propertyFile: ${propertyFile} in job: ${name}" + + " in application: ${appName}, in account: ${account}, location: ${location}." + + " It is either missing, empty or contains invalid syntax" + ) + + // For succeeded kubernetes jobs, let's try one more time to get property file contents. + if (status == ExecutionStatus.SUCCEEDED) { + // let's try one more time to get properties from a kubernetes pod + properties = getPropertyFileContentsForSucceededKubernetesJob( + job, + appName, + account, + location, + propertyFile + ) + if (properties.size() == 0) { + // since we didn't get any properties, we fail with this exception + throw new ConfigurationException("Expected properties file: ${propertyFile} in " + + "job: ${name}, application: ${appName}, location: ${location}, account: ${account} " + + "but it was either missing, empty or contained invalid syntax") + } + } + } + return properties + } + + /** + * This method is supposed to be called from getPropertyFileContents(). This is only applicable for + * Kubernetes jobs. It finds a successful pod in the job and directly queries it for property file + * contents. + * + *

It is meant to handle the following case: + * + *

if ExecutionStatus == SUCCEEDED, and especially for kubernetes run jobs, it can so happen that a + * user has configured the job spec to run 1 pod, have completions and parallelism == 1, and + * restartPolicy == Never. Despite that, kubernetes may end up running another pod as stated here: + * https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures + * In such a scenario, it may so happen that two pods are created for that job. The first pod may still be + * around, such as in a PodInitializing state and the second pod could complete before the first one is + * terminated. This leads to the getFileContents() call failing, since under the covers, kubernetes job + * provider runs kubectl logs job/ command, which picks one out of the two pods to obtain the + * logs as seen here: + * + *

kubectl -n test logs job/test-run-job-5j2vl -c parser + * Found 2 pods, using pod/test-run-job-5j2vl-fj8hd + * Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is PodInitializing + * + *

That means, even if kubernetes and clouddriver marked the job as successful, since number of + * succeeded pods >= number of completions, the kubectl command shown above could still end using + * the failed pod for obtaining the logs. + * + *

To handle this case, if we get an error while making the getFileContents() call or if we don't receive + * any properties, then for kubernetes jobs, we figure out if the job status has any pod with phase + * SUCCEEDED. If we find such a pod, then we directly get the logs from this succeeded pod. Otherwise, + * we throw an exception as before. + * + *

To keep it simple, and not worry about how to deal with property file + * contents obtained from various successful pods in a job, if that may happen, we simply query the first + * successful pod in that job. + * + * @param job - job status returned by clouddriver + * @param appName - application in which this job is run + * @param account - account under which this job is run + * @param namespace - where this job is run + * @param propertyFile - file name to query from the job + * @return map of property file contents + */ + private Map getPropertyFileContentsForSucceededKubernetesJob( + Map job, + String appName, + String account, + String namespace, + String propertyFile + ) { + Map properties = [:] + if (job.get("provider", "unknown") == "kubernetes") { + Optional succeededPod = job.get("pods", []) + .stream() + .filter({ Map pod -> pod.get("status", [:]).get("phase", "Running") == "Succeeded" + }) + .findFirst() + + if (succeededPod.isPresent()) { + String podName = (succeededPod.get() as Map).get("name") + retrySupport.retry({ + properties = katoRestService.getFileContentsFromKubernetesPod( + appName, + account, + namespace, + podName, + propertyFile + ) + }, + configProperties.getFileContentRetry().maxAttempts, + Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), + configProperties.getFileContentRetry().exponentialBackoffEnabled + ) + } + } + return properties + } } diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java index 9b3e823542..f480a72b67 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java @@ -71,6 +71,13 @@ public Map getFileContents( return getService().getFileContents(app, account, region, id, fileName); } + @Override + public Map getFileContentsFromKubernetesPod( + String app, String account, String namespace, String podName, String fileName) { + return getService() + .getFileContentsFromKubernetesPod(app, account, namespace, podName, fileName); + } + @Override public Task lookupTask(String id) { return getService().lookupTask(id); diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java index eac1ebce17..ebbb51af4d 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java @@ -73,6 +73,14 @@ Map getFileContents( @Path("id") String id, @Path("fileName") String fileName); + @GET("/applications/{app}/kubernetes/pods/{account}/{namespace}/{podName}/{fileName}") + Map getFileContentsFromKubernetesPod( + @Path("app") String app, + @Path("account") String account, + @Path("namespace") String namespace, + @Path("podName") String podName, + @Path("fileName") String fileName); + /** * This should _only_ be called if there is a problem retrieving the Task from * CloudDriverTaskStatusService (ie. a clouddriver replica). diff --git a/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java b/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java index 09faa83b66..a03fad6349 100644 --- a/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java +++ b/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java @@ -264,7 +264,9 @@ void testPropertyFileContentsHandlingForASuccessfulK8sRunJob(boolean isPropertyF thrown .getMessage() .matches( - "Expected properties file testrep but it was either missing, empty or contained invalid syntax")); + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or" + + " contained invalid syntax")); } else { assertNotNull(result); assertThat(result.getContext().containsKey("propertyFileContents")).isTrue(); @@ -315,8 +317,137 @@ void testPropertyFileContentsErrorHandlingForASuccessfulK8sRunJob() throws IOExc thrown .getMessage() .matches( - "Property File: testrep contents could not be retrieved. " - + "Error: java.lang.RuntimeException: some exception")); + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or contained" + + " invalid syntax. Error: java.lang.RuntimeException: some exception")); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " properties file should be obtained from the getFileContents() call, if that is successful") + @Test + void + testParsePropertiesFileContentsForSuccessfulK8sJobWith2PodsWithSuccessfulGetFileContentsCall() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of("some-key", "some-value")); + + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); + + // then + assertThat(result.getOutputs()).isNotEmpty(); + assertThat(result.getContext()).isNotEmpty(); + + assertThat(result.getOutputs().containsKey("some-key")); + assertThat(result.getOutputs().containsValue("some-value")); + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + // no need to get file contents from a specific pod if the getFileContents call was successful + verify(mockKatoRestService, never()) + .getFileContentsFromKubernetesPod( + anyString(), anyString(), anyString(), anyString(), anyString()); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " the properties file should be read directly from the succeeded pod if the getFileContents() call fails") + @Test + void testParsePropertiesFileContentsForSuccessfulK8sJobWith2PodsWithFailedGetFileContentsCall() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of()); + + when(mockKatoRestService.getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep")) + .thenReturn(Map.of("some-key", "some-value")); + + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); + + // then + assertThat(result.getOutputs()).isNotEmpty(); + assertThat(result.getContext()).isNotEmpty(); + + assertThat(result.getOutputs().containsKey("some-key")); + assertThat(result.getOutputs().containsValue("some-value")); + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + verify(mockKatoRestService) + .getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep"); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " the properties file should be read from the getFileContents() call first. If that fails, a call to " + + " get the properties from the getFileContentsFromPod() should be made. If that fails," + + " an exception should be thrown") + @Test + void testParsePropertiesFileContentsErrorHandlingForSuccessfulK8sJobWith2Pods() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of()); + + when(mockKatoRestService.getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep")) + .thenReturn(Map.of()); + + // then + ConfigurationException thrown = + assertThrows( + ConfigurationException.class, + () -> + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"))); + + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + verify(mockKatoRestService) + .getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep"); + assertTrue( + thrown + .getMessage() + .matches( + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or contained" + + " invalid syntax")); } @DisplayName( diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json index 1a375774aa..479e9e92aa 100644 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json @@ -5,5 +5,6 @@ ] }, "account": "test-account", - "propertyFile": "testrep" + "propertyFile": "testrep", + "application": "test-app" } diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json index 15f1120a56..31177b1b5e 100644 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json @@ -4,5 +4,6 @@ "job testrep" ] }, - "account": "test-account" + "account": "test-account", + "application": "test-app" } diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json new file mode 100644 index 0000000000..092fe2cfc5 --- /dev/null +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json @@ -0,0 +1,786 @@ +{ + "account": "test-account", + "completionDetails": { + "exitCode": "", + "message": "", + "reason": "", + "signal": "" + }, + "createdTime": 1633792127000, + "jobState": "Succeeded", + "location": "test", + "name": "testrep", + "pods": [ + { + "name": "testrep-bf2g7", + "status": { + "conditions": [ + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with incomplete status: [sidecar1 sidecar2]", + "reason": "ContainersNotInitialized", + "status": "False", + "type": "Initialized" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with unready status: [testrep]", + "reason": "ContainersNotReady", + "status": "False", + "type": "Ready" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with unready status: [testrep]", + "reason": "ContainersNotReady", + "status": "False", + "type": "ContainersReady" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "image": "some-main-app-image:1", + "imageID": "", + "lastState": {}, + "name": "testrep", + "ready": false, + "restartCount": 0, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "hostIP": "1.1.1.1", + "initContainerStatuses": [ + { + "containerID": "docker://2ed", + "image": "some-init-container-image:1", + "imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", + "lastState": {}, + "name": "sidecar1", + "ready": false, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://2ed", + "exitCode": 137, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792167000, + "millisOfDay": 54567000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54567, + "secondOfMinute": 27, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "OOMKilled", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792166000, + "millisOfDay": 54566000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54566, + "secondOfMinute": 26, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + }, + { + "image": "some-init-container-other-image:1", + "imageID": "", + "lastState": {}, + "name": "sidecar2", + "ready": false, + "restartCount": 0, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "phase": "Failed", + "podIP": "1.1.1.1", + "qosClass": "Burstable", + "startTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + }, + { + "name": "testrep-rn5qt", + "status": { + "conditions": [ + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792204000, + "millisOfDay": 54604000, + "millisOfSecond": 0, + "minuteOfDay": 910, + "minuteOfHour": 10, + "monthOfYear": 10, + "secondOfDay": 54604, + "secondOfMinute": 4, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "Initialized" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792321000, + "millisOfDay": 54721000, + "millisOfSecond": 0, + "minuteOfDay": 912, + "minuteOfHour": 12, + "monthOfYear": 10, + "secondOfDay": 54721, + "secondOfMinute": 1, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "Ready" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792321000, + "millisOfDay": 54721000, + "millisOfSecond": 0, + "minuteOfDay": 912, + "minuteOfHour": 12, + "monthOfYear": 10, + "secondOfDay": 54721, + "secondOfMinute": 1, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "ContainersReady" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792170000, + "millisOfDay": 54570000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54570, + "secondOfMinute": 30, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://451", + "image": "some-main-app-image:1", + "imageID": "docker-pullable://some-main-app-repo/some-main-app-image:1", + "lastState": {}, + "name": "testrep", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://451", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633791818000, + "millisOfDay": 54218000, + "millisOfSecond": 0, + "minuteOfDay": 903, + "minuteOfHour": 3, + "monthOfYear": 10, + "secondOfDay": 54218, + "secondOfMinute": 38, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633791758000, + "millisOfDay": 54158000, + "millisOfSecond": 0, + "minuteOfDay": 902, + "minuteOfHour": 2, + "monthOfYear": 10, + "secondOfDay": 54158, + "secondOfMinute": 38, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + } + ], + "hostIP": "1.1.1.1", + "initContainerStatuses": [ + { + "containerID": "docker://51b", + "image": "some-init-container-image:1", + "imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", + "lastState": {}, + "name": "casam-sidecar", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://51b", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792194000, + "millisOfDay": 54594000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54594, + "secondOfMinute": 54, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792194000, + "millisOfDay": 54594000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54594, + "secondOfMinute": 54, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + }, + { + "containerID": "docker://b5b", + "image": "some-other-init-container-image:1", + "imageID": "docker-pullable://some-other-init-container-repo/some-other-init-container-image:1", + "lastState": {}, + "name": "sidecar2", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://b5b", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792195000, + "millisOfDay": 54595000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54595, + "secondOfMinute": 55, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792195000, + "millisOfDay": 54595000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54595, + "secondOfMinute": 55, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + } + ], + "phase": "Succeeded", + "podIP": "1.1.1.1", + "qosClass": "Burstable", + "startTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792170000, + "millisOfDay": 54570000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54570, + "secondOfMinute": 30, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + ], + "provider": "kubernetes" +}