From 7b27660e172c00bc7750230dc277779a96e1eb5a Mon Sep 17 00:00:00 2001 From: Apoorv Mahajan Date: Fri, 22 Oct 2021 00:57:23 +0530 Subject: [PATCH] feat(clouddriver): make fetching properties file more resilient for k8s jobs If a k8s run job is marked as succeeded, and property file is defined in the stage context, then it can so happen that multiple pods are created for that job. See https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures In extreme edge cases, the first pod may be around before the second one succeeds. That leads to kubectl logs job/ command failing as seen below: kubectl -n test logs job/test-run-job-5j2vl -c parser Found 2 pods, using pod/test-run-job-5j2vl-fj8hd Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is terminated or Found 2 pods, using pod/test-run-job-5j2vl-fj8hd Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is waiting to start: PodInitializing where that commands defaults to using one of the two pods. To fix this issue, if we encounter an error from the kubectl logs job/ command, we find a successful pod in the job and directly query it for logs. --- .../tasks/job/WaitOnJobCompletion.groovy | 225 ++++- .../DelegatingKatoRestService.java | 7 + .../orca/clouddriver/KatoRestService.java | 8 + .../tasks/job/WaitOnJobCompletionTest.java | 137 ++- ...njob-stage-context-with-property-file.json | 3 +- ...b-stage-context-without-property-file.json | 3 +- ...sful-runjob-status-with-multiple-pods.json | 786 ++++++++++++++++++ 7 files changed, 1142 insertions(+), 27 deletions(-) create mode 100644 orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy index 252100e98d..4ef895c188 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy @@ -169,28 +169,16 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo if ((status == ExecutionStatus.SUCCEEDED) || (status == ExecutionStatus.TERMINAL)) { if (stage.context.propertyFile) { - Map properties = [:] - try { - retrySupport.retry({ - properties = katoRestService.getFileContents(appName, account, location, name, stage.context.propertyFile) - }, - configProperties.getFileContentRetry().maxAttempts, - Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), - configProperties.getFileContentRetry().exponentialBackoffEnabled - ) - } catch (Exception e) { - if (status == ExecutionStatus.SUCCEEDED) { - throw new ConfigurationException("Property File: ${stage.context.propertyFile} contents could not be retrieved. Error: " + e) - } - log.warn("failed to get file contents for ${appName}, account: ${account}, namespace: ${location}, " + - "manifest: ${name} from propertyFile: ${stage.context.propertyFile}. Error: ", e) - } - - if (properties.size() == 0) { - if (status == ExecutionStatus.SUCCEEDED) { - throw new ConfigurationException("Expected properties file ${stage.context.propertyFile} but it was either missing, empty or contained invalid syntax") - } - } else if (properties.size() > 0) { + Map properties = getPropertyFileContents( + job, + appName, + status, + account, + location, + name, + stage.context.propertyFile as String) + + if (properties.size() > 0) { outputs << properties outputs.propertyFileContents = properties } @@ -259,4 +247,197 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo } throw new JobFailedException(errorMessage) } + + /** + *

this method attempts to get property file from clouddriver and then parses its contents. Depending + * on the job itself, it could be handled by any job provider in clouddriver. This method should only be + * called for jobs with ExecutionStatus as either SUCCEEDED or TERMINAL. + * + *

If property file contents could not be retrieved from clouddriver, then the error handling depends + * on the job's ExecutionStatus. If it is SUCCEEDED, then an exception is thrown. Otherwise, no exception + * is thrown since we don't want to mask the real reason behind the job failure. + * + *

If ExecutionStatus == SUCCEEDED, and especially for kubernetes run jobs, it can so happen that a user + * has configured the job spec to run 1 pod, have completions and parallelism == 1, and + * restartPolicy == Never. Despite that, kubernetes may end up running another pod as stated here: + * https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures + * In such a scenario, it may so happen that two pods are created for that job. The first pod may still be + * around, such as in a PodInitializing state and the second pod could complete before the first one is + * terminated. This leads to the getFileContents() call failing, since under the covers, kubernetes job + * provider runs kubectl logs job/ command, which picks one out of the two pods to obtain the + * logs as seen here: + * + *

kubectl -n test logs job/test-run-job-5j2vl -c parser + * Found 2 pods, using pod/test-run-job-5j2vl-fj8hd + * Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is PodInitializing + * + *

That means, even if kubernetes and clouddriver marked the job as successful, since number of + * succeeded pods >= number of completions, the kubectl command shown above could still end using + * the failed pod for obtaining the logs. + * + *

To handle this case, if we get an error while making the getFileContents() call or if we don't receive + * any properties, then for kubernetes jobs, we figure out if the job status has any pod with phase + * SUCCEEDED. If we find such a pod, then we directly get the logs from this succeeded pod. Otherwise, + * we throw an exception as before. + * + *

we aren't handling the above case for ExecutionStatus == TERMINAL, because at that point, we wouldn't + * know which pod to query for properties file contents. It could so happen that all the pods in such a job + * have failed, then we would have to loop over each pod and see what it generated. Then if say, two pods + * generated different property values for the same key, which one do we choose? Bearing this complexity + * in mind, and knowing that for succeeded jobs, this solution actually helps prevent a pipeline failure, + * we are limiting this logic to succeeded jobs only for now. + * + * @param job - job status returned by clouddriver + * @param appName - application name where the job is run + * @param status - Execution status of the job. Should either be SUCCEEDED or TERMINAL + * @param account - account under which this job is run + * @param location - where this job is run + * @param name - name of the job + * @param propertyFile - file name to query from the job + * @return map of property file contents + */ + private Map getPropertyFileContents( + Map job, + String appName, + ExecutionStatus status, + String account, + String location, + String name, + String propertyFile + ) { + Map properties = [:] + try { + retrySupport.retry({ + properties = katoRestService.getFileContents(appName, account, location, name, propertyFile) + }, + configProperties.getFileContentRetry().maxAttempts, + Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), + configProperties.getFileContentRetry().exponentialBackoffEnabled + ) + } catch (Exception e) { + log.warn("Error occurred while retrieving property file contents from job: ${name}" + + " in application: ${appName}, in account: ${account}, location: ${location}," + + " using propertyFile: ${propertyFile}. Error: ", e + ) + + // For succeeded kubernetes jobs, let's try one more time to get property file contents. + if (status == ExecutionStatus.SUCCEEDED) { + properties = getPropertyFileContentsForSucceededKubernetesJob( + job, + appName, + account, + location, + propertyFile + ) + if (properties.size() == 0) { + // since we didn't get any properties, we fail with this exception + throw new ConfigurationException("Expected properties file: ${propertyFile} in " + + "job: ${name}, application: ${appName}, location: ${location}, account: ${account} " + + "but it was either missing, empty or contained invalid syntax. Error: ${e}") + } + } + } + + if (properties.size() == 0) { + log.warn("Could not parse propertyFile: ${propertyFile} in job: ${name}" + + " in application: ${appName}, in account: ${account}, location: ${location}." + + " It is either missing, empty or contains invalid syntax" + ) + + // For succeeded kubernetes jobs, let's try one more time to get property file contents. + if (status == ExecutionStatus.SUCCEEDED) { + // let's try one more time to get properties from a kubernetes pod + properties = getPropertyFileContentsForSucceededKubernetesJob( + job, + appName, + account, + location, + propertyFile + ) + if (properties.size() == 0) { + // since we didn't get any properties, we fail with this exception + throw new ConfigurationException("Expected properties file: ${propertyFile} in " + + "job: ${name}, application: ${appName}, location: ${location}, account: ${account} " + + "but it was either missing, empty or contained invalid syntax") + } + } + } + return properties + } + + /** + * This method is supposed to be called from getPropertyFileContents(). This is only applicable for + * Kubernetes jobs. It finds a successful pod in the job and directly queries it for property file + * contents. + * + *

It is meant to handle the following case: + * + *

if ExecutionStatus == SUCCEEDED, and especially for kubernetes run jobs, it can so happen that a + * user has configured the job spec to run 1 pod, have completions and parallelism == 1, and + * restartPolicy == Never. Despite that, kubernetes may end up running another pod as stated here: + * https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures + * In such a scenario, it may so happen that two pods are created for that job. The first pod may still be + * around, such as in a PodInitializing state and the second pod could complete before the first one is + * terminated. This leads to the getFileContents() call failing, since under the covers, kubernetes job + * provider runs kubectl logs job/ command, which picks one out of the two pods to obtain the + * logs as seen here: + * + *

kubectl -n test logs job/test-run-job-5j2vl -c parser + * Found 2 pods, using pod/test-run-job-5j2vl-fj8hd + * Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is PodInitializing + * + *

That means, even if kubernetes and clouddriver marked the job as successful, since number of + * succeeded pods >= number of completions, the kubectl command shown above could still end using + * the failed pod for obtaining the logs. + * + *

To handle this case, if we get an error while making the getFileContents() call or if we don't receive + * any properties, then for kubernetes jobs, we figure out if the job status has any pod with phase + * SUCCEEDED. If we find such a pod, then we directly get the logs from this succeeded pod. Otherwise, + * we throw an exception as before. + * + *

To keep it simple, and not worry about how to deal with property file + * contents obtained from various successful pods in a job, if that may happen, we simply query the first + * successful pod in that job. + * + * @param job - job status returned by clouddriver + * @param appName - application in which this job is run + * @param account - account under which this job is run + * @param namespace - where this job is run + * @param propertyFile - file name to query from the job + * @return map of property file contents + */ + private Map getPropertyFileContentsForSucceededKubernetesJob( + Map job, + String appName, + String account, + String namespace, + String propertyFile + ) { + Map properties = [:] + if (job.get("provider", "unknown") == "kubernetes") { + Optional succeededPod = job.get("pods", []) + .stream() + .filter({ Map pod -> pod.get("status", [:]).get("phase", "Running") == "Succeeded" + }) + .findFirst() + + if (succeededPod.isPresent()) { + String podName = (succeededPod.get() as Map).get("name") + retrySupport.retry({ + properties = katoRestService.getFileContentsFromKubernetesPod( + appName, + account, + namespace, + podName, + propertyFile + ) + }, + configProperties.getFileContentRetry().maxAttempts, + Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), + configProperties.getFileContentRetry().exponentialBackoffEnabled + ) + } + } + return properties + } } diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java index 9b3e823542..f480a72b67 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java @@ -71,6 +71,13 @@ public Map getFileContents( return getService().getFileContents(app, account, region, id, fileName); } + @Override + public Map getFileContentsFromKubernetesPod( + String app, String account, String namespace, String podName, String fileName) { + return getService() + .getFileContentsFromKubernetesPod(app, account, namespace, podName, fileName); + } + @Override public Task lookupTask(String id) { return getService().lookupTask(id); diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java index eac1ebce17..ebbb51af4d 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java @@ -73,6 +73,14 @@ Map getFileContents( @Path("id") String id, @Path("fileName") String fileName); + @GET("/applications/{app}/kubernetes/pods/{account}/{namespace}/{podName}/{fileName}") + Map getFileContentsFromKubernetesPod( + @Path("app") String app, + @Path("account") String account, + @Path("namespace") String namespace, + @Path("podName") String podName, + @Path("fileName") String fileName); + /** * This should _only_ be called if there is a problem retrieving the Task from * CloudDriverTaskStatusService (ie. a clouddriver replica). diff --git a/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java b/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java index 09faa83b66..a03fad6349 100644 --- a/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java +++ b/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java @@ -264,7 +264,9 @@ void testPropertyFileContentsHandlingForASuccessfulK8sRunJob(boolean isPropertyF thrown .getMessage() .matches( - "Expected properties file testrep but it was either missing, empty or contained invalid syntax")); + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or" + + " contained invalid syntax")); } else { assertNotNull(result); assertThat(result.getContext().containsKey("propertyFileContents")).isTrue(); @@ -315,8 +317,137 @@ void testPropertyFileContentsErrorHandlingForASuccessfulK8sRunJob() throws IOExc thrown .getMessage() .matches( - "Property File: testrep contents could not be retrieved. " - + "Error: java.lang.RuntimeException: some exception")); + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or contained" + + " invalid syntax. Error: java.lang.RuntimeException: some exception")); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " properties file should be obtained from the getFileContents() call, if that is successful") + @Test + void + testParsePropertiesFileContentsForSuccessfulK8sJobWith2PodsWithSuccessfulGetFileContentsCall() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of("some-key", "some-value")); + + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); + + // then + assertThat(result.getOutputs()).isNotEmpty(); + assertThat(result.getContext()).isNotEmpty(); + + assertThat(result.getOutputs().containsKey("some-key")); + assertThat(result.getOutputs().containsValue("some-value")); + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + // no need to get file contents from a specific pod if the getFileContents call was successful + verify(mockKatoRestService, never()) + .getFileContentsFromKubernetesPod( + anyString(), anyString(), anyString(), anyString(), anyString()); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " the properties file should be read directly from the succeeded pod if the getFileContents() call fails") + @Test + void testParsePropertiesFileContentsForSuccessfulK8sJobWith2PodsWithFailedGetFileContentsCall() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of()); + + when(mockKatoRestService.getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep")) + .thenReturn(Map.of("some-key", "some-value")); + + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); + + // then + assertThat(result.getOutputs()).isNotEmpty(); + assertThat(result.getContext()).isNotEmpty(); + + assertThat(result.getOutputs().containsKey("some-key")); + assertThat(result.getOutputs().containsValue("some-value")); + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + verify(mockKatoRestService) + .getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep"); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " the properties file should be read from the getFileContents() call first. If that fails, a call to " + + " get the properties from the getFileContentsFromPod() should be made. If that fails," + + " an exception should be thrown") + @Test + void testParsePropertiesFileContentsErrorHandlingForSuccessfulK8sJobWith2Pods() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of()); + + when(mockKatoRestService.getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep")) + .thenReturn(Map.of()); + + // then + ConfigurationException thrown = + assertThrows( + ConfigurationException.class, + () -> + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"))); + + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + verify(mockKatoRestService) + .getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep"); + assertTrue( + thrown + .getMessage() + .matches( + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or contained" + + " invalid syntax")); } @DisplayName( diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json index 1a375774aa..479e9e92aa 100644 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json @@ -5,5 +5,6 @@ ] }, "account": "test-account", - "propertyFile": "testrep" + "propertyFile": "testrep", + "application": "test-app" } diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json index 15f1120a56..31177b1b5e 100644 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json @@ -4,5 +4,6 @@ "job testrep" ] }, - "account": "test-account" + "account": "test-account", + "application": "test-app" } diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json new file mode 100644 index 0000000000..092fe2cfc5 --- /dev/null +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json @@ -0,0 +1,786 @@ +{ + "account": "test-account", + "completionDetails": { + "exitCode": "", + "message": "", + "reason": "", + "signal": "" + }, + "createdTime": 1633792127000, + "jobState": "Succeeded", + "location": "test", + "name": "testrep", + "pods": [ + { + "name": "testrep-bf2g7", + "status": { + "conditions": [ + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with incomplete status: [sidecar1 sidecar2]", + "reason": "ContainersNotInitialized", + "status": "False", + "type": "Initialized" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with unready status: [testrep]", + "reason": "ContainersNotReady", + "status": "False", + "type": "Ready" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with unready status: [testrep]", + "reason": "ContainersNotReady", + "status": "False", + "type": "ContainersReady" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "image": "some-main-app-image:1", + "imageID": "", + "lastState": {}, + "name": "testrep", + "ready": false, + "restartCount": 0, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "hostIP": "1.1.1.1", + "initContainerStatuses": [ + { + "containerID": "docker://2ed", + "image": "some-init-container-image:1", + "imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", + "lastState": {}, + "name": "sidecar1", + "ready": false, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://2ed", + "exitCode": 137, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792167000, + "millisOfDay": 54567000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54567, + "secondOfMinute": 27, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "OOMKilled", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792166000, + "millisOfDay": 54566000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54566, + "secondOfMinute": 26, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + }, + { + "image": "some-init-container-other-image:1", + "imageID": "", + "lastState": {}, + "name": "sidecar2", + "ready": false, + "restartCount": 0, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "phase": "Failed", + "podIP": "1.1.1.1", + "qosClass": "Burstable", + "startTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + }, + { + "name": "testrep-rn5qt", + "status": { + "conditions": [ + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792204000, + "millisOfDay": 54604000, + "millisOfSecond": 0, + "minuteOfDay": 910, + "minuteOfHour": 10, + "monthOfYear": 10, + "secondOfDay": 54604, + "secondOfMinute": 4, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "Initialized" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792321000, + "millisOfDay": 54721000, + "millisOfSecond": 0, + "minuteOfDay": 912, + "minuteOfHour": 12, + "monthOfYear": 10, + "secondOfDay": 54721, + "secondOfMinute": 1, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "Ready" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792321000, + "millisOfDay": 54721000, + "millisOfSecond": 0, + "minuteOfDay": 912, + "minuteOfHour": 12, + "monthOfYear": 10, + "secondOfDay": 54721, + "secondOfMinute": 1, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "ContainersReady" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792170000, + "millisOfDay": 54570000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54570, + "secondOfMinute": 30, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://451", + "image": "some-main-app-image:1", + "imageID": "docker-pullable://some-main-app-repo/some-main-app-image:1", + "lastState": {}, + "name": "testrep", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://451", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633791818000, + "millisOfDay": 54218000, + "millisOfSecond": 0, + "minuteOfDay": 903, + "minuteOfHour": 3, + "monthOfYear": 10, + "secondOfDay": 54218, + "secondOfMinute": 38, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633791758000, + "millisOfDay": 54158000, + "millisOfSecond": 0, + "minuteOfDay": 902, + "minuteOfHour": 2, + "monthOfYear": 10, + "secondOfDay": 54158, + "secondOfMinute": 38, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + } + ], + "hostIP": "1.1.1.1", + "initContainerStatuses": [ + { + "containerID": "docker://51b", + "image": "some-init-container-image:1", + "imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", + "lastState": {}, + "name": "casam-sidecar", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://51b", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792194000, + "millisOfDay": 54594000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54594, + "secondOfMinute": 54, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792194000, + "millisOfDay": 54594000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54594, + "secondOfMinute": 54, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + }, + { + "containerID": "docker://b5b", + "image": "some-other-init-container-image:1", + "imageID": "docker-pullable://some-other-init-container-repo/some-other-init-container-image:1", + "lastState": {}, + "name": "sidecar2", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://b5b", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792195000, + "millisOfDay": 54595000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54595, + "secondOfMinute": 55, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792195000, + "millisOfDay": 54595000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54595, + "secondOfMinute": 55, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + } + ], + "phase": "Succeeded", + "podIP": "1.1.1.1", + "qosClass": "Burstable", + "startTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792170000, + "millisOfDay": 54570000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54570, + "secondOfMinute": 30, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + ], + "provider": "kubernetes" +}