Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(controller): online log with the correct pod #2565

Merged
merged 2 commits into from
Jul 27, 2023
Merged

fix(controller): online log with the correct pod #2565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a51a166
fix(controller): online log with the correct pod
jialeicui Jul 26, 2023
de55cc8
fix ut
jialeicui Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion ...ller/src/main/java/ai/starwhale/mlops/schedule/k8s/log/CancellableJobLogK8sCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,20 @@ private String getPodName(String taskId) throws ApiException {
if (podList.getItems().isEmpty()) {
throw new ApiException("get empty pod list by job name " + taskId);
}
return podList.getItems().get(0).getMetadata().getName();
// returns the running pod
var thePod = podList.getItems().stream().filter(pod -> {
if (pod.getStatus() == null) {
return false;
}
if (pod.getStatus().getPhase() == null) {
return false;
}
return pod.getStatus().getPhase().equals("Running");
}).findFirst();
if (thePod.isEmpty()) {
throw new ApiException("get empty running pod by job name " + taskId);
}
return thePod.get().getMetadata().getName();
}

@Override
Expand Down
4 changes: 3 additions & 1 deletion ...st/java/ai/starwhale/mlops/schedule/k8s/log/CancellableJobLogK8sCollectorFactoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1Pod;
import io.kubernetes.client.openapi.models.V1PodList;
import io.kubernetes.client.openapi.models.V1PodStatus;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
Expand All @@ -39,8 +40,9 @@ public class CancellableJobLogK8sCollectorFactoryTest {
@Test
public void testMake() throws IOException, ApiException {
var k8sClient = mock(K8sClient.class);
var pod = new V1Pod().metadata(new V1ObjectMeta().name("1-xx")).status(new V1PodStatus().phase("Running"));
when(k8sClient.getPodsByJobName("1"))
.thenReturn(new V1PodList().items(List.of(new V1Pod().metadata(new V1ObjectMeta().name("1-xx")))));
.thenReturn(new V1PodList().items(List.of(pod)));
var call = mock(Call.class);
var resp = mock(Response.class);
var respBody = mock(ResponseBody.class);
Expand Down
11 changes: 8 additions & 3 deletions .../src/test/java/ai/starwhale/mlops/schedule/k8s/log/CancellableJobLogK8sCollectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static org.hamcrest.Matchers.is;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
Expand All @@ -29,6 +30,7 @@
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1Pod;
import io.kubernetes.client.openapi.models.V1PodList;
import io.kubernetes.client.openapi.models.V1PodStatus;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
Expand All @@ -48,8 +50,11 @@ public void setup() {

@Test
public void testInitAndRead() throws IOException, ApiException {
var podMeta = new V1ObjectMeta().name("1-xxx");
when(k8sClient.getPodsByJobName("1")).thenReturn(new V1PodList().items(List.of(new V1Pod().metadata(podMeta))));
var runningPodMeta = new V1ObjectMeta().name("running-pod");
var runningPod = new V1Pod().metadata(runningPodMeta).status(new V1PodStatus().phase("Running"));
var failedPodMeta = new V1ObjectMeta().name("failed-pod");
var failedPod = new V1Pod().metadata(failedPodMeta).status(new V1PodStatus().phase("Failed"));
when(k8sClient.getPodsByJobName("1")).thenReturn(new V1PodList().items(List.of(runningPod, failedPod)));

var line = "abc";

Expand All @@ -60,7 +65,7 @@ public void testInitAndRead() throws IOException, ApiException {
when(resp.body()).thenReturn(respBody);
when(call.execute()).thenReturn(resp);

when(k8sClient.readLog(anyString(), anyString(), anyBoolean())).thenReturn(call);
when(k8sClient.readLog(eq("running-pod"), anyString(), anyBoolean())).thenReturn(call);
var ins = new CancellableJobLogK8sCollector(k8sClient, "1");

assertThat(ins.readLine(), is(line));
Expand Down
Loading