From 40b6d0bce3e4366828787a18a1985b25f488baec Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 14 Sep 2018 19:31:23 +0100 Subject: [PATCH] Update status for failed and creating conditions --- .../seldon/clustermanager/k8s/Constants.java | 3 ++ .../clustermanager/k8s/KubeCRDHandler.java | 1 + .../k8s/KubeCRDHandlerImpl.java | 24 +++++++++- .../k8s/SeldonDeploymentControllerImpl.java | 6 +-- .../k8s/SeldonDeploymentOperatorImpl.java | 5 +++ .../k8s/SeldonDeploymentStatusUpdateImpl.java | 14 ++++++ .../k8s/SeldonDeploymentWatcher.java | 44 +++++++++++++++++-- 7 files changed, 88 insertions(+), 9 deletions(-) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/Constants.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/Constants.java index d11ffc213e..fbd4b2c27e 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/Constants.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/Constants.java @@ -17,4 +17,7 @@ public class Constants { public static final String LABEL_SELDON_ID = "seldon-deployment-id"; + public static final String STATE_CREATING = "Creating"; + public static final String STATE_FAILED = "Failed"; + public static final String STATE_AVAILABLE = "Available"; } diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java index 8105903d40..fcccc186d5 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java @@ -21,6 +21,7 @@ public interface KubeCRDHandler { + public void updateRaw(String json,String seldonDeploymentName); public void updateSeldonDeployment(SeldonDeployment mlDep); public SeldonDeployment getSeldonDeployment(String name); public ExtensionsV1beta1DeploymentList getOwnedDeployments(String seldonDeploymentName); diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java index 146df3fd28..992e3a6367 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.fasterxml.jackson.databind.JsonNode; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.protobuf.InvalidProtocolBufferException; @@ -55,6 +56,25 @@ public class KubeCRDHandlerImpl implements KubeCRDHandler { public KubeCRDHandlerImpl(ClusterManagerProperites clusterManagerProperites) { this.namespace = StringUtils.isEmpty(clusterManagerProperites.getNamespace()) ? "default" : clusterManagerProperites.getNamespace(); } + + @Override + public void updateRaw(String json,String seldonDeploymentName) { + try + { + logger.info(json); + ApiClient client = Config.defaultClient(); + CustomObjectsApi api = new CustomObjectsApi(client); + api.replaceNamespacedCustomObject(GROUP, VERSION, namespace, KIND_PLURAL, seldonDeploymentName,json.getBytes()); + } catch (InvalidProtocolBufferException e) { + logger.error("Failed to update deployment in kubernetes ",e); + } catch (ApiException e) { + logger.error("Failed to update deployment in kubernetes : {}",e.getResponseBody()); + } catch (IOException e) { + logger.error("Failed to get client ",e); + } + + + } @Override public void updateSeldonDeployment(SeldonDeployment mldep) { @@ -84,7 +104,7 @@ public void updateSeldonDeployment(SeldonDeployment mldep) { } catch (IOException e) { logger.error("Failed to get client ",e); } - finally{} + } @Override @@ -144,6 +164,8 @@ public V1ServiceList getOwnedServices(String seldonDeploymentName) { return null; } } + + diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java index a4ea766a15..4fc15fbcd1 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java @@ -53,8 +53,6 @@ public class SeldonDeploymentControllerImpl implements SeldonDeploymentControlle private final KubeCRDHandler crdHandler; private final SeldonDeploymentCache mlCache; - private static final String FAILED_STATE_MSG = "FAILED"; - private static final String DEPLOYMENT_API_VERSION = "extensions/v1beta1"; @@ -253,14 +251,14 @@ private String getNamespace(SeldonDeployment d) private void failDeployment(SeldonDeployment mlDep,Exception e) { SeldonDeployment.Builder mlBuilder = SeldonDeployment.newBuilder(mlDep); - mlBuilder.getStatusBuilder().setState(FAILED_STATE_MSG).setDescription(e.getMessage()); + mlBuilder.getStatusBuilder().setState(Constants.STATE_FAILED).setDescription(e.getMessage()); crdHandler.updateSeldonDeployment(mlBuilder.build()); } @Override public void createOrReplaceSeldonDeployment(SeldonDeployment mlDep) { - if (mlDep.hasStatus() && mlDep.getStatus().hasState() && mlDep.getStatus().getState().equals(FAILED_STATE_MSG)) + if (mlDep.hasStatus() && mlDep.getStatus().hasState() && mlDep.getStatus().getState().equals(Constants.STATE_FAILED)) { logger.warn("Ignoring failed deployment "+mlDep.getMetadata().getName()); return; diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index 11a2d376e8..683807c9c9 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -399,6 +399,11 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) { } } + if (!mlBuilder.hasStatus()) + { + mlBuilder.getStatusBuilder().setState(Constants.STATE_CREATING); + } + return mlBuilder.build(); } diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentStatusUpdateImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentStatusUpdateImpl.java index 5f353461fa..ebd8581f1e 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentStatusUpdateImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentStatusUpdateImpl.java @@ -45,6 +45,16 @@ private void update(PredictorStatus.Builder b,Integer replicas, Integer replicas b.setReplicasAvailable(0); } + private boolean isAvailable(SeldonDeployment.Builder mlBuilder) + { + for (PredictorStatus.Builder b : mlBuilder.getStatusBuilder().getPredictorStatusBuilderList()) + { + if (b.getReplicas() != b.getReplicasAvailable()) + return false; + } + return true; + } + @Override public void updateStatus(String mlDepName, String depName, Integer replicas, Integer replicasAvailable) { if (replicas == null || replicas == 0) @@ -73,6 +83,10 @@ public void updateStatus(String mlDepName, String depName, Integer replicas, Int update(b,replicas,replicasAvailable); mlBuilder.getStatusBuilder().addPredictorStatus(b); } + if (isAvailable(mlBuilder)) + { + mlBuilder.getStatusBuilder().setState(Constants.STATE_AVAILABLE); + } crdHandler.updateSeldonDeployment(mlBuilder.build()); } else diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java index 9923ac64d5..43cd757021 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java @@ -28,10 +28,12 @@ import org.springframework.stereotype.Component; import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; @@ -43,6 +45,9 @@ import io.kubernetes.client.util.Config; import io.kubernetes.client.util.Watch; import io.seldon.clustermanager.ClusterManagerProperites; +import io.seldon.clustermanager.pb.JsonFormat; +import io.seldon.clustermanager.pb.JsonFormat.Printer; +import io.seldon.protos.DeploymentProtos.DeploymentStatus; import io.seldon.protos.DeploymentProtos.SeldonDeployment; @Component @@ -52,16 +57,18 @@ public class SeldonDeploymentWatcher { private final SeldonDeploymentController seldonDeploymentController; private final SeldonDeploymentCache mlCache; private final ClusterManagerProperites clusterManagerProperites; + private final KubeCRDHandler crdHandler; private int resourceVersion = 0; private int resourceVersionProcessed = 0; @Autowired - public SeldonDeploymentWatcher(ClusterManagerProperites clusterManagerProperites,SeldonDeploymentController seldonDeploymentController,SeldonDeploymentCache mlCache) throws IOException, ApiException + public SeldonDeploymentWatcher(ClusterManagerProperites clusterManagerProperites,SeldonDeploymentController seldonDeploymentController,SeldonDeploymentCache mlCache,KubeCRDHandler crdHandler) throws IOException, ApiException { this.seldonDeploymentController = seldonDeploymentController; this.mlCache = mlCache; this.clusterManagerProperites = clusterManagerProperites; + this.crdHandler = crdHandler; CRDCreator crdCreator = new CRDCreator(); crdCreator.createCRD(); } @@ -84,7 +91,33 @@ private void processWatch(SeldonDeployment mldep,String action) throws InvalidPr } } - + private void failDeployment(JsonNode mlDep,Exception e) + { + try + { + //Create status message + DeploymentStatus.Builder statusBuilder = DeploymentStatus.newBuilder(); + statusBuilder.setState(Constants.STATE_FAILED).setDescription(e.getMessage()); + //Get JSON for status message + Printer jsonPrinter = JsonFormat.printer().preservingProtoFieldNames(); + ObjectMapper mapper = new ObjectMapper(); + JsonFactory factory = mapper.getFactory(); + JsonParser parser = factory.createParser(jsonPrinter.print(statusBuilder)); + JsonNode statusObj = mapper.readTree(parser); + //Update deployment json with status + ((ObjectNode) mlDep).set("status", statusObj); + String json = mapper.writeValueAsString(mlDep); + String name = mlDep.get("metadata").get("name").asText(); + //Update seldon deployment + crdHandler.updateRaw(json, name); + } catch (JsonParseException e1) { + logger.error("Fasile to create status for failed parse",e); + } catch (InvalidProtocolBufferException e1) { + logger.error("Fasile to create status for failed parse",e); + } catch (IOException e1) { + logger.error("Fasile to create status for failed parse",e); + } + } public int watchSeldonMLDeployments(int resourceVersion,int resourceVersionProcessed) throws ApiException, JsonProcessingException, IOException { @@ -133,8 +166,11 @@ public int watchSeldonMLDeployments(int resourceVersion,int resourceVersionProce } catch (InvalidProtocolBufferException e) { - //TODO : update status of seldondeployment to show error - logger.warn("Failed to parse SeldonDelployment " + jsonInString, e); + if ("ADDED".equals(item.type)) + { + failDeployment(actualObj, e); + logger.warn("Failed to parse SeldonDelployment " + jsonInString, e); + } } } }