Skip to content

Commit

Permalink
Merge pull request #223 from cliveseldon/status_messages
Browse files Browse the repository at this point in the history 
Update SeldonDeployment status for lifecycle conditions
  • Loading branch information
@ukclivecox
ukclivecox authored Sep 17, 2018
2 parents 04ecaa1 + 40b6d0b commit ea2b77f
Show file tree
Hide file tree
Showing 7 changed files with 88 additions and 9 deletions.
3 changes: 3 additions & 0 deletions cluster-manager/src/main/java/io/seldon/clustermanager/k8s/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,7 @@

public class Constants {
public static final String LABEL_SELDON_ID = "seldon-deployment-id";
public static final String STATE_CREATING = "Creating";
public static final String STATE_FAILED = "Failed";
public static final String STATE_AVAILABLE = "Available";
}
1 change: 1 addition & 0 deletions cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

public interface KubeCRDHandler {

public void updateRaw(String json,String seldonDeploymentName);
public void updateSeldonDeployment(SeldonDeployment mlDep);
public SeldonDeployment getSeldonDeployment(String name);
public ExtensionsV1beta1DeploymentList getOwnedDeployments(String seldonDeploymentName);
Expand Down
24 changes: 23 additions & 1 deletion cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.protobuf.InvalidProtocolBufferException;
Expand Down Expand Up @@ -55,6 +56,25 @@ public class KubeCRDHandlerImpl implements KubeCRDHandler {
public KubeCRDHandlerImpl(ClusterManagerProperites clusterManagerProperites) {
this.namespace = StringUtils.isEmpty(clusterManagerProperites.getNamespace()) ? "default" : clusterManagerProperites.getNamespace();
}

@Override
public void updateRaw(String json,String seldonDeploymentName) {
try
{
logger.info(json);
ApiClient client = Config.defaultClient();
CustomObjectsApi api = new CustomObjectsApi(client);
api.replaceNamespacedCustomObject(GROUP, VERSION, namespace, KIND_PLURAL, seldonDeploymentName,json.getBytes());
} catch (InvalidProtocolBufferException e) {
logger.error("Failed to update deployment in kubernetes ",e);
} catch (ApiException e) {
logger.error("Failed to update deployment in kubernetes : {}",e.getResponseBody());
} catch (IOException e) {
logger.error("Failed to get client ",e);
}


}

@Override
public void updateSeldonDeployment(SeldonDeployment mldep) {
Expand Down Expand Up @@ -84,7 +104,7 @@ public void updateSeldonDeployment(SeldonDeployment mldep) {
} catch (IOException e) {
logger.error("Failed to get client ",e);
}
finally{}

}

@Override
Expand Down Expand Up @@ -144,6 +164,8 @@ public V1ServiceList getOwnedServices(String seldonDeploymentName) {
return null;
}
}





Expand Down
6 changes: 2 additions & 4 deletions ...er-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,6 @@ public class SeldonDeploymentControllerImpl implements SeldonDeploymentControlle
private final KubeCRDHandler crdHandler;
private final SeldonDeploymentCache mlCache;

private static final String FAILED_STATE_MSG = "FAILED";

private static final String DEPLOYMENT_API_VERSION = "extensions/v1beta1";


Expand Down Expand Up @@ -253,14 +251,14 @@ private String getNamespace(SeldonDeployment d)
private void failDeployment(SeldonDeployment mlDep,Exception e)
{
SeldonDeployment.Builder mlBuilder = SeldonDeployment.newBuilder(mlDep);
mlBuilder.getStatusBuilder().setState(FAILED_STATE_MSG).setDescription(e.getMessage());
mlBuilder.getStatusBuilder().setState(Constants.STATE_FAILED).setDescription(e.getMessage());
crdHandler.updateSeldonDeployment(mlBuilder.build());
}

@Override
public void createOrReplaceSeldonDeployment(SeldonDeployment mlDep) {

if (mlDep.hasStatus() && mlDep.getStatus().hasState() && mlDep.getStatus().getState().equals(FAILED_STATE_MSG))
if (mlDep.hasStatus() && mlDep.getStatus().hasState() && mlDep.getStatus().getState().equals(Constants.STATE_FAILED))
{
logger.warn("Ignoring failed deployment "+mlDep.getMetadata().getName());
return;
Expand Down
5 changes: 5 additions & 0 deletions cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,11 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) {
}
}

if (!mlBuilder.hasStatus())
{
mlBuilder.getStatusBuilder().setState(Constants.STATE_CREATING);
}

return mlBuilder.build();
}

Expand Down
14 changes: 14 additions & 0 deletions ...-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentStatusUpdateImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,16 @@ private void update(PredictorStatus.Builder b,Integer replicas, Integer replicas
b.setReplicasAvailable(0);
}

private boolean isAvailable(SeldonDeployment.Builder mlBuilder)
{
for (PredictorStatus.Builder b : mlBuilder.getStatusBuilder().getPredictorStatusBuilderList())
{
if (b.getReplicas() != b.getReplicasAvailable())
return false;
}
return true;
}

@Override
public void updateStatus(String mlDepName, String depName, Integer replicas, Integer replicasAvailable) {
if (replicas == null || replicas == 0)
Expand Down Expand Up @@ -73,6 +83,10 @@ public void updateStatus(String mlDepName, String depName, Integer replicas, Int
update(b,replicas,replicasAvailable);
mlBuilder.getStatusBuilder().addPredictorStatus(b);
}
if (isAvailable(mlBuilder))
{
mlBuilder.getStatusBuilder().setState(Constants.STATE_AVAILABLE);
}
crdHandler.updateSeldonDeployment(mlBuilder.build());
}
else
Expand Down
44 changes: 40 additions & 4 deletions cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@
import org.springframework.stereotype.Component;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
Expand All @@ -43,6 +45,9 @@
import io.kubernetes.client.util.Config;
import io.kubernetes.client.util.Watch;
import io.seldon.clustermanager.ClusterManagerProperites;
import io.seldon.clustermanager.pb.JsonFormat;
import io.seldon.clustermanager.pb.JsonFormat.Printer;
import io.seldon.protos.DeploymentProtos.DeploymentStatus;
import io.seldon.protos.DeploymentProtos.SeldonDeployment;

@Component
Expand All @@ -52,16 +57,18 @@ public class SeldonDeploymentWatcher {
private final SeldonDeploymentController seldonDeploymentController;
private final SeldonDeploymentCache mlCache;
private final ClusterManagerProperites clusterManagerProperites;
private final KubeCRDHandler crdHandler;

private int resourceVersion = 0;
private int resourceVersionProcessed = 0;

@Autowired
public SeldonDeploymentWatcher(ClusterManagerProperites clusterManagerProperites,SeldonDeploymentController seldonDeploymentController,SeldonDeploymentCache mlCache) throws IOException, ApiException
public SeldonDeploymentWatcher(ClusterManagerProperites clusterManagerProperites,SeldonDeploymentController seldonDeploymentController,SeldonDeploymentCache mlCache,KubeCRDHandler crdHandler) throws IOException, ApiException
{
this.seldonDeploymentController = seldonDeploymentController;
this.mlCache = mlCache;
this.clusterManagerProperites = clusterManagerProperites;
this.crdHandler = crdHandler;
CRDCreator crdCreator = new CRDCreator();
crdCreator.createCRD();
}
Expand All @@ -84,7 +91,33 @@ private void processWatch(SeldonDeployment mldep,String action) throws InvalidPr
}
}


private void failDeployment(JsonNode mlDep,Exception e)
{
try
{
//Create status message
DeploymentStatus.Builder statusBuilder = DeploymentStatus.newBuilder();
statusBuilder.setState(Constants.STATE_FAILED).setDescription(e.getMessage());
//Get JSON for status message
Printer jsonPrinter = JsonFormat.printer().preservingProtoFieldNames();
ObjectMapper mapper = new ObjectMapper();
JsonFactory factory = mapper.getFactory();
JsonParser parser = factory.createParser(jsonPrinter.print(statusBuilder));
JsonNode statusObj = mapper.readTree(parser);
//Update deployment json with status
((ObjectNode) mlDep).set("status", statusObj);
String json = mapper.writeValueAsString(mlDep);
String name = mlDep.get("metadata").get("name").asText();
//Update seldon deployment
crdHandler.updateRaw(json, name);
} catch (JsonParseException e1) {
logger.error("Fasile to create status for failed parse",e);
} catch (InvalidProtocolBufferException e1) {
logger.error("Fasile to create status for failed parse",e);
} catch (IOException e1) {
logger.error("Fasile to create status for failed parse",e);
}
}

public int watchSeldonMLDeployments(int resourceVersion,int resourceVersionProcessed) throws ApiException, JsonProcessingException, IOException
{
Expand Down Expand Up @@ -133,8 +166,11 @@ public int watchSeldonMLDeployments(int resourceVersion,int resourceVersionProce
}
catch (InvalidProtocolBufferException e)
{
//TODO : update status of seldondeployment to show error
logger.warn("Failed to parse SeldonDelployment " + jsonInString, e);
if ("ADDED".equals(item.type))
{
failDeployment(actualObj, e);
logger.warn("Failed to parse SeldonDelployment " + jsonInString, e);
}
}
}
}
Expand Down

0 comments on commit ea2b77f

Please sign in to comment.