Skip to content

Commit

Permalink
Merge pull request #3280 from ryandawsonuk/3279-req-logger-tf-batch
Browse files Browse the repository at this point in the history 
elements for batch and tf protocol
  • Loading branch information
@seldondev
seldondev authored Jun 11, 2021
2 parents 706d795 + e24c223 commit 1e504dc
Show file tree
Hide file tree
Showing 7 changed files with 100 additions and 24 deletions.
1 change: 0 additions & 1 deletion components/seldon-request-logger/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ Here is a truncated version of an example output for an image use-case with outl


TODO: HANDLE GRPC
TODO: ELEMENTS ARE CURRENTLY CREATED TO SPLIT FEATURES AND MAKE SEARCHABLE BY FEATURE VALUE. BUT ONLY FOR NON-BATCHED.
TODO: FEEDBACK - IF SENT WITH CUSTOM ID HEADER COULD SUPPORT A/B TESTS WITH RECORDED RESULTS

# Scaling
Expand Down
46 changes: 31 additions & 15 deletions components/seldon-request-logger/app/default_logger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,19 @@ def process_and_update_elastic_doc(

no_items_in_batch = len(new_content_part["instance"])
index = 0
for item in new_content_part["instance"]:
elements = None
if "elements" in new_content_part:
elements = new_content_part["elements"]

for num, item in enumerate(new_content_part["instance"],start=0):

item_body = doc_body.copy()

item_body[message_type]["instance"] = item

if type(elements) == type([]) and len(elements) >= num:
item_body[message_type]["elements"] = elements[num]

item_request_id = build_request_id_batched(
request_id, no_items_in_batch, index
)
Expand All @@ -148,6 +156,10 @@ def process_and_update_elastic_doc(
)
index = index + 1
else:
#not batch so don't batch elements either
if "elements" in new_content_part and type(new_content_part["elements"]) == type([]):
new_content_part["elements"] = new_content_part["elements"][0]

item_request_id = build_request_id_batched(request_id, 1, 0)
upsert_doc_to_elastic(
elastic_object, message_type, doc_body, item_request_id, index_name
Expand Down Expand Up @@ -288,6 +300,11 @@ def create_np_from_v2(data: list,ty: str, shape: list) -> np.array:
def extract_data_part(content, headers, message_type):
copy = content.copy()

namespace = log_helper.get_header(log_helper.NAMESPACE_HEADER_NAME, headers)
inferenceservice_name = log_helper.get_header(log_helper.INFERENCESERVICE_HEADER_NAME, headers)
endpoint_name = log_helper.get_header(log_helper.ENDPOINT_HEADER_NAME, headers)
serving_engine = log_helper.serving_engine(headers)

# if 'instances' in body then tensorflow request protocol
# if 'predictions' then tensorflow response
# if 'model_name' and 'outputs' then v2 dataplane response
Expand Down Expand Up @@ -342,6 +359,9 @@ def extract_data_part(content, headers, message_type):
first_element = content_np.item(0)

set_datatype_from_numpy(content_np, copy, first_element)
elements = createElelmentsArray(content_np, None, namespace, serving_engine, inferenceservice_name, endpoint_name, message_type)
copy["elements"] = elements

del copy["instances"]
elif "predictions" in copy:
copy["instance"] = copy["predictions"]
Expand All @@ -350,6 +370,8 @@ def extract_data_part(content, headers, message_type):
copy["dataType"] = "tabular"
first_element = content_np.item(0)
set_datatype_from_numpy(content_np, copy, first_element)
elements = createElelmentsArray(content_np, None, namespace, serving_engine, inferenceservice_name, endpoint_name, message_type)
copy["elements"] = elements

del copy["predictions"]
else:
Expand All @@ -367,21 +389,9 @@ def extract_data_part(content, headers, message_type):
copy["dataType"] = "image"

if isinstance(req_features, Iterable):
namespace = log_helper.get_header(log_helper.NAMESPACE_HEADER_NAME, headers)
inferenceservice_name = log_helper.get_header(log_helper.INFERENCESERVICE_HEADER_NAME, headers)
endpoint_name = log_helper.get_header(log_helper.ENDPOINT_HEADER_NAME, headers)
serving_engine = log_helper.serving_engine(headers)

elements = createElelmentsArray(req_features, list(req_datadef.names), namespace, serving_engine, inferenceservice_name, endpoint_name, message_type)

if isinstance(elements, Iterable):

for i, e in enumerate(elements):
reqJson = extractRow(
i, requestMsg, req_datatype, req_features, req_datadef
)
reqJson["elements"] = e
copy = reqJson
copy["elements"] = elements

copy["instance"] = json.loads(
json.dumps(req_features, cls=log_helper.NumpyEncoder)
Expand Down Expand Up @@ -458,16 +468,20 @@ def extractRow(


def createElelmentsArray(X: np.ndarray, names: list, namespace_name, serving_engine, inferenceservice_name, endpoint_name, message_type):
metadata_schema = None

if namespace_name is not None and inferenceservice_name is not None and serving_engine is not None and endpoint_name is not None:
metadata_schema = log_mapping.fetch_metadata(namespace_name, serving_engine, inferenceservice_name, endpoint_name)
else:
print('missing a param required for metadata lookup')
sys.stdout.flush()

results = None
if metadata_schema is None:
if not metadata_schema or metadata_schema is None:
results = createElementsNoMetadata(X, names, results)
else:
results = createElementsWithMetadata(X, names, results, metadata_schema, message_type)

return results


Expand Down Expand Up @@ -600,6 +614,8 @@ def lookupValueWithMetadata(name, metadata_dict, raw_value):
return raw_value

def createElementsNoMetadata(X, names, results):
if not names:
return results
if isinstance(X, np.ndarray):
if len(X.shape) == 1:
results = []
Expand Down
22 changes: 19 additions & 3 deletions components/seldon-request-logger/app/log_mapping.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def get_index_mapping(index_name, upsert_body):

metadata = fetch_metadata(
namespace_name, serving_engine, inferenceservice_name, endpoint_name)
if not metadata:
if not metadata or metadata is None:
return index_mapping
else:
print("Retrieved metadata for index", index_name)
Expand Down Expand Up @@ -167,6 +167,17 @@ def fetch_user():

def fetch_metadata(namespace, serving_engine, inferenceservice_name, predictor_name):

deployment_type = None
if serving_engine == 'seldon':
deployment_type = 'SeldonDeployment'
if serving_engine == 'inferenceservice':
deployment_type = 'InferenceService'

if not deployment_type:
print('unknown deployment type for '+namespace+' / '+inferenceservice_name)
print(deployment_type)
#TODO: should be sending deployment_type in call below but currently sdk says unexpected keyword argument

if metadata_api is None:
print('metadata service not configured')
return None
Expand All @@ -175,7 +186,9 @@ def fetch_metadata(namespace, serving_engine, inferenceservice_name, predictor_n
# was expcting to set deployment_type=serving_engine but deployment_type does not seem to be a param
runtime_metadata = metadata_api.model_metadata_service_list_runtime_metadata_for_model(
deployment_name=inferenceservice_name,deployment_namespace=namespace,predictor_name=predictor_name)
if runtime_metadata is not None and runtime_metadata.runtime_metadata is not None:

if runtime_metadata is not None and runtime_metadata and \
runtime_metadata.runtime_metadata is not None and runtime_metadata.runtime_metadata:
print(runtime_metadata.runtime_metadata)
if len(runtime_metadata.runtime_metadata) == 0:
print('no runtime metadata for '+namespace+'/'+inferenceservice_name)
Expand All @@ -193,7 +206,10 @@ def fetch_metadata(namespace, serving_engine, inferenceservice_name, predictor_n

print('prediction schema for '+namespace+'/'+inferenceservice_name)
print(model_metadata.models[0].prediction_schema)
return model_metadata.models[0].prediction_schema.to_dict()
if model_metadata.models[0].prediction_schema:
return model_metadata.models[0].prediction_schema.to_dict()
else:
return None
else:
print('no metadata found for '+namespace+' / '+inferenceservice_name+' / '+predictor_name)
return None
Expand Down
2 changes: 1 addition & 1 deletion components/seldon-request-logger/requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ dict_digger==0.2.1
seldon_core
elasticsearch==7.12.1
click==8.0.0a1
seldon-deploy-sdk==1.3.0.dev1
seldon-deploy-sdk==1.3.0.dev2
0 .../seldon-request-logger/app/cifardata.json → ...don-request-logger/testing/cifardata.json
View file
File renamed without changes.
31 changes: 31 additions & 0 deletions components/seldon-request-logger/testing/create_dummy_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@


models = [
# To test e2e have to use wizard to deploy some of below AFTER running this script
# Use names, uris and artifact types below when filling in wizard.
# Same model different versions
{
"uri": "gs://test-model-beta-v2.0.0",
Expand All @@ -35,6 +37,7 @@
"artifact_type": "SKLEARN",
"task_type": "classification",
"tags": {"author": "Jon"},
"prediction_schema": {"requests":[{"name":"Sepal Length","type":"REAL","data_type":"FLOAT"},{"name":"Sepal Width","type":"REAL","data_type":"FLOAT"},{"name":"Petal Length","type":"REAL","data_type":"FLOAT"},{"name":"Petal Width","type":"REAL","data_type":"FLOAT"}],"responses":[{"name":"Iris Species","type":"PROBA","data_type":"FLOAT","schema":[{"name":"Setosa"},{"name":"Versicolor"},{"name":"Virginica"}]}]}
},
{
"uri": "gs://seldon-models/sklearn/iris",
Expand All @@ -43,6 +46,16 @@
"artifact_type": "SKLEARN",
"task_type": "classification",
"tags": {"author": "Bob"},
"prediction_schema": {"requests":[{"name":"Sepal Length","type":"REAL","data_type":"FLOAT"},{"name":"Sepal Width","type":"REAL","data_type":"FLOAT"},{"name":"Petal Length","type":"REAL","data_type":"FLOAT"},{"name":"Petal Width","type":"REAL","data_type":"FLOAT"}],"responses":[{"name":"Iris Species","type":"PROBA","data_type":"FLOAT","schema":[{"name":"Setosa"},{"name":"Versicolor"},{"name":"Virginica"}]}]}
},
{ #kfserving iris
"uri": "gs://kfserving-samples/models/sklearn/iris",
"name": "iris-kf",
"version": "v2.0.0",
"artifact_type": "SKLEARN",
"task_type": "classification",
"tags": {"author": "Jeff"},
"prediction_schema": {"requests":[{"name":"Sepal Length","type":"REAL","data_type":"FLOAT"},{"name":"Sepal Width","type":"REAL","data_type":"FLOAT"},{"name":"Petal Length","type":"REAL","data_type":"FLOAT"},{"name":"Petal Width","type":"REAL","data_type":"FLOAT"}],"responses":[{"name":"Iris Species","type":"PROBA","data_type":"FLOAT","schema":[{"name":"Setosa"},{"name":"Versicolor"},{"name":"Virginica"}]}]}
},
{ #schema from https://github.com/SeldonIO/ml-prediction-schema/blob/master/examples/income-classifier.json
"uri": "gs://seldon-models/sklearn/income/model-0.23.2",
Expand All @@ -53,6 +66,15 @@
"tags": {"author": "Fred"},
"prediction_schema": {"requests":[{"name":"Age","type":"REAL","data_type":"FLOAT"},{"name":"Workclass","type":"CATEGORICAL","data_type":"INT","n_categories":9,"category_map":{"0":"?","1":"Federal-gov","2":"Local-gov","3":"Never-worked","4":"Private","5":"Self-emp-inc","6":"Self-emp-not-inc","7":"State-gov","8":"Without-pay"}},{"name":"Education","type":"CATEGORICAL","data_type":"INT","n_categories":7,"category_map":{"0":"Associates","1":"Bachelors","2":"Doctorate","3":"Dropout","4":"High School grad","5":"Masters","6":"Prof-School"}},{"name":"Marital Status","type":"CATEGORICAL","data_type":"INT","n_categories":4,"category_map":{"0":"Married","1":"Never-Married","2":"Separated","3":"Widowed"}},{"name":"Occupation","type":"CATEGORICAL","data_type":"INT","n_categories":9,"category_map":{"0":"?","1":"Admin","2":"Blue-Collar","3":"Military","4":"Other","5":"Professional","6":"Sales","7":"Service","8":"White-Collar"}},{"name":"Relationship","type":"CATEGORICAL","data_type":"INT","n_categories":6,"category_map":{"0":"Husband","1":"Not-in-family","2":"Other-relative","3":"Own-child","4":"Unmarried","5":"Wife"}},{"name":"Race","type":"CATEGORICAL","data_type":"INT","n_categories":5,"category_map":{"0":"Amer-Indian-Eskimo","1":"Asian-Pac-Islander","2":"Black","3":"Other","4":"White"}},{"name":"Sex","type":"CATEGORICAL","data_type":"INT","n_categories":2,"category_map":{"0":"Female","1":"Male"}},{"name":"Capital Gain","type":"REAL","data_type":"FLOAT"},{"name":"Capital Loss","type":"REAL","data_type":"FLOAT"},{"name":"Hours per week","type":"REAL","data_type":"FLOAT"},{"name":"Country","type":"CATEGORICAL","data_type":"INT","n_categories":11,"category_map":{"0":"?","1":"British-Commonwealth","2":"China","3":"Euro_1","4":"Euro_2","5":"Latin-America","6":"Other","7":"SE-Asia","8":"South-America","9":"United-States","10":"Yugoslavia"}}],"responses":[{"name":"Income","type":"PROBA","data_type":"FLOAT","schema":[{"name":"<=$50K"},{"name":">$50K"}]}]}
},
{ # kfserving income
"uri": "gs://seldon-models/sklearn/income/model",
"name": "income-kf",
"version": "v2.0.0",
"artifact_type": "SKLEARN",
"task_type": "classification",
"tags": {"author": "Jim"},
"prediction_schema": {"requests":[{"name":"Age","type":"REAL","data_type":"FLOAT"},{"name":"Workclass","type":"CATEGORICAL","data_type":"INT","n_categories":9,"category_map":{"0":"?","1":"Federal-gov","2":"Local-gov","3":"Never-worked","4":"Private","5":"Self-emp-inc","6":"Self-emp-not-inc","7":"State-gov","8":"Without-pay"}},{"name":"Education","type":"CATEGORICAL","data_type":"INT","n_categories":7,"category_map":{"0":"Associates","1":"Bachelors","2":"Doctorate","3":"Dropout","4":"High School grad","5":"Masters","6":"Prof-School"}},{"name":"Marital Status","type":"CATEGORICAL","data_type":"INT","n_categories":4,"category_map":{"0":"Married","1":"Never-Married","2":"Separated","3":"Widowed"}},{"name":"Occupation","type":"CATEGORICAL","data_type":"INT","n_categories":9,"category_map":{"0":"?","1":"Admin","2":"Blue-Collar","3":"Military","4":"Other","5":"Professional","6":"Sales","7":"Service","8":"White-Collar"}},{"name":"Relationship","type":"CATEGORICAL","data_type":"INT","n_categories":6,"category_map":{"0":"Husband","1":"Not-in-family","2":"Other-relative","3":"Own-child","4":"Unmarried","5":"Wife"}},{"name":"Race","type":"CATEGORICAL","data_type":"INT","n_categories":5,"category_map":{"0":"Amer-Indian-Eskimo","1":"Asian-Pac-Islander","2":"Black","3":"Other","4":"White"}},{"name":"Sex","type":"CATEGORICAL","data_type":"INT","n_categories":2,"category_map":{"0":"Female","1":"Male"}},{"name":"Capital Gain","type":"REAL","data_type":"FLOAT"},{"name":"Capital Loss","type":"REAL","data_type":"FLOAT"},{"name":"Hours per week","type":"REAL","data_type":"FLOAT"},{"name":"Country","type":"CATEGORICAL","data_type":"INT","n_categories":11,"category_map":{"0":"?","1":"British-Commonwealth","2":"China","3":"Euro_1","4":"Euro_2","5":"Latin-America","6":"Other","7":"SE-Asia","8":"South-America","9":"United-States","10":"Yugoslavia"}}],"responses":[{"name":"Income","type":"PROBA","data_type":"FLOAT","schema":[{"name":"<=$50K"},{"name":">$50K"}]}]}
},
{ # schema made up to test edge cases
"uri": "gs://seldon-models/sklearn/iris2",
"name": "dummy",
Expand All @@ -61,6 +83,15 @@
"task_type": "classification",
"tags": {"author": "Noname"},
"prediction_schema": {"requests":[{"name":"dummy_one_hot","type":"ONE_HOT","data_type":"INT","schema":[{"name":"dummy_one_hot_1"},{"name":"dummy_one_hot_2"}]},{"name":"dummy_categorical","type":"CATEGORICAL","data_type":"INT","n_categories":2,"category_map":{"0":"dummy_cat_0","1":"dummy_cat_1"}},{"name":"dummy_float","type":"REAL","data_type":"FLOAT"}],"responses":[{"name":"dummy_proba","type":"PROBA","data_type":"FLOAT","schema":[{"name":"dummy_proba_0"},{"name":"dummy_proba_1"}]},{"name":"dummy_float","type":"REAL","data_type":"FLOAT"}]}
},
{ # cifar10
"uri": "gs://seldon-models/tfserving/cifar10/resnet32",
"name": "cifar10",
"version": "v1.0.0",
"artifact_type": "TENSORFLOW",
"task_type": "classification",
"tags": {"author": "Noname"},
"prediction_schema": {"requests":[{"name":"Input Image","type":"TENSOR","data_type":"FLOAT","shape":[32,32,3]}],"responses":[{"name":"Image Class","type":"PROBA","data_type":"FLOAT","schema":[{"name":"Airplane"},{"name":"Automobile"},{"name":"Bird"},{"name":"Cat"},{"name":"Deer"},{"name":"Dog"},{"name":"Frog"},{"name":"Horse"},{"name":"Ship"},{"name":"Truck"}]}]}
}
]

Expand Down
Loading

0 comments on commit 1e504dc

Please sign in to comment.