From e2d731cc70ffb37ebab66f92945ab8393f4c5e5f Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 23 Mar 2020 03:49:38 +0700 Subject: [PATCH 01/23] Add labels column to feature field --- core/src/main/java/feast/core/model/Field.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index cb23e4eceb..e22b72b0d4 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -47,6 +47,10 @@ public class Field { @Column(name = "project") private String project; + // Labels that this field belongs to + @Column(name = "labels", columnDefinition = "text") + private String labels; + // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec) // Only one of them can be set. private byte[] presence; From 0cdb63b7886bcc81d54af301946ccdb9f8d98212 Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 23 Mar 2020 03:50:36 +0700 Subject: [PATCH 02/23] Add labels to feature spec proto --- protos/feast/core/FeatureSet.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index 429d99c854..adacb06f46 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -156,6 +156,9 @@ message FeatureSpec { tensorflow.metadata.v0.TimeDomain time_domain = 17; tensorflow.metadata.v0.TimeOfDayDomain time_of_day_domain = 18; } + + // Labels for user defined metadata on a feature + map labels=19; } message FeatureSetMeta { From 6802e0a511383986a8745dbb9efe4b48064944c3 Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 23 Mar 2020 04:32:40 +0700 Subject: [PATCH 03/23] Implement labels on feature model --- core/src/main/java/feast/core/model/FeatureSet.java | 4 ++++ core/src/main/java/feast/core/model/Field.java | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index 232a5f67d1..d134845837 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -352,6 +352,10 @@ private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Field featureSpecBuilder.setTimeOfDayDomain( TimeOfDayDomain.parseFrom(featureField.getTimeOfDayDomain())); } + + if (featureField.getLabels() != null) { + featureSpecBuilder.putAllLabels(featureField.getLabelsJSON()); + } } /** diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index e22b72b0d4..ef67d94275 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -18,8 +18,10 @@ import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.util.TypeConversion; import feast.types.ValueProto.ValueType; import java.util.Arrays; +import java.util.Map; import java.util.Objects; import javax.persistence.Column; import javax.persistence.Embeddable; @@ -86,6 +88,7 @@ public Field(String name, ValueType.Enum type) { public Field(FeatureSpec featureSpec) { this.name = featureSpec.getName(); this.type = featureSpec.getValueType().toString(); + this.labels = TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap()); switch (featureSpec.getPresenceConstraintsCase()) { case PRESENCE: @@ -219,6 +222,10 @@ public Field(EntitySpec entitySpec) { } } + public Map getLabelsJSON() { + return TypeConversion.convertJsonStringToMap(this.labels); + } + @Override public boolean equals(Object o) { if (this == o) { From 60e4cc8f95790b6a91e3749895bd309490dcbb4f Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 23 Mar 2020 04:50:15 +0700 Subject: [PATCH 04/23] Implement list labels --- sdk/python/feast/feature.py | 2 +- sdk/python/feast/field.py | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index c9fc1cbff4..ada1df4b0b 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -29,4 +29,4 @@ def to_proto(self) -> FeatureProto: @classmethod def from_proto(cls, feature_proto: FeatureProto): """Converts Protobuf Feature to its SDK equivalent""" - return cls(name=feature_proto.name, dtype=ValueType(feature_proto.value_type)) + return cls(name=feature_proto.name, dtype=ValueType(feature_proto.value_type), labels=feature_proto.labels) diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 2efd4587ff..7642f5c9a4 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -21,14 +21,22 @@ class Field: features. """ - def __init__(self, name: str, dtype: ValueType): + def __init__(self, name: str, dtype: ValueType, labels=None): self._name = name if not isinstance(dtype, ValueType): raise ValueError("dtype is not a valid ValueType") self._dtype = dtype + if labels is None: + self._labels = dict() + else: + self._labels = labels def __eq__(self, other): - if self.name != other.name or self.dtype != other.dtype: + if ( + self.name != other.name + or self.dtype != other.dtype + or self.labels != other.labels + ): return False return True @@ -46,6 +54,14 @@ def dtype(self) -> ValueType: """ return self._dtype + @property + def labels(self): + """ + Getter for labels of this filed + """ + + return self._labels + def to_proto(self): """ Unimplemented to_proto method for a field. This should be extended. From b5977990795542e6dc5c3bed2621e283fbbb58ac Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 23 Mar 2020 05:11:08 +0700 Subject: [PATCH 05/23] Implement set_label and remove_label from feature client --- sdk/python/feast/feature.py | 20 ++++++++++++++++++-- sdk/python/tests/test_client.py | 31 ++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index ada1df4b0b..f17446a4b4 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -24,9 +24,25 @@ class Feature(Field): def to_proto(self) -> FeatureProto: """Converts Feature object to its Protocol Buffer representation""" value_type = ValueTypeProto.ValueType.Enum.Value(self.dtype.name) - return FeatureProto(name=self.name, value_type=value_type) + return FeatureProto(name=self.name, value_type=value_type, labels=self.labels) @classmethod def from_proto(cls, feature_proto: FeatureProto): """Converts Protobuf Feature to its SDK equivalent""" - return cls(name=feature_proto.name, dtype=ValueType(feature_proto.value_type), labels=feature_proto.labels) + return cls( + name=feature_proto.name, + dtype=ValueType(feature_proto.value_type), + labels=feature_proto.labels, + ) + + def set_label(self, key, value): + """ + Set label for feature + """ + self._labels[key] = value + + def remove_label(self, key): + """ + Remove label for a feature + """ + del self._labels[key] diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 3c1e8bef0f..5495c4eae9 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -431,12 +431,24 @@ def test_apply_feature_set_success(self, test_client): # Create Feature Sets fs1 = FeatureSet("my-feature-set-1") - fs1.add(Feature(name="fs1-my-feature-1", dtype=ValueType.INT64)) - fs1.add(Feature(name="fs1-my-feature-2", dtype=ValueType.STRING)) + fs1.add( + Feature( + name="fs1-my-feature-1", + dtype=ValueType.INT64, + labels={"fs1": "feature_1", "fs1_2": "feature_1"}, + ) + ) + fs1_f2 = Feature(name="fs1-my-feature-2", dtype=ValueType.STRING) + fs1_f2.set_label("fs1", "feature_2") + fs1.add(fs1_f2) fs1.add(Entity(name="fs1-my-entity-1", dtype=ValueType.INT64)) fs2 = FeatureSet("my-feature-set-2") - fs2.add(Feature(name="fs2-my-feature-1", dtype=ValueType.STRING_LIST)) + fs2_f1 = Feature(name="fs2-my-feature-1", dtype=ValueType.STRING_LIST) + fs2_f1.set_label("fs2", "feature_1") + fs2_f1.set_label("fs3", "feature_1") + fs2_f1.remove_label("fs3") + fs2.add(fs2_f1) fs2.add(Feature(name="fs2-my-feature-2", dtype=ValueType.BYTES_LIST)) fs2.add(Entity(name="fs2-my-entity-1", dtype=ValueType.INT64)) @@ -452,7 +464,20 @@ def test_apply_feature_set_success(self, test_client): and feature_sets[0].name == "my-feature-set-1" and feature_sets[0].features[0].name == "fs1-my-feature-1" and feature_sets[0].features[0].dtype == ValueType.INT64 + and feature_sets[0].features[0].labels + == {"fs1": "feature_1", "fs1_2": "feature_1"} + and feature_sets[0].features[1].name == "fs1-my-feature-2" + and feature_sets[0].features[1].dtype == ValueType.STRING + and feature_sets[0].features[1].labels == {"fs1": "feature_2"} + and feature_sets[0].entities[0].name == "fs1-my-entity-1" + and feature_sets[0].entities[0].dtype == ValueType.INT64 + and feature_sets[1].features[0].name == "fs2-my-feature-1" + and feature_sets[1].features[0].dtype == ValueType.STRING_LIST + and feature_sets[1].features[0].labels == {"fs2": "feature_1"} + and feature_sets[1].features[1].name == "fs2-my-feature-2" and feature_sets[1].features[1].dtype == ValueType.BYTES_LIST + and feature_sets[1].entities[0].name == "fs2-my-entity-1" + and feature_sets[1].entities[0].dtype == ValueType.INT64 ) @pytest.mark.parametrize( From 4e69b2e20a1b99959ef98c06f051913194f4c84a Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 23 Mar 2020 05:11:15 +0700 Subject: [PATCH 06/23] Refactor ingest to only accept featureset and dtype --- sdk/python/feast/loaders/ingest.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index b4490f025c..073aa0cdbc 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -25,7 +25,9 @@ KAFKA_CHUNK_PRODUCTION_TIMEOUT = 120 # type: int -def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[bytes]: +def _encode_pa_tables( + file: str, feature_set: str, fields: dict, row_group_idx: int +) -> List[bytes]: """ Helper function to encode a PyArrow table(s) read from parquet file(s) into FeatureRows. @@ -41,8 +43,11 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt File directory of all the parquet file to encode. Parquet file must have more than one row group. - fs (feast.feature_set.FeatureSet): - FeatureSet describing parquet files. + feature_set (str): + FeatureSet describing the full feature set refference in the format f"{project}/{name}:{version}". + + fields (dict[str, enum.Enum.ValueType]): + Fields represent dictionary of field name and field value type. row_group_idx(int): Row group index to read and encode into byte like FeatureRow @@ -61,12 +66,10 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt # Preprocess the columns by converting all its values to Proto values proto_columns = { - field_name: pa_column_to_proto_column(field.dtype, table.column(field_name)) - for field_name, field in fs.fields.items() + field_name: pa_column_to_proto_column(dtype, table.column(field_name)) + for field_name, dtype in fields.items() } - feature_set = f"{fs.project}/{fs.name}:{fs.version}" - # List to store result feature_rows = [] @@ -93,6 +96,10 @@ def _encode_pa_tables(file: str, fs: FeatureSet, row_group_idx: int) -> List[byt return feature_rows +def normalize_fields(fields): + return {field.name: field.dtype for field in fields.values()} + + def get_feature_row_chunks( file: str, row_groups: List[int], fs: FeatureSet, max_workers: int ) -> Iterable[List[bytes]]: @@ -120,8 +127,12 @@ def get_feature_row_chunks( Iterable list of byte encoded FeatureRow(s). """ + feature_set = f"{fs.project}/{fs.name}:{fs.version}" + + fields = normalize_fields(fs.fields) + pool = Pool(max_workers) - func = partial(_encode_pa_tables, file, fs) + func = partial(_encode_pa_tables, file, feature_set, fields) for chunk in pool.imap(func, row_groups): yield chunk return From 258560c88c5d2945c170948230f790e921ca3720 Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Tue, 24 Mar 2020 14:08:45 +0700 Subject: [PATCH 07/23] Add equality on labels field --- core/src/main/java/feast/core/model/Field.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index ef67d94275..0379d6654b 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -89,6 +89,9 @@ public Field(FeatureSpec featureSpec) { this.name = featureSpec.getName(); this.type = featureSpec.getValueType().toString(); this.labels = TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap()); + if (this.labels.equals("{}")) { + this.labels = null; + } switch (featureSpec.getPresenceConstraintsCase()) { case PRESENCE: @@ -238,6 +241,7 @@ public boolean equals(Object o) { return Objects.equals(name, field.name) && Objects.equals(type, field.type) && Objects.equals(project, field.project) + && Objects.equals(labels, field.labels) && Arrays.equals(presence, field.presence) && Arrays.equals(groupPresence, field.groupPresence) && Arrays.equals(shape, field.shape) @@ -258,6 +262,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), name, type); + return Objects.hash(super.hashCode(), name, type, project, labels); } } From ced6609d9b2de6b49a730e08394d057b5eaef7c4 Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Tue, 24 Mar 2020 15:48:00 +0700 Subject: [PATCH 08/23] Add tests for labels apply --- .../feast/core/service/SpecServiceTest.java | 76 ++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 1eb56caac2..937e7df091 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -142,7 +142,6 @@ public void setUp() { Arrays.asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1); when(featureSetRepository.findAll()).thenReturn(featureSets); when(featureSetRepository.findAllByOrderByNameAscVersionAsc()).thenReturn(featureSets); - when(featureSetRepository.findFeatureSetByNameAndProject_NameAndVersion("f1", "project1", 1)) .thenReturn(featureSets.get(0)); when(featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc( @@ -706,6 +705,81 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() } } + @Test + public void applyFeatureSetShouldAcceptLabels() throws InvalidProtocolBufferException { + List entitySpecs = new ArrayList<>(); + entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build()); + + Map featureLabels0 = + new HashMap<>() { + { + put("label1", "feast1"); + } + }; + + Map featureLabels1 = + new HashMap<>() { + { + put("label1", "feast1"); + put("label2", "feast2"); + } + }; + + List> featureLabels = new ArrayList<>(); + featureLabels.add(featureLabels0); + featureLabels.add(featureLabels1); + + List featureSpecs = new ArrayList<>(); + featureSpecs.add( + FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(Enum.INT64) + .putAllLabels(featureLabels.get(0)) + .build()); + featureSpecs.add( + FeatureSpec.newBuilder() + .setName("feature2") + .setValueType(Enum.INT64) + .putAllLabels(featureLabels.get(1)) + .build()); + + FeatureSetSpec featureSetSpec = + FeatureSetSpec.newBuilder() + .setProject("project1") + .setName("featureSetWithConstraints") + .addAllEntities(entitySpecs) + .addAllFeatures(featureSpecs) + .build(); + FeatureSetProto.FeatureSet featureSet = + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); + + ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(featureSet); + FeatureSetSpec appliedFeatureSetSpec = applyFeatureSetResponse.getFeatureSet().getSpec(); + + // appliedEntitySpecs needs to be sorted because the list returned by specService may not + // follow the order in the request + List appliedEntitySpecs = new ArrayList<>(appliedFeatureSetSpec.getEntitiesList()); + appliedEntitySpecs.sort(Comparator.comparing(EntitySpec::getName)); + + // appliedFeatureSpecs needs to be sorted because the list returned by specService may not + // follow the order in the request + List appliedFeatureSpecs = + new ArrayList<>(appliedFeatureSetSpec.getFeaturesList()); + appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName)); + + assertEquals(appliedEntitySpecs.size(), entitySpecs.size()); + assertEquals(appliedFeatureSpecs.size(), featureSpecs.size()); + + for (int i = 0; i < appliedEntitySpecs.size(); i++) { + assertEquals(entitySpecs.get(i), appliedEntitySpecs.get(i)); + } + + for (int i = 0; i < appliedFeatureSpecs.size(); i++) { + assertEquals(featureSpecs.get(i), appliedFeatureSpecs.get(i)); + assertEquals(featureSpecs.get(i).getLabelsMap(), featureLabels.get(i)); + } + } + @Test public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException { when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0))); From 1a068339000de2293095a850aac57182276889ba Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 30 Mar 2020 14:18:27 +0700 Subject: [PATCH 09/23] Add Optional type hint to labels --- sdk/python/feast/field.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index 7642f5c9a4..ecf7202de2 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, Optional + from feast.value_type import ValueType @@ -21,7 +23,7 @@ class Field: features. """ - def __init__(self, name: str, dtype: ValueType, labels=None): + def __init__(self, name: str, dtype: ValueType, labels: Optional[Dict] = None): self._name = name if not isinstance(dtype, ValueType): raise ValueError("dtype is not a valid ValueType") From 9e768a1f93997966a224bedd36babb1d47eb526d Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Mon, 30 Mar 2020 16:14:41 +0700 Subject: [PATCH 10/23] Add empty labels key validation --- .../core/validators/FeatureSetValidator.java | 4 +++ .../feast/core/service/SpecServiceTest.java | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/core/src/main/java/feast/core/validators/FeatureSetValidator.java b/core/src/main/java/feast/core/validators/FeatureSetValidator.java index 213e3898d5..552aca6db9 100644 --- a/core/src/main/java/feast/core/validators/FeatureSetValidator.java +++ b/core/src/main/java/feast/core/validators/FeatureSetValidator.java @@ -44,6 +44,10 @@ public static void validateSpec(FeatureSet featureSet) { } for (FeatureSpec featureSpec : featureSet.getSpec().getFeaturesList()) { checkValidCharacters(featureSpec.getName(), "features::name"); + if (featureSpec.getLabelsMap().containsKey("")) { + throw new IllegalArgumentException( + "Labels key on Feature spec should not be an empty string"); + } } } diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index 937e7df091..f87d8b113e 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -780,6 +780,42 @@ public void applyFeatureSetShouldAcceptLabels() throws InvalidProtocolBufferExce } } + @Test + public void applyFeatureSetShouldNotAcceptLabelsWithEmptyKey() + throws InvalidProtocolBufferException { + List entitySpecs = new ArrayList<>(); + entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build()); + + Map featureLabels = + new HashMap<>() { + { + put("", "empty_key"); + } + }; + + List featureSpecs = new ArrayList<>(); + featureSpecs.add( + FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(Enum.INT64) + .putAllLabels(featureLabels) + .build()); + + FeatureSetSpec featureSetSpec = + FeatureSetSpec.newBuilder() + .setProject("project1") + .setName("featureSetWithConstraints") + .addAllEntities(entitySpecs) + .addAllFeatures(featureSpecs) + .build(); + FeatureSetProto.FeatureSet featureSet = + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Labels key on Feature spec should not be an empty string"); + specService.applyFeatureSet(featureSet); + } + @Test public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException { when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0))); From 108de14ba45486f61238d09e1ef9d3b4d21a4a64 Mon Sep 17 00:00:00 2001 From: Julio Anthony Leonard Date: Wed, 1 Apr 2020 12:58:46 +0700 Subject: [PATCH 11/23] Add label when generating feature spec for specServiceTest to test equality --- .../src/test/java/feast/core/service/SpecServiceTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index f87d8b113e..c0a0f51140 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -863,7 +863,13 @@ public void shouldFailIfGetFeatureSetWithoutProject() throws InvalidProtocolBuff } private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Field feature = new Field("feature", Enum.INT64); + FeatureSpec f1 = + FeatureSpec.newBuilder() + .setName("feature") + .setValueType(Enum.STRING) + .putLabels("key", "value") + .build(); + Field feature = new Field(f1); Field entity = new Field("entity", Enum.STRING); FeatureSet fs = From c6cf42cc3826de9dac260faea6333c6b469ba323 Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Fri, 24 Apr 2020 16:02:49 +0700 Subject: [PATCH 12/23] corrected convention (push test) --- protos/feast/core/FeatureSet.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index adacb06f46..ebac862ffb 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -158,7 +158,7 @@ message FeatureSpec { } // Labels for user defined metadata on a feature - map labels=19; + map labels = 19; } message FeatureSetMeta { From b9fd9af2cf001835d86147b2d1d9b0d62245f19b Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Sun, 26 Apr 2020 14:52:19 +0700 Subject: [PATCH 13/23] corrected review comments --- .../java/feast/core/model/FeatureSet.java | 2 +- .../src/main/java/feast/core/model/Field.java | 2 +- .../core/validators/FeatureSetValidator.java | 3 +- .../feast/core/service/SpecServiceTest.java | 64 ++-------------- .../validators/FeatureSetValidatorTest.java | 73 +++++++++++++++++++ sdk/python/feast/loaders/ingest.py | 15 ++-- 6 files changed, 90 insertions(+), 69 deletions(-) create mode 100644 core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index d134845837..bd205e0ca2 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -354,7 +354,7 @@ private void setFeatureSpecFields(FeatureSpec.Builder featureSpecBuilder, Field } if (featureField.getLabels() != null) { - featureSpecBuilder.putAllLabels(featureField.getLabelsJSON()); + featureSpecBuilder.putAllLabels(featureField.getLabels()); } } diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index 0379d6654b..a8702667b6 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -225,7 +225,7 @@ public Field(EntitySpec entitySpec) { } } - public Map getLabelsJSON() { + public Map getLabels() { return TypeConversion.convertJsonStringToMap(this.labels); } diff --git a/core/src/main/java/feast/core/validators/FeatureSetValidator.java b/core/src/main/java/feast/core/validators/FeatureSetValidator.java index 552aca6db9..95a6a7136a 100644 --- a/core/src/main/java/feast/core/validators/FeatureSetValidator.java +++ b/core/src/main/java/feast/core/validators/FeatureSetValidator.java @@ -45,8 +45,7 @@ public static void validateSpec(FeatureSet featureSet) { for (FeatureSpec featureSpec : featureSet.getSpec().getFeaturesList()) { checkValidCharacters(featureSpec.getName(), "features::name"); if (featureSpec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException( - "Labels key on Feature spec should not be an empty string"); + throw new IllegalArgumentException("Label keys must not be empty"); } } } diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index c0a0f51140..dc082c8ed9 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -625,16 +625,8 @@ public void applyFeatureSetShouldAcceptPresenceShapeAndDomainConstraints() new ArrayList<>(appliedFeatureSetSpec.getFeaturesList()); appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName)); - assertEquals(appliedEntitySpecs.size(), entitySpecs.size()); - assertEquals(appliedFeatureSpecs.size(), featureSpecs.size()); - - for (int i = 0; i < appliedEntitySpecs.size(); i++) { - assertEquals(entitySpecs.get(i), appliedEntitySpecs.get(i)); - } - - for (int i = 0; i < appliedFeatureSpecs.size(); i++) { - assertEquals(featureSpecs.get(i), appliedFeatureSpecs.get(i)); - } + assertEquals(appliedEntitySpecs, entitySpecs); + assertEquals(appliedFeatureSpecs, featureSpecs); } @Test @@ -767,53 +759,11 @@ public void applyFeatureSetShouldAcceptLabels() throws InvalidProtocolBufferExce new ArrayList<>(appliedFeatureSetSpec.getFeaturesList()); appliedFeatureSpecs.sort(Comparator.comparing(FeatureSpec::getName)); - assertEquals(appliedEntitySpecs.size(), entitySpecs.size()); - assertEquals(appliedFeatureSpecs.size(), featureSpecs.size()); - - for (int i = 0; i < appliedEntitySpecs.size(); i++) { - assertEquals(entitySpecs.get(i), appliedEntitySpecs.get(i)); - } - - for (int i = 0; i < appliedFeatureSpecs.size(); i++) { - assertEquals(featureSpecs.get(i), appliedFeatureSpecs.get(i)); - assertEquals(featureSpecs.get(i).getLabelsMap(), featureLabels.get(i)); - } - } - - @Test - public void applyFeatureSetShouldNotAcceptLabelsWithEmptyKey() - throws InvalidProtocolBufferException { - List entitySpecs = new ArrayList<>(); - entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build()); - - Map featureLabels = - new HashMap<>() { - { - put("", "empty_key"); - } - }; - - List featureSpecs = new ArrayList<>(); - featureSpecs.add( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .putAllLabels(featureLabels) - .build()); - - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Labels key on Feature spec should not be an empty string"); - specService.applyFeatureSet(featureSet); + var featureSpecsLabels = + featureSpecs.stream().map(e -> e.getLabelsMap()).collect(Collectors.toList()); + assertEquals(appliedEntitySpecs, entitySpecs); + assertEquals(appliedFeatureSpecs, featureSpecs); + assertEquals(featureSpecsLabels, featureLabels); } @Test diff --git a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java new file mode 100644 index 0000000000..39124495fa --- /dev/null +++ b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.validators; + +import com.google.protobuf.InvalidProtocolBufferException; +import feast.core.FeatureSetProto; +import feast.types.ValueProto; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class FeatureSetValidatorTest { + + @Rule public final ExpectedException expectedException = ExpectedException.none(); + + @Test + public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() + throws InvalidProtocolBufferException { + List entitySpecs = new ArrayList<>(); + entitySpecs.add( + FeatureSetProto.EntitySpec.newBuilder() + .setName("entity1") + .setValueType(ValueProto.ValueType.Enum.INT64) + .build()); + + Map featureLabels = + new HashMap<>() { + { + put("", "empty_key"); + } + }; + + List featureSpecs = new ArrayList<>(); + featureSpecs.add( + FeatureSetProto.FeatureSpec.newBuilder() + .setName("feature1") + .setValueType(ValueProto.ValueType.Enum.INT64) + .putAllLabels(featureLabels) + .build()); + + FeatureSetProto.FeatureSetSpec featureSetSpec = + FeatureSetProto.FeatureSetSpec.newBuilder() + .setProject("project1") + .setName("featureSetWithConstraints") + .addAllEntities(entitySpecs) + .addAllFeatures(featureSpecs) + .build(); + FeatureSetProto.FeatureSet featureSet = + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Label keys must not be empty"); + FeatureSetValidator.validateSpec(featureSet); + } +} diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 073aa0cdbc..8187fd8728 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -44,10 +44,10 @@ def _encode_pa_tables( Parquet file must have more than one row group. feature_set (str): - FeatureSet describing the full feature set refference in the format f"{project}/{name}:{version}". + Feature set reference in the format f"{project}/{name}:{version}". fields (dict[str, enum.Enum.ValueType]): - Fields represent dictionary of field name and field value type. + A mapping of field names to their value types. row_group_idx(int): Row group index to read and encode into byte like FeatureRow @@ -96,10 +96,6 @@ def _encode_pa_tables( return feature_rows -def normalize_fields(fields): - return {field.name: field.dtype for field in fields.values()} - - def get_feature_row_chunks( file: str, row_groups: List[int], fs: FeatureSet, max_workers: int ) -> Iterable[List[bytes]]: @@ -127,12 +123,15 @@ def get_feature_row_chunks( Iterable list of byte encoded FeatureRow(s). """ + def normalize_fields(fields): + return {field.name: field.dtype for field in fields.values()} + feature_set = f"{fs.project}/{fs.name}:{fs.version}" - fields = normalize_fields(fs.fields) + normalized_fields = normalize_fields(fs.fields) pool = Pool(max_workers) - func = partial(_encode_pa_tables, file, feature_set, fields) + func = partial(_encode_pa_tables, file, feature_set, normalized_fields) for chunk in pool.imap(func, row_groups): yield chunk return From dd88b6c86490d946437b69ff8ad51a4f9c017f55 Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Mon, 27 Apr 2020 11:37:48 +0700 Subject: [PATCH 14/23] corrected lint-python check --- sdk/python/feast/feature.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index d41f6939e4..1b38f3f0ce 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -1,4 +1,4 @@ -# Copyright 2019 The Feast Authors + # Copyright 2019 The Feast Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -59,7 +59,7 @@ def from_proto(cls, feature_proto: FeatureProto): feature = cls( name=feature_proto.name, dtype=ValueType(feature_proto.value_type), - labels=feature_proto.labels + labels=feature_proto.labels, ) feature.update_presence_constraints(feature_proto) feature.update_shape_type(feature_proto) @@ -76,4 +76,4 @@ def remove_label(self, key): """ Remove label for a feature """ - del self._labels[key] \ No newline at end of file + del self._labels[key] From 2f7c86ae41c0a792abdb6f2e2698213cf4c5a170 Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Mon, 27 Apr 2020 11:40:52 +0700 Subject: [PATCH 15/23] corrected lint-python 2 --- sdk/python/feast/feature.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index 1b38f3f0ce..8bb36eba3d 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -1,4 +1,4 @@ - # Copyright 2019 The Feast Authors +# Copyright 2019 The Feast Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 30d63fa0fa94244f3acb461801457de4ad2142e2 Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Tue, 28 Apr 2020 18:04:29 +0700 Subject: [PATCH 16/23] back out python SDK changes --- sdk/python/feast/feature.py | 14 -------------- sdk/python/feast/field.py | 21 ++------------------- sdk/python/tests/test_client.py | 22 +++------------------- 3 files changed, 5 insertions(+), 52 deletions(-) diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index 8bb36eba3d..d45d1ab4c8 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -43,7 +43,6 @@ def to_proto(self) -> FeatureProto: url_domain=self.url_domain, time_domain=self.time_domain, time_of_day_domain=self.time_of_day_domain, - labels=self.labels, ) @classmethod @@ -59,21 +58,8 @@ def from_proto(cls, feature_proto: FeatureProto): feature = cls( name=feature_proto.name, dtype=ValueType(feature_proto.value_type), - labels=feature_proto.labels, ) feature.update_presence_constraints(feature_proto) feature.update_shape_type(feature_proto) feature.update_domain_info(feature_proto) return feature - - def set_label(self, key, value): - """ - Set label for feature - """ - self._labels[key] = value - - def remove_label(self, key): - """ - Remove label for a feature - """ - del self._labels[key] diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index da84906190..be56823489 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. from typing import Union -from typing import Dict, Optional from feast.core.FeatureSet_pb2 import EntitySpec, FeatureSpec from feast.value_type import ValueType @@ -25,7 +24,7 @@ class Field: features. """ - def __init__(self, name: str, dtype: ValueType, labels: Optional[Dict] = None): + def __init__(self, name: str, dtype: ValueType): self._name = name if not isinstance(dtype, ValueType): raise ValueError("dtype is not a valid ValueType") @@ -46,17 +45,9 @@ def __init__(self, name: str, dtype: ValueType, labels: Optional[Dict] = None): self._url_domain = None self._time_domain = None self._time_of_day_domain = None - if labels is None: - self._labels = dict() - else: - self._labels = labels def __eq__(self, other): - if ( - self.name != other.name - or self.dtype != other.dtype - or self.labels != other.labels - ): + if self.name != other.name or self.dtype != other.dtype: return False return True @@ -422,14 +413,6 @@ def update_domain_info( elif domain_info_case == "time_of_day_domain": self.time_of_day_domain = feature.time_of_day_domain - @property - def labels(self): - """ - Getter for labels of this filed - """ - - return self._labels - def to_proto(self): """ Unimplemented to_proto method for a field. This should be extended. diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 02b104da2c..3082265ecc 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -538,24 +538,12 @@ def test_apply_feature_set_success(self, test_client): # Create Feature Sets fs1 = FeatureSet("my-feature-set-1") - fs1.add( - Feature( - name="fs1-my-feature-1", - dtype=ValueType.INT64, - labels={"fs1": "feature_1", "fs1_2": "feature_1"}, - ) - ) - fs1_f2 = Feature(name="fs1-my-feature-2", dtype=ValueType.STRING) - fs1_f2.set_label("fs1", "feature_2") - fs1.add(fs1_f2) + fs1.add(Feature(name="fs1-my-feature-1", dtype=ValueType.INT64)) + fs1.add(Feature(name="fs1-my-feature-2", dtype=ValueType.STRING)) fs1.add(Entity(name="fs1-my-entity-1", dtype=ValueType.INT64)) fs2 = FeatureSet("my-feature-set-2") - fs2_f1 = Feature(name="fs2-my-feature-1", dtype=ValueType.STRING_LIST) - fs2_f1.set_label("fs2", "feature_1") - fs2_f1.set_label("fs3", "feature_1") - fs2_f1.remove_label("fs3") - fs2.add(fs2_f1) + fs2.add(Feature(name="fs2-my-feature-1", dtype=ValueType.STRING_LIST)) fs2.add(Feature(name="fs2-my-feature-2", dtype=ValueType.BYTES_LIST)) fs2.add(Entity(name="fs2-my-entity-1", dtype=ValueType.INT64)) @@ -571,16 +559,12 @@ def test_apply_feature_set_success(self, test_client): and feature_sets[0].name == "my-feature-set-1" and feature_sets[0].features[0].name == "fs1-my-feature-1" and feature_sets[0].features[0].dtype == ValueType.INT64 - and feature_sets[0].features[0].labels - == {"fs1": "feature_1", "fs1_2": "feature_1"} and feature_sets[0].features[1].name == "fs1-my-feature-2" and feature_sets[0].features[1].dtype == ValueType.STRING - and feature_sets[0].features[1].labels == {"fs1": "feature_2"} and feature_sets[0].entities[0].name == "fs1-my-entity-1" and feature_sets[0].entities[0].dtype == ValueType.INT64 and feature_sets[1].features[0].name == "fs2-my-feature-1" and feature_sets[1].features[0].dtype == ValueType.STRING_LIST - and feature_sets[1].features[0].labels == {"fs2": "feature_1"} and feature_sets[1].features[1].name == "fs2-my-feature-2" and feature_sets[1].features[1].dtype == ValueType.BYTES_LIST and feature_sets[1].entities[0].name == "fs2-my-entity-1" From 4b7ec663431ebb001a82e945711cc6de3b5e539e Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Wed, 29 Apr 2020 01:03:28 +0700 Subject: [PATCH 17/23] Implemented labels on a feature set level --- .../java/feast/core/model/FeatureSet.java | 9 ++ .../src/main/java/feast/core/model/Field.java | 3 - .../java/feast/core/util/TypeConversion.java | 6 +- .../feast/core/service/JobServiceTest.java | 23 +---- .../feast/core/service/SpecServiceTest.java | 95 ++++++++----------- .../feast/core/service/TestObjectFactory.java | 51 ++++++++++ .../feast/core/util/TypeConversionTest.java | 11 ++- protos/feast/core/FeatureSet.proto | 3 + sdk/python/feast/feature.py | 3 +- 9 files changed, 115 insertions(+), 89 deletions(-) create mode 100644 core/src/test/java/feast/core/service/TestObjectFactory.java diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java index bd205e0ca2..ec8da77c5f 100644 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ b/core/src/main/java/feast/core/model/FeatureSet.java @@ -25,6 +25,7 @@ import feast.core.FeatureSetProto.FeatureSetSpec; import feast.core.FeatureSetProto.FeatureSetStatus; import feast.core.FeatureSetProto.FeatureSpec; +import feast.core.util.TypeConversion; import feast.types.ValueProto.ValueType.Enum; import java.util.ArrayList; import java.util.HashMap; @@ -116,6 +117,10 @@ public class FeatureSet extends AbstractTimestampEntity implements Comparable entities, List features, Source source, + Map labels, FeatureSetStatus status) { this.maxAgeSeconds = maxAgeSeconds; this.source = source; @@ -137,6 +143,7 @@ public FeatureSet( this.name = name; this.project = new Project(project); this.version = version; + this.labels = TypeConversion.convertMapToJsonString(labels); this.setId(project, name, version); addEntities(entities); addFeatures(features); @@ -191,6 +198,7 @@ public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { entitySpecs, featureSpecs, source, + featureSetProto.getSpec().getLabelsMap(), featureSetProto.getMeta().getStatus()); } @@ -247,6 +255,7 @@ public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferExceptio .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSeconds)) .addAllEntities(entitySpecs) .addAllFeatures(featureSpecs) + .putAllLabels(TypeConversion.convertJsonStringToMap(labels)) .setSource(source.toProto()); return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build(); diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index a8702667b6..8d39ac7ea7 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -89,9 +89,6 @@ public Field(FeatureSpec featureSpec) { this.name = featureSpec.getName(); this.type = featureSpec.getValueType().toString(); this.labels = TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap()); - if (this.labels.equals("{}")) { - this.labels = null; - } switch (featureSpec.getPresenceConstraintsCase()) { case PRESENCE: diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java index e01a551135..f54de2675c 100644 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ b/core/src/main/java/feast/core/util/TypeConversion.java @@ -67,12 +67,10 @@ public static Map convertJsonStringToMap(String jsonString) { * Marshals a given map into its corresponding json string * * @param map - * @return json string corresponding to given map + * @return json string corresponding to given map or null if the map is empty */ public static String convertMapToJsonString(Map map) { - if (map.isEmpty()) { - return "{}"; - } + if (map.isEmpty()) return null; return gson.toJson(map); } diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java index c0e90ca43f..d8b69e037c 100644 --- a/core/src/test/java/feast/core/service/JobServiceTest.java +++ b/core/src/test/java/feast/core/service/JobServiceTest.java @@ -34,11 +34,8 @@ import feast.core.CoreServiceProto.RestartIngestionJobResponse; import feast.core.CoreServiceProto.StopIngestionJobRequest; import feast.core.CoreServiceProto.StopIngestionJobResponse; -import feast.core.FeatureSetProto.FeatureSetStatus; import feast.core.FeatureSetReferenceProto.FeatureSetReference; import feast.core.IngestionJobProto.IngestionJob; -import feast.core.SourceProto.KafkaSourceConfig; -import feast.core.SourceProto.SourceType; import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; import feast.core.dao.JobRepository; @@ -84,14 +81,7 @@ public void setup() { // create mock objects for testing // fake data source - this.dataSource = - new Source( - SourceType.KAFKA, - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("my-topic") - .build(), - true); + this.dataSource = TestObjectFactory.defaultSource; // fake data store this.dataStore = new Store( @@ -162,15 +152,8 @@ private FeatureSet newDummyFeatureSet(String name, int version, String project) Field entity = new Field(name + "_entity", Enum.STRING); FeatureSet fs = - new FeatureSet( - name, - project, - version, - 100L, - Arrays.asList(entity), - Arrays.asList(feature), - this.dataSource, - FeatureSetStatus.STATUS_READY); + TestObjectFactory.CreateFeatureSet( + name, project, version, Arrays.asList(entity), Arrays.asList(feature)); fs.setCreated(Date.from(Instant.ofEpochSecond(10L))); return fs; } diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index de605a46e3..e40375e497 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -39,10 +39,7 @@ import feast.core.FeatureSetProto; import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSetSpec; -import feast.core.FeatureSetProto.FeatureSetStatus; import feast.core.FeatureSetProto.FeatureSpec; -import feast.core.SourceProto.KafkaSourceConfig; -import feast.core.SourceProto.SourceType; import feast.core.StoreProto; import feast.core.StoreProto.Store.RedisConfig; import feast.core.StoreProto.Store.StoreType; @@ -110,14 +107,7 @@ public class SpecServiceTest { @Before public void setUp() { initMocks(this); - defaultSource = - new Source( - SourceType.KAFKA, - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("my-topic") - .build(), - true); + defaultSource = TestObjectFactory.defaultSource; FeatureSet featureSet1v1 = newDummyFeatureSet("f1", 1, "project1"); FeatureSet featureSet1v2 = newDummyFeatureSet("f1", 2, "project1"); @@ -128,15 +118,8 @@ public void setUp() { Field f3f2 = new Field("f3f2", Enum.INT64); Field f3e1 = new Field("f3e1", Enum.STRING); FeatureSet featureSet3v1 = - new FeatureSet( - "f3", - "project1", - 1, - 100L, - Arrays.asList(f3e1), - Arrays.asList(f3f2, f3f1), - defaultSource, - FeatureSetStatus.STATUS_READY); + TestObjectFactory.CreateFeatureSet( + "f3", "project1", 1, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1)); featureSets = Arrays.asList(featureSet1v1, featureSet1v2, featureSet1v3, featureSet2v1, featureSet3v1); @@ -493,15 +476,8 @@ public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() Field f3f2 = new Field("f3f2", Enum.INT64); Field f3e1 = new Field("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = - (new FeatureSet( - "f3", - "project1", - 5, - 100L, - Arrays.asList(f3e1), - Arrays.asList(f3f2, f3f1), - defaultSource, - FeatureSetStatus.STATUS_READY)) + (TestObjectFactory.CreateFeatureSet( + "f3", "project1", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) .toProto(); ApplyFeatureSetResponse applyFeatureSetResponse = @@ -708,15 +684,8 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() Field f3f2 = new Field("f3f2", Enum.INT64); Field f3e1 = new Field("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = - (new FeatureSet( - "f3", - "newproject", - 5, - 100L, - Arrays.asList(f3e1), - Arrays.asList(f3f2, f3f1), - defaultSource, - FeatureSetStatus.STATUS_READY)) + (TestObjectFactory.CreateFeatureSet( + "f3", "newproject", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) .toProto(); ApplyFeatureSetResponse applyFeatureSetResponse = @@ -734,15 +703,8 @@ public void applyFeatureSetShouldFailWhenProjectIsArchived() Field f3f2 = new Field("f3f2", Enum.INT64); Field f3e1 = new Field("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = - (new FeatureSet( - "f3", - "archivedproject", - 5, - 100L, - Arrays.asList(f3e1), - Arrays.asList(f3f2, f3f1), - defaultSource, - FeatureSetStatus.STATUS_READY)) + (TestObjectFactory.CreateFeatureSet( + "f3", "archivedproject", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) .toProto(); expectedException.expect(IllegalArgumentException.class); @@ -751,7 +713,7 @@ public void applyFeatureSetShouldFailWhenProjectIsArchived() } @Test - public void applyFeatureSetShouldAcceptLabels() throws InvalidProtocolBufferException { + public void applyFeatureSetShouldAcceptFeatureLabels() throws InvalidProtocolBufferException { List entitySpecs = new ArrayList<>(); entitySpecs.add(EntitySpec.newBuilder().setName("entity1").setValueType(Enum.INT64).build()); @@ -819,6 +781,32 @@ public void applyFeatureSetShouldAcceptLabels() throws InvalidProtocolBufferExce assertEquals(featureSpecsLabels, featureLabels); } + @Test + public void applyFeatureSetShouldAcceptFeatureSetLabels() throws InvalidProtocolBufferException { + Map featureSetLabels = + new HashMap<>() { + { + put("description", "My precious feature set"); + } + }; + + FeatureSetSpec featureSetSpec = + FeatureSetSpec.newBuilder() + .setProject("project1") + .setName("preciousFeatureSet") + .putAllLabels(featureSetLabels) + .build(); + FeatureSetProto.FeatureSet featureSet = + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); + + ApplyFeatureSetResponse applyFeatureSetResponse = specService.applyFeatureSet(featureSet); + FeatureSetSpec appliedFeatureSetSpec = applyFeatureSetResponse.getFeatureSet().getSpec(); + + var appliedLabels = appliedFeatureSetSpec.getLabelsMap(); + + assertEquals(featureSetLabels, appliedLabels); + } + @Test public void shouldUpdateStoreIfConfigChanges() throws InvalidProtocolBufferException { when(storeRepository.findById("SERVING")).thenReturn(Optional.of(stores.get(0))); @@ -876,15 +864,8 @@ private FeatureSet newDummyFeatureSet(String name, int version, String project) Field entity = new Field("entity", Enum.STRING); FeatureSet fs = - new FeatureSet( - name, - project, - version, - 100L, - Arrays.asList(entity), - Arrays.asList(feature), - defaultSource, - FeatureSetStatus.STATUS_READY); + TestObjectFactory.CreateFeatureSet( + name, project, version, Arrays.asList(entity), Arrays.asList(feature)); fs.setCreated(Date.from(Instant.ofEpochSecond(10L))); return fs; } diff --git a/core/src/test/java/feast/core/service/TestObjectFactory.java b/core/src/test/java/feast/core/service/TestObjectFactory.java new file mode 100644 index 0000000000..886b47341a --- /dev/null +++ b/core/src/test/java/feast/core/service/TestObjectFactory.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.core.service; + +import feast.core.FeatureSetProto; +import feast.core.SourceProto; +import feast.core.model.FeatureSet; +import feast.core.model.Field; +import feast.core.model.Source; +import java.util.HashMap; +import java.util.List; + +public class TestObjectFactory { + + public static Source defaultSource = + new Source( + SourceProto.SourceType.KAFKA, + SourceProto.KafkaSourceConfig.newBuilder() + .setBootstrapServers("kafka:9092") + .setTopic("my-topic") + .build(), + true); + + public static FeatureSet CreateFeatureSet( + String name, String project, int version, List entities, List features) { + return new FeatureSet( + name, + project, + version, + 100L, + entities, + features, + defaultSource, + new HashMap<>(), + FeatureSetProto.FeatureSetStatus.STATUS_READY); + } +} diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java index 75548f3465..ed59aeaa0b 100644 --- a/core/src/test/java/feast/core/util/TypeConversionTest.java +++ b/core/src/test/java/feast/core/util/TypeConversionTest.java @@ -18,8 +18,7 @@ import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import com.google.protobuf.Timestamp; import java.util.*; @@ -56,7 +55,7 @@ public void convertJsonStringToMapShouldConvertJsonStringToMap() { } @Test - public void convertJsonStringToMapShouldReturnEmptyMapForEmptyJson() { + public void convertJsonStringToMapShouldReturnNullForEmptyJson() { String input = "{}"; Map expected = Collections.emptyMap(); assertThat(TypeConversion.convertJsonStringToMap(input), equalTo(expected)); @@ -70,6 +69,12 @@ public void convertMapToJsonStringShouldReturnJsonStringForGivenMap() { TypeConversion.convertMapToJsonString(input), hasJsonPath("$.key", equalTo("value"))); } + @Test + public void convertMapToJsonStringShouldReturnNullForEmptyMap() { + Map input = new HashMap<>(); + assertNull(TypeConversion.convertMapToJsonString(input)); + } + @Test public void convertJsonStringToArgsShouldReturnCorrectListOfArgs() { Map input = new HashMap<>(); diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto index ebac862ffb..73173f0991 100644 --- a/protos/feast/core/FeatureSet.proto +++ b/protos/feast/core/FeatureSet.proto @@ -60,6 +60,9 @@ message FeatureSetSpec { // Optional. Source on which feature rows can be found. // If not set, source will be set to the default value configured in Feast Core. Source source = 6; + + // User defined metadata + map labels = 8; } message EntitySpec { diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index d45d1ab4c8..f5c07070b0 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -56,8 +56,7 @@ def from_proto(cls, feature_proto: FeatureProto): Feature object """ feature = cls( - name=feature_proto.name, - dtype=ValueType(feature_proto.value_type), + name=feature_proto.name, dtype=ValueType(feature_proto.value_type), ) feature.update_presence_constraints(feature_proto) feature.update_shape_type(feature_proto) From b3a6317004373dbef4a20487e03a56b2c26b2e3b Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Wed, 29 Apr 2020 01:28:19 +0700 Subject: [PATCH 18/23] added empty keys validation --- .../core/validators/FeatureSetValidator.java | 8 ++++- .../feast/core/util/TypeConversionTest.java | 2 +- .../validators/FeatureSetValidatorTest.java | 36 +++++++++++++------ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/feast/core/validators/FeatureSetValidator.java b/core/src/main/java/feast/core/validators/FeatureSetValidator.java index 95a6a7136a..5fc9efb33b 100644 --- a/core/src/main/java/feast/core/validators/FeatureSetValidator.java +++ b/core/src/main/java/feast/core/validators/FeatureSetValidator.java @@ -27,6 +27,9 @@ import java.util.stream.Collectors; public class FeatureSetValidator { + + private static final String EMPTY_LABEL_KEYS_MESSAGE = "Label keys must not be empty"; + public static void validateSpec(FeatureSet featureSet) { if (featureSet.getSpec().getProject().isEmpty()) { throw new IllegalArgumentException("Project name must be provided"); @@ -34,6 +37,9 @@ public static void validateSpec(FeatureSet featureSet) { if (featureSet.getSpec().getName().isEmpty()) { throw new IllegalArgumentException("Feature set name must be provided"); } + if (featureSet.getSpec().getLabelsMap().containsKey("")) { + throw new IllegalArgumentException(EMPTY_LABEL_KEYS_MESSAGE); + } checkValidCharacters(featureSet.getSpec().getProject(), "project"); checkValidCharacters(featureSet.getSpec().getName(), "name"); @@ -45,7 +51,7 @@ public static void validateSpec(FeatureSet featureSet) { for (FeatureSpec featureSpec : featureSet.getSpec().getFeaturesList()) { checkValidCharacters(featureSpec.getName(), "features::name"); if (featureSpec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Label keys must not be empty"); + throw new IllegalArgumentException(EMPTY_LABEL_KEYS_MESSAGE); } } } diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java index ed59aeaa0b..5736ff8d92 100644 --- a/core/src/test/java/feast/core/util/TypeConversionTest.java +++ b/core/src/test/java/feast/core/util/TypeConversionTest.java @@ -55,7 +55,7 @@ public void convertJsonStringToMapShouldConvertJsonStringToMap() { } @Test - public void convertJsonStringToMapShouldReturnNullForEmptyJson() { + public void convertJsonStringToMapShouldReturnEmptyMapForEmptyJson() { String input = "{}"; Map expected = Collections.emptyMap(); assertThat(TypeConversion.convertJsonStringToMap(input), equalTo(expected)); diff --git a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java index 39124495fa..d888d865ed 100644 --- a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java +++ b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java @@ -16,7 +16,6 @@ */ package feast.core.validators; -import com.google.protobuf.InvalidProtocolBufferException; import feast.core.FeatureSetProto; import feast.types.ValueProto; import java.util.ArrayList; @@ -32,15 +31,7 @@ public class FeatureSetValidatorTest { @Rule public final ExpectedException expectedException = ExpectedException.none(); @Test - public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() - throws InvalidProtocolBufferException { - List entitySpecs = new ArrayList<>(); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()); - + public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() { Map featureLabels = new HashMap<>() { { @@ -60,7 +51,6 @@ public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() FeatureSetProto.FeatureSetSpec.newBuilder() .setProject("project1") .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) .addAllFeatures(featureSpecs) .build(); FeatureSetProto.FeatureSet featureSet = @@ -70,4 +60,28 @@ public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() expectedException.expectMessage("Label keys must not be empty"); FeatureSetValidator.validateSpec(featureSet); } + + @Test + public void shouldThrowExceptionForFeatureSetLabelsWithAnEmptyKey() { + + Map featureSetLabels = + new HashMap<>() { + { + put("", "empty_key"); + } + }; + + FeatureSetProto.FeatureSetSpec featureSetSpec = + FeatureSetProto.FeatureSetSpec.newBuilder() + .setProject("project1") + .setName("featureSetWithConstraints") + .putAllLabels(featureSetLabels) + .build(); + FeatureSetProto.FeatureSet featureSet = + FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Label keys must not be empty"); + FeatureSetValidator.validateSpec(featureSet); + } } From de1153e9e9f499de44cc3ae1897400220ae64f1a Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Wed, 29 Apr 2020 12:38:35 +0700 Subject: [PATCH 19/23] corrected review comments (storing empty json for features if labels map is empty) --- .../src/main/java/feast/core/model/Field.java | 6 ----- .../java/feast/core/util/TypeConversion.java | 1 - .../core/validators/FeatureSetValidator.java | 6 ++--- .../feast/core/service/JobServiceTest.java | 4 +-- .../feast/core/service/SpecServiceTest.java | 26 +++++++++---------- .../feast/core/service/TestObjectFactory.java | 11 ++++++++ .../feast/core/util/TypeConversionTest.java | 4 +-- .../validators/FeatureSetValidatorTest.java | 4 +-- sdk/python/feast/loaders/ingest.py | 7 ++--- 9 files changed, 34 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/feast/core/model/Field.java b/core/src/main/java/feast/core/model/Field.java index 8d39ac7ea7..213c17f954 100644 --- a/core/src/main/java/feast/core/model/Field.java +++ b/core/src/main/java/feast/core/model/Field.java @@ -19,7 +19,6 @@ import feast.core.FeatureSetProto.EntitySpec; import feast.core.FeatureSetProto.FeatureSpec; import feast.core.util.TypeConversion; -import feast.types.ValueProto.ValueType; import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -80,11 +79,6 @@ public class Field { public Field() {} - public Field(String name, ValueType.Enum type) { - this.name = name; - this.type = type.toString(); - } - public Field(FeatureSpec featureSpec) { this.name = featureSpec.getName(); this.type = featureSpec.getValueType().toString(); diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java index f54de2675c..7481a3c574 100644 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ b/core/src/main/java/feast/core/util/TypeConversion.java @@ -70,7 +70,6 @@ public static Map convertJsonStringToMap(String jsonString) { * @return json string corresponding to given map or null if the map is empty */ public static String convertMapToJsonString(Map map) { - if (map.isEmpty()) return null; return gson.toJson(map); } diff --git a/core/src/main/java/feast/core/validators/FeatureSetValidator.java b/core/src/main/java/feast/core/validators/FeatureSetValidator.java index 5fc9efb33b..ca0f1ec035 100644 --- a/core/src/main/java/feast/core/validators/FeatureSetValidator.java +++ b/core/src/main/java/feast/core/validators/FeatureSetValidator.java @@ -28,8 +28,6 @@ public class FeatureSetValidator { - private static final String EMPTY_LABEL_KEYS_MESSAGE = "Label keys must not be empty"; - public static void validateSpec(FeatureSet featureSet) { if (featureSet.getSpec().getProject().isEmpty()) { throw new IllegalArgumentException("Project name must be provided"); @@ -38,7 +36,7 @@ public static void validateSpec(FeatureSet featureSet) { throw new IllegalArgumentException("Feature set name must be provided"); } if (featureSet.getSpec().getLabelsMap().containsKey("")) { - throw new IllegalArgumentException(EMPTY_LABEL_KEYS_MESSAGE); + throw new IllegalArgumentException("Feature set label keys must not be empty"); } checkValidCharacters(featureSet.getSpec().getProject(), "project"); @@ -51,7 +49,7 @@ public static void validateSpec(FeatureSet featureSet) { for (FeatureSpec featureSpec : featureSet.getSpec().getFeaturesList()) { checkValidCharacters(featureSpec.getName(), "features::name"); if (featureSpec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException(EMPTY_LABEL_KEYS_MESSAGE); + throw new IllegalArgumentException("Feature label keys must not be empty"); } } } diff --git a/core/src/test/java/feast/core/service/JobServiceTest.java b/core/src/test/java/feast/core/service/JobServiceTest.java index d8b69e037c..e98d66c3b7 100644 --- a/core/src/test/java/feast/core/service/JobServiceTest.java +++ b/core/src/test/java/feast/core/service/JobServiceTest.java @@ -148,8 +148,8 @@ public void setupJobManager() { // dummy model constructorss private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Field feature = new Field(name + "_feature", Enum.INT64); - Field entity = new Field(name + "_entity", Enum.STRING); + Field feature = TestObjectFactory.CreateFeatureField(name + "_feature", Enum.INT64); + Field entity = TestObjectFactory.CreateEntityField(name + "_entity", Enum.STRING); FeatureSet fs = TestObjectFactory.CreateFeatureSet( diff --git a/core/src/test/java/feast/core/service/SpecServiceTest.java b/core/src/test/java/feast/core/service/SpecServiceTest.java index e40375e497..bb9f832bd7 100644 --- a/core/src/test/java/feast/core/service/SpecServiceTest.java +++ b/core/src/test/java/feast/core/service/SpecServiceTest.java @@ -114,9 +114,9 @@ public void setUp() { FeatureSet featureSet1v3 = newDummyFeatureSet("f1", 3, "project1"); FeatureSet featureSet2v1 = newDummyFeatureSet("f2", 1, "project1"); - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Field f3f1 = TestObjectFactory.CreateFeatureField("f3f1", Enum.INT64); + Field f3f2 = TestObjectFactory.CreateFeatureField("f3f2", Enum.INT64); + Field f3e1 = TestObjectFactory.CreateEntityField("f3e1", Enum.STRING); FeatureSet featureSet3v1 = TestObjectFactory.CreateFeatureSet( "f3", "project1", 1, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1)); @@ -472,9 +472,9 @@ public void applyFeatureSetShouldIncrementFeatureSetVersionIfAlreadyExists() public void applyFeatureSetShouldNotCreateFeatureSetIfFieldsUnordered() throws InvalidProtocolBufferException { - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Field f3f1 = TestObjectFactory.CreateFeatureField("f3f1", Enum.INT64); + Field f3f2 = TestObjectFactory.CreateFeatureField("f3f2", Enum.INT64); + Field f3e1 = TestObjectFactory.CreateEntityField("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (TestObjectFactory.CreateFeatureSet( "f3", "project1", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) @@ -680,9 +680,9 @@ public void applyFeatureSetShouldUpdateFeatureSetWhenConstraintsAreUpdated() @Test public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() throws InvalidProtocolBufferException { - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Field f3f1 = TestObjectFactory.CreateFeatureField("f3f1", Enum.INT64); + Field f3f2 = TestObjectFactory.CreateFeatureField("f3f2", Enum.INT64); + Field f3e1 = TestObjectFactory.CreateEntityField("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (TestObjectFactory.CreateFeatureSet( "f3", "newproject", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) @@ -699,9 +699,9 @@ public void applyFeatureSetShouldCreateProjectWhenNotAlreadyExists() @Test public void applyFeatureSetShouldFailWhenProjectIsArchived() throws InvalidProtocolBufferException { - Field f3f1 = new Field("f3f1", Enum.INT64); - Field f3f2 = new Field("f3f2", Enum.INT64); - Field f3e1 = new Field("f3e1", Enum.STRING); + Field f3f1 = TestObjectFactory.CreateFeatureField("f3f1", Enum.INT64); + Field f3f2 = TestObjectFactory.CreateFeatureField("f3f2", Enum.INT64); + Field f3e1 = TestObjectFactory.CreateEntityField("f3e1", Enum.STRING); FeatureSetProto.FeatureSet incomingFeatureSet = (TestObjectFactory.CreateFeatureSet( "f3", "archivedproject", 5, Arrays.asList(f3e1), Arrays.asList(f3f2, f3f1))) @@ -861,7 +861,7 @@ private FeatureSet newDummyFeatureSet(String name, int version, String project) .putLabels("key", "value") .build(); Field feature = new Field(f1); - Field entity = new Field("entity", Enum.STRING); + Field entity = TestObjectFactory.CreateEntityField("entity", Enum.STRING); FeatureSet fs = TestObjectFactory.CreateFeatureSet( diff --git a/core/src/test/java/feast/core/service/TestObjectFactory.java b/core/src/test/java/feast/core/service/TestObjectFactory.java index 886b47341a..966cb8d816 100644 --- a/core/src/test/java/feast/core/service/TestObjectFactory.java +++ b/core/src/test/java/feast/core/service/TestObjectFactory.java @@ -21,6 +21,7 @@ import feast.core.model.FeatureSet; import feast.core.model.Field; import feast.core.model.Source; +import feast.types.ValueProto; import java.util.HashMap; import java.util.List; @@ -48,4 +49,14 @@ public static FeatureSet CreateFeatureSet( new HashMap<>(), FeatureSetProto.FeatureSetStatus.STATUS_READY); } + + public static Field CreateFeatureField(String name, ValueProto.ValueType.Enum valueType) { + return new Field( + FeatureSetProto.FeatureSpec.newBuilder().setName(name).setValueType(valueType).build()); + } + + public static Field CreateEntityField(String name, ValueProto.ValueType.Enum valueType) { + return new Field( + FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build()); + } } diff --git a/core/src/test/java/feast/core/util/TypeConversionTest.java b/core/src/test/java/feast/core/util/TypeConversionTest.java index 5736ff8d92..02f0a7cee4 100644 --- a/core/src/test/java/feast/core/util/TypeConversionTest.java +++ b/core/src/test/java/feast/core/util/TypeConversionTest.java @@ -70,9 +70,9 @@ public void convertMapToJsonStringShouldReturnJsonStringForGivenMap() { } @Test - public void convertMapToJsonStringShouldReturnNullForEmptyMap() { + public void convertMapToJsonStringShouldReturnEmptyJsonForAnEmptyMap() { Map input = new HashMap<>(); - assertNull(TypeConversion.convertMapToJsonString(input)); + assertThat(TypeConversion.convertMapToJsonString(input), equalTo("{}")); } @Test diff --git a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java index d888d865ed..2e1e4e381a 100644 --- a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java +++ b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java @@ -57,7 +57,7 @@ public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() { FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Label keys must not be empty"); + expectedException.expectMessage("Feature label keys must not be empty"); FeatureSetValidator.validateSpec(featureSet); } @@ -81,7 +81,7 @@ public void shouldThrowExceptionForFeatureSetLabelsWithAnEmptyKey() { FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Label keys must not be empty"); + expectedException.expectMessage("Feature set label keys must not be empty"); FeatureSetValidator.validateSpec(featureSet); } } diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 8187fd8728..4d215cc990 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -123,15 +123,12 @@ def get_feature_row_chunks( Iterable list of byte encoded FeatureRow(s). """ - def normalize_fields(fields): - return {field.name: field.dtype for field in fields.values()} - feature_set = f"{fs.project}/{fs.name}:{fs.version}" - normalized_fields = normalize_fields(fs.fields) + field_map = {field.name: field.dtype for field in fs.fields.values()} pool = Pool(max_workers) - func = partial(_encode_pa_tables, file, feature_set, normalized_fields) + func = partial(_encode_pa_tables, file, feature_set, field_map) for chunk in pool.imap(func, row_groups): yield chunk return From a756929ad968c85a25fdb1d05734678c5796640c Mon Sep 17 00:00:00 2001 From: suwik Date: Wed, 29 Apr 2020 12:52:01 +0700 Subject: [PATCH 20/23] Updated the comment to match the logic --- core/src/main/java/feast/core/util/TypeConversion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/feast/core/util/TypeConversion.java b/core/src/main/java/feast/core/util/TypeConversion.java index 7481a3c574..6ee990fc1c 100644 --- a/core/src/main/java/feast/core/util/TypeConversion.java +++ b/core/src/main/java/feast/core/util/TypeConversion.java @@ -67,7 +67,7 @@ public static Map convertJsonStringToMap(String jsonString) { * Marshals a given map into its corresponding json string * * @param map - * @return json string corresponding to given map or null if the map is empty + * @return json string corresponding to given map */ public static String convertMapToJsonString(Map map) { return gson.toJson(map); From 18edc12b8f0fd0cf2b636deabbe0a5f7b2def26f Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Thu, 30 Apr 2020 13:06:49 +0700 Subject: [PATCH 21/23] added e2e tests for feature and feature set labels --- tests/e2e/grpc-based-registration.py | 90 ++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/e2e/grpc-based-registration.py diff --git a/tests/e2e/grpc-based-registration.py b/tests/e2e/grpc-based-registration.py new file mode 100644 index 0000000000..367891a4cc --- /dev/null +++ b/tests/e2e/grpc-based-registration.py @@ -0,0 +1,90 @@ +import pytest +import grpc +import uuid + +import feast.core.CoreService_pb2 +from feast import Feature +from feast.core.CoreService_pb2_grpc import CoreServiceStub +from feast.feature_set import FeatureSet +from feast import ValueType + +PROJECT_NAME = 'batch_' + uuid.uuid4().hex.upper()[0:6] +LAST_VERSION = 0 +GRPC_CONNECTION_TIMEOUT = 3 +LABEL_KEY = "my" +LABEL_VALUE = "label" + + +@pytest.fixture(scope="module") +def core_url(pytestconfig): + return pytestconfig.getoption("core_url") + + +@pytest.fixture(scope="module") +def core_service_stub(core_url): + if core_url.endswith(":443"): + core_channel = grpc.secure_channel( + core_url, grpc.ssl_channel_credentials() + ) + else: + core_channel = grpc.insecure_channel(core_url) + + try: + grpc.channel_ready_future(core_channel).result(timeout=GRPC_CONNECTION_TIMEOUT) + except grpc.FutureTimeoutError: + raise ConnectionError( + f"Connection timed out while attempting to connect to Feast " + f"Core gRPC server {core_url} " + ) + core_service_stub = CoreServiceStub(core_channel) + return core_service_stub + + +def apply_feature_set(core_service_stub, feature_set_proto): + try: + apply_fs_response = core_service_stub.ApplyFeatureSet( + feast.core.CoreService_pb2.ApplyFeatureSetRequest(feature_set=feature_set_proto), + timeout=GRPC_CONNECTION_TIMEOUT, + ) # type: ApplyFeatureSetResponse + except grpc.RpcError as e: + raise grpc.RpcError(e.details()) + return apply_fs_response.feature_set + + +def get_feature_set(core_service_stub, name, project): + try: + get_feature_set_response = core_service_stub.GetFeatureSet( + feast.core.CoreService_pb2.GetFeatureSetRequest( + project=project, name=name.strip(), version=LAST_VERSION + ) + ) # type: GetFeatureSetResponse + except grpc.RpcError as e: + raise grpc.RpcError(e.details()) + return get_feature_set_response.feature_set + + +@pytest.mark.timeout(45) +def test_feature_set_labels(core_service_stub): + feature_set_name = "test_feature_set_labels" + feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME).to_proto() + feature_set_proto.spec.labels[LABEL_KEY] = LABEL_VALUE + apply_feature_set(core_service_stub, feature_set_proto) + + retrieved_feature_set = get_feature_set(core_service_stub, feature_set_name, PROJECT_NAME) + + assert LABEL_KEY in retrieved_feature_set.spec.labels + assert retrieved_feature_set.spec.labels[LABEL_KEY] == LABEL_VALUE + + +def test_feature_labels(core_service_stub): + feature_set_name = "test_feature_labels" + feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME, features=[Feature("rating", ValueType.INT64)])\ + .to_proto() + feature_set_proto.spec.features[0].labels[LABEL_KEY] = LABEL_VALUE + apply_feature_set(core_service_stub, feature_set_proto) + + retrieved_feature_set = get_feature_set(core_service_stub, feature_set_name, PROJECT_NAME) + retrieved_feature = retrieved_feature_set.spec.features[0] + + assert LABEL_KEY in retrieved_feature.labels + assert retrieved_feature.labels[LABEL_KEY] == LABEL_VALUE From 02197f86741365a478206f3647934f5c73a0a7f4 Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Thu, 30 Apr 2020 14:47:11 +0700 Subject: [PATCH 22/23] moved e2e tests for feature and feature set labels --- tests/e2e/basic-ingest-redis-serving.py | 114 ++++++++++++++++++++---- tests/e2e/grpc-based-registration.py | 90 ------------------- 2 files changed, 99 insertions(+), 105 deletions(-) delete mode 100644 tests/e2e/grpc-based-registration.py diff --git a/tests/e2e/basic-ingest-redis-serving.py b/tests/e2e/basic-ingest-redis-serving.py index 8e40794344..be40be3eea 100644 --- a/tests/e2e/basic-ingest-redis-serving.py +++ b/tests/e2e/basic-ingest-redis-serving.py @@ -2,12 +2,15 @@ import math import random import time +import grpc from feast.entity import Entity from feast.serving.ServingService_pb2 import ( GetOnlineFeaturesRequest, GetOnlineFeaturesResponse, ) from feast.core.IngestionJob_pb2 import IngestionJobStatus +from feast.core.CoreService_pb2_grpc import CoreServiceStub +from feast.core import CoreService_pb2 from feast.types.Value_pb2 import Value as Value from feast.client import Client from feast.feature_set import FeatureSet, FeatureSetRef @@ -26,6 +29,7 @@ FLOAT_TOLERANCE = 0.00001 PROJECT_NAME = 'basic_' + uuid.uuid4().hex.upper()[0:6] + @pytest.fixture(scope='module') def core_url(pytestconfig): return pytestconfig.getoption("core_url") @@ -75,7 +79,7 @@ def basic_dataframe(): @pytest.mark.timeout(45) -@pytest.mark.run(order=10) +@pytest.mark.run(order=8) def test_basic_register_feature_set_success(client): # Load feature set from file cust_trans_fs_expected = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") @@ -109,6 +113,7 @@ def test_basic_ingest_success(client, basic_dataframe): client.ingest(cust_trans_fs, basic_dataframe) time.sleep(5) + @pytest.mark.timeout(45) @pytest.mark.run(order=12) def test_basic_retrieve_online_success(client, basic_dataframe): @@ -146,12 +151,13 @@ def test_basic_retrieve_online_success(client, basic_dataframe): basic_dataframe.iloc[0]["daily_transactions"]) if math.isclose( - sent_daily_transactions, - returned_daily_transactions, - abs_tol=FLOAT_TOLERANCE, + sent_daily_transactions, + returned_daily_transactions, + abs_tol=FLOAT_TOLERANCE, ): break + @pytest.mark.timeout(300) @pytest.mark.run(order=19) def test_basic_ingest_jobs(client, basic_dataframe): @@ -319,20 +325,20 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): if response is None: continue - returned_float_list = ( response.field_values[0] - .fields[PROJECT_NAME+"/float_list_feature"] + .fields[PROJECT_NAME + "/float_list_feature"] .float_list_val.val ) sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] if math.isclose( - returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE + returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE ): break + @pytest.mark.timeout(300) @pytest.mark.run(order=29) def test_all_types_ingest_jobs(client, all_types_dataframe): @@ -355,6 +361,7 @@ def test_all_types_ingest_jobs(client, all_types_dataframe): ingest_job.wait(IngestionJobStatus.ABORTED) assert ingest_job.status == IngestionJobStatus.ABORTED + @pytest.fixture(scope='module') def large_volume_dataframe(): ROW_COUNT = 100000 @@ -445,9 +452,9 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): large_volume_dataframe.iloc[0]["daily_transactions_large"]) if math.isclose( - sent_daily_transactions, - returned_daily_transactions, - abs_tol=FLOAT_TOLERANCE, + sent_daily_transactions, + returned_daily_transactions, + abs_tol=FLOAT_TOLERANCE, ): break @@ -462,14 +469,14 @@ def all_types_parquet_file(): "customer_id": [np.int32(random.randint(0, 10000)) for _ in range(COUNT)], "int32_feature_parquet": [np.int32(random.randint(0, 10000)) for _ in - range(COUNT)], + range(COUNT)], "int64_feature_parquet": [np.int64(random.randint(0, 10000)) for _ in - range(COUNT)], + range(COUNT)], "float_feature_parquet": [np.float(random.random()) for _ in range(COUNT)], "double_feature_parquet": [np.float64(random.random()) for _ in - range(COUNT)], + range(COUNT)], "string_feature_parquet": ["one" + str(random.random()) for _ in - range(COUNT)], + range(COUNT)], "bytes_feature_parquet": [b"one" for _ in range(COUNT)], "int32_list_feature_parquet": [ np.array([1, 2, 3, random.randint(0, 10000)], dtype=np.int32) @@ -509,6 +516,7 @@ def all_types_parquet_file(): df.to_parquet(file_path, allow_truncated_timestamps=True) return file_path + @pytest.mark.timeout(300) @pytest.mark.run(order=40) def test_all_types_parquet_register_feature_set_success(client): @@ -539,10 +547,86 @@ def test_all_types_parquet_register_feature_set_success(client): @pytest.mark.timeout(600) @pytest.mark.run(order=41) def test_all_types_infer_register_ingest_file_success(client, - all_types_parquet_file): + all_types_parquet_file): # Get feature set all_types_fs = client.get_feature_set(name="all_types_parquet") # Ingest user embedding data client.ingest(feature_set=all_types_fs, source=all_types_parquet_file, force_update=True) + + +# TODO: rewrite these using python SDK once the labels are implemented there +class TestsBasedOnGrpc: + LAST_VERSION = 0 + GRPC_CONNECTION_TIMEOUT = 3 + LABEL_KEY = "my" + LABEL_VALUE = "label" + + @pytest.fixture(scope="module") + def core_service_stub(self, core_url): + if core_url.endswith(":443"): + core_channel = grpc.secure_channel( + core_url, grpc.ssl_channel_credentials() + ) + else: + core_channel = grpc.insecure_channel(core_url) + + try: + grpc.channel_ready_future(core_channel).result(timeout=self.GRPC_CONNECTION_TIMEOUT) + except grpc.FutureTimeoutError: + raise ConnectionError( + f"Connection timed out while attempting to connect to Feast " + f"Core gRPC server {core_url} " + ) + core_service_stub = CoreServiceStub(core_channel) + return core_service_stub + + def apply_feature_set(self, core_service_stub, feature_set_proto): + try: + apply_fs_response = core_service_stub.ApplyFeatureSet( + CoreService_pb2.ApplyFeatureSetRequest(feature_set=feature_set_proto), + timeout=self.GRPC_CONNECTION_TIMEOUT, + ) # type: ApplyFeatureSetResponse + except grpc.RpcError as e: + raise grpc.RpcError(e.details()) + return apply_fs_response.feature_set + + def get_feature_set(self, core_service_stub, name, project): + try: + get_feature_set_response = core_service_stub.GetFeatureSet( + CoreService_pb2.GetFeatureSetRequest( + project=project, name=name.strip(), version=self.LAST_VERSION + ) + ) # type: GetFeatureSetResponse + except grpc.RpcError as e: + raise grpc.RpcError(e.details()) + return get_feature_set_response.feature_set + + @pytest.mark.timeout(45) + @pytest.mark.run(order=9) + def test_register_feature_set_with_labels(self, core_service_stub): + feature_set_name = "test_feature_set_labels" + feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME).to_proto() + feature_set_proto.spec.labels[self.LABEL_KEY] = self.LABEL_VALUE + self.apply_feature_set(core_service_stub, feature_set_proto) + + retrieved_feature_set = self.get_feature_set(core_service_stub, feature_set_name, PROJECT_NAME) + + assert self.LABEL_KEY in retrieved_feature_set.spec.labels + assert retrieved_feature_set.spec.labels[self.LABEL_KEY] == self.LABEL_VALUE + + @pytest.mark.timeout(45) + @pytest.mark.run(order=10) + def test_register_feature_with_labels(self, core_service_stub): + feature_set_name = "test_feature_labels" + feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME, features=[Feature("rating", ValueType.INT64)]) \ + .to_proto() + feature_set_proto.spec.features[0].labels[self.LABEL_KEY] = self.LABEL_VALUE + self.apply_feature_set(core_service_stub, feature_set_proto) + + retrieved_feature_set = self.get_feature_set(core_service_stub, feature_set_name, PROJECT_NAME) + retrieved_feature = retrieved_feature_set.spec.features[0] + + assert self.LABEL_KEY in retrieved_feature.labels + assert retrieved_feature.labels[self.LABEL_KEY] == self.LABEL_VALUE diff --git a/tests/e2e/grpc-based-registration.py b/tests/e2e/grpc-based-registration.py deleted file mode 100644 index 367891a4cc..0000000000 --- a/tests/e2e/grpc-based-registration.py +++ /dev/null @@ -1,90 +0,0 @@ -import pytest -import grpc -import uuid - -import feast.core.CoreService_pb2 -from feast import Feature -from feast.core.CoreService_pb2_grpc import CoreServiceStub -from feast.feature_set import FeatureSet -from feast import ValueType - -PROJECT_NAME = 'batch_' + uuid.uuid4().hex.upper()[0:6] -LAST_VERSION = 0 -GRPC_CONNECTION_TIMEOUT = 3 -LABEL_KEY = "my" -LABEL_VALUE = "label" - - -@pytest.fixture(scope="module") -def core_url(pytestconfig): - return pytestconfig.getoption("core_url") - - -@pytest.fixture(scope="module") -def core_service_stub(core_url): - if core_url.endswith(":443"): - core_channel = grpc.secure_channel( - core_url, grpc.ssl_channel_credentials() - ) - else: - core_channel = grpc.insecure_channel(core_url) - - try: - grpc.channel_ready_future(core_channel).result(timeout=GRPC_CONNECTION_TIMEOUT) - except grpc.FutureTimeoutError: - raise ConnectionError( - f"Connection timed out while attempting to connect to Feast " - f"Core gRPC server {core_url} " - ) - core_service_stub = CoreServiceStub(core_channel) - return core_service_stub - - -def apply_feature_set(core_service_stub, feature_set_proto): - try: - apply_fs_response = core_service_stub.ApplyFeatureSet( - feast.core.CoreService_pb2.ApplyFeatureSetRequest(feature_set=feature_set_proto), - timeout=GRPC_CONNECTION_TIMEOUT, - ) # type: ApplyFeatureSetResponse - except grpc.RpcError as e: - raise grpc.RpcError(e.details()) - return apply_fs_response.feature_set - - -def get_feature_set(core_service_stub, name, project): - try: - get_feature_set_response = core_service_stub.GetFeatureSet( - feast.core.CoreService_pb2.GetFeatureSetRequest( - project=project, name=name.strip(), version=LAST_VERSION - ) - ) # type: GetFeatureSetResponse - except grpc.RpcError as e: - raise grpc.RpcError(e.details()) - return get_feature_set_response.feature_set - - -@pytest.mark.timeout(45) -def test_feature_set_labels(core_service_stub): - feature_set_name = "test_feature_set_labels" - feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME).to_proto() - feature_set_proto.spec.labels[LABEL_KEY] = LABEL_VALUE - apply_feature_set(core_service_stub, feature_set_proto) - - retrieved_feature_set = get_feature_set(core_service_stub, feature_set_name, PROJECT_NAME) - - assert LABEL_KEY in retrieved_feature_set.spec.labels - assert retrieved_feature_set.spec.labels[LABEL_KEY] == LABEL_VALUE - - -def test_feature_labels(core_service_stub): - feature_set_name = "test_feature_labels" - feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME, features=[Feature("rating", ValueType.INT64)])\ - .to_proto() - feature_set_proto.spec.features[0].labels[LABEL_KEY] = LABEL_VALUE - apply_feature_set(core_service_stub, feature_set_proto) - - retrieved_feature_set = get_feature_set(core_service_stub, feature_set_name, PROJECT_NAME) - retrieved_feature = retrieved_feature_set.spec.features[0] - - assert LABEL_KEY in retrieved_feature.labels - assert retrieved_feature.labels[LABEL_KEY] == LABEL_VALUE From 8d6d4f444832278ff2bf8051de4c892262c7e4d3 Mon Sep 17 00:00:00 2001 From: Krzysztof Suwinski Date: Thu, 30 Apr 2020 15:26:35 +0700 Subject: [PATCH 23/23] changed tests ordering --- tests/e2e/basic-ingest-redis-serving.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/basic-ingest-redis-serving.py b/tests/e2e/basic-ingest-redis-serving.py index be40be3eea..e80c5b7af6 100644 --- a/tests/e2e/basic-ingest-redis-serving.py +++ b/tests/e2e/basic-ingest-redis-serving.py @@ -79,7 +79,7 @@ def basic_dataframe(): @pytest.mark.timeout(45) -@pytest.mark.run(order=8) +@pytest.mark.run(order=10) def test_basic_register_feature_set_success(client): # Load feature set from file cust_trans_fs_expected = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") @@ -604,7 +604,7 @@ def get_feature_set(self, core_service_stub, name, project): return get_feature_set_response.feature_set @pytest.mark.timeout(45) - @pytest.mark.run(order=9) + @pytest.mark.run(order=51) def test_register_feature_set_with_labels(self, core_service_stub): feature_set_name = "test_feature_set_labels" feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME).to_proto() @@ -617,7 +617,7 @@ def test_register_feature_set_with_labels(self, core_service_stub): assert retrieved_feature_set.spec.labels[self.LABEL_KEY] == self.LABEL_VALUE @pytest.mark.timeout(45) - @pytest.mark.run(order=10) + @pytest.mark.run(order=52) def test_register_feature_with_labels(self, core_service_stub): feature_set_name = "test_feature_labels" feature_set_proto = FeatureSet(feature_set_name, PROJECT_NAME, features=[Feature("rating", ValueType.INT64)]) \