From 37d76cc07413a4330702680bd10fe6340adf731b Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Mon, 30 Dec 2019 15:26:30 -0500 Subject: [PATCH 1/4] support lazy loading --- .../datastore/core/DatastoreTemplate.java | 77 ++++++++----- .../mapping/DatastorePersistentProperty.java | 6 ++ .../DatastorePersistentPropertyImpl.java | 6 ++ .../core/mapping/ReferenceCollection.java | 47 ++++++++ .../data/datastore/core/util/LazyUtil.java | 102 ++++++++++++++++++ .../core/DatastoreTemplateTests.java | 101 +++++++++++++++-- .../DatastorePersistentPropertyImplTests.java | 18 +++- .../it/DatastoreIntegrationTests.java | 26 ++++- .../gcp/data/datastore/it/ReferenceEntry.java | 5 +- .../it/TransactionalTemplateService.java | 12 +++ 10 files changed, 358 insertions(+), 42 deletions(-) create mode 100644 spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java create mode 100644 spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java index 5306822f6c..e9688b33a0 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java @@ -66,6 +66,7 @@ import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeSaveEvent; import org.springframework.cloud.gcp.data.datastore.core.util.KeyUtil; +import org.springframework.cloud.gcp.data.datastore.core.util.LazyUtil; import org.springframework.cloud.gcp.data.datastore.core.util.SliceUtil; import org.springframework.cloud.gcp.data.datastore.core.util.ValueUtil; import org.springframework.context.ApplicationEvent; @@ -463,12 +464,17 @@ private List getReferenceEntitiesForSave(Object entity, Builder builder, } Value value; if (persistentProperty.isCollectionLike()) { - Iterable iterableVal = (Iterable) ValueUtil.toListIfArray(val); - entitiesToSave.addAll(getEntitiesForSave(iterableVal, persistedEntities)); - List keyValues = StreamSupport.stream((iterableVal).spliterator(), false) - .map((o) -> KeyValue.of(this.getKey(o, false))) - .collect(Collectors.toList()); - value = ListValue.of(keyValues); + if (LazyUtil.hasUsableKeys(val)) { + value = ListValue.of(LazyUtil.getKeys(val)); + } + else { + Iterable iterableVal = (Iterable) ValueUtil.toListIfArray(val); + entitiesToSave.addAll(getEntitiesForSave(iterableVal, persistedEntities)); + List keyValues = StreamSupport.stream((iterableVal).spliterator(), false) + .map((o) -> KeyValue.of(this.getKey(o, false))) + .collect(Collectors.toList()); + value = ListValue.of(keyValues); + } } else { entitiesToSave.addAll(getEntitiesForSave(Collections.singletonList(val), persistedEntities)); @@ -593,14 +599,31 @@ private void resolveReferenceProperties(DatastorePersistentEntity datastoreP BaseEntity entity, T convertedObject, ReadContext context) { datastorePersistentEntity.doWithAssociations( (AssociationHandler) (association) -> { - DatastorePersistentProperty referencePersistentProperty = (DatastorePersistentProperty) association + DatastorePersistentProperty referenceProperty = (DatastorePersistentProperty) association .getInverse(); - Object referenced = findReferenced(entity, referencePersistentProperty, context); - if (referenced != null) { - datastorePersistentEntity.getPropertyAccessor(convertedObject) - .setProperty(referencePersistentProperty, referenced); + String fieldName = referenceProperty.getFieldName(); + if (entity.contains(fieldName) && !entity.isNull(fieldName)) { + Class type = referenceProperty.getType(); + Object referenced; + if (referenceProperty.isLazyLoaded() && referenceProperty.isCollectionLike()) { + List keyList = entity.getList(fieldName); + DatastoreReaderWriter originalTx = getDatastoreReadWriter(); + referenced = LazyUtil.wrapSimpleLazyProxy(() -> { + if (getDatastoreReadWriter() != originalTx) { + throw new DatastoreDataException("Lazy load should be invoked within the same transaction"); + } + return fetchReferenced(referenceProperty, context, + valuesToKeys(keyList)); + }, type, keyList); + } + else { + referenced = findReferenced(entity, referenceProperty, context); + } + if (referenced != null) { + datastorePersistentEntity.getPropertyAccessor(convertedObject) + .setProperty(referenceProperty, referenced); + } } - }); } @@ -609,19 +632,8 @@ private Object findReferenced(BaseEntity entity, DatastorePersistentProperty ref String fieldName = referencePersistentProperty.getFieldName(); try { Object referenced; - if (!entity.contains(fieldName)) { - referenced = null; - } - else if (referencePersistentProperty.isCollectionLike()) { - Class referencedType = referencePersistentProperty.getComponentType(); - List> keyValues = entity.getList(fieldName); - referenced = this.datastoreEntityConverter.getConversions() - .convertOnRead( - findAllById( - keyValues.stream().map(Value::get).collect(Collectors.toSet()), - referencedType, context), - referencePersistentProperty.getType(), - referencedType); + if (referencePersistentProperty.isCollectionLike()) { + referenced = fetchReferenced(referencePersistentProperty, context, valuesToKeys(entity.getList(fieldName))); } else { List referencedList = findAllById(Collections.singleton(entity.getKey(fieldName)), @@ -638,6 +650,21 @@ else if (referencePersistentProperty.isCollectionLike()) { } } + private Object fetchReferenced(DatastorePersistentProperty referencePersistentProperty, ReadContext context, Set keys) { + Class referencedType = referencePersistentProperty.getComponentType(); + return this.datastoreEntityConverter.getConversions() + .convertOnRead( + findAllById( + keys, + referencedType, context), + referencePersistentProperty.getType(), + referencedType); + } + + private Set valuesToKeys(List> keyValues) { + return keyValues.stream().map(Value::get).collect(Collectors.toSet()); + } + private void resolveDescendantProperties(DatastorePersistentEntity datastorePersistentEntity, BaseEntity entity, T convertedObject, ReadContext context) { datastorePersistentEntity diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentProperty.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentProperty.java index 3961db5a6f..455b489ab6 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentProperty.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentProperty.java @@ -60,4 +60,10 @@ public interface DatastorePersistentProperty * @return true if the property is stored within Datastore entity */ boolean isColumnBacked(); + + /** + * Return whether this property is a lazily-fetched one. + * @return {@code true} if the property is lazily-fetched. {@code false} otherwise. + */ + boolean isLazyLoaded(); } diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java index 4db74e2dd8..34fc8817c6 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java @@ -140,4 +140,10 @@ public Iterable> getPersistentEntityTypes() { .filter((typeInfo) -> typeInfo.getType().isAnnotationPresent(Entity.class)) .collect(Collectors.toList()); } + + @Override + public boolean isLazyLoaded() { + ReferenceCollection annotation = findAnnotation(ReferenceCollection.class); + return annotation != null && annotation.lazy(); + } } diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java new file mode 100644 index 0000000000..1638639d88 --- /dev/null +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java @@ -0,0 +1,47 @@ +/* + * Copyright 2017-2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.data.datastore.core.mapping; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.data.annotation.Reference; + +/** + * Annotation for a class that indicates it is an entity stored in a Datastore Entity. + * + * @author Dmitry Solomakha + * + * @since 1.3 + */ +@Documented +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +@Reference +public @interface ReferenceCollection { + + /** + * Controls whether the referenced entity should be loaded lazily. This defaults to + * {@literal false}. + * + * @return whether the interleaved property is retrieved lazily. + */ + boolean lazy() default false; +} diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java new file mode 100644 index 0000000000..dea2ac1cd2 --- /dev/null +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.gcp.data.datastore.core.util; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.function.Supplier; + +import org.springframework.cloud.gcp.data.datastore.core.mapping.DatastoreDataException; +import org.springframework.util.Assert; + +/** + * @author Dmitry Solomakha + */ +final public class LazyUtil { + + private LazyUtil() { + } + + public static T wrapSimpleLazyProxy(Supplier supplierFunc, Class type, List keys) { + return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, + new SimpleLazyDynamicInvocationHandler(supplierFunc, keys)); + } + + public static boolean hasUsableKeys(Object object) { + if (Proxy.isProxyClass(object.getClass()) + && (Proxy.getInvocationHandler(object) instanceof SimpleLazyDynamicInvocationHandler)) { + SimpleLazyDynamicInvocationHandler handler = (SimpleLazyDynamicInvocationHandler) Proxy + .getInvocationHandler(object); + return !handler.isEvaluated() && handler.getKeys() != null; + } + return false; + } + + public static List getKeys(Object object) { + if (Proxy.isProxyClass(object.getClass()) + && (Proxy.getInvocationHandler(object) instanceof SimpleLazyDynamicInvocationHandler)) { + SimpleLazyDynamicInvocationHandler handler = (SimpleLazyDynamicInvocationHandler) Proxy + .getInvocationHandler(object); + if (!handler.isEvaluated()) { + return handler.getKeys(); + } + } + return null; + } + + public static final class SimpleLazyDynamicInvocationHandler implements InvocationHandler { + + private final Supplier supplierFunc; + + private final List keys; + + private boolean isEvaluated = false; + + private T value; + + private SimpleLazyDynamicInvocationHandler(Supplier supplierFunc, List keys) { + Assert.notNull(supplierFunc, "A non-null supplier function is required for a lazy proxy."); + this.supplierFunc = supplierFunc; + this.keys = keys; + } + + private boolean isEvaluated() { + return this.isEvaluated; + } + + public List getKeys() { + return this.keys; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (!this.isEvaluated) { + T value = this.supplierFunc.get(); + if (value == null) { + throw new DatastoreDataException("Can't load referenced entity"); + } + this.value = value; + + this.isEvaluated = true; + } + return method.invoke(this.value, args); + } + } + +} diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java index 0776b35a87..ec25f8fb44 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java @@ -42,6 +42,7 @@ import com.google.cloud.datastore.KeyFactory; import com.google.cloud.datastore.KeyQuery; import com.google.cloud.datastore.KeyValue; +import com.google.cloud.datastore.ListValue; import com.google.cloud.datastore.LongValue; import com.google.cloud.datastore.NullValue; import com.google.cloud.datastore.PathElement; @@ -68,12 +69,14 @@ import org.springframework.cloud.gcp.data.datastore.core.mapping.DiscriminatorField; import org.springframework.cloud.gcp.data.datastore.core.mapping.DiscriminatorValue; import org.springframework.cloud.gcp.data.datastore.core.mapping.Field; +import org.springframework.cloud.gcp.data.datastore.core.mapping.ReferenceCollection; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterFindByKeyEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterQueryEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterSaveEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeSaveEvent; +import org.springframework.cloud.gcp.data.datastore.core.util.LazyUtil; import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationEventPublisher; import org.springframework.data.annotation.Id; @@ -139,11 +142,13 @@ public class DatastoreTemplateTests { private ChildEntity childEntity4; private ChildEntity childEntity5; private ChildEntity childEntity6; + private ChildEntity childEntity7; private Key childKey2; private Key childKey3; private Key childKey4; private Key childKey5; private Key childKey6; + private Key childKey7; private SimpleTestEntity simpleTestEntity = new SimpleTestEntity(); private SimpleTestEntity simpleTestEntityNullVallues = new SimpleTestEntity(); private TestEntity ob1; @@ -199,6 +204,11 @@ this.datastoreEntityConverter, new DatastoreMappingContext(), this.childEntity6 = new ChildEntity(); this.ob1.multipleReference.add(this.childEntity6); + this.ob1.lazyMultipleReference = new LinkedList<>(); + + this.childEntity7 = new ChildEntity(); + this.ob1.lazyMultipleReference.add(this.childEntity7); + // mocked query results for entities and child entities. QueryResults childTestEntityQueryResults = mock(QueryResults.class); doAnswer((invocation) -> { @@ -306,6 +316,11 @@ else if (key == this.keyChild1) { .thenReturn(this.childKey6); when(this.objectToKeyFactory.getKeyFromObject(same(this.childEntity6), any())) .thenReturn(this.childKey6); + this.childKey7 = createFakeKey("child_id7"); + when(this.objectToKeyFactory.allocateKeyForObject(same(this.childEntity7), any(), any())) + .thenReturn(this.childKey7); + when(this.objectToKeyFactory.getKeyFromObject(same(this.childEntity7), any())) + .thenReturn(this.childKey7); } @Test @@ -448,16 +463,25 @@ public void findAllReferenceLoopTest() { Entity referenceTestDatastoreEntity = Entity.newBuilder(this.key1) .set("sibling", this.key1) + .set("lazyChildren", ListValue.of(this.key2)) .build(); + Entity child = Entity.newBuilder(this.key1).build(); + when(this.datastore.fetch(eq(this.key1))) .thenReturn(Collections.singletonList(referenceTestDatastoreEntity)); + when(this.datastore.fetch(eq(this.key2))) + .thenReturn(Collections.singletonList(child)); ReferenceTestEntity referenceTestEntity = new ReferenceTestEntity(); + ReferenceTestEntity childEntity = new ReferenceTestEntity(); when(this.datastoreEntityConverter.read(eq(ReferenceTestEntity.class), same(referenceTestDatastoreEntity))) .thenAnswer(invocationOnMock -> referenceTestEntity); + when(this.datastoreEntityConverter.read(eq(ReferenceTestEntity.class), same(child))) + .thenAnswer(invocationOnMock -> childEntity); + verifyBeforeAndAfterEvents(null, new AfterFindByKeyEvent(Collections.singletonList(referenceTestEntity), Collections.singleton(this.key1)), @@ -466,6 +490,13 @@ public void findAllReferenceLoopTest() { ReferenceTestEntity.class); assertThat(readReferenceTestEntity.sibling).isSameAs(readReferenceTestEntity); + verify(this.datastore, times(1)).fetch(any()); + + assertThat(readReferenceTestEntity.lazyChildren.size()).isEqualTo(1); + verify(this.datastore, times(2)).fetch(any()); + verify(this.datastore, times(1)).fetch(eq(this.key1)); + verify(this.datastore, times(1)).fetch(eq(this.key2)); + }, x -> { }); } @@ -493,9 +524,22 @@ public void saveReferenceLoopTest() { @Test public void saveTest() { + saveTestCommon(this.ob1, false); + } + + @Test + public void saveTestLazy() { + this.ob1.lazyMultipleReference = LazyUtil.wrapSimpleLazyProxy( + () -> Collections.singletonList(this.childEntity7), List.class, + Collections.singletonList(KeyValue.of(this.childKey7))); + saveTestCommon(this.ob1, true); + } + + void saveTestCommon(TestEntity parent, boolean lazy) { Entity writtenEntity = Entity.newBuilder(this.key1) .set("singularReference", this.childKey4) .set("multipleReference", Arrays.asList(KeyValue.of(this.childKey5), KeyValue.of(this.childKey6))) + .set("lazyMultipleReference", Collections.singletonList(KeyValue.of(this.childKey7))) .build(); Entity writtenChildEntity2 = Entity.newBuilder(this.childKey2).build(); @@ -503,23 +547,44 @@ public void saveTest() { Entity writtenChildEntity4 = Entity.newBuilder(this.childKey4).build(); Entity writtenChildEntity5 = Entity.newBuilder(this.childKey5).build(); Entity writtenChildEntity6 = Entity.newBuilder(this.childKey6).build(); + Entity writtenChildEntity7 = Entity.newBuilder(this.childKey7).build(); doAnswer(invocation -> { - assertThat(invocation.getArguments()).containsExactlyInAnyOrder(writtenChildEntity2, writtenChildEntity3, - writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity); + Object[] arguments = invocation.getArguments(); + assertThat(arguments).contains(writtenEntity); + assertThat(arguments).contains(writtenChildEntity2); + assertThat(arguments).contains(writtenChildEntity3); + assertThat(arguments).contains(writtenChildEntity4); + assertThat(arguments).contains(writtenChildEntity5); + assertThat(arguments).contains(writtenChildEntity6); + if (lazy) { + assertThat(arguments).hasSize(6); + } + else { + assertThat(arguments).contains(writtenChildEntity7); + assertThat(arguments).hasSize(7); + } + return null; }).when(this.datastore).put(ArgumentMatchers.any()); - assertThat(this.datastoreTemplate.save(this.ob1)).isInstanceOf(TestEntity.class); + assertThat(this.datastoreTemplate.save(parent)).isInstanceOf(TestEntity.class); verify(this.datastore, times(1)).put(ArgumentMatchers.any()); - verify(this.datastoreEntityConverter, times(1)).write(same(this.ob1), notNull()); + verify(this.datastoreEntityConverter, times(1)).write(same(parent), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity2), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity3), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity4), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity5), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity6), notNull()); + if (lazy) { + verify(this.datastoreEntityConverter, times(0)).write(same(this.childEntity7), notNull()); + } + else { + verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity7), notNull()); + } } + private List gatherVarArgCallsArgs(Object methodCall, List returnVal) { List callsArgs = new ArrayList<>(); when(methodCall).thenAnswer(invocationOnMock -> { @@ -598,15 +663,17 @@ public void saveAndAllocateIdTest() { Entity writtenEntity1 = Entity.newBuilder(this.key1) .set("singularReference", this.childKey4) .set("multipleReference", Arrays.asList(KeyValue.of(this.childKey5), KeyValue.of(this.childKey6))) + .set("lazyMultipleReference", Collections.singletonList(KeyValue.of(this.childKey7))) .build(); Entity writtenChildEntity2 = Entity.newBuilder(this.childKey2).build(); Entity writtenChildEntity3 = Entity.newBuilder(this.childKey3).build(); Entity writtenChildEntity4 = Entity.newBuilder(this.childKey4).build(); Entity writtenChildEntity5 = Entity.newBuilder(this.childKey5).build(); Entity writtenChildEntity6 = Entity.newBuilder(this.childKey6).build(); + Entity writtenChildEntity7 = Entity.newBuilder(this.childKey7).build(); doAnswer(invocation -> { assertThat(invocation.getArguments()).containsExactlyInAnyOrder(writtenChildEntity2, writtenChildEntity3, - writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1); + writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1, writtenChildEntity7); return null; }).when(this.datastore).put(ArgumentMatchers.any()); @@ -626,20 +693,24 @@ public void saveAllTest() { Entity writtenEntity1 = Entity.newBuilder(this.key1) .set("singularReference", this.childKey4) .set("multipleReference", Arrays.asList(KeyValue.of(this.childKey5), KeyValue.of(this.childKey6))) + .set("lazyMultipleReference", Collections.singletonList(KeyValue.of(this.childKey7))) .build(); + Entity writtenEntity2 = Entity.newBuilder(this.key2).build(); Entity writtenChildEntity2 = Entity.newBuilder(this.childKey2).build(); Entity writtenChildEntity3 = Entity.newBuilder(this.childKey3).build(); Entity writtenChildEntity4 = Entity.newBuilder(this.childKey4).build(); Entity writtenChildEntity5 = Entity.newBuilder(this.childKey5).build(); Entity writtenChildEntity6 = Entity.newBuilder(this.childKey6).build(); + Entity writtenChildEntity7 = Entity.newBuilder(this.childKey7).build(); doAnswer(invocation -> { assertThat(invocation.getArguments()).containsExactlyInAnyOrder(writtenChildEntity2, writtenChildEntity3, - writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1, writtenEntity2); + writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1, writtenEntity2, + writtenChildEntity7); return null; }).when(this.datastore).put(ArgumentMatchers.any()); - List expected = Arrays.asList(writtenChildEntity2, writtenChildEntity3, + List expected = Arrays.asList(writtenChildEntity2, writtenChildEntity3, writtenChildEntity7, writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1, writtenEntity2); List javaExpected = Arrays.asList(this.ob1, this.ob2); @@ -668,6 +739,7 @@ public void saveAllMaxWriteSizeTest() { Entity writtenEntity1 = Entity.newBuilder(this.key1) .set("singularReference", this.childKey4) .set("multipleReference", Arrays.asList(KeyValue.of(this.childKey5), KeyValue.of(this.childKey6))) + .set("lazyMultipleReference", Collections.singletonList(KeyValue.of(this.childKey7))) .build(); Entity writtenEntity2 = Entity.newBuilder(this.key2).build(); Entity writtenChildEntity2 = Entity.newBuilder(this.childKey2).build(); @@ -675,8 +747,9 @@ public void saveAllMaxWriteSizeTest() { Entity writtenChildEntity4 = Entity.newBuilder(this.childKey4).build(); Entity writtenChildEntity5 = Entity.newBuilder(this.childKey5).build(); Entity writtenChildEntity6 = Entity.newBuilder(this.childKey6).build(); + Entity writtenChildEntity7 = Entity.newBuilder(this.childKey7).build(); Set entities = new HashSet<>(); - entities.addAll(Arrays.asList(writtenChildEntity2, writtenChildEntity3, + entities.addAll(Arrays.asList(writtenChildEntity2, writtenChildEntity3, writtenChildEntity7, writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1, writtenEntity2)); doAnswer(invocation -> { assertThat(invocation.getArguments()).hasSize(1); @@ -685,7 +758,7 @@ public void saveAllMaxWriteSizeTest() { return null; }).when(this.datastore).put(ArgumentMatchers.any()); - List expected = Arrays.asList(writtenChildEntity2, writtenChildEntity3, + List expected = Arrays.asList(writtenChildEntity2, writtenChildEntity3, writtenChildEntity7, writtenChildEntity4, writtenChildEntity5, writtenChildEntity6, writtenEntity1, writtenEntity2); List javaExpected = Arrays.asList(this.ob1, this.ob2); @@ -705,8 +778,9 @@ public void saveAllMaxWriteSizeTest() { verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity4), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity5), notNull()); verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity6), notNull()); + verify(this.datastoreEntityConverter, times(1)).write(same(this.childEntity7), notNull()); - verify(this.datastore, times(7)).put(ArgumentMatchers.any()); + verify(this.datastore, times(8)).put(ArgumentMatchers.any()); } @Test @@ -1107,6 +1181,10 @@ private static class TestEntity { @Reference LinkedList multipleReference; + @ReferenceCollection(lazy = true) + List lazyMultipleReference; + + @Override public boolean equals(Object o) { if (this == o) { @@ -1176,6 +1254,9 @@ class ReferenceTestEntity { @Reference ReferenceTestEntity sibling; + + @ReferenceCollection(lazy = true) + List lazyChildren; } } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java index 7357ae2c04..50f237be53 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java @@ -122,6 +122,15 @@ public void fieldReferenceAnnotatedTest() { .getPersistentEntity(FieldReferenceAnnotatedEntity.class); } + @Test + public void fieldReferencedAnnotatedTest() { + this.expectedException.expect(DatastoreDataException.class); + this.expectedException.expectMessage("Property cannot be annotated as @Field if it is " + + "annotated @Descendants or @Reference: name"); + this.datastoreMappingContext + .getPersistentEntity(FieldReferencedAnnotatedEntity.class); + } + @Entity(name = "custom_test_kind") private static class TestEntity { @Id @@ -146,7 +155,7 @@ private static class TestEntity { @Descendants List linkedEntity; - @Reference + @ReferenceCollection TestSubEntity linkedEntityRef; } @@ -171,6 +180,13 @@ private static class FieldReferenceAnnotatedEntity { TestSubEntity[] subEntity; } + private static class FieldReferencedAnnotatedEntity { + @Field(name = "name") + @ReferenceCollection + TestSubEntity[] subEntity; + } + + private static class DescendantFieldAnnotatedEntity { @Descendants @Field(name = "name") diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java index 543356905b..609b736d2b 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java @@ -539,6 +539,24 @@ public void referenceTest() { ReferenceEntry loadedParentAfterUpdate = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); assertThat(loadedParentAfterUpdate).isEqualTo(parent); + + //Saving an entity with not loaded lazy field + parent = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); + + this.datastoreTemplate.save(parent); + + loadedParent = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); + assertThat(loadedParent).isEqualTo(parent); + + //Exception should be produced if a lazy loaded property accessed outside of the initial transaction + ReferenceEntry finalLoadedParent = this.transactionalTemplateService.findByIdLazy(parent.id); + assertThatThrownBy(() -> finalLoadedParent.childeren.size()).isInstanceOf(DatastoreDataException.class) + .hasMessage("Lazy load should be invoked within the same transaction"); + + //No exception should be produced if a lazy loaded property accessed within the initial transaction + ReferenceEntry finalLoadedParentLazyLoaded = this.transactionalTemplateService.findByIdLazyLoad(parent.id); + assertThat(finalLoadedParentLazyLoaded).isEqualTo(parent); + } @Test @@ -715,7 +733,7 @@ public void readOnlyDeleteTest() { @Test public void readOnlyCountTest() { - assertThat(this.transactionalTemplateService.findByIdInReadOnly(1)).isEqualTo(testEntityA); + assertThat(this.transactionalTemplateService.findByIdInReadOnly(1)).isEqualTo(this.testEntityA); } @Test @@ -936,10 +954,10 @@ public int hashCode() { @Override public String toString() { return "Employee{" + - "id=" + id.getNameOrId() + + "id=" + this.id.getNameOrId() + ", subordinates=" - + (subordinates != null - ? subordinates.stream() + + (this.subordinates != null + ? this.subordinates.stream() .map(employee -> employee.id.getNameOrId()) .collect(Collectors.toList()) : null) diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java index dace6d24fb..2e3ea4a2a2 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Objects; +import org.springframework.cloud.gcp.data.datastore.core.mapping.ReferenceCollection; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Reference; @@ -29,7 +30,7 @@ * * @author Dmitry Solomakha */ -class ReferenceEntry { +public class ReferenceEntry { @Id Long id; @@ -38,7 +39,7 @@ class ReferenceEntry { @Reference ReferenceEntry sibling; - @Reference + @ReferenceCollection(lazy = true) List childeren; ReferenceEntry(String name, ReferenceEntry sibling, List childeren) { diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java index f9d38c8328..ee91b79edd 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java @@ -75,4 +75,16 @@ public void deleteInReadOnly() { public TestEntity findByIdInReadOnly(long id) { return this.datastoreTemplate.findById(id, TestEntity.class); } + + @Transactional(readOnly = true) + public ReferenceEntry findByIdLazy(long id) { + return this.datastoreTemplate.findById(id, ReferenceEntry.class); + } + + @Transactional(readOnly = true) + public ReferenceEntry findByIdLazyLoad(long id) { + ReferenceEntry entry = this.datastoreTemplate.findById(id, ReferenceEntry.class); + entry.childeren.size(); + return entry; + } } From a35ccf919bad5c1ed094927b704f174dd0268036 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Fri, 3 Jan 2020 16:20:46 -0500 Subject: [PATCH 2/4] support lazy loading --- docs/src/main/asciidoc/datastore.adoc | 1 + .../datastore/core/DatastoreTemplate.java | 50 +++++++++------- .../datastore/core/{util => }/LazyUtil.java | 60 +++++++++++++------ .../DatastorePersistentPropertyImpl.java | 7 ++- ...tion.java => LazyReferenceCollection.java} | 12 +--- .../core/DatastoreTemplateTests.java | 28 +++++++-- .../DatastorePersistentPropertyImplTests.java | 4 +- .../it/DatastoreIntegrationTests.java | 41 ++++++++----- .../gcp/data/datastore/it/ReferenceEntry.java | 18 +++--- .../it/TransactionalTemplateService.java | 8 +-- 10 files changed, 144 insertions(+), 85 deletions(-) rename spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/{util => }/LazyUtil.java (61%) rename spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/{ReferenceCollection.java => LazyReferenceCollection.java} (75%) diff --git a/docs/src/main/asciidoc/datastore.adoc b/docs/src/main/asciidoc/datastore.adoc index d0a192e58d..8598d841b7 100644 --- a/docs/src/main/asciidoc/datastore.adoc +++ b/docs/src/main/asciidoc/datastore.adoc @@ -425,6 +425,7 @@ There are three ways to represent relationships between entities that are descri * Embedded entities stored directly in the field of the containing entity * `@Descendant` annotated properties for one-to-many relationships * `@Reference` annotated properties for general relationships without hierarchy +* `@LazyReferenceCollection` similar to `@Reference`, but the entities are lazy loaded when the property is accessed (note that the children keys retrieved when the parent entity is loaded) ==== Embedded Entities diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java index e9688b33a0..a0053db24d 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplate.java @@ -66,7 +66,6 @@ import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeSaveEvent; import org.springframework.cloud.gcp.data.datastore.core.util.KeyUtil; -import org.springframework.cloud.gcp.data.datastore.core.util.LazyUtil; import org.springframework.cloud.gcp.data.datastore.core.util.SliceUtil; import org.springframework.cloud.gcp.data.datastore.core.util.ValueUtil; import org.springframework.context.ApplicationEvent; @@ -463,8 +462,8 @@ private List getReferenceEntitiesForSave(Object entity, Builder builder, return; } Value value; - if (persistentProperty.isCollectionLike()) { - if (LazyUtil.hasUsableKeys(val)) { + if (persistentProperty.isCollectionLike() || persistentProperty.getType() == Optional.class) { + if (LazyUtil.isLazyAndNotLoaded(val)) { value = ListValue.of(LazyUtil.getKeys(val)); } else { @@ -603,22 +602,8 @@ private void resolveReferenceProperties(DatastorePersistentEntity datastoreP .getInverse(); String fieldName = referenceProperty.getFieldName(); if (entity.contains(fieldName) && !entity.isNull(fieldName)) { - Class type = referenceProperty.getType(); - Object referenced; - if (referenceProperty.isLazyLoaded() && referenceProperty.isCollectionLike()) { - List keyList = entity.getList(fieldName); - DatastoreReaderWriter originalTx = getDatastoreReadWriter(); - referenced = LazyUtil.wrapSimpleLazyProxy(() -> { - if (getDatastoreReadWriter() != originalTx) { - throw new DatastoreDataException("Lazy load should be invoked within the same transaction"); - } - return fetchReferenced(referenceProperty, context, - valuesToKeys(keyList)); - }, type, keyList); - } - else { - referenced = findReferenced(entity, referenceProperty, context); - } + Class type = referenceProperty.getType(); + Object referenced = computeReferencedField(entity, context, referenceProperty, fieldName, type); if (referenced != null) { datastorePersistentEntity.getPropertyAccessor(convertedObject) .setProperty(referenceProperty, referenced); @@ -627,13 +612,34 @@ private void resolveReferenceProperties(DatastorePersistentEntity datastoreP }); } + private T computeReferencedField(BaseEntity entity, ReadContext context, + DatastorePersistentProperty referenceProperty, String fieldName, Class type) { + T referenced; + if (referenceProperty.isLazyLoaded() && referenceProperty.isCollectionLike()) { + List> keyList = entity.getList(fieldName); + DatastoreReaderWriter originalTx = getDatastoreReadWriter(); + referenced = LazyUtil.wrapSimpleLazyProxy((List> storedKeys) -> { + if (getDatastoreReadWriter() != originalTx) { + throw new DatastoreDataException("Lazy load should be invoked within the same transaction"); + } + return (T) fetchReferenced(referenceProperty, context, valuesToKeys(storedKeys)); + }, type, keyList); + } + else { + referenced = (T) findReferenced(entity, referenceProperty, context); + } + return referenced; + } + + // Extracts key(s) from a property, fetches and if necessary, converts values to the required type private Object findReferenced(BaseEntity entity, DatastorePersistentProperty referencePersistentProperty, ReadContext context) { String fieldName = referencePersistentProperty.getFieldName(); try { Object referenced; if (referencePersistentProperty.isCollectionLike()) { - referenced = fetchReferenced(referencePersistentProperty, context, valuesToKeys(entity.getList(fieldName))); + referenced = fetchReferenced(referencePersistentProperty, context, + valuesToKeys(entity.getList(fieldName))); } else { List referencedList = findAllById(Collections.singleton(entity.getKey(fieldName)), @@ -650,7 +656,9 @@ private Object findReferenced(BaseEntity entity, DatastorePersistentProperty ref } } - private Object fetchReferenced(DatastorePersistentProperty referencePersistentProperty, ReadContext context, Set keys) { + // Given keys, fetches and converts values to the required collection type + private Object fetchReferenced(DatastorePersistentProperty referencePersistentProperty, ReadContext context, + Set keys) { Class referencedType = referencePersistentProperty.getComponentType(); return this.datastoreEntityConverter.getConversions() .convertOnRead( diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/LazyUtil.java similarity index 61% rename from spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java rename to spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/LazyUtil.java index dea2ac1cd2..b09697c1e8 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/util/LazyUtil.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/LazyUtil.java @@ -14,45 +14,58 @@ * limitations under the License. */ -package org.springframework.cloud.gcp.data.datastore.core.util; +package org.springframework.cloud.gcp.data.datastore.core; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.List; -import java.util.function.Supplier; +import java.util.function.Function; + +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.Value; import org.springframework.cloud.gcp.data.datastore.core.mapping.DatastoreDataException; import org.springframework.util.Assert; /** + * Utilities used to support lazy loaded properties. + * * @author Dmitry Solomakha + * + * @since 1.3 */ -final public class LazyUtil { +final class LazyUtil { private LazyUtil() { } - public static T wrapSimpleLazyProxy(Supplier supplierFunc, Class type, List keys) { + public static T wrapSimpleLazyProxy(Function>, T> supplierFunc, Class type, List keys) { return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class[] {type}, new SimpleLazyDynamicInvocationHandler(supplierFunc, keys)); } - public static boolean hasUsableKeys(Object object) { - if (Proxy.isProxyClass(object.getClass()) - && (Proxy.getInvocationHandler(object) instanceof SimpleLazyDynamicInvocationHandler)) { - SimpleLazyDynamicInvocationHandler handler = (SimpleLazyDynamicInvocationHandler) Proxy - .getInvocationHandler(object); + /** + * Check if the object is a lazy loaded proxy that hasn't been evaluated. + * @param object an object + * @return true if the object is a proxy that was not evaluated + */ + public static boolean isLazyAndNotLoaded(Object object) { + SimpleLazyDynamicInvocationHandler handler = getProxy(object); + if (handler != null) { return !handler.isEvaluated() && handler.getKeys() != null; } return false; } + /** + * Extract keys from a proxy object. + * @param object a proxy object + * @return list of keys if the object is a proxy, null otherwise + */ public static List getKeys(Object object) { - if (Proxy.isProxyClass(object.getClass()) - && (Proxy.getInvocationHandler(object) instanceof SimpleLazyDynamicInvocationHandler)) { - SimpleLazyDynamicInvocationHandler handler = (SimpleLazyDynamicInvocationHandler) Proxy - .getInvocationHandler(object); + SimpleLazyDynamicInvocationHandler handler = getProxy(object); + if (handler != null) { if (!handler.isEvaluated()) { return handler.getKeys(); } @@ -60,17 +73,29 @@ public static List getKeys(Object object) { return null; } + private static SimpleLazyDynamicInvocationHandler getProxy(Object object) { + if (Proxy.isProxyClass(object.getClass()) + && (Proxy.getInvocationHandler(object) instanceof SimpleLazyDynamicInvocationHandler)) { + return (SimpleLazyDynamicInvocationHandler) Proxy + .getInvocationHandler(object); + } + return null; + } + + /** + * Proxy class used for lazy loading. + */ public static final class SimpleLazyDynamicInvocationHandler implements InvocationHandler { - private final Supplier supplierFunc; + private final Function>, T> supplierFunc; - private final List keys; + private final List> keys; private boolean isEvaluated = false; private T value; - private SimpleLazyDynamicInvocationHandler(Supplier supplierFunc, List keys) { + private SimpleLazyDynamicInvocationHandler(Function>, T> supplierFunc, List> keys) { Assert.notNull(supplierFunc, "A non-null supplier function is required for a lazy proxy."); this.supplierFunc = supplierFunc; this.keys = keys; @@ -87,7 +112,7 @@ public List getKeys() { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { if (!this.isEvaluated) { - T value = this.supplierFunc.get(); + T value = this.supplierFunc.apply(this.keys); if (value == null) { throw new DatastoreDataException("Can't load referenced entity"); } @@ -98,5 +123,4 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl return method.invoke(this.value, args); } } - } diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java index 34fc8817c6..b90d10ad8a 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java @@ -80,6 +80,10 @@ private void verify() { "Only collection-like properties can contain the " + "descendant entity objects can be annotated @Descendants."); } + if (isLazyLoaded() && !isCollectionLike()) { + throw new DatastoreDataException( + "Only collection-like properties can be lazy-loaded"); + } } @Override @@ -143,7 +147,6 @@ public Iterable> getPersistentEntityTypes() { @Override public boolean isLazyLoaded() { - ReferenceCollection annotation = findAnnotation(ReferenceCollection.class); - return annotation != null && annotation.lazy(); + return findAnnotation(LazyReferenceCollection.class) != null; } } diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReferenceCollection.java similarity index 75% rename from spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java rename to spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReferenceCollection.java index 1638639d88..f8975c5d68 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/ReferenceCollection.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReferenceCollection.java @@ -25,7 +25,7 @@ import org.springframework.data.annotation.Reference; /** - * Annotation for a class that indicates it is an entity stored in a Datastore Entity. + * Annotation for a class that indicates that a property is a collection of lazy loaded Datastore entities. * * @author Dmitry Solomakha * @@ -35,13 +35,5 @@ @Target(ElementType.FIELD) @Retention(RetentionPolicy.RUNTIME) @Reference -public @interface ReferenceCollection { - - /** - * Controls whether the referenced entity should be loaded lazily. This defaults to - * {@literal false}. - * - * @return whether the interleaved property is retrieved lazily. - */ - boolean lazy() default false; +public @interface LazyReferenceCollection { } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java index ec25f8fb44..f80dc603cf 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java @@ -69,14 +69,13 @@ import org.springframework.cloud.gcp.data.datastore.core.mapping.DiscriminatorField; import org.springframework.cloud.gcp.data.datastore.core.mapping.DiscriminatorValue; import org.springframework.cloud.gcp.data.datastore.core.mapping.Field; -import org.springframework.cloud.gcp.data.datastore.core.mapping.ReferenceCollection; +import org.springframework.cloud.gcp.data.datastore.core.mapping.LazyReferenceCollection; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterFindByKeyEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterQueryEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterSaveEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.BeforeSaveEvent; -import org.springframework.cloud.gcp.data.datastore.core.util.LazyUtil; import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationEventPublisher; import org.springframework.data.annotation.Id; @@ -501,6 +500,17 @@ public void findAllReferenceLoopTest() { }); } + @Test + public void nonCollectionLazyException() { + this.expectedEx.expect(DatastoreDataException.class); + this.expectedEx.expectMessage("Only collection-like properties can be lazy-loaded"); + + when(this.objectToKeyFactory.allocateKeyForObject(any(), any(), any())).thenReturn(createFakeKey("fakeKey")); + BadLazyReferenceTestEntity entity = new BadLazyReferenceTestEntity(); + entity.lazyChild = new ReferenceTestEntity(); + this.datastoreTemplate.save(entity); + } + @Test public void saveReferenceLoopTest() { ReferenceTestEntity referenceTestEntity = new ReferenceTestEntity(); @@ -530,7 +540,7 @@ public void saveTest() { @Test public void saveTestLazy() { this.ob1.lazyMultipleReference = LazyUtil.wrapSimpleLazyProxy( - () -> Collections.singletonList(this.childEntity7), List.class, + (keys) -> Collections.singletonList(this.childEntity7), List.class, Collections.singletonList(KeyValue.of(this.childKey7))); saveTestCommon(this.ob1, true); } @@ -1181,7 +1191,7 @@ private static class TestEntity { @Reference LinkedList multipleReference; - @ReferenceCollection(lazy = true) + @LazyReferenceCollection List lazyMultipleReference; @@ -1255,8 +1265,16 @@ class ReferenceTestEntity { @Reference ReferenceTestEntity sibling; - @ReferenceCollection(lazy = true) + @LazyReferenceCollection List lazyChildren; } + class BadLazyReferenceTestEntity { + @Id + Long id; + + @LazyReferenceCollection + ReferenceTestEntity lazyChild; + } + } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java index 50f237be53..18715bd513 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java @@ -155,7 +155,7 @@ private static class TestEntity { @Descendants List linkedEntity; - @ReferenceCollection + @Reference TestSubEntity linkedEntityRef; } @@ -182,7 +182,7 @@ private static class FieldReferenceAnnotatedEntity { private static class FieldReferencedAnnotatedEntity { @Field(name = "name") - @ReferenceCollection + @LazyReferenceCollection TestSubEntity[] subEntity; } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java index 609b736d2b..b89fdc0894 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/DatastoreIntegrationTests.java @@ -516,19 +516,13 @@ public void ancestorsTest() { @Test public void referenceTest() { - ReferenceEntry child1 = new ReferenceEntry("child1", null, null); - ReferenceEntry child2 = new ReferenceEntry("child2", null, null); - ReferenceEntry sibling = new ReferenceEntry("sibling", null, null); - ReferenceEntry parent = new ReferenceEntry("parent", sibling, Arrays.asList(child1, child2)); - - this.datastoreTemplate.save(parent); - waitUntilTrue(() -> this.datastoreTemplate.findAll(ReferenceEntry.class).size() == 4); + ReferenceEntry parent = saveEntitiesGraph(); ReferenceEntry loadedParent = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); assertThat(loadedParent).isEqualTo(parent); parent.name = "parent updated"; - parent.childeren.forEach((child) -> child.name = child.name + " updated"); + parent.children.forEach((child) -> child.name = child.name + " updated"); parent.sibling.name = "sibling updated"; this.datastoreTemplate.save(parent); @@ -539,24 +533,43 @@ public void referenceTest() { ReferenceEntry loadedParentAfterUpdate = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); assertThat(loadedParentAfterUpdate).isEqualTo(parent); + } + + @Test + public void lazyReferenceTest() { + ReferenceEntry parent = saveEntitiesGraph(); //Saving an entity with not loaded lazy field - parent = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); + ReferenceEntry lazyParent = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); - this.datastoreTemplate.save(parent); + this.datastoreTemplate.save(lazyParent); - loadedParent = this.datastoreTemplate.findById(parent.id, ReferenceEntry.class); - assertThat(loadedParent).isEqualTo(parent); + ReferenceEntry loadedParent = this.datastoreTemplate.findById(lazyParent.id, ReferenceEntry.class); + assertThat(loadedParent.children).containsExactlyInAnyOrder(parent.children.toArray(new ReferenceEntry[0])); + } + + @Test + public void lazyReferenceTransactionTest() { + ReferenceEntry parent = saveEntitiesGraph(); //Exception should be produced if a lazy loaded property accessed outside of the initial transaction ReferenceEntry finalLoadedParent = this.transactionalTemplateService.findByIdLazy(parent.id); - assertThatThrownBy(() -> finalLoadedParent.childeren.size()).isInstanceOf(DatastoreDataException.class) + assertThatThrownBy(() -> finalLoadedParent.children.size()).isInstanceOf(DatastoreDataException.class) .hasMessage("Lazy load should be invoked within the same transaction"); //No exception should be produced if a lazy loaded property accessed within the initial transaction - ReferenceEntry finalLoadedParentLazyLoaded = this.transactionalTemplateService.findByIdLazyLoad(parent.id); + ReferenceEntry finalLoadedParentLazyLoaded = this.transactionalTemplateService.findByIdLazyAndLoad(parent.id); assertThat(finalLoadedParentLazyLoaded).isEqualTo(parent); + } + private ReferenceEntry saveEntitiesGraph() { + ReferenceEntry child1 = new ReferenceEntry("child1", null, null); + ReferenceEntry child2 = new ReferenceEntry("child2", null, null); + ReferenceEntry sibling = new ReferenceEntry("sibling", null, null); + ReferenceEntry parent = new ReferenceEntry("parent", sibling, Arrays.asList(child1, child2)); + this.datastoreTemplate.save(parent); + waitUntilTrue(() -> this.datastoreTemplate.findAll(ReferenceEntry.class).size() == 4); + return parent; } @Test diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java index 2e3ea4a2a2..bf9c4056e7 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Objects; -import org.springframework.cloud.gcp.data.datastore.core.mapping.ReferenceCollection; +import org.springframework.cloud.gcp.data.datastore.core.mapping.LazyReferenceCollection; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Reference; @@ -39,13 +39,13 @@ public class ReferenceEntry { @Reference ReferenceEntry sibling; - @ReferenceCollection(lazy = true) - List childeren; + @LazyReferenceCollection + List children; - ReferenceEntry(String name, ReferenceEntry sibling, List childeren) { + ReferenceEntry(String name, ReferenceEntry sibling, List children) { this.name = name; this.sibling = sibling; - this.childeren = childeren; + this.children = children; } @Override @@ -60,14 +60,14 @@ public boolean equals(Object o) { return Objects.equals(this.id, that.id) && Objects.equals(this.name, that.name) && Objects.equals(this.sibling, that.sibling) && - new HashSet<>((this.childeren != null) ? this.childeren : Collections.emptyList()) - .equals(new HashSet<>((that.childeren != null) ? that.childeren : Collections.emptyList())); + new HashSet<>((this.children != null) ? this.children : Collections.emptyList()) + .equals(new HashSet<>((that.children != null) ? that.children : Collections.emptyList())); } @Override public int hashCode() { - return Objects.hash(this.id, this.name, this.sibling, this.childeren); + return Objects.hash(this.id, this.name, this.sibling, this.children); } @Override @@ -76,7 +76,7 @@ public String toString() { "id=" + this.id + ", name='" + this.name + '\'' + ", sibling=" + this.sibling + - ", childeren=" + this.childeren + + ", childeren=" + this.children + '}'; } } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java index ee91b79edd..5db1ea5343 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/TransactionalTemplateService.java @@ -76,15 +76,15 @@ public TestEntity findByIdInReadOnly(long id) { return this.datastoreTemplate.findById(id, TestEntity.class); } - @Transactional(readOnly = true) + @Transactional public ReferenceEntry findByIdLazy(long id) { return this.datastoreTemplate.findById(id, ReferenceEntry.class); } - @Transactional(readOnly = true) - public ReferenceEntry findByIdLazyLoad(long id) { + @Transactional + public ReferenceEntry findByIdLazyAndLoad(long id) { ReferenceEntry entry = this.datastoreTemplate.findById(id, ReferenceEntry.class); - entry.childeren.size(); + entry.children.size(); return entry; } } From 5d7b9035f93aed25e30f4f65a44207310140daf9 Mon Sep 17 00:00:00 2001 From: dmitry-s Date: Mon, 6 Jan 2020 13:54:35 -0500 Subject: [PATCH 3/4] Update docs/src/main/asciidoc/datastore.adoc Co-Authored-By: Mike Eltsufin --- docs/src/main/asciidoc/datastore.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/main/asciidoc/datastore.adoc b/docs/src/main/asciidoc/datastore.adoc index 8598d841b7..c824efbd09 100644 --- a/docs/src/main/asciidoc/datastore.adoc +++ b/docs/src/main/asciidoc/datastore.adoc @@ -425,7 +425,7 @@ There are three ways to represent relationships between entities that are descri * Embedded entities stored directly in the field of the containing entity * `@Descendant` annotated properties for one-to-many relationships * `@Reference` annotated properties for general relationships without hierarchy -* `@LazyReferenceCollection` similar to `@Reference`, but the entities are lazy loaded when the property is accessed (note that the children keys retrieved when the parent entity is loaded) +* `@LazyReferenceCollection` similar to `@Reference`, but the entities are lazy-loaded when the property is accessed. (Note that the keys of the children are retrieved when the parent entity is loaded.) ==== Embedded Entities From aa9f81acce91587edb08b7bfc408ecd024cdcb72 Mon Sep 17 00:00:00 2001 From: dsolomakha Date: Mon, 6 Jan 2020 16:10:37 -0500 Subject: [PATCH 4/4] support lazy loading --- docs/src/main/asciidoc/datastore.adoc | 4 +++- .../core/mapping/DatastorePersistentPropertyImpl.java | 2 +- .../{LazyReferenceCollection.java => LazyReference.java} | 2 +- .../gcp/data/datastore/core/DatastoreTemplateTests.java | 8 ++++---- .../mapping/DatastorePersistentPropertyImplTests.java | 2 +- .../cloud/gcp/data/datastore/it/ReferenceEntry.java | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) rename spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/{LazyReferenceCollection.java => LazyReference.java} (96%) diff --git a/docs/src/main/asciidoc/datastore.adoc b/docs/src/main/asciidoc/datastore.adoc index c824efbd09..d917e3370b 100644 --- a/docs/src/main/asciidoc/datastore.adoc +++ b/docs/src/main/asciidoc/datastore.adoc @@ -425,7 +425,9 @@ There are three ways to represent relationships between entities that are descri * Embedded entities stored directly in the field of the containing entity * `@Descendant` annotated properties for one-to-many relationships * `@Reference` annotated properties for general relationships without hierarchy -* `@LazyReferenceCollection` similar to `@Reference`, but the entities are lazy-loaded when the property is accessed. (Note that the keys of the children are retrieved when the parent entity is loaded.) +* `@LazyReference` similar to `@Reference`, but the entities are lazy-loaded when the property is accessed. + Supports collections only. + (Note that the keys of the children are retrieved when the parent entity is loaded.) ==== Embedded Entities diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java index b90d10ad8a..512e323f3d 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImpl.java @@ -147,6 +147,6 @@ public Iterable> getPersistentEntityTypes() { @Override public boolean isLazyLoaded() { - return findAnnotation(LazyReferenceCollection.class) != null; + return findAnnotation(LazyReference.class) != null; } } diff --git a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReferenceCollection.java b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReference.java similarity index 96% rename from spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReferenceCollection.java rename to spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReference.java index f8975c5d68..d0be111345 100644 --- a/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReferenceCollection.java +++ b/spring-cloud-gcp-data-datastore/src/main/java/org/springframework/cloud/gcp/data/datastore/core/mapping/LazyReference.java @@ -35,5 +35,5 @@ @Target(ElementType.FIELD) @Retention(RetentionPolicy.RUNTIME) @Reference -public @interface LazyReferenceCollection { +public @interface LazyReference { } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java index f80dc603cf..1f42505cac 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/DatastoreTemplateTests.java @@ -69,7 +69,7 @@ import org.springframework.cloud.gcp.data.datastore.core.mapping.DiscriminatorField; import org.springframework.cloud.gcp.data.datastore.core.mapping.DiscriminatorValue; import org.springframework.cloud.gcp.data.datastore.core.mapping.Field; -import org.springframework.cloud.gcp.data.datastore.core.mapping.LazyReferenceCollection; +import org.springframework.cloud.gcp.data.datastore.core.mapping.LazyReference; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterDeleteEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterFindByKeyEvent; import org.springframework.cloud.gcp.data.datastore.core.mapping.event.AfterQueryEvent; @@ -1191,7 +1191,7 @@ private static class TestEntity { @Reference LinkedList multipleReference; - @LazyReferenceCollection + @LazyReference List lazyMultipleReference; @@ -1265,7 +1265,7 @@ class ReferenceTestEntity { @Reference ReferenceTestEntity sibling; - @LazyReferenceCollection + @LazyReference List lazyChildren; } @@ -1273,7 +1273,7 @@ class BadLazyReferenceTestEntity { @Id Long id; - @LazyReferenceCollection + @LazyReference ReferenceTestEntity lazyChild; } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java index 18715bd513..c6f6148145 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/core/mapping/DatastorePersistentPropertyImplTests.java @@ -182,7 +182,7 @@ private static class FieldReferenceAnnotatedEntity { private static class FieldReferencedAnnotatedEntity { @Field(name = "name") - @LazyReferenceCollection + @LazyReference TestSubEntity[] subEntity; } diff --git a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java index bf9c4056e7..e2a627b090 100644 --- a/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java +++ b/spring-cloud-gcp-data-datastore/src/test/java/org/springframework/cloud/gcp/data/datastore/it/ReferenceEntry.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.Objects; -import org.springframework.cloud.gcp.data.datastore.core.mapping.LazyReferenceCollection; +import org.springframework.cloud.gcp.data.datastore.core.mapping.LazyReference; import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Reference; @@ -39,7 +39,7 @@ public class ReferenceEntry { @Reference ReferenceEntry sibling; - @LazyReferenceCollection + @LazyReference List children; ReferenceEntry(String name, ReferenceEntry sibling, List children) {