From e0784699b1320f36b7df456666ea52e656f31d5b Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Tue, 22 Sep 2015 18:42:33 -0700 Subject: [PATCH 1/5] Updates to write-related code for datastore v1beta3 update --- .../datastore/BaseDatastoreBatchWriter.java | 34 ++-- .../google/gcloud/datastore/BaseEntity.java | 32 ++-- .../google/gcloud/datastore/BatchImpl.java | 54 +++--- .../google/gcloud/datastore/BatchOption.java | 58 ------ .../google/gcloud/datastore/Datastore.java | 7 +- .../gcloud/datastore/DatastoreHelper.java | 5 +- .../gcloud/datastore/DatastoreImpl.java | 165 ++++++++++-------- .../gcloud/datastore/DatastoreOptions.java | 26 +-- .../com/google/gcloud/datastore/Entity.java | 3 +- .../google/gcloud/datastore/EntityValue.java | 7 +- .../google/gcloud/datastore/FullEntity.java | 4 +- .../gcloud/datastore/ProjectionEntity.java | 3 +- .../com/google/gcloud/datastore/Query.java | 12 +- .../gcloud/datastore/QueryResultsImpl.java | 3 +- .../gcloud/datastore/TransactionImpl.java | 65 +++---- .../gcloud/datastore/TransactionOption.java | 114 ------------ .../com/google/gcloud/spi/DatastoreRpc.java | 31 ++-- .../gcloud/spi/DefaultDatastoreRpc.java | 94 +++++----- .../BaseDatastoreBatchWriterTest.java | 156 +++++++---------- .../google/gcloud/datastore/BaseKeyTest.java | 1 - .../datastore/DatastoreOptionsTest.java | 2 +- .../gcloud/datastore/DatastoreTest.java | 30 ++-- .../gcloud/datastore/KeyFactoryTest.java | 1 - .../gcloud/datastore/LocalGcdHelper.java | 12 +- 24 files changed, 349 insertions(+), 570 deletions(-) delete mode 100644 gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchOption.java delete mode 100644 gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionOption.java diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java index d94eafbb42f6..6d8ac86834de 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java @@ -16,11 +16,11 @@ package com.google.gcloud.datastore; -import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; @@ -88,9 +88,7 @@ public final List add(FullEntity... entities) { for (FullEntity entity : entities) { IncompleteKey key = entity.key(); Preconditions.checkArgument(key != null, "Entity must have a key"); - if (key instanceof Key) { - addInternal((FullEntity) entity); - } else { + if (!(key instanceof Key)) { incompleteKeys.add(key); } } @@ -104,6 +102,7 @@ public final List add(FullEntity... entities) { List answer = Lists.newArrayListWithExpectedSize(entities.length); for (FullEntity entity : entities) { if (entity.key() instanceof Key) { + addInternal((FullEntity) entity); answer.add(Entity.convert((FullEntity) entity)); } else { Entity entityWithAllocatedId = Entity.builder(allocated.next(), entity).build(); @@ -185,6 +184,10 @@ protected Set toDelete() { return toDelete; } + protected int numAutoAllocatedIds() { + return toAddAutoId.size(); + } + protected void deactivate() { active = false; } @@ -199,25 +202,30 @@ protected DatastoreException newInvalidRequest(String msg, Object... params) { return DatastoreException.throwInvalidRequest(String.format(msg, params)); } - protected DatastoreV1.Mutation.Builder toMutationPb() { - DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + protected List toMutationPbList() { + List mutationsPb = + new ArrayList<>(); for (FullEntity entity : toAddAutoId()) { - mutationPb.addInsertAutoId(entity.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setInsert(entity.toPb()).build()); } for (FullEntity entity : toAdd().values()) { - mutationPb.addInsert(entity.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setInsert(entity.toPb()).build()); } for (FullEntity entity : toUpdate().values()) { - mutationPb.addUpdate(entity.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setUpdate(entity.toPb()).build()); } for (FullEntity entity : toPut().values()) { - mutationPb.addUpsert(entity.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); } for (Key key : toDelete()) { - // TODO(ajaykannan): fix me! - //mutationPb.addDelete(key.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(key.toPb()).build()); } - return mutationPb; + return mutationsPb; } protected abstract Datastore datastore(); diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java index 8608e4fcedc9..97718e708e83 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java @@ -27,7 +27,6 @@ import static com.google.gcloud.datastore.NullValue.of; import static com.google.gcloud.datastore.StringValue.of; -import com.google.api.services.datastore.DatastoreV1; import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Maps; import com.google.protobuf.InvalidProtocolBufferException; @@ -48,7 +47,8 @@ * @see Google Cloud Datastore * Entities, Properties, and Keys */ -public abstract class BaseEntity extends Serializable { +public abstract class BaseEntity + extends Serializable { private static final long serialVersionUID = 8175618724683792766L; @@ -90,16 +90,15 @@ private B self() { } @SuppressWarnings("unchecked") - protected B fill(DatastoreV1.Entity entityPb) { + protected B fill(com.google.datastore.v1beta3.Entity entityPb) { Map> copiedProperties = Maps.newHashMap(); - for (DatastoreV1.Property property : entityPb.getPropertyList()) { - // TODO(ajaykannan): fix me! - //copiedProperties.put(property.getName(), Value.fromPb(property.getValue())); + for (Map.Entry entry : + entityPb.getProperties().entrySet()) { + copiedProperties.put(entry.getKey(), Value.fromPb(entry.getValue())); } properties(copiedProperties); if (entityPb.hasKey()) { - // TODO(ajaykannan): fix me! - //key((K) IncompleteKey.fromPb(entityPb.getKey())); + key((K) IncompleteKey.fromPb(entityPb.getKey())); } return self(); } @@ -379,25 +378,22 @@ ImmutableSortedMap> properties() { @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { Builder builder = emptyBuilder(); - builder.fill(DatastoreV1.Entity.parseFrom(bytesPb)); + builder.fill(com.google.datastore.v1beta3.Entity.parseFrom(bytesPb)); return builder.build(); } protected abstract Builder emptyBuilder(); @Override - protected final DatastoreV1.Entity toPb() { - DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); + protected final com.google.datastore.v1beta3.Entity toPb() { + com.google.datastore.v1beta3.Entity.Builder entityPb = + com.google.datastore.v1beta3.Entity.newBuilder(); + Map propertiesPb = entityPb.getMutableProperties(); for (Map.Entry> entry : properties.entrySet()) { - DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); - propertyPb.setName(entry.getKey()); - // TODO(ajaykannan): fix me! - //propertyPb.setValue(entry.getValue().toPb()); - entityPb.addProperty(propertyPb.build()); + propertiesPb.put(entry.getKey(), entry.getValue().toPb()); } if (key != null) { - // TODO(ajaykannan): fix me! - //entityPb.setKey(key.toPb()); + entityPb.setKey(key.toPb()); } return entityPb.build(); } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java index c86ae28b3344..a9eab9e9c3f1 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java @@ -16,65 +16,53 @@ package com.google.gcloud.datastore; -import com.google.api.services.datastore.DatastoreV1; -import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.gcloud.datastore.BatchOption.ForceWrites; - +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; -import java.util.Map; class BatchImpl extends BaseDatastoreBatchWriter implements Batch { private final DatastoreImpl datastore; - private final boolean force; static class ResponseImpl implements Batch.Response { - private final DatastoreV1.CommitResponse response; + private final com.google.datastore.v1beta3.CommitResponse response; + private final int numAutoAllocatedIds; - ResponseImpl(DatastoreV1.CommitResponse response) { + ResponseImpl(com.google.datastore.v1beta3.CommitResponse response, int numAutoAllocatedIds) { this.response = response; + this.numAutoAllocatedIds = numAutoAllocatedIds; } @Override public List generatedKeys() { - return Lists.transform(response.getMutationResult().getInsertAutoIdKeyList(), - new Function() { - @Override public Key apply(DatastoreV1.Key keyPb) { - // TODO(ajaykannan): fix me! - //return Key.fromPb(keyPb); - return Key.builder(null).build(); // TODO(ajaykannan): fix me! - } - }); + Iterator results = + response.getMutationResultsList().iterator(); + List generated = new LinkedList(); + for (int i = 0; i < numAutoAllocatedIds; i++) { + generated.add(Key.fromPb(results.next().getKey())); + } + return generated; } } - BatchImpl(DatastoreImpl datastore, BatchOption... options) { + BatchImpl(DatastoreImpl datastore) { super("batch"); this.datastore = datastore; - Map, BatchOption> optionsMap = BatchOption.asImmutableMap(options); - if (optionsMap.containsKey(ForceWrites.class)) { - force = ((ForceWrites) optionsMap.get(ForceWrites.class)).force(); - } else { - force = datastore.options().force(); - } } @Override public Batch.Response submit() { validateActive(); - DatastoreV1.Mutation.Builder mutationPb = toMutationPb(); - if (force) { - mutationPb.setForce(force); - } - DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); - requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); - requestPb.setMutation(mutationPb); - DatastoreV1.CommitResponse responsePb = datastore.commit(requestPb.build()); + List mutationsPb = toMutationPbList(); + com.google.datastore.v1beta3.CommitRequest.Builder requestPb = + com.google.datastore.v1beta3.CommitRequest.newBuilder(); + requestPb.setMode(com.google.datastore.v1beta3.CommitRequest.Mode.NON_TRANSACTIONAL); + requestPb.addAllMutations(mutationsPb); + com.google.datastore.v1beta3.CommitResponse responsePb = datastore.commit(requestPb.build()); deactivate(); - return new ResponseImpl(responsePb); + return new ResponseImpl(responsePb, numAutoAllocatedIds()); } @Override diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchOption.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchOption.java deleted file mode 100644 index 362a74e96c79..000000000000 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchOption.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.datastore; - -import com.google.common.collect.ImmutableMap; - -import java.util.Map; - -public abstract class BatchOption implements java.io.Serializable { - - private static final long serialVersionUID = -3932758377282659839L; - - public static final class ForceWrites extends BatchOption { - - private static final long serialVersionUID = 2555054296046232799L; - - private final boolean force; - - public ForceWrites(boolean force) { - this.force = force; - } - - public boolean force() { - return force; - } - } - - BatchOption() { - // package protected - } - - public static ForceWrites forceWrites() { - return new ForceWrites(true); - } - - static Map, BatchOption> asImmutableMap(BatchOption... options) { - ImmutableMap.Builder, BatchOption> builder = - ImmutableMap.builder(); - for (BatchOption option : options) { - builder.put(option.getClass(), option); - } - return builder.build(); - } -} diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java index 870ed8d9474f..b49db1cacdfe 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java @@ -30,7 +30,7 @@ public interface Datastore extends Service, DatastoreReaderWri * * @throws DatastoreException upon failure */ - Transaction newTransaction(TransactionOption... options); + Transaction newTransaction(); /** @@ -54,15 +54,14 @@ interface TransactionCallable { * as a {@link DatastoreException} with the original exception as its root cause. * * @param callable the callback to call with a newly created transactional readerWriter - * @param options the options for the created transaction * @throws DatastoreException upon failure */ - T runInTransaction(TransactionCallable callable, TransactionOption... options); + T runInTransaction(TransactionCallable callable); /** * Returns a new Batch for processing multiple write operations in one request. */ - Batch newBatch(BatchOption... options); + Batch newBatch(); /** * Allocate a unique id for the given key. diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java index a74d06642740..5ea0f755d2f8 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java @@ -69,9 +69,8 @@ static List fetch(DatastoreReader reader, Key... keys) { return list; } - static T runInTransaction(Datastore datastore, - Datastore.TransactionCallable callable, TransactionOption... options) { - Transaction transaction = datastore.newTransaction(options); + static T runInTransaction(Datastore datastore, Datastore.TransactionCallable callable) { + Transaction transaction = datastore.newTransaction(); try { T value = callable.run(transaction); transaction.commit(); diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java index c21c0c2a7087..262098e82f96 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java @@ -16,7 +16,6 @@ package com.google.gcloud.datastore; -import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.AbstractIterator; @@ -33,6 +32,7 @@ import com.google.protobuf.ByteString; import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; @@ -79,18 +79,18 @@ public RetryResult beforeEval(Exception exception) { } @Override - public Batch newBatch(BatchOption... options) { - return new BatchImpl(this, options); + public Batch newBatch() { + return new BatchImpl(this); } @Override - public Transaction newTransaction(TransactionOption... options) { - return new TransactionImpl(this, options); + public Transaction newTransaction() { + return new TransactionImpl(this); } @Override - public T runInTransaction(TransactionCallable callable, TransactionOption... options) { - return DatastoreHelper.runInTransaction(this, callable, options); + public T runInTransaction(TransactionCallable callable) { + return DatastoreHelper.runInTransaction(this, callable); } @Override @@ -98,14 +98,19 @@ public QueryResults run(Query query) { return run(null, query); } - QueryResults run(DatastoreV1.ReadOptions readOptionsPb, Query query) { - return new QueryResultsImpl<>(this, readOptionsPb, query); + QueryResults run(com.google.datastore.v1beta3.ReadOptions readOptionsPb, Query query) { + // TODO(ajaykannan): fix me! + //return new QueryResultsImpl<>(this, readOptionsPb, query); + return null; // TODO(ajaykannan): fix me! } - DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestPb) { + com.google.datastore.v1beta3.RunQueryResponse runQuery( + final com.google.datastore.v1beta3.RunQueryRequest requestPb) { try { - return RetryHelper.runWithRetries(new Callable() { - @Override public DatastoreV1.RunQueryResponse call() throws DatastoreRpcException { + return RetryHelper.runWithRetries( + new Callable() { + @Override public com.google.datastore.v1beta3.RunQueryResponse call() + throws DatastoreRpcException { return datastoreRpc.runQuery(requestPb); } }, retryParams, EXCEPTION_HANDLER); @@ -124,24 +129,26 @@ public List allocateId(IncompleteKey... keys) { if (keys.length == 0) { return Collections.emptyList(); } - DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder(); + com.google.datastore.v1beta3.AllocateIdsRequest.Builder requestPb = + com.google.datastore.v1beta3.AllocateIdsRequest.newBuilder(); for (IncompleteKey key : keys) { - // TODO(ajaykannan): fix me! - //requestPb.addKey(trimNameOrId(key).toPb()); + requestPb.addKeys(trimNameOrId(key).toPb()); } - DatastoreV1.AllocateIdsResponse responsePb = allocateIds(requestPb.build()); + com.google.datastore.v1beta3.AllocateIdsResponse responsePb = allocateIds(requestPb.build()); ImmutableList.Builder keyList = ImmutableList.builder(); - for (DatastoreV1.Key keyPb : responsePb.getKeyList()) { - // TODO(ajaykannan): fix me! - // keyList.add(Key.fromPb(keyPb)); + for (com.google.datastore.v1beta3.Key keyPb : responsePb.getKeysList()) { + keyList.add(Key.fromPb(keyPb)); } return keyList.build(); } - DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest requestPb) { + com.google.datastore.v1beta3.AllocateIdsResponse allocateIds( + final com.google.datastore.v1beta3.AllocateIdsRequest requestPb) { try { - return RetryHelper.runWithRetries(new Callable() { - @Override public DatastoreV1.AllocateIdsResponse call() throws DatastoreRpcException { + return RetryHelper.runWithRetries( + new Callable() { + @Override public com.google.datastore.v1beta3.AllocateIdsResponse call() + throws DatastoreRpcException { return datastoreRpc.allocateIds(requestPb); } }, retryParams, EXCEPTION_HANDLER); @@ -168,7 +175,7 @@ public List add(FullEntity... entities) { if (entities.length == 0) { return Collections.emptyList(); } - DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + List mutationsPb = new ArrayList<>(); Map completeEntities = new LinkedHashMap<>(); for (FullEntity entity : entities) { Entity completeEntity = null; @@ -180,23 +187,26 @@ public List add(FullEntity... entities) { throw DatastoreException.throwInvalidRequest( "Duplicate entity with the key %s", entity.key()); } - mutationPb.addInsert(completeEntity.toPb()); + mutationsPb.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(completeEntity.toPb()).build()); } else { Preconditions.checkArgument(entity.hasKey(), "entity %s is missing a key", entity); - mutationPb.addInsertAutoId(entity.toPb()); + mutationsPb.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(entity.toPb()).build()); } } - DatastoreV1.CommitResponse commitResponse = commitMutation(mutationPb); - Iterator allocatedKeys = - commitResponse.getMutationResult().getInsertAutoIdKeyList().iterator(); + com.google.datastore.v1beta3.CommitResponse commitResponse = commitMutation(mutationsPb); + Iterator mutationResults = + commitResponse.getMutationResultsList().iterator(); ImmutableList.Builder responseBuilder = ImmutableList.builder(); for (FullEntity entity : entities) { Entity completeEntity = completeEntities.get(entity.key()); if (completeEntity != null) { responseBuilder.add(completeEntity); + mutationResults.next(); } else { - // TODO(ajaykannan): fix me! - //responseBuilder.add(Entity.builder(Key.fromPb(allocatedKeys.next()), entity).build()); + responseBuilder.add( + Entity.builder(Key.fromPb(mutationResults.next().getKey()), entity).build()); } } return responseBuilder.build(); @@ -217,37 +227,37 @@ public List fetch(Key... keys) { return DatastoreHelper.fetch(this, keys); } - Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) { + Iterator get(com.google.datastore.v1beta3.ReadOptions readOptionsPb, final Key... keys) { if (keys.length == 0) { return Collections.emptyIterator(); } - DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder(); + com.google.datastore.v1beta3.LookupRequest.Builder requestPb = + com.google.datastore.v1beta3.LookupRequest.newBuilder(); if (readOptionsPb != null) { requestPb.setReadOptions(readOptionsPb); } for (Key k : Sets.newLinkedHashSet(Arrays.asList(keys))) { - // TODO(ajaykannan): fix me! - //requestPb.addKey(k.toPb()); + requestPb.addKeys(k.toPb()); } return new ResultsIterator(requestPb); } final class ResultsIterator extends AbstractIterator { - private final DatastoreV1.LookupRequest.Builder requestPb; - Iterator iter; + private final com.google.datastore.v1beta3.LookupRequest.Builder requestPb; + Iterator iter; - ResultsIterator(DatastoreV1.LookupRequest.Builder requestPb) { + ResultsIterator(com.google.datastore.v1beta3.LookupRequest.Builder requestPb) { this.requestPb = requestPb; loadResults(); } private void loadResults() { - DatastoreV1.LookupResponse responsePb = lookup(requestPb.build()); + com.google.datastore.v1beta3.LookupResponse responsePb = lookup(requestPb.build()); iter = responsePb.getFoundList().iterator(); - requestPb.clearKey(); + requestPb.clearKeys(); if (responsePb.getDeferredCount() > 0) { - requestPb.addAllKey(responsePb.getDeferredList()); + requestPb.addAllKeys(responsePb.getDeferredList()); } } @@ -255,7 +265,7 @@ private void loadResults() { @Override protected Entity computeNext() { while (!iter.hasNext()) { - if (requestPb.getKeyCount() == 0) { + if (requestPb.getKeysCount() == 0) { return endOfData(); } loadResults(); @@ -264,10 +274,13 @@ protected Entity computeNext() { } } - DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) { + com.google.datastore.v1beta3.LookupResponse lookup( + final com.google.datastore.v1beta3.LookupRequest requestPb) { try { - return RetryHelper.runWithRetries(new Callable() { - @Override public DatastoreV1.LookupResponse call() throws DatastoreRpcException { + return RetryHelper.runWithRetries( + new Callable() { + @Override public com.google.datastore.v1beta3.LookupResponse call() + throws DatastoreRpcException { return datastoreRpc.lookup(requestPb); } }, retryParams, EXCEPTION_HANDLER); @@ -280,15 +293,17 @@ DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) { @Override public final void update(Entity... entities) { if (entities.length > 0) { - DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + List mutationsPb = + new ArrayList<>(); Map dedupEntities = new LinkedHashMap<>(); for (Entity entity : entities) { dedupEntities.put(entity.key(), entity); } for (Entity entity : dedupEntities.values()) { - mutationPb.addUpdate(entity.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setUpdate(entity.toPb()).build()); } - commitMutation(mutationPb); + commitMutation(mutationsPb); } } @@ -296,28 +311,30 @@ public final void update(Entity... entities) { @Override public final void put(Entity... entities) { if (entities.length > 0) { - DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + List mutationsPb = + new ArrayList<>(); Map dedupEntities = new LinkedHashMap<>(); for (Entity entity : entities) { dedupEntities.put(entity.key(), entity); } for (Entity e : dedupEntities.values()) { - mutationPb.addUpsert(e.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(e.toPb()).build()); } - commitMutation(mutationPb); + commitMutation(mutationsPb); } } @Override public void delete(Key... keys) { if (keys.length > 0) { - DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + List mutationsPb = new ArrayList<>(); Set dedupKeys = new LinkedHashSet<>(Arrays.asList(keys)); for (Key key : dedupKeys) { - // TODO(ajaykannan): fix me! - //mutationPb.addDelete(key.toPb()); + mutationsPb.add( + com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(key.toPb()).build()); } - commitMutation(mutationPb); + commitMutation(mutationsPb); } } @@ -326,20 +343,22 @@ public KeyFactory newKeyFactory() { return DatastoreHelper.newKeyFactory(options()); } - private DatastoreV1.CommitResponse commitMutation(DatastoreV1.Mutation.Builder mutationPb) { - if (options().force()) { - mutationPb.setForce(true); - } - DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); - requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); - requestPb.setMutation(mutationPb); + private com.google.datastore.v1beta3.CommitResponse commitMutation( + List mutationsPb) { + com.google.datastore.v1beta3.CommitRequest.Builder requestPb = + com.google.datastore.v1beta3.CommitRequest.newBuilder(); + requestPb.setMode(com.google.datastore.v1beta3.CommitRequest.Mode.NON_TRANSACTIONAL); + requestPb.addAllMutations(mutationsPb); return commit(requestPb.build()); } - DatastoreV1.CommitResponse commit(final DatastoreV1.CommitRequest requestPb) { + com.google.datastore.v1beta3.CommitResponse commit( + final com.google.datastore.v1beta3.CommitRequest requestPb) { try { - return RetryHelper.runWithRetries(new Callable() { - @Override public DatastoreV1.CommitResponse call() throws DatastoreRpcException { + return RetryHelper.runWithRetries( + new Callable() { + @Override public com.google.datastore.v1beta3.CommitResponse call() + throws DatastoreRpcException { return datastoreRpc.commit(requestPb); } }, retryParams, EXCEPTION_HANDLER); @@ -348,16 +367,19 @@ DatastoreV1.CommitResponse commit(final DatastoreV1.CommitRequest requestPb) { } } - ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requestPb) { + ByteString requestTransactionId( + com.google.datastore.v1beta3.BeginTransactionRequest.Builder requestPb) { return beginTransaction(requestPb.build()).getTransaction(); } - DatastoreV1.BeginTransactionResponse beginTransaction( - final DatastoreV1.BeginTransactionRequest requestPb) { + com.google.datastore.v1beta3.BeginTransactionResponse beginTransaction( + final com.google.datastore.v1beta3.BeginTransactionRequest requestPb) { try { - return RetryHelper.runWithRetries(new Callable() { + return RetryHelper.runWithRetries( + new Callable() { @Override - public DatastoreV1.BeginTransactionResponse call() throws DatastoreRpcException { + public com.google.datastore.v1beta3.BeginTransactionResponse call() + throws DatastoreRpcException { return datastoreRpc.beginTransaction(requestPb); } }, retryParams, EXCEPTION_HANDLER); @@ -367,12 +389,13 @@ public DatastoreV1.BeginTransactionResponse call() throws DatastoreRpcException } void rollbackTransaction(ByteString transaction) { - DatastoreV1.RollbackRequest.Builder requestPb = DatastoreV1.RollbackRequest.newBuilder(); + com.google.datastore.v1beta3.RollbackRequest.Builder requestPb = + com.google.datastore.v1beta3.RollbackRequest.newBuilder(); requestPb.setTransaction(transaction); rollback(requestPb.build()); } - void rollback(final DatastoreV1.RollbackRequest requestPb) { + void rollback(final com.google.datastore.v1beta3.RollbackRequest requestPb) { try { RetryHelper.runWithRetries(new Callable() { @Override public Void call() throws DatastoreRpcException { diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java index 7b0dd3a2a606..23128a856761 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java @@ -18,9 +18,7 @@ import static com.google.gcloud.datastore.Validator.validateNamespace; -import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.EntityResult; -import com.google.api.services.datastore.DatastoreV1.LookupResponse; +import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.gcloud.ServiceOptions; @@ -102,21 +100,23 @@ private DatastoreOptions normalize() { Builder builder = toBuilder(); builder.normalizeDataset(false); // Replace provided project-id with full project-id (s~xxx, e~xxx,...) - DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder(); - DatastoreV1.Key key = DatastoreV1.Key.newBuilder() - .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar")) + com.google.datastore.v1beta3.LookupRequest.Builder requestPb = + com.google.datastore.v1beta3.LookupRequest.newBuilder(); + com.google.datastore.v1beta3.Key key = com.google.datastore.v1beta3.Key.newBuilder() + .addPath(com.google.datastore.v1beta3.Key.PathElement.newBuilder() + .setKind("__foo__").setName("bar")) .build(); - requestPb.addKey(key); + requestPb.addKeys(key); try { - LookupResponse responsePb = datastoreRpc().lookup(requestPb.build()); + com.google.datastore.v1beta3.LookupResponse responsePb = datastoreRpc().lookup(requestPb.build()); if (responsePb.getDeferredCount() > 0) { key = responsePb.getDeferred(0); } else { - Iterator combinedIter = + Iterator combinedIter = Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator(); key = combinedIter.next().getEntity().getKey(); } - builder.projectId(key.getPartitionId().getDatasetId()); + builder.projectId(key.getPartitionId().getProjectId()); return new DatastoreOptions(builder); } catch (DatastoreRpcException e) { throw DatastoreException.translateAndThrow(e); @@ -149,10 +149,10 @@ private static String defaultNamespace() { Class clazz = Class.forName("com.google.appengine.api.NamespaceManager"); Method method = clazz.getMethod("get"); String namespace = (String) method.invoke(null); - return namespace == null || namespace.isEmpty() ? null : namespace; + return MoreObjects.firstNonNull(namespace, ""); } catch (Exception ignore) { - // return null (Datastore default namespace) if could not automatically determine - return null; + // return empty string (Datastore default namespace) if could not automatically determine + return ""; } } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java index dc1af5b8a2d9..d012eff14422 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull; -import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; /** @@ -93,7 +92,7 @@ public static Builder builder(Key key, FullEntity copyFrom) { return new Builder(key, copyFrom); } - static Entity fromPb(DatastoreV1.Entity entityPb) { + static Entity fromPb(com.google.datastore.v1beta3.Entity entityPb) { return new Builder().fill(entityPb).build(); } } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java index 16ca55aec4a6..4a327383afd4 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java @@ -39,15 +39,12 @@ public Builder newBuilder(FullEntity value) { @Override protected FullEntity getValue(com.google.datastore.v1beta3.Value from) { - // TODO(ajaykannan): fix me! - //return FullEntity.fromPb(from.getEntityValue()); - return null; // TODO(ajaykannan): fix me! + return FullEntity.fromPb(from.getEntityValue()); } @Override protected void setValue(EntityValue from, com.google.datastore.v1beta3.Value.Builder to) { - // TODO(ajaykannan): fix me! - //to.setEntityValue(from.get().toPb()); + to.setEntityValue(from.get().toPb()); } }; diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java index bb08fca12e3c..b1534984aeb0 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java @@ -16,8 +16,6 @@ package com.google.gcloud.datastore; -import com.google.api.services.datastore.DatastoreV1; - /** * A full entity is a {@link BaseEntity} that with a complete set of properties. */ @@ -70,7 +68,7 @@ public static Builder builder(FullEntity copyFro } - static FullEntity fromPb(DatastoreV1.Entity entityPb) { + static FullEntity fromPb(com.google.datastore.v1beta3.Entity entityPb) { return new Builder<>().fill(entityPb).build(); } } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java index 46a528617a0c..f8af814245ab 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java @@ -16,7 +16,6 @@ package com.google.gcloud.datastore; -import com.google.api.services.datastore.DatastoreV1; import com.google.protobuf.ByteString; /** @@ -72,7 +71,7 @@ public Blob getBlob(String name) { return ((Value) value).get(); } - static ProjectionEntity fromPb(DatastoreV1.Entity entityPb) { + static ProjectionEntity fromPb(com.google.datastore.v1beta3.Entity entityPb) { return new Builder().fill(entityPb).build(); } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java index f3dd254ce9a3..7fae120dd402 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java @@ -69,7 +69,9 @@ public abstract static class ResultType implements java.io.Serializable { //TODO(ajaykannan): fix me! //return Key.fromPb(entityPb.getKey()); } - return ProjectionEntity.fromPb(entityPb); + // TODO(ajaykannan): fix me! + //return ProjectionEntity.fromPb(entityPb); + return ProjectionEntity.fromPb((com.google.datastore.v1beta3.Entity) null); // TODO(ajaykannan): fix me! } }; @@ -79,7 +81,9 @@ public abstract static class ResultType implements java.io.Serializable { private static final long serialVersionUID = 7712959777507168274L; @Override protected Entity convert(DatastoreV1.Entity entityPb) { - return Entity.fromPb(entityPb); + // TODO(ajaykannan): fix me! + //return Entity.fromPb(entityPb); + return Entity.fromPb((com.google.datastore.v1beta3.Entity) null); // TODO(ajaykannan): fix me! } }; @@ -102,7 +106,9 @@ public abstract static class ResultType implements java.io.Serializable { private static final long serialVersionUID = -7591409419690650246L; @Override protected ProjectionEntity convert(DatastoreV1.Entity entityPb) { - return ProjectionEntity.fromPb(entityPb); + // TODO(ajaykannan): fix me! + //return ProjectionEntity.fromPb(entityPb); + return ProjectionEntity.fromPb((com.google.datastore.v1beta3.Entity) null); // TODO(ajaykannan): fix me! } }; diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java index cd3fe9dd776b..caf5d7f8a84f 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java @@ -63,7 +63,8 @@ private void sendRequest() { } requestPb.setPartitionId(partitionIdPb); query.populatePb(requestPb); - queryResultBatchPb = datastore.runQuery(requestPb.build()).getBatch(); + // TODO(ajaykannan): fix me! + //queryResultBatchPb = datastore.runQuery(requestPb.build()).getBatch(); lastBatch = queryResultBatchPb.getMoreResults() != MoreResultsType.NOT_FINISHED; entityResultPbIter = queryResultBatchPb.getEntityResultList().iterator(); // cursor = resultPb.getSkippedCursor(); // available in v1beta3, use startCursor if not skipped diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java index ae9d0e50f0d7..ff9ce5e50e9d 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java @@ -16,58 +16,45 @@ package com.google.gcloud.datastore; -import com.google.api.services.datastore.DatastoreV1; -import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.gcloud.datastore.TransactionOption.ForceWrites; -import com.google.gcloud.datastore.TransactionOption.IsolationLevel; import com.google.protobuf.ByteString; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; -import java.util.Map; final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction { private final DatastoreImpl datastore; private final ByteString transaction; - private final boolean force; private boolean rolledback; static class ResponseImpl implements Transaction.Response { - private final DatastoreV1.CommitResponse response; + private final com.google.datastore.v1beta3.CommitResponse response; + private final int numAutoAllocatedIds; - ResponseImpl(DatastoreV1.CommitResponse response) { + ResponseImpl(com.google.datastore.v1beta3.CommitResponse response, int numAutoAllocatedIds) { this.response = response; + this.numAutoAllocatedIds = numAutoAllocatedIds; } @Override public List generatedKeys() { - return Lists.transform(response.getMutationResult().getInsertAutoIdKeyList(), - new Function() { - @Override public Key apply(DatastoreV1.Key keyPb) { - // TODO(ajaykannan): fix me! - //return Key.fromPb(keyPb); - return Key.builder(null).build(); // TODO(ajaykannan): fix me! - } - }); + Iterator results = + response.getMutationResultsList().iterator(); + List generated = new LinkedList(); + for (int i = 0; i < numAutoAllocatedIds; i++) { + generated.add(Key.fromPb(results.next().getKey())); + } + return generated; } } - TransactionImpl(DatastoreImpl datastore, TransactionOption... options) { + TransactionImpl(DatastoreImpl datastore) { super("transaction"); this.datastore = datastore; - DatastoreV1.BeginTransactionRequest.Builder requestPb = - DatastoreV1.BeginTransactionRequest.newBuilder(); - Map, TransactionOption> optionsMap = - TransactionOption.asImmutableMap(options); - IsolationLevel isolationLevel = (IsolationLevel) optionsMap.get(IsolationLevel.class); - if (isolationLevel != null) { - requestPb.setIsolationLevel(isolationLevel.level().toPb()); - } - ForceWrites forceWrites = (ForceWrites) optionsMap.get(TransactionOption.ForceWrites.class); - force = forceWrites != null && forceWrites.force(); + com.google.datastore.v1beta3.BeginTransactionRequest.Builder requestPb = + com.google.datastore.v1beta3.BeginTransactionRequest.newBuilder(); transaction = datastore.requestTransactionId(requestPb); } @@ -79,7 +66,8 @@ public Entity get(Key key) { @Override public Iterator get(Key... keys) { validateActive(); - DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder(); + com.google.datastore.v1beta3.ReadOptions.Builder readOptionsPb = + com.google.datastore.v1beta3.ReadOptions.newBuilder(); readOptionsPb.setTransaction(transaction); return datastore.get(readOptionsPb.build(), keys); } @@ -93,7 +81,8 @@ public List fetch(Key... keys) { @Override public QueryResults run(Query query) { validateActive(); - DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder(); + com.google.datastore.v1beta3.ReadOptions.Builder readOptionsPb = + com.google.datastore.v1beta3.ReadOptions.newBuilder(); readOptionsPb.setTransaction(transaction); return datastore.run(readOptionsPb.build(), query); } @@ -101,17 +90,15 @@ public QueryResults run(Query query) { @Override public Transaction.Response commit() { validateActive(); - DatastoreV1.Mutation.Builder mutationPb = toMutationPb(); - if (force) { - mutationPb.setForce(force); - } - DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); - requestPb.setMode(DatastoreV1.CommitRequest.Mode.TRANSACTIONAL); + List mutationsPb = toMutationPbList(); + com.google.datastore.v1beta3.CommitRequest.Builder requestPb = + com.google.datastore.v1beta3.CommitRequest.newBuilder(); + requestPb.setMode(com.google.datastore.v1beta3.CommitRequest.Mode.TRANSACTIONAL); requestPb.setTransaction(transaction); - requestPb.setMutation(mutationPb); - DatastoreV1.CommitResponse responsePb = datastore.commit(requestPb.build()); + requestPb.addAllMutations(mutationsPb); + com.google.datastore.v1beta3.CommitResponse responsePb = datastore.commit(requestPb.build()); deactivate(); - return new ResponseImpl(responsePb); + return new ResponseImpl(responsePb, numAutoAllocatedIds()); } @Override diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionOption.java deleted file mode 100644 index c1c8368213dc..000000000000 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionOption.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2015 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.gcloud.datastore; - -import com.google.api.services.datastore.DatastoreV1; -import com.google.common.collect.ImmutableMap; - -import java.io.Serializable; -import java.util.Map; - - -public abstract class TransactionOption implements Serializable { - - private static final long serialVersionUID = -1862234444015690375L; - - public static final class ForceWrites extends TransactionOption { - - private static final long serialVersionUID = -6873967516988380886L; - - private final boolean force; - - public ForceWrites(boolean force) { - this.force = force; - } - - public boolean force() { - return force; - } - - @Override - BatchOption toBatchWriteOption() { - return new BatchOption.ForceWrites(force); - } - } - - public static final class IsolationLevel extends TransactionOption { - - private static final long serialVersionUID = -5592165378565409515L; - - private final Level level; - - public enum Level { - - SERIALIZABLE(DatastoreV1.BeginTransactionRequest.IsolationLevel.SERIALIZABLE), - SNAPSHOT(DatastoreV1.BeginTransactionRequest.IsolationLevel.SNAPSHOT); - - private final DatastoreV1.BeginTransactionRequest.IsolationLevel levelPb; - - Level(DatastoreV1.BeginTransactionRequest.IsolationLevel levelPb) { - this.levelPb = levelPb; - } - - DatastoreV1.BeginTransactionRequest.IsolationLevel toPb() { - return levelPb; - } - } - - public IsolationLevel(Level level) { - this.level = level; - } - - - public Level level() { - return level; - } - - @Override - BatchOption toBatchWriteOption() { - return null; - } - } - - TransactionOption() { - // package protected - } - - public static ForceWrites forceWrites() { - return new ForceWrites(true); - } - - public static IsolationLevel serializable() { - return new IsolationLevel(IsolationLevel.Level.SERIALIZABLE); - } - - public static IsolationLevel snapshot() { - return new IsolationLevel(IsolationLevel.Level.SNAPSHOT); - } - - static Map, TransactionOption> asImmutableMap( - TransactionOption... options) { - ImmutableMap.Builder, TransactionOption> builder = - ImmutableMap.builder(); - for (TransactionOption option : options) { - builder.put(option.getClass(), option); - } - return builder.build(); - } - - abstract BatchOption toBatchWriteOption(); -} diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java index dffcc3f0e16f..329623a565f6 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java @@ -15,19 +15,6 @@ */ package com.google.gcloud.spi; -import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest; -import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; -import com.google.api.services.datastore.DatastoreV1.CommitRequest; -import com.google.api.services.datastore.DatastoreV1.CommitResponse; -import com.google.api.services.datastore.DatastoreV1.LookupRequest; -import com.google.api.services.datastore.DatastoreV1.LookupResponse; -import com.google.api.services.datastore.DatastoreV1.RollbackRequest; -import com.google.api.services.datastore.DatastoreV1.RollbackResponse; -import com.google.api.services.datastore.DatastoreV1.RunQueryRequest; -import com.google.api.services.datastore.DatastoreV1.RunQueryResponse; - /** * Provides access to the remote Datastore service. */ @@ -103,16 +90,22 @@ public boolean retryable() { } } - AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException; + com.google.datastore.v1beta3.AllocateIdsResponse allocateIds( + com.google.datastore.v1beta3.AllocateIdsRequest request) throws DatastoreRpcException; - BeginTransactionResponse beginTransaction(BeginTransactionRequest request) + com.google.datastore.v1beta3.BeginTransactionResponse beginTransaction( + com.google.datastore.v1beta3.BeginTransactionRequest request) throws DatastoreRpcException; - CommitResponse commit(CommitRequest request) throws DatastoreRpcException; + com.google.datastore.v1beta3.CommitResponse commit( + com.google.datastore.v1beta3.CommitRequest request) throws DatastoreRpcException; - LookupResponse lookup(LookupRequest request) throws DatastoreRpcException; + com.google.datastore.v1beta3.LookupResponse lookup( + com.google.datastore.v1beta3.LookupRequest request) throws DatastoreRpcException; - RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException; + com.google.datastore.v1beta3.RollbackResponse rollback( + com.google.datastore.v1beta3.RollbackRequest request) throws DatastoreRpcException; - RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException; + com.google.datastore.v1beta3.RunQueryResponse runQuery( + com.google.datastore.v1beta3.RunQueryRequest request) throws DatastoreRpcException; } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java index 2f245260b325..1ae0b6eea22e 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java @@ -16,36 +16,16 @@ package com.google.gcloud.spi; -import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest; -import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; -import com.google.api.services.datastore.DatastoreV1.CommitRequest; -import com.google.api.services.datastore.DatastoreV1.CommitResponse; -import com.google.api.services.datastore.DatastoreV1.LookupRequest; -import com.google.api.services.datastore.DatastoreV1.LookupResponse; -import com.google.api.services.datastore.DatastoreV1.RollbackRequest; -import com.google.api.services.datastore.DatastoreV1.RollbackResponse; -import com.google.api.services.datastore.DatastoreV1.RunQueryRequest; -import com.google.api.services.datastore.DatastoreV1.RunQueryResponse; -import com.google.api.services.datastore.client.Datastore; -import com.google.api.services.datastore.client.DatastoreException; -import com.google.api.services.datastore.client.DatastoreFactory; -import com.google.api.services.datastore.client.DatastoreOptions.Builder; import com.google.common.collect.ImmutableMap; import com.google.gcloud.datastore.DatastoreOptions; import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONTokener; - import java.util.HashMap; import java.util.Map; public class DefaultDatastoreRpc implements DatastoreRpc { - private final Datastore client; + private final com.google.datastore.v1beta3.client.Datastore client; private static final ImmutableMap STR_TO_REASON; private static final ImmutableMap HTTP_STATUS_TO_REASON; @@ -62,26 +42,27 @@ public class DefaultDatastoreRpc implements DatastoreRpc { } public DefaultDatastoreRpc(DatastoreOptions options) { - client = DatastoreFactory.get().create( - new Builder() - .dataset(options.projectId()) - .host(options.host()) - .initializer(options.httpRequestInitializer()) - .build()); + if (options.host().contains("localhost")) { + client = com.google.datastore.v1beta3.client.DatastoreFactory.get().create( + new com.google.datastore.v1beta3.client.DatastoreOptions.Builder() + .projectId(options.projectId()) + .localHost(options.host()) + .initializer(options.httpRequestInitializer()) + .build()); + } else { + client = com.google.datastore.v1beta3.client.DatastoreFactory.get().create( + new com.google.datastore.v1beta3.client.DatastoreOptions.Builder() + .projectId(options.projectId()) + .initializer(options.httpRequestInitializer()) + .build()); + } } - private static DatastoreRpcException translate(DatastoreException exception) { - String message = exception.getMessage(); + private static DatastoreRpcException translate( + com.google.datastore.v1beta3.client.DatastoreException exception) { String reasonStr = ""; - if (message != null) { - try { - JSONObject json = new JSONObject(new JSONTokener(message)); - JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0); - reasonStr = error.getString("reason"); - message = error.getString("message"); - } catch (JSONException ignore) { - // ignore - will be converted to unknown - } + if (exception.getCode() != null) { + reasonStr = exception.getCode().name(); } Reason reason = STR_TO_REASON.get(reasonStr); if (reason == null) { @@ -89,61 +70,68 @@ private static DatastoreRpcException translate(DatastoreException exception) { } return reason != null ? new DatastoreRpcException(reason) - : new DatastoreRpcException("Unknown", exception.getCode(), false, message); + : new DatastoreRpcException("Unknown", + exception.getCode().ordinal(), + false, + exception.getMessage()); } @Override - public AllocateIdsResponse allocateIds(AllocateIdsRequest request) - throws DatastoreRpcException { + public com.google.datastore.v1beta3.AllocateIdsResponse allocateIds( + com.google.datastore.v1beta3.AllocateIdsRequest request) throws DatastoreRpcException { try { return client.allocateIds(request); - } catch (DatastoreException ex) { + } catch (com.google.datastore.v1beta3.client.DatastoreException ex) { throw translate(ex); } } @Override - public BeginTransactionResponse beginTransaction(BeginTransactionRequest request) - throws DatastoreRpcException { + public com.google.datastore.v1beta3.BeginTransactionResponse beginTransaction( + com.google.datastore.v1beta3.BeginTransactionRequest request) throws DatastoreRpcException { try { return client.beginTransaction(request); - } catch (DatastoreException ex) { + } catch (com.google.datastore.v1beta3.client.DatastoreException ex) { throw translate(ex); } } @Override - public CommitResponse commit(CommitRequest request) throws DatastoreRpcException { + public com.google.datastore.v1beta3.CommitResponse commit( + com.google.datastore.v1beta3.CommitRequest request) throws DatastoreRpcException { try { return client.commit(request); - } catch (DatastoreException ex) { + } catch (com.google.datastore.v1beta3.client.DatastoreException ex) { throw translate(ex); } } @Override - public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException { + public com.google.datastore.v1beta3.LookupResponse lookup( + com.google.datastore.v1beta3.LookupRequest request) throws DatastoreRpcException { try { return client.lookup(request); - } catch (DatastoreException ex) { + } catch (com.google.datastore.v1beta3.client.DatastoreException ex) { throw translate(ex); } } @Override - public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException { + public com.google.datastore.v1beta3.RollbackResponse rollback( + com.google.datastore.v1beta3.RollbackRequest request) throws DatastoreRpcException { try { return client.rollback(request); - } catch (DatastoreException ex) { + } catch (com.google.datastore.v1beta3.client.DatastoreException ex) { throw translate(ex); } } @Override - public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException { + public com.google.datastore.v1beta3.RunQueryResponse runQuery( + com.google.datastore.v1beta3.RunQueryRequest request) throws DatastoreRpcException { try { return client.runQuery(request); - } catch (DatastoreException ex) { + } catch (com.google.datastore.v1beta3.client.DatastoreException ex) { throw translate(ex); } } diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java index 43df45d7a04e..8996f4536931 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java @@ -21,7 +21,6 @@ import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; -import com.google.api.services.datastore.DatastoreV1; import com.google.common.collect.ImmutableList; import org.easymock.EasyMock; @@ -30,6 +29,7 @@ import org.junit.Test; import java.util.List; +import java.util.LinkedList; public class BaseDatastoreBatchWriterTest { @@ -80,36 +80,33 @@ public void tearDown() { batchWriter.finish(); } - // TODO(ajaykannan): fix me! - /* @Test public void testAdd() throws Exception { Entity entity2 = Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addInsert(ENTITY1.toPb()) - .addInsert(entity2.toPb()) - .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb()) - .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setInsert(ENTITY1.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setInsert(entity2.toPb()).build()); List entities = batchWriter .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); assertEquals(ENTITY1, entities.get(0)); assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1)); assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2)); assertEquals(entity2, entities.get(3)); } - */ @Test public void testAddAfterDelete() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(ENTITY1.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(ENTITY1.toPb()).build()); batchWriter.delete(KEY1); batchWriter.add(ENTITY1); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test(expected = DatastoreException.class) @@ -136,18 +133,17 @@ public void testAddWhenNotActive() throws Exception { batchWriter.add(ENTITY1); } - // TODO(ajaykannan): fix me! - /* @Test public void testAddWithDeferredAllocation() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addInsert(ENTITY1.toPb()) - .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb()) - .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(INCOMPLETE_ENTITY_1.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(INCOMPLETE_ENTITY_2.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setInsert(ENTITY1.toPb()).build()); batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1); batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test(expected = DatastoreException.class) @@ -158,49 +154,44 @@ public void testAddWithDeferredAllocationWhenNotActive() throws Exception { @Test public void testUpdate() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpdate(ENTITY1.toPb()) - .addUpdate(ENTITY2.toPb()) - .addUpdate(ENTITY3.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpdate(ENTITY1.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpdate(ENTITY2.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpdate(ENTITY3.toPb()).build()); batchWriter.update(ENTITY1, ENTITY2); batchWriter.update(ENTITY3); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testUpdateAfterUpdate() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpdate(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpdate(entity.toPb()).build()); batchWriter.update(ENTITY1); batchWriter.update(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testUpdateAfterAdd() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); batchWriter.add(ENTITY1); batchWriter.update(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testUpdateAfterPut() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); batchWriter.put(ENTITY1); batchWriter.update(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } - */ @Test(expected = DatastoreException.class) public void testUpdateAfterDelete() throws Exception { @@ -214,117 +205,104 @@ public void testUpdateWhenNotActive() throws Exception { batchWriter.update(ENTITY1); } - // TODO(ajaykannan): fix me! - /* @Test public void testPut() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(ENTITY1.toPb()) - .addUpsert(ENTITY2.toPb()) - .addUpsert(ENTITY3.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(ENTITY1.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(ENTITY2.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(ENTITY3.toPb()).build()); batchWriter.put(ENTITY1, ENTITY2); batchWriter.put(ENTITY3); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testPutAfterPut() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); batchWriter.put(ENTITY1); batchWriter.put(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testPutAfterAdd() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); batchWriter.add(ENTITY1); batchWriter.put(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testPutAfterUpdate() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); batchWriter.update(ENTITY1); batchWriter.put(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testPutAfterDelete() throws Exception { Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build(); - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addUpsert(entity.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setUpsert(entity.toPb()).build()); batchWriter.delete(KEY1); batchWriter.put(entity); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } - */ @Test(expected = DatastoreException.class) public void testPutWhenNotActive() throws Exception { batchWriter.deactivate(); batchWriter.put(ENTITY1); } - // TODO(ajaykannan): fix me! - /* + @Test public void testDelete() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addDelete(KEY1.toPb()) - .addDelete(KEY2.toPb()) - .addDelete(KEY3.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(KEY1.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(KEY2.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(KEY3.toPb()).build()); batchWriter.delete(KEY1, KEY2); batchWriter.delete(KEY3); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testDeleteAfterAdd() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb()) - .addDelete(KEY1.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(INCOMPLETE_ENTITY_1.toPb()).build()); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(KEY1.toPb()).build()); batchWriter.add(ENTITY1); batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1); batchWriter.delete(KEY1); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testDeleteAfterUpdate() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addDelete(KEY1.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(KEY1.toPb()).build()); batchWriter.update(ENTITY1); batchWriter.delete(KEY1); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } @Test public void testDeleteAfterPut() throws Exception { - DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder() - .addDelete(KEY1.toPb()) - .build(); + List pbs = new LinkedList<>(); + pbs.add(com.google.datastore.v1beta3.Mutation.newBuilder().setDelete(KEY1.toPb()).build()); batchWriter.put(ENTITY1); batchWriter.delete(KEY1); - assertEquals(pb, batchWriter.toMutationPb().build()); + assertEquals(pbs, batchWriter.toMutationPbList()); } - */ @Test(expected = DatastoreException.class) public void testDeleteWhenNotActive() throws Exception { diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java index 79576748bb58..c8773243ed69 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java @@ -17,7 +17,6 @@ package com.google.gcloud.datastore; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import com.google.common.collect.ImmutableList; diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java index e7dc71c50ff6..97aeec3ee129 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java @@ -65,7 +65,7 @@ public void testHost() throws Exception { @Test public void testNamespace() throws Exception { - assertNull(options.build().namespace()); + assertTrue(options.build().namespace().isEmpty()); assertEquals("ns1", options.namespace("ns1").build().namespace()); } diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java index 57342ed3e26d..689d2a7b8b6b 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java @@ -14,8 +14,6 @@ * limitations under the License. */ -// TODO(ajaykannan): fix me! -/* package com.google.gcloud.datastore; import static org.junit.Assert.assertEquals; @@ -27,8 +25,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.EntityResult; import com.google.common.collect.Iterators; import com.google.gcloud.RetryParams; import com.google.gcloud.datastore.Query.ResultType; @@ -116,12 +112,12 @@ public static void beforeClass() throws IOException, InterruptedException { public void setUp() throws IOException, InterruptedException { options = DatastoreOptions.builder() .projectId(PROJECT_ID) - .host("http://localhost:" + LocalGcdHelper.PORT) + .host("localhost:" + LocalGcdHelper.PORT) .build(); datastore = DatastoreFactory.instance().get(options); - StructuredQuery query = Query.keyQueryBuilder().build(); - QueryResults result = datastore.run(query); - datastore.delete(Iterators.toArray(result, Key.class)); + //StructuredQuery query = Query.keyQueryBuilder().build(); + //QueryResults result = datastore.run(query); + //datastore.delete(Iterators.toArray(result, Key.class)); datastore.add(ENTITY1, ENTITY2); } @@ -192,7 +188,7 @@ public void testTransactionWithRead() { assertEquals(DatastoreException.Code.ABORTED, expected.code()); } } - + /* TODO(ajaykannan): fix me! @Test public void testTransactionWithQuery() { Query query = Query.entityQueryBuilder() @@ -220,7 +216,7 @@ public void testTransactionWithQuery() { assertEquals(DatastoreException.Code.ABORTED, expected.code()); } } - + */ @Test public void testNewTransactionRollback() { Transaction transaction = datastore.newTransaction(); @@ -335,7 +331,7 @@ public void testNewBatch() { assertNull(entities.get(4)); assertEquals(5, entities.size()); } - + /* TODO(ajaykannan): fix me! @Test public void testRunGqlQueryNoCasting() { Query query1 = Query.gqlQueryBuilder(ResultType.ENTITY, "select * from " + KIND1).build(); @@ -452,7 +448,7 @@ public void testRunStructuredQuery() { assertFalse(results4.hasNext()); // TODO(ozarov): construct a test to verify nextQuery/pagination } - + */ @Test public void testAllocateId() { KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1); @@ -570,7 +566,6 @@ public void testAddEntity() { assertNotNull(datastore.get(entities.get(2).key())); } - @Test public void testUpdate() { List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key()); @@ -639,10 +634,10 @@ public void testKeyFactory() { @Test public void testRetires() throws Exception { - DatastoreV1.LookupRequest requestPb = - DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build(); - DatastoreV1.LookupResponse responsePb = DatastoreV1.LookupResponse.newBuilder() - .addFound(EntityResult.newBuilder().setEntity(ENTITY1.toPb())).build(); + com.google.datastore.v1beta3.LookupRequest requestPb = + com.google.datastore.v1beta3.LookupRequest.newBuilder().addKeys(KEY1.toPb()).build(); + com.google.datastore.v1beta3.LookupResponse responsePb = com.google.datastore.v1beta3.LookupResponse.newBuilder() + .addFound(com.google.datastore.v1beta3.EntityResult.newBuilder().setEntity(ENTITY1.toPb())).build(); DatastoreRpcFactory rpcFactoryMock = EasyMock.createStrictMock(DatastoreRpcFactory.class); DatastoreRpc rpcMock = EasyMock.createStrictMock(DatastoreRpc.class); EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(DatastoreOptions.class))) @@ -661,4 +656,3 @@ public void testRetires() throws Exception { EasyMock.verify(rpcFactoryMock, rpcMock); } } -*/ diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java index 7acf8abbbf19..dacb348c2172 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java @@ -17,7 +17,6 @@ package com.google.gcloud.datastore; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import org.junit.Before; diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 2d7f5802f247..0625f2afb38d 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -61,9 +61,9 @@ public class LocalGcdHelper { public static final String DEFAULT_PROJECT_ID = "projectid1"; public static final int PORT = 8080; - private static final String GCD = "gcd-v1beta2-rev1-2.1.2b"; + private static final String GCD = "gcd-v1beta3-0.0.1"; private static final String GCD_FILENAME = GCD + ".zip"; - private static final String MD5_CHECKSUM = "d84384cdfa8658e1204f4f8be51300e8"; + private static final String MD5_CHECKSUM = "496b16f32473d0de0c7a974bd0ee1461"; private static final URL GCD_URL; static { @@ -159,13 +159,13 @@ public void start() throws IOException, InterruptedException { } } // cleanup any possible data for the same project - File datasetFolder = new File(gcdFolder, GCD + '/' + projectId); + File datasetFolder = new File(gcdFolder, "gcd/" + projectId); deleteRecurse(datasetFolder.toPath()); // create the datastore for the project ProcessBuilder processBuilder = new ProcessBuilder() .redirectError(ProcessBuilder.Redirect.INHERIT) - .directory(new File(gcdFolder, GCD)); + .directory(new File(gcdFolder, "gcd")); if (isWindows()) { processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId); processBuilder.redirectOutput(new File("NULL:")); @@ -179,7 +179,7 @@ public void start() throws IOException, InterruptedException { // start the datastore for the project processBuilder = new ProcessBuilder() - .directory(new File(gcdFolder, GCD)) + .directory(new File(gcdFolder, "gcd")) .redirectErrorStream(true); if (isWindows()) { processBuilder.command("cmd", "/C", "gcd.cmd", "start", "--testing", @@ -309,7 +309,7 @@ public static void main(String... args) throws IOException, InterruptedException public static boolean isActive(String projectId) { try { StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT); - urlBuilder.append("/datastore/v1beta2/datasets/").append(projectId).append("/lookup"); + urlBuilder.append("/datastore/v1beta3/datasets/").append(projectId).append(":lookup"); URL url = new URL(urlBuilder.toString()); try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) { From 959d67d9f9f940b3db62b556076b2a90ba2fb263 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Fri, 25 Sep 2015 09:22:16 -0700 Subject: [PATCH 2/5] cleanup v1beta3 changes and fix LocalGcdHelper bugs --- .../datastore/BaseDatastoreBatchWriter.java | 4 --- .../google/gcloud/datastore/BatchImpl.java | 6 ++-- .../gcloud/datastore/DatastoreImpl.java | 17 +++++---- .../gcloud/datastore/DatastoreOptions.java | 3 +- .../gcloud/datastore/TransactionImpl.java | 6 ++-- .../gcloud/spi/DefaultDatastoreRpc.java | 36 ++++++++++++------- .../datastore/DatastoreOptionsTest.java | 1 - .../gcloud/datastore/LocalGcdHelper.java | 2 +- 8 files changed, 40 insertions(+), 35 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java index 6d8ac86834de..3aa0d38f45f1 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java @@ -184,10 +184,6 @@ protected Set toDelete() { return toDelete; } - protected int numAutoAllocatedIds() { - return toAddAutoId.size(); - } - protected void deactivate() { active = false; } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java index a9eab9e9c3f1..303e9703f4cc 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java @@ -16,8 +16,8 @@ package com.google.gcloud.datastore; +import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; @@ -39,7 +39,7 @@ static class ResponseImpl implements Batch.Response { public List generatedKeys() { Iterator results = response.getMutationResultsList().iterator(); - List generated = new LinkedList(); + List generated = new ArrayList<>(numAutoAllocatedIds); for (int i = 0; i < numAutoAllocatedIds; i++) { generated.add(Key.fromPb(results.next().getKey())); } @@ -62,7 +62,7 @@ public Batch.Response submit() { requestPb.addAllMutations(mutationsPb); com.google.datastore.v1beta3.CommitResponse responsePb = datastore.commit(requestPb.build()); deactivate(); - return new ResponseImpl(responsePb, numAutoAllocatedIds()); + return new ResponseImpl(responsePb, toAddAutoId().size()); } @Override diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java index 262098e82f96..f0238083b5fa 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java @@ -176,33 +176,32 @@ public List add(FullEntity... entities) { return Collections.emptyList(); } List mutationsPb = new ArrayList<>(); - Map completeEntities = new LinkedHashMap<>(); + Set completeEntities = new LinkedHashSet<>(); for (FullEntity entity : entities) { Entity completeEntity = null; if (entity.key() instanceof Key) { completeEntity = Entity.convert((FullEntity) entity); } if (completeEntity != null) { - if (completeEntities.put(completeEntity.key(), completeEntity) != null) { + if (completeEntities.contains(completeEntity)) { throw DatastoreException.throwInvalidRequest( "Duplicate entity with the key %s", entity.key()); } - mutationsPb.add(com.google.datastore.v1beta3.Mutation.newBuilder() - .setInsert(completeEntity.toPb()).build()); + completeEntities.add(completeEntity); } else { Preconditions.checkArgument(entity.hasKey(), "entity %s is missing a key", entity); - mutationsPb.add(com.google.datastore.v1beta3.Mutation.newBuilder() - .setInsert(entity.toPb()).build()); } + mutationsPb.add(com.google.datastore.v1beta3.Mutation.newBuilder() + .setInsert(entity.toPb()).build()); } com.google.datastore.v1beta3.CommitResponse commitResponse = commitMutation(mutationsPb); Iterator mutationResults = commitResponse.getMutationResultsList().iterator(); + Iterator completeEntitiesIt = completeEntities.iterator(); ImmutableList.Builder responseBuilder = ImmutableList.builder(); for (FullEntity entity : entities) { - Entity completeEntity = completeEntities.get(entity.key()); - if (completeEntity != null) { - responseBuilder.add(completeEntity); + if (completeEntities.contains(entity)) { + responseBuilder.add(completeEntitiesIt.next()); mutationResults.next(); } else { responseBuilder.add( diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java index 23128a856761..a354f7328959 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java @@ -108,7 +108,8 @@ private DatastoreOptions normalize() { .build(); requestPb.addKeys(key); try { - com.google.datastore.v1beta3.LookupResponse responsePb = datastoreRpc().lookup(requestPb.build()); + com.google.datastore.v1beta3.LookupResponse responsePb = + datastoreRpc().lookup(requestPb.build()); if (responsePb.getDeferredCount() > 0) { key = responsePb.getDeferred(0); } else { diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java index ff9ce5e50e9d..5a422927005a 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java @@ -18,8 +18,8 @@ import com.google.protobuf.ByteString; +import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction { @@ -42,7 +42,7 @@ static class ResponseImpl implements Transaction.Response { public List generatedKeys() { Iterator results = response.getMutationResultsList().iterator(); - List generated = new LinkedList(); + List generated = new ArrayList<>(numAutoAllocatedIds); for (int i = 0; i < numAutoAllocatedIds; i++) { generated.add(Key.fromPb(results.next().getKey())); } @@ -98,7 +98,7 @@ public Transaction.Response commit() { requestPb.addAllMutations(mutationsPb); com.google.datastore.v1beta3.CommitResponse responsePb = datastore.commit(requestPb.build()); deactivate(); - return new ResponseImpl(responsePb, numAutoAllocatedIds()); + return new ResponseImpl(responsePb, toAddAutoId().size()); } @Override diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java index 1ae0b6eea22e..3c82e2a93cc4 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java @@ -20,6 +20,10 @@ import com.google.gcloud.datastore.DatastoreOptions; import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.UnknownHostException; +import java.net.URL; import java.util.HashMap; import java.util.Map; @@ -42,20 +46,26 @@ public class DefaultDatastoreRpc implements DatastoreRpc { } public DefaultDatastoreRpc(DatastoreOptions options) { - if (options.host().contains("localhost")) { - client = com.google.datastore.v1beta3.client.DatastoreFactory.get().create( - new com.google.datastore.v1beta3.client.DatastoreOptions.Builder() - .projectId(options.projectId()) - .localHost(options.host()) - .initializer(options.httpRequestInitializer()) - .build()); - } else { - client = com.google.datastore.v1beta3.client.DatastoreFactory.get().create( - new com.google.datastore.v1beta3.client.DatastoreOptions.Builder() - .projectId(options.projectId()) - .initializer(options.httpRequestInitializer()) - .build()); + com.google.datastore.v1beta3.client.DatastoreOptions.Builder clientBuilder = + new com.google.datastore.v1beta3.client.DatastoreOptions.Builder() + .projectId(options.projectId()) + .initializer(options.httpRequestInitializer()); + if (options.host() != null) { + try { + String normalizedHost = options.host(); + if (!normalizedHost.startsWith("http")) { + normalizedHost = "http://" + normalizedHost; + } + InetAddress hostAddr = InetAddress.getByName(new URL(normalizedHost).getHost()); + if (hostAddr.isAnyLocalAddress() || hostAddr.isLoopbackAddress()) { + clientBuilder = clientBuilder.localHost(options.host()); + } + } catch (UnknownHostException | MalformedURLException e) { + // ignore + } } + client = com.google.datastore.v1beta3.client.DatastoreFactory.get() + .create(clientBuilder.build()); } private static DatastoreRpcException translate( diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java index 97aeec3ee129..63657902d737 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 0625f2afb38d..1d692040c015 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -309,7 +309,7 @@ public static void main(String... args) throws IOException, InterruptedException public static boolean isActive(String projectId) { try { StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT); - urlBuilder.append("/datastore/v1beta3/datasets/").append(projectId).append(":lookup"); + urlBuilder.append("/datastore/v1beta3/projects/").append(projectId).append(":lookup"); URL url = new URL(urlBuilder.toString()); try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) { From 9463c5592e230e7384960664119781e642a87c45 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Wed, 30 Sep 2015 18:33:27 -0700 Subject: [PATCH 3/5] Fix handling of host in DatastoreOptions.java --- .../gcloud/datastore/DatastoreImpl.java | 11 +++---- .../gcloud/datastore/DatastoreOptions.java | 26 +++++----------- .../gcloud/spi/DefaultDatastoreRpc.java | 31 +++++++++++++++---- .../datastore/DatastoreOptionsTest.java | 9 +----- .../gcloud/datastore/SerializationTest.java | 1 - 5 files changed, 38 insertions(+), 40 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java index f0238083b5fa..59804831872c 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java @@ -176,18 +176,17 @@ public List add(FullEntity... entities) { return Collections.emptyList(); } List mutationsPb = new ArrayList<>(); - Set completeEntities = new LinkedHashSet<>(); + Map completeEntities = new LinkedHashMap<>(); for (FullEntity entity : entities) { Entity completeEntity = null; if (entity.key() instanceof Key) { completeEntity = Entity.convert((FullEntity) entity); } if (completeEntity != null) { - if (completeEntities.contains(completeEntity)) { + if (completeEntities.put(completeEntity.key(), completeEntity) != null) { throw DatastoreException.throwInvalidRequest( "Duplicate entity with the key %s", entity.key()); } - completeEntities.add(completeEntity); } else { Preconditions.checkArgument(entity.hasKey(), "entity %s is missing a key", entity); } @@ -197,11 +196,11 @@ public List add(FullEntity... entities) { com.google.datastore.v1beta3.CommitResponse commitResponse = commitMutation(mutationsPb); Iterator mutationResults = commitResponse.getMutationResultsList().iterator(); - Iterator completeEntitiesIt = completeEntities.iterator(); ImmutableList.Builder responseBuilder = ImmutableList.builder(); for (FullEntity entity : entities) { - if (completeEntities.contains(entity)) { - responseBuilder.add(completeEntitiesIt.next()); + Entity completeEntity = completeEntities.get(entity.key()); + if (completeEntity != null) { + responseBuilder.add(completeEntity); mutationResults.next(); } else { responseBuilder.add( diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java index a354f7328959..fe318ef8720f 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java @@ -35,14 +35,11 @@ public class DatastoreOptions extends ServiceOptions { private static final long serialVersionUID = -8636602944160689193L; - private static final String DATASET_ENV_NAME = "DATASTORE_DATASET"; - private static final String HOST_ENV_NAME = "DATASTORE_HOST"; private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore"; private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email"; private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE); private final String namespace; - private final boolean force; private final boolean normalizeDataset; private transient DatastoreRpc datastoreRpc; @@ -50,7 +47,6 @@ public static class Builder extends ServiceOptions.Builder { private String namespace; - private boolean force; private boolean normalizeDataset = true; private Builder() { @@ -58,7 +54,6 @@ private Builder() { private Builder(DatastoreOptions options) { super(options); - force = options.force; namespace = options.namespace; normalizeDataset = options.normalizeDataset; } @@ -74,11 +69,6 @@ public Builder namespace(String namespace) { return this; } - public Builder force(boolean force) { - this.force = force; - return this; - } - Builder normalizeDataset(boolean normalizeDataset) { this.normalizeDataset = normalizeDataset; return this; @@ -89,7 +79,6 @@ private DatastoreOptions(Builder builder) { super(builder); normalizeDataset = builder.normalizeDataset; namespace = builder.namespace != null ? builder.namespace : defaultNamespace(); - force = builder.force; } private DatastoreOptions normalize() { @@ -126,13 +115,17 @@ private DatastoreOptions normalize() { @Override protected String defaultHost() { - String host = System.getProperty(HOST_ENV_NAME, System.getenv(HOST_ENV_NAME)); + String host = System.getProperty( + com.google.datastore.v1beta3.client.DatastoreHelper.LOCAL_HOST_ENV_VAR, + System.getenv(com.google.datastore.v1beta3.client.DatastoreHelper.LOCAL_HOST_ENV_VAR)); return host != null ? host : super.defaultHost(); } @Override protected String defaultProject() { - String projectId = System.getProperty(DATASET_ENV_NAME, System.getenv(DATASET_ENV_NAME)); + String projectId = System.getProperty( + com.google.datastore.v1beta3.client.DatastoreHelper.PROJECT_ID_ENV_VAR, + System.getenv(com.google.datastore.v1beta3.client.DatastoreHelper.PROJECT_ID_ENV_VAR)); if (projectId == null) { projectId = appEngineAppId(); } @@ -157,10 +150,6 @@ private static String defaultNamespace() { } } - public boolean force() { - return force; - } - @Override protected Set scopes() { return SCOPES; @@ -173,7 +162,7 @@ public Builder toBuilder() { @Override public int hashCode() { - return baseHashCode() ^ Objects.hash(namespace, force, normalizeDataset); + return baseHashCode() ^ Objects.hash(namespace, normalizeDataset); } @Override @@ -183,7 +172,6 @@ public boolean equals(Object obj) { } DatastoreOptions other = (DatastoreOptions) obj; return baseEquals(other) && Objects.equals(namespace, other.namespace) - && Objects.equals(force, other.force) && Objects.equals(normalizeDataset, other.normalizeDataset); } diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java index 3c82e2a93cc4..31fe2f8e0d66 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java @@ -50,22 +50,41 @@ public DefaultDatastoreRpc(DatastoreOptions options) { new com.google.datastore.v1beta3.client.DatastoreOptions.Builder() .projectId(options.projectId()) .initializer(options.httpRequestInitializer()); - if (options.host() != null) { + if (isLocalHost(options.host())) { + clientBuilder = clientBuilder.localHost(options.host()); + } else if (!options.host() + .equals(com.google.datastore.v1beta3.client.DatastoreFactory.DEFAULT_HOST)) { + String fullURL = options.host(); + if (fullURL.charAt(fullURL.length() - 1) != '/') { + fullURL = fullURL + '/'; + } + fullURL = fullURL + "datastore/v1beta3/projects/" + options.projectId(); + clientBuilder = clientBuilder.projectId(null).projectEndpoint(fullURL); + } + client = com.google.datastore.v1beta3.client.DatastoreFactory.get() + .create(clientBuilder.build()); + } + + private static boolean isLocalHost(String host) { + if (host != null) { try { - String normalizedHost = options.host(); - if (!normalizedHost.startsWith("http")) { + String normalizedHost = host; + if (!includesScheme(normalizedHost)) { normalizedHost = "http://" + normalizedHost; } InetAddress hostAddr = InetAddress.getByName(new URL(normalizedHost).getHost()); if (hostAddr.isAnyLocalAddress() || hostAddr.isLoopbackAddress()) { - clientBuilder = clientBuilder.localHost(options.host()); + return true; } } catch (UnknownHostException | MalformedURLException e) { // ignore } } - client = com.google.datastore.v1beta3.client.DatastoreFactory.get() - .create(clientBuilder.build()); + return false; + } + + private static boolean includesScheme(String url) { + return url.startsWith("http://") || url.startsWith("https://"); } private static DatastoreRpcException translate( diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java index 63657902d737..161dfc078f6c 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java @@ -68,12 +68,6 @@ public void testNamespace() throws Exception { assertEquals("ns1", options.namespace("ns1").build().namespace()); } - @Test - public void testForce() throws Exception { - assertFalse(options.build().force()); - assertTrue(options.force(true).build().force()); - } - @Test public void testDatastore() throws Exception { assertSame(datastoreRpcFactory, options.build().serviceRpcFactory()); @@ -82,12 +76,11 @@ public void testDatastore() throws Exception { @Test public void testToBuilder() throws Exception { - DatastoreOptions original = options.namespace("ns1").force(true).build(); + DatastoreOptions original = options.namespace("ns1").build(); DatastoreOptions copy = original.toBuilder().build(); assertEquals(original.projectId(), copy.projectId()); assertEquals(original.namespace(), copy.namespace()); assertEquals(original.host(), copy.host()); - assertEquals(original.force(), copy.force()); assertEquals(original.retryParams(), copy.retryParams()); assertEquals(original.authCredentials(), copy.authCredentials()); } diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 624689cd420e..bd4a683f9c9c 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -148,7 +148,6 @@ public void testServiceOptions() throws Exception { .namespace("ns1") .retryParams(RetryParams.getDefaultInstance()) .authCredentials(AuthCredentials.noCredentials()) - .force(true) .build(); serializedCopy = serializeAndDeserialize(options); assertEquals(options, serializedCopy); From 4d2fe25b88f32abb9cc4c3d4bc01fbfde1221a79 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 5 Oct 2015 08:03:49 -0700 Subject: [PATCH 4/5] Remove scheme from localhost and check if user-provided host is null/empty --- .../gcloud/spi/DefaultDatastoreRpc.java | 24 +++++++++++++------ .../gcloud/datastore/LocalGcdHelper.java | 3 ++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java index 31fe2f8e0d66..c6e3f396e574 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java @@ -16,6 +16,7 @@ package com.google.gcloud.spi; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.gcloud.datastore.DatastoreOptions; import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason; @@ -51,14 +52,16 @@ public DefaultDatastoreRpc(DatastoreOptions options) { .projectId(options.projectId()) .initializer(options.httpRequestInitializer()); if (isLocalHost(options.host())) { - clientBuilder = clientBuilder.localHost(options.host()); - } else if (!options.host() - .equals(com.google.datastore.v1beta3.client.DatastoreFactory.DEFAULT_HOST)) { + clientBuilder = clientBuilder.localHost(removeScheme(options.host())); + } else if (!com.google.datastore.v1beta3.client.DatastoreFactory.DEFAULT_HOST + .equals(options.host()) && !Strings.isNullOrEmpty(options.host())) { String fullURL = options.host(); if (fullURL.charAt(fullURL.length() - 1) != '/') { fullURL = fullURL + '/'; } - fullURL = fullURL + "datastore/v1beta3/projects/" + options.projectId(); + fullURL = fullURL + "datastore/" + + com.google.datastore.v1beta3.client.DatastoreFactory.VERSION + "/projects/" + + options.projectId(); clientBuilder = clientBuilder.projectId(null).projectEndpoint(fullURL); } client = com.google.datastore.v1beta3.client.DatastoreFactory.get() @@ -73,9 +76,7 @@ private static boolean isLocalHost(String host) { normalizedHost = "http://" + normalizedHost; } InetAddress hostAddr = InetAddress.getByName(new URL(normalizedHost).getHost()); - if (hostAddr.isAnyLocalAddress() || hostAddr.isLoopbackAddress()) { - return true; - } + return hostAddr.isAnyLocalAddress() || hostAddr.isLoopbackAddress(); } catch (UnknownHostException | MalformedURLException e) { // ignore } @@ -87,6 +88,15 @@ private static boolean includesScheme(String url) { return url.startsWith("http://") || url.startsWith("https://"); } + private static String removeScheme(String url) { + if (url.startsWith("https://")) { + return url.substring("https://".length()); + } else if (url.startsWith("http://")) { + return url.substring("http://".length()); + } + return url; + } + private static DatastoreRpcException translate( com.google.datastore.v1beta3.client.DatastoreException exception) { String reasonStr = ""; diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 1d692040c015..1054b461ab0d 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -309,7 +309,8 @@ public static void main(String... args) throws IOException, InterruptedException public static boolean isActive(String projectId) { try { StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT); - urlBuilder.append("/datastore/v1beta3/projects/").append(projectId).append(":lookup"); + urlBuilder.append("/datastore/" + com.google.datastore.v1beta3.client.DatastoreFactory.VERSION + + "/projects/").append(projectId).append(":lookup"); URL url = new URL(urlBuilder.toString()); try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) { From e0aaaba871960a006fb82dc5972c015ea987c980 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Mon, 5 Oct 2015 09:27:23 -0700 Subject: [PATCH 5/5] Fix version in LocalGcdHelper and check for null values when removing URL scheme --- .../com/google/gcloud/spi/DefaultDatastoreRpc.java | 10 ++++++---- .../com/google/gcloud/datastore/LocalGcdHelper.java | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java index c6e3f396e574..3063dda6eca3 100644 --- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java +++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java @@ -89,10 +89,12 @@ private static boolean includesScheme(String url) { } private static String removeScheme(String url) { - if (url.startsWith("https://")) { - return url.substring("https://".length()); - } else if (url.startsWith("http://")) { - return url.substring("http://".length()); + if (url != null) { + if (url.startsWith("https://")) { + return url.substring("https://".length()); + } else if (url.startsWith("http://")) { + return url.substring("http://".length()); + } } return url; } diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java index 1054b461ab0d..1d692040c015 100644 --- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java +++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java @@ -309,8 +309,7 @@ public static void main(String... args) throws IOException, InterruptedException public static boolean isActive(String projectId) { try { StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT); - urlBuilder.append("/datastore/" + com.google.datastore.v1beta3.client.DatastoreFactory.VERSION - + "/projects/").append(projectId).append(":lookup"); + urlBuilder.append("/datastore/v1beta3/projects/").append(projectId).append(":lookup"); URL url = new URL(urlBuilder.toString()); try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) {